diff --git a/pkg/internal/services/analyzer.go b/pkg/internal/services/analyzer.go index 5a7cbfd..d1160e8 100644 --- a/pkg/internal/services/analyzer.go +++ b/pkg/internal/services/analyzer.go @@ -13,6 +13,7 @@ import ( "git.solsynth.dev/hydrogen/paperclip/pkg/internal/database" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/models" jsoniter "github.com/json-iterator/go" + "github.com/rs/zerolog/log" "github.com/spf13/viper" _ "image/gif" @@ -26,6 +27,15 @@ func PublishAnalyzeTask(file models.Attachment) { fileAnalyzeQueue <- file } +func StartConsumeAnalyzeTask() { + for { + task := <-fileAnalyzeQueue + if err := AnalyzeAttachment(task); err != nil { + log.Error().Err(err).Any("task", task).Msg("A file analyze task failed...") + } + } +} + func AnalyzeAttachment(file models.Attachment) error { if file.Destination != models.AttachmentDstTemporary { return fmt.Errorf("attachment isn't in temporary storage, unable to analyze") diff --git a/pkg/internal/services/attachments.go b/pkg/internal/services/attachments.go index 0af79bf..51bd62c 100644 --- a/pkg/internal/services/attachments.go +++ b/pkg/internal/services/attachments.go @@ -154,7 +154,7 @@ func DeleteAttachment(item models.Attachment) error { tx.Commit() if dat.RefCount == 0 { - return DeleteFile(dat) + PublishDeleteFileTask(dat) } return nil diff --git a/pkg/internal/services/recycler.go b/pkg/internal/services/recycler.go index 10ebbb8..bc92e10 100644 --- a/pkg/internal/services/recycler.go +++ b/pkg/internal/services/recycler.go @@ -10,9 +10,25 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) +var fileDeletionQueue = make(chan models.Attachment, 256) + +func PublishDeleteFileTask(file models.Attachment) { + fileDeletionQueue <- file +} + +func StartConsumeDeletionTask() { + for { + task := <-fileDeletionQueue + if err := DeleteFile(task); err != nil { + log.Error().Err(err).Any("task", task).Msg("A file deletion task failed...") + } + } +} + func DeleteFile(meta models.Attachment) error { var destMap map[string]any if meta.Destination == models.AttachmentDstTemporary { diff --git a/pkg/main.go b/pkg/main.go index 61e61db..6b9bdd2 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -48,6 +48,14 @@ func main() { log.Error().Err(err).Msg("An error occurred when registering service to dealer...") } + // Setup some workers + for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ { + go services.StartConsumeDeletionTask() + } + for idx := 0; idx < viper.GetInt("workers.files_analyze"); idx++ { + go services.StartConsumeAnalyzeTask() + } + // Configure timed tasks quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger))) quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup) diff --git a/settings.toml b/settings.toml index fc75ae9..b46da6f 100644 --- a/settings.toml +++ b/settings.toml @@ -7,6 +7,10 @@ secret = "LtTjzAGFLshwXhN4ZD4nG5KlMv1MWcsvfv03TSZYnT1VhiAnLIZFTnHUwR0XhGgi" accepts_usage = ["p.avatar", "p.banner", "i.attachment", "m.attachment"] +[workers] +files_deletion = 4 +files_analyze = 4 + [debug] database = false print_routes = false