✨ File operations queue

pkg/internal/services/analyzer.go

@@ -13,6 +13,7 @@ import (
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 
 	_ "image/gif"
@@ -26,6 +27,15 @@ func PublishAnalyzeTask(file models.Attachment) {
 	fileAnalyzeQueue <- file
 }
 
+func StartConsumeAnalyzeTask() {
+	for {
+		task := <-fileAnalyzeQueue
+		if err := AnalyzeAttachment(task); err != nil {
+			log.Error().Err(err).Any("task", task).Msg("A file analyze task failed...")
+		}
+	}
+}
+
 func AnalyzeAttachment(file models.Attachment) error {
 	if file.Destination != models.AttachmentDstTemporary {
 		return fmt.Errorf("attachment isn't in temporary storage, unable to analyze")

pkg/internal/services/attachments.go

@@ -154,7 +154,7 @@ func DeleteAttachment(item models.Attachment) error {
 	tx.Commit()
 
 	if dat.RefCount == 0 {
-		return DeleteFile(dat)
+		PublishDeleteFileTask(dat)
 	}
 
 	return nil

pkg/internal/services/recycler.go

@@ -10,9 +10,25 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )
 
+var fileDeletionQueue = make(chan models.Attachment, 256)
+
+func PublishDeleteFileTask(file models.Attachment) {
+	fileDeletionQueue <- file
+}
+
+func StartConsumeDeletionTask() {
+	for {
+		task := <-fileDeletionQueue
+		if err := DeleteFile(task); err != nil {
+			log.Error().Err(err).Any("task", task).Msg("A file deletion task failed...")
+		}
+	}
+}
+
 func DeleteFile(meta models.Attachment) error {
 	var destMap map[string]any
 	if meta.Destination == models.AttachmentDstTemporary {

pkg/main.go

@@ -48,6 +48,14 @@ func main() {
 		log.Error().Err(err).Msg("An error occurred when registering service to dealer...")
 	}
 
+	// Setup some workers
+	for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ {
+		go services.StartConsumeDeletionTask()
+	}
+	for idx := 0; idx < viper.GetInt("workers.files_analyze"); idx++ {
+		go services.StartConsumeAnalyzeTask()
+	}
+
 	// Configure timed tasks
 	quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
 	quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)

settings.toml

@@ -7,6 +7,10 @@ secret = "LtTjzAGFLshwXhN4ZD4nG5KlMv1MWcsvfv03TSZYnT1VhiAnLIZFTnHUwR0XhGgi"
 
 accepts_usage = ["p.avatar", "p.banner", "i.attachment", "m.attachment"]
 
+[workers]
+files_deletion = 4
+files_analyze = 4
+
 [debug]
 database = false
 print_routes = false