From 98cf753f66d6901a6b8c160fa936557c05810a80 Mon Sep 17 00:00:00 2001
From: LittleSheep <littlesheep.code@hotmail.com>
Date: Sun, 18 Aug 2024 15:58:07 +0800
Subject: [PATCH] :recycle: Split mark and delete file

---
 .idea/workspace.xml               |  6 ++--
 pkg/internal/services/recycler.go | 47 +++++++++++++++++--------------
 pkg/main.go                       |  5 ++--
 3 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 9b4c0c4..60abead 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -4,7 +4,7 @@
     <option name="autoReloadType" value="ALL" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Attachment has pool">
+    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":recycle: Split mark and delete file">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pkg/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/main.go" afterDir="false" />
@@ -122,7 +122,9 @@
     <MESSAGE value=":sparkles: List attachment original filter" />
     <MESSAGE value=":sparkles: Attachment pool basis" />
     <MESSAGE value=":sparkles: Attachment has pool" />
-    <option name="LAST_COMMIT_MESSAGE" value=":sparkles: Attachment has pool" />
+    <MESSAGE value=":sparkles: Pool clean by lifecycle config" />
+    <MESSAGE value=":recycle: Split mark and delete file" />
+    <option name="LAST_COMMIT_MESSAGE" value=":recycle: Split mark and delete file" />
   </component>
   <component name="VgoProject">
     <settings-migrated>true</settings-migrated>
diff --git a/pkg/internal/services/recycler.go b/pkg/internal/services/recycler.go
index a6f0107..683a66b 100644
--- a/pkg/internal/services/recycler.go
+++ b/pkg/internal/services/recycler.go
@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
 	"github.com/samber/lo"
-	"gorm.io/gorm/clause"
 	"os"
 	"path/filepath"
 	"time"
@@ -36,7 +35,7 @@ func StartConsumeDeletionTask() {
 	}
 }
 
-func RunScheduleDeletionTask() {
+func RunMarkDeletionTask() {
 	var pools []models.AttachmentPool
 	if err := database.C.Find(&pools).Error; err != nil {
 		return
@@ -51,30 +50,36 @@ func RunScheduleDeletionTask() {
 
 	for _, pool := range pendingPools {
 		lifecycle := fmt.Sprintf("%d seconds", *pool.Config.Data().ExistLifecycle)
-		var attachments []models.Attachment
-		if err := database.C.Where("pool_id = ? AND created_at < NOW() - INTERVAL ?", pool.ID, lifecycle).Find(&attachments).Error; err != nil {
-			continue
-		}
+		tx := database.C.
+			Where("pool_id = ? AND created_at < NOW() - INTERVAL ?", pool.ID, lifecycle).
+			Updates(&models.Attachment{CleanedAt: lo.ToPtr(time.Now())})
 		log.Info().
 			Str("pool", pool.Alias).
-			Int("count", len(attachments)).
-			Msg("Deleting attachments due to pool's lifecycle configuration...")
-		for idx, attachment := range attachments {
-			if err := DeleteFile(attachment); err != nil {
-				log.Error().
-					Str("pool", pool.Alias).
-					Uint("id", attachment.ID).
-					Msg("An error occurred when deleting attachment due to pool's lifecycle configuration...")
-			} else {
-				attachments[idx].CleanedAt = lo.ToPtr(time.Now())
-			}
-		}
-		database.C.Clauses(clause.OnConflict{
-			UpdateAll: true,
-		}).CreateInBatches(attachments, 1000)
+			Int64("count", tx.RowsAffected).
+			Err(tx.Error).
+			Msg("Marking attachments as clean needed due to pool's lifecycle configuration...")
 	}
 }
 
+func RunScheduleDeletionTask() {
+	var attachments []models.Attachment
+	if err := database.C.Where("cleaned_at IS NOT NULL").Find(&attachments).Error; err != nil {
+		return
+	}
+
+	for idx, attachment := range attachments {
+		if err := DeleteFile(attachment); err != nil {
+			log.Error().
+				Uint("id", attachment.ID).
+				Msg("An error occurred when deleting marked clean up attachments...")
+		} else {
+			attachments[idx].CleanedAt = lo.ToPtr(time.Now())
+		}
+	}
+
+	database.C.Where("cleaned_at IS NOT NULL").Delete(&models.Attachment{})
+}
+
 func DeleteFile(meta models.Attachment) error {
 	var destMap map[string]any
 	if meta.Destination == models.AttachmentDstTemporary {
diff --git a/pkg/main.go b/pkg/main.go
index 88247cf..3987dc7 100644
--- a/pkg/main.go
+++ b/pkg/main.go
@@ -48,7 +48,7 @@ func main() {
 		log.Error().Err(err).Msg("An error occurred when registering service to dealer...")
 	}
 
-	// Setup some workers
+	// Set up some workers
 	for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ {
 		go services.StartConsumeDeletionTask()
 	}
@@ -59,6 +59,7 @@ func main() {
 	// Configure timed tasks
 	quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
 	quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
+	quartz.AddFunc("@every 60m", services.RunMarkDeletionTask)
 	quartz.AddFunc("@midnight", services.RunScheduleDeletionTask)
 	quartz.Start()
 
@@ -74,7 +75,7 @@ func main() {
 	log.Info().Msgf("Paperclip v%s is started...", pkg.AppVersion)
 
 	services.ScanUnanalyzedFileFromDatabase()
-	services.RunScheduleDeletionTask()
+	services.RunMarkDeletionTask()
 
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)