♻️ Split mark and delete file
This commit is contained in:
parent
922a76ee7f
commit
98cf753f66
6
.idea/workspace.xml
generated
6
.idea/workspace.xml
generated
@ -4,7 +4,7 @@
|
|||||||
<option name="autoReloadType" value="ALL" />
|
<option name="autoReloadType" value="ALL" />
|
||||||
</component>
|
</component>
|
||||||
<component name="ChangeListManager">
|
<component name="ChangeListManager">
|
||||||
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Attachment has pool">
|
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":recycle: Split mark and delete file">
|
||||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||||
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
|
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
|
||||||
<change beforePath="$PROJECT_DIR$/pkg/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/main.go" afterDir="false" />
|
<change beforePath="$PROJECT_DIR$/pkg/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/main.go" afterDir="false" />
|
||||||
@ -122,7 +122,9 @@
|
|||||||
<MESSAGE value=":sparkles: List attachment original filter" />
|
<MESSAGE value=":sparkles: List attachment original filter" />
|
||||||
<MESSAGE value=":sparkles: Attachment pool basis" />
|
<MESSAGE value=":sparkles: Attachment pool basis" />
|
||||||
<MESSAGE value=":sparkles: Attachment has pool" />
|
<MESSAGE value=":sparkles: Attachment has pool" />
|
||||||
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Attachment has pool" />
|
<MESSAGE value=":sparkles: Pool clean by lifecycle config" />
|
||||||
|
<MESSAGE value=":recycle: Split mark and delete file" />
|
||||||
|
<option name="LAST_COMMIT_MESSAGE" value=":recycle: Split mark and delete file" />
|
||||||
</component>
|
</component>
|
||||||
<component name="VgoProject">
|
<component name="VgoProject">
|
||||||
<settings-migrated>true</settings-migrated>
|
<settings-migrated>true</settings-migrated>
|
||||||
|
@ -5,7 +5,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
|
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
|
||||||
"github.com/samber/lo"
|
"github.com/samber/lo"
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
@ -36,7 +35,7 @@ func StartConsumeDeletionTask() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func RunScheduleDeletionTask() {
|
func RunMarkDeletionTask() {
|
||||||
var pools []models.AttachmentPool
|
var pools []models.AttachmentPool
|
||||||
if err := database.C.Find(&pools).Error; err != nil {
|
if err := database.C.Find(&pools).Error; err != nil {
|
||||||
return
|
return
|
||||||
@ -51,30 +50,36 @@ func RunScheduleDeletionTask() {
|
|||||||
|
|
||||||
for _, pool := range pendingPools {
|
for _, pool := range pendingPools {
|
||||||
lifecycle := fmt.Sprintf("%d seconds", *pool.Config.Data().ExistLifecycle)
|
lifecycle := fmt.Sprintf("%d seconds", *pool.Config.Data().ExistLifecycle)
|
||||||
var attachments []models.Attachment
|
tx := database.C.
|
||||||
if err := database.C.Where("pool_id = ? AND created_at < NOW() - INTERVAL ?", pool.ID, lifecycle).Find(&attachments).Error; err != nil {
|
Where("pool_id = ? AND created_at < NOW() - INTERVAL ?", pool.ID, lifecycle).
|
||||||
continue
|
Updates(&models.Attachment{CleanedAt: lo.ToPtr(time.Now())})
|
||||||
}
|
|
||||||
log.Info().
|
log.Info().
|
||||||
Str("pool", pool.Alias).
|
Str("pool", pool.Alias).
|
||||||
Int("count", len(attachments)).
|
Int64("count", tx.RowsAffected).
|
||||||
Msg("Deleting attachments due to pool's lifecycle configuration...")
|
Err(tx.Error).
|
||||||
for idx, attachment := range attachments {
|
Msg("Marking attachments as clean needed due to pool's lifecycle configuration...")
|
||||||
if err := DeleteFile(attachment); err != nil {
|
|
||||||
log.Error().
|
|
||||||
Str("pool", pool.Alias).
|
|
||||||
Uint("id", attachment.ID).
|
|
||||||
Msg("An error occurred when deleting attachment due to pool's lifecycle configuration...")
|
|
||||||
} else {
|
|
||||||
attachments[idx].CleanedAt = lo.ToPtr(time.Now())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
database.C.Clauses(clause.OnConflict{
|
|
||||||
UpdateAll: true,
|
|
||||||
}).CreateInBatches(attachments, 1000)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RunScheduleDeletionTask() {
|
||||||
|
var attachments []models.Attachment
|
||||||
|
if err := database.C.Where("cleaned_at IS NOT NULL").Find(&attachments).Error; err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for idx, attachment := range attachments {
|
||||||
|
if err := DeleteFile(attachment); err != nil {
|
||||||
|
log.Error().
|
||||||
|
Uint("id", attachment.ID).
|
||||||
|
Msg("An error occurred when deleting marked clean up attachments...")
|
||||||
|
} else {
|
||||||
|
attachments[idx].CleanedAt = lo.ToPtr(time.Now())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
database.C.Where("cleaned_at IS NOT NULL").Delete(&models.Attachment{})
|
||||||
|
}
|
||||||
|
|
||||||
func DeleteFile(meta models.Attachment) error {
|
func DeleteFile(meta models.Attachment) error {
|
||||||
var destMap map[string]any
|
var destMap map[string]any
|
||||||
if meta.Destination == models.AttachmentDstTemporary {
|
if meta.Destination == models.AttachmentDstTemporary {
|
||||||
|
@ -48,7 +48,7 @@ func main() {
|
|||||||
log.Error().Err(err).Msg("An error occurred when registering service to dealer...")
|
log.Error().Err(err).Msg("An error occurred when registering service to dealer...")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Setup some workers
|
// Set up some workers
|
||||||
for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ {
|
for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ {
|
||||||
go services.StartConsumeDeletionTask()
|
go services.StartConsumeDeletionTask()
|
||||||
}
|
}
|
||||||
@ -59,6 +59,7 @@ func main() {
|
|||||||
// Configure timed tasks
|
// Configure timed tasks
|
||||||
quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
|
quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
|
||||||
quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
|
quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
|
||||||
|
quartz.AddFunc("@every 60m", services.RunMarkDeletionTask)
|
||||||
quartz.AddFunc("@midnight", services.RunScheduleDeletionTask)
|
quartz.AddFunc("@midnight", services.RunScheduleDeletionTask)
|
||||||
quartz.Start()
|
quartz.Start()
|
||||||
|
|
||||||
@ -74,7 +75,7 @@ func main() {
|
|||||||
log.Info().Msgf("Paperclip v%s is started...", pkg.AppVersion)
|
log.Info().Msgf("Paperclip v%s is started...", pkg.AppVersion)
|
||||||
|
|
||||||
services.ScanUnanalyzedFileFromDatabase()
|
services.ScanUnanalyzedFileFromDatabase()
|
||||||
services.RunScheduleDeletionTask()
|
services.RunMarkDeletionTask()
|
||||||
|
|
||||||
quit := make(chan os.Signal, 1)
|
quit := make(chan os.Signal, 1)
|
||||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
Loading…
Reference in New Issue
Block a user