Hypernet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Pool clean by lifecycle config

Browse Source
This commit is contained in:
LittleSheep 2024-08-18 14:50:12 +08:00
parent a82fb3a49c
commit 922a76ee7f
3 changed files with 49 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.idea/workspace.xml
View File

@ -4,18 +4,10 @@
<option name="autoReloadType" value="ALL" />
</component>
<component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Attachment pool basis">
<change afterPath="$PROJECT_DIR$/pkg/internal/services/random_id.go" afterDir="false" />
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Attachment has pool">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/go.mod" beforeDir="false" afterPath="$PROJECT_DIR$/go.mod" afterDir="false" />
<change beforePath="$PROJECT_DIR$/go.sum" beforeDir="false" afterPath="$PROJECT_DIR$/go.sum" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/pools.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/pools.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/pools.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/pools.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/main.go" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -129,7 +121,8 @@
<MESSAGE value=":sparkles: Self reference detection" />
<MESSAGE value=":sparkles: List attachment original filter" />
<MESSAGE value=":sparkles: Attachment pool basis" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Attachment pool basis" />
<MESSAGE value=":sparkles: Attachment has pool" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Attachment has pool" />
</component>
<component name="VgoProject">
<settings-migrated>true</settings-migrated>

42
pkg/internal/services/recycler.go
View File

@ -3,6 +3,9 @@ package services
import (
"context"
"fmt"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
"github.com/samber/lo"
"gorm.io/gorm/clause"
"os"
"path/filepath"
"time"
@ -33,6 +36,45 @@ func StartConsumeDeletionTask() {
}
}
func RunScheduleDeletionTask() {
var pools []models.AttachmentPool
if err := database.C.Find(&pools).Error; err != nil {
return
}
var pendingPools []models.AttachmentPool
for _, pool := range pendingPools {
if pool.Config.Data().ExistLifecycle != nil {
pendingPools = append(pendingPools, pool)
}
}
for _, pool := range pendingPools {
lifecycle := fmt.Sprintf("%d seconds", *pool.Config.Data().ExistLifecycle)
var attachments []models.Attachment
if err := database.C.Where("pool_id = ? AND created_at < NOW() - INTERVAL ?", pool.ID, lifecycle).Find(&attachments).Error; err != nil {
continue
}
log.Info().
Str("pool", pool.Alias).
Int("count", len(attachments)).
Msg("Deleting attachments due to pool's lifecycle configuration...")
for idx, attachment := range attachments {
if err := DeleteFile(attachment); err != nil {
log.Error().
Str("pool", pool.Alias).
Uint("id", attachment.ID).
Msg("An error occurred when deleting attachment due to pool's lifecycle configuration...")
} else {
attachments[idx].CleanedAt = lo.ToPtr(time.Now())
}
}
database.C.Clauses(clause.OnConflict{
UpdateAll: true,
}).CreateInBatches(attachments, 1000)
}
}
func DeleteFile(meta models.Attachment) error {
var destMap map[string]any
if meta.Destination == models.AttachmentDstTemporary {

2
pkg/main.go
View File

@ -59,6 +59,7 @@ func main() {
// Configure timed tasks
quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
quartz.AddFunc("@midnight", services.RunScheduleDeletionTask)
quartz.Start()
// Server
@ -73,6 +74,7 @@ func main() {
log.Info().Msgf("Paperclip v%s is started...", pkg.AppVersion)
services.ScanUnanalyzedFileFromDatabase()
services.RunScheduleDeletionTask()
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)