Unused attachment cleanup

This commit is contained in:
LittleSheep 2025-03-10 22:30:22 +08:00
parent e3ca50c4ae
commit 38aa06cc00
3 changed files with 96 additions and 13 deletions

View File

@ -239,6 +239,85 @@ func DeleteAttachment(item models.Attachment, txs ...*gorm.DB) error {
return nil
}
func DeleteAttachmentInBatch(items []models.Attachment, txs ...*gorm.DB) error {
if len(items) == 0 {
return nil
}
var tx *gorm.DB
if len(txs) == 0 {
tx = database.C.Begin()
} else {
tx = txs[0]
}
refIDs := []uint{}
for _, item := range items {
if item.RefID != nil {
refIDs = append(refIDs, *item.RefID)
}
}
if len(refIDs) > 0 {
var refTargets []models.Attachment
if err := tx.Where("id IN ?", refIDs).Find(&refTargets).Error; err == nil {
for i := range refTargets {
refTargets[i].RefCount--
}
if err := tx.Save(&refTargets).Error; err != nil {
tx.Rollback()
return fmt.Errorf("unable to update ref count: %v", err)
}
}
}
var subAttachments []models.Attachment
for _, item := range items {
if item.Thumbnail != nil {
subAttachments = append(subAttachments, *item.Thumbnail)
}
if item.Compressed != nil {
subAttachments = append(subAttachments, *item.Compressed)
}
}
if len(subAttachments) > 0 {
if err := DeleteAttachmentInBatch(subAttachments, tx); err != nil {
tx.Rollback()
return err
}
}
rids := make([]string, len(items))
for i, item := range items {
rids[i] = item.Rid
}
if err := tx.Where("rid IN ?", rids).Delete(&models.Attachment{}).Error; err != nil {
tx.Rollback()
return err
}
cacheManager := cache.New[any](localCache.S)
marshal := marshaler.New(cacheManager)
contx := context.Background()
for _, rid := range rids {
_ = marshal.Delete(contx, GetAttachmentCacheKey(rid))
}
tx.Commit()
go func() {
for _, item := range items {
if item.RefCount == 0 {
fs.DeleteFile(item)
}
}
}()
return nil
}
func CountAttachmentUsage(id []uint, delta int) (int64, error) {
if tx := database.C.Model(&models.Attachment{}).
Where("id IN ?", id).

View File

@ -1,24 +1,28 @@
package services
import (
database2 "git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"time"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
"github.com/rs/zerolog/log"
)
func DoAutoDatabaseCleanup() {
deadline := time.Now().Add(60 * time.Minute)
log.Debug().Time("deadline", deadline).Msg("Now cleaning up entire database...")
func DoUnusedAttachmentCleanup() {
deadline := time.Now().Add(-60 * time.Minute)
var count int64
for _, model := range database2.AutoMaintainRange {
tx := database2.C.Unscoped().Delete(model, "deleted_at >= ?", deadline)
if tx.Error != nil {
log.Error().Err(tx.Error).Msg("An error occurred when running auth context cleanup...")
}
count += tx.RowsAffected
var result []models.Attachment
if err := database.C.Where("created_at < ? AND used_count = 0", deadline).
Find(&result).Error; err != nil {
log.Error().Err(err).Msg("An error occurred when getting unused attachments...")
return
}
log.Debug().Int64("affected", count).Msg("Clean up entire database accomplished.")
if err := DeleteAttachmentInBatch(result); err != nil {
log.Error().Err(err).Msg("An error occurred when deleting unused attachments...")
return
}
log.Info().Int("count", len(result)).Msg("Deleted unused attachments...")
}

View File

@ -80,7 +80,7 @@ func main() {
// Configure timed tasks
quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
quartz.AddFunc("@every 60m", services.DoUnusedAttachmentCleanup)
quartz.AddFunc("@every 60m", fs.RunMarkLifecycleDeletionTask)
quartz.AddFunc("@every 60m", fs.RunMarkMultipartDeletionTask)
quartz.AddFunc("@midnight", fs.RunScheduleDeletionTask)