From c46d7fa3120cb92a13bd56eba7c4acaf696aa6bd Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Mon, 29 Jul 2024 13:22:57 +0800 Subject: [PATCH] :bug: Bug fixes and logging --- pkg/internal/models/attachments.go | 5 +++-- pkg/internal/models/destination.go | 13 +++++------ pkg/internal/server/api/attachments_api.go | 25 +++++++++++++++------- pkg/internal/services/analyzer.go | 22 ++++++++++++++++--- pkg/internal/services/recycler.go | 2 +- pkg/internal/services/uploader.go | 9 ++++++++ settings.toml | 1 + 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/pkg/internal/models/attachments.go b/pkg/internal/models/attachments.go index bd5d5b0..bcd187d 100644 --- a/pkg/internal/models/attachments.go +++ b/pkg/internal/models/attachments.go @@ -22,8 +22,9 @@ type Attachment struct { Destination AttachmentDst `json:"destination"` RefCount int `json:"ref_count"` - Metadata datatypes.JSONMap `json:"metadata"` - IsMature bool `json:"is_mature"` + Metadata datatypes.JSONMap `json:"metadata"` + IsMature bool `json:"is_mature"` + IsAnalyzed bool `json:"is_analyzed"` Ref *Attachment `json:"ref"` RefID *uint `json:"ref_id"` diff --git a/pkg/internal/models/destination.go b/pkg/internal/models/destination.go index 88750fc..9c67eb6 100644 --- a/pkg/internal/models/destination.go +++ b/pkg/internal/models/destination.go @@ -18,10 +18,11 @@ type LocalDestination struct { type S3Destination struct { BaseDestination - Path string `json:"path"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - SecretID string `json:"secret_id"` - SecretKey string `json:"secret_key"` - EnableSSL bool `json:"enable_ssl"` + Path string `json:"path"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + SecretID string `json:"secret_id"` + SecretKey string `json:"secret_key"` + AccessBaseURL string `json:"access_baseurl"` + EnableSSL bool `json:"enable_ssl"` } diff --git a/pkg/internal/server/api/attachments_api.go b/pkg/internal/server/api/attachments_api.go index 66df753..988e977 100644 --- a/pkg/internal/server/api/attachments_api.go +++ b/pkg/internal/server/api/attachments_api.go @@ -47,14 +47,22 @@ func openAttachment(c *fiber.Ctx) error { case models.DestinationTypeS3: var destConfigured models.S3Destination _ = jsoniter.Unmarshal(rawDest, &destConfigured) - protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http") - return c.Redirect(fmt.Sprintf( - "%s://%s.%s/%s", - protocol, - destConfigured.Bucket, - destConfigured.Endpoint, - url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)), - ), fiber.StatusMovedPermanently) + if len(destConfigured.AccessBaseURL) > 0 { + return c.Redirect(fmt.Sprintf( + "%s/%s", + destConfigured.AccessBaseURL, + url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)), + ), fiber.StatusMovedPermanently) + } else { + protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http") + return c.Redirect(fmt.Sprintf( + "%s://%s.%s/%s", + protocol, + destConfigured.Bucket, + destConfigured.Endpoint, + url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)), + ), fiber.StatusMovedPermanently) + } default: return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) } @@ -103,6 +111,7 @@ func createAttachment(c *fiber.Ctx) error { MimeType: c.FormValue("mimetype"), Metadata: usermeta, IsMature: len(c.FormValue("mature")) > 0, + IsAnalyzed: false, Destination: models.AttachmentDstTemporary, }) if err != nil { diff --git a/pkg/internal/services/analyzer.go b/pkg/internal/services/analyzer.go index 2a269d4..abce957 100644 --- a/pkg/internal/services/analyzer.go +++ b/pkg/internal/services/analyzer.go @@ -35,7 +35,7 @@ func StartConsumeAnalyzeTask() { if err := AnalyzeAttachment(task); err != nil { log.Error().Err(err).Any("task", task).Msg("A file analyze task failed...") } else { - log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file analyze task was completed.") + log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file analyze task was completed.") } } } @@ -51,6 +51,8 @@ func AnalyzeAttachment(file models.Attachment) error { rawDest, _ := jsoniter.Marshal(destMap) _ = jsoniter.Unmarshal(rawDest, &dest) + start := time.Now() + dst := filepath.Join(dest.Path, file.Uuid) if _, err := os.Stat(dst); os.IsNotExist(err) { return fmt.Errorf("attachment doesn't exists in temporary storage: %v", err) @@ -85,24 +87,38 @@ func AnalyzeAttachment(file models.Attachment) error { tx := database.C.Begin() + file.IsAnalyzed = true + linked, err := TryLinkAttachment(tx, file, file.HashCode) if linked && err != nil { return fmt.Errorf("unable to link file record: %v", err) } else if !linked { + metadataCache[file.ID] = file if err := tx.Save(&file).Error; err != nil { tx.Rollback() return fmt.Errorf("unable to save file record: %v", err) } } + tx.Commit() + + log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Msg("A file analyze task was finished, starting uploading...") + + start = time.Now() + + // Move temporary to permanet if !linked { if err := ReUploadFileToPermanent(file); err != nil { - tx.Rollback() return fmt.Errorf("unable to move file to permanet storage: %v", err) } } - tx.Commit() + // Recycle the temporary file + file.Destination = models.AttachmentDstTemporary + PublishDeleteFileTask(file) + + // Finish + log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Bool("linked", linked).Msg("A file post-analyze upload task was finished.") return nil } diff --git a/pkg/internal/services/recycler.go b/pkg/internal/services/recycler.go index c9362ca..7f2c23c 100644 --- a/pkg/internal/services/recycler.go +++ b/pkg/internal/services/recycler.go @@ -28,7 +28,7 @@ func StartConsumeDeletionTask() { if err := DeleteFile(task); err != nil { log.Error().Err(err).Any("task", task).Msg("A file deletion task failed...") } else { - log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file deletion task was completed.") + log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file deletion task was completed.") } } } diff --git a/pkg/internal/services/uploader.go b/pkg/internal/services/uploader.go index e4405fd..400f296 100644 --- a/pkg/internal/services/uploader.go +++ b/pkg/internal/services/uploader.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" + "git.solsynth.dev/hydrogen/paperclip/pkg/internal/database" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/models" "github.com/gofiber/fiber/v2" jsoniter "github.com/json-iterator/go" @@ -39,6 +40,8 @@ func ReUploadFileToPermanent(meta models.Attachment) error { return fmt.Errorf("attachment isn't in temporary storage, unable to process") } + meta.Destination = models.AttachmentDstPermanent + destMap := viper.GetStringMap("destinations.permanent") var dest models.BaseDestination @@ -76,6 +79,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error { if err != nil { return fmt.Errorf("unable to copy data to dest file: %v", err) } + + database.C.Save(&meta) + metadataCache[meta.ID] = meta return nil case models.DestinationTypeS3: var destConfigured models.S3Destination @@ -97,6 +103,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error { if err != nil { return fmt.Errorf("unable to upload file to s3: %v", err) } + + database.C.Save(&meta) + metadataCache[meta.ID] = meta return nil default: return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) diff --git a/settings.toml b/settings.toml index b46da6f..1ca9999 100644 --- a/settings.toml +++ b/settings.toml @@ -38,4 +38,5 @@ bucket = "bucket" endpoint = "s3.ap-east-1.amazonaws.com" secret_id = "secret" secret_key = "secret" +access_baseurl = "https://raw.sn.solsynth.dev" enable_ssl = true