🐛 Bug fixes and logging

pkg/internal/models/attachments.go

@@ -22,8 +22,9 @@ type Attachment struct {
 	Destination AttachmentDst `json:"destination"`
 	RefCount    int           `json:"ref_count"`
 
-	Metadata datatypes.JSONMap `json:"metadata"`
+	Metadata   datatypes.JSONMap `json:"metadata"`
-	IsMature bool              `json:"is_mature"`
+	IsMature   bool              `json:"is_mature"`
+	IsAnalyzed bool              `json:"is_analyzed"`
 
 	Ref   *Attachment `json:"ref"`
 	RefID *uint       `json:"ref_id"`

pkg/internal/models/destination.go

@@ -18,10 +18,11 @@ type LocalDestination struct {
 type S3Destination struct {
 	BaseDestination
 
-	Path      string `json:"path"`
+	Path          string `json:"path"`
-	Bucket    string `json:"bucket"`
+	Bucket        string `json:"bucket"`
-	Endpoint  string `json:"endpoint"`
+	Endpoint      string `json:"endpoint"`
-	SecretID  string `json:"secret_id"`
+	SecretID      string `json:"secret_id"`
-	SecretKey string `json:"secret_key"`
+	SecretKey     string `json:"secret_key"`
-	EnableSSL bool   `json:"enable_ssl"`
+	AccessBaseURL string `json:"access_baseurl"`
+	EnableSSL     bool   `json:"enable_ssl"`
 }

pkg/internal/server/api/attachments_api.go

@@ -47,14 +47,22 @@ func openAttachment(c *fiber.Ctx) error {
 	case models.DestinationTypeS3:
 		var destConfigured models.S3Destination
 		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
-		protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
+		if len(destConfigured.AccessBaseURL) > 0 {
-		return c.Redirect(fmt.Sprintf(
+			return c.Redirect(fmt.Sprintf(
-			"%s://%s.%s/%s",
+				"%s/%s",
-			protocol,
+				destConfigured.AccessBaseURL,
-			destConfigured.Bucket,
+				url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
-			destConfigured.Endpoint,
+			), fiber.StatusMovedPermanently)
-			url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
+		} else {
-		), fiber.StatusMovedPermanently)
+			protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
+			return c.Redirect(fmt.Sprintf(
+				"%s://%s.%s/%s",
+				protocol,
+				destConfigured.Bucket,
+				destConfigured.Endpoint,
+				url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
+			), fiber.StatusMovedPermanently)
+		}
 	default:
 		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
 	}
@@ -103,6 +111,7 @@ func createAttachment(c *fiber.Ctx) error {
 		MimeType:    c.FormValue("mimetype"),
 		Metadata:    usermeta,
 		IsMature:    len(c.FormValue("mature")) > 0,
+		IsAnalyzed:  false,
 		Destination: models.AttachmentDstTemporary,
 	})
 	if err != nil {

pkg/internal/services/analyzer.go

@@ -35,7 +35,7 @@ func StartConsumeAnalyzeTask() {
 		if err := AnalyzeAttachment(task); err != nil {
 			log.Error().Err(err).Any("task", task).Msg("A file analyze task failed...")
 		} else {
-			log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file analyze task was completed.")
+			log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file analyze task was completed.")
 		}
 	}
 }
@@ -51,6 +51,8 @@ func AnalyzeAttachment(file models.Attachment) error {
 	rawDest, _ := jsoniter.Marshal(destMap)
 	_ = jsoniter.Unmarshal(rawDest, &dest)
 
+	start := time.Now()
+
 	dst := filepath.Join(dest.Path, file.Uuid)
 	if _, err := os.Stat(dst); os.IsNotExist(err) {
 		return fmt.Errorf("attachment doesn't exists in temporary storage: %v", err)
@@ -85,24 +87,38 @@ func AnalyzeAttachment(file models.Attachment) error {
 
 	tx := database.C.Begin()
 
+	file.IsAnalyzed = true
+
 	linked, err := TryLinkAttachment(tx, file, file.HashCode)
 	if linked && err != nil {
 		return fmt.Errorf("unable to link file record: %v", err)
 	} else if !linked {
+		metadataCache[file.ID] = file
 		if err := tx.Save(&file).Error; err != nil {
 			tx.Rollback()
 			return fmt.Errorf("unable to save file record: %v", err)
 		}
 	}
 
+	tx.Commit()
+
+	log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Msg("A file analyze task was finished, starting uploading...")
+
+	start = time.Now()
+
+	// Move temporary to permanet
 	if !linked {
 		if err := ReUploadFileToPermanent(file); err != nil {
-			tx.Rollback()
 			return fmt.Errorf("unable to move file to permanet storage: %v", err)
 		}
 	}
 
-	tx.Commit()
+	// Recycle the temporary file
+	file.Destination = models.AttachmentDstTemporary
+	PublishDeleteFileTask(file)
+
+	// Finish
+	log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Bool("linked", linked).Msg("A file post-analyze upload task was finished.")
 
 	return nil
 }

pkg/internal/services/recycler.go

@@ -28,7 +28,7 @@ func StartConsumeDeletionTask() {
 		if err := DeleteFile(task); err != nil {
 			log.Error().Err(err).Any("task", task).Msg("A file deletion task failed...")
 		} else {
-			log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file deletion task was completed.")
+			log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file deletion task was completed.")
 		}
 	}
 }

pkg/internal/services/uploader.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
 	"github.com/gofiber/fiber/v2"
 	jsoniter "github.com/json-iterator/go"
@@ -39,6 +40,8 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
 		return fmt.Errorf("attachment isn't in temporary storage, unable to process")
 	}
 
+	meta.Destination = models.AttachmentDstPermanent
+
 	destMap := viper.GetStringMap("destinations.permanent")
 
 	var dest models.BaseDestination
@@ -76,6 +79,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
 		if err != nil {
 			return fmt.Errorf("unable to copy data to dest file: %v", err)
 		}
+
+		database.C.Save(&meta)
+		metadataCache[meta.ID] = meta
 		return nil
 	case models.DestinationTypeS3:
 		var destConfigured models.S3Destination
@@ -97,6 +103,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
 		if err != nil {
 			return fmt.Errorf("unable to upload file to s3: %v", err)
 		}
+
+		database.C.Save(&meta)
+		metadataCache[meta.ID] = meta
 		return nil
 	default:
 		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)

settings.toml

@@ -38,4 +38,5 @@ bucket = "bucket"
 endpoint = "s3.ap-east-1.amazonaws.com"
 secret_id = "secret"
 secret_key = "secret"
+access_baseurl = "https://raw.sn.solsynth.dev"
 enable_ssl = true