HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Bug fixes and logging

Browse Source
This commit is contained in:
LittleSheep
2024-07-29 13:22:57 +08:00
parent 8f08d85fb1
commit c46d7fa312
7 changed files with 57 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/internal/models/attachments.go
View File

@@ -22,8 +22,9 @@ type Attachment struct {
Destination AttachmentDst `json:"destination"`
RefCount int `json:"ref_count"`
Metadata datatypes.JSONMap `json:"metadata"`
IsMature bool `json:"is_mature"`
Metadata datatypes.JSONMap `json:"metadata"`
IsMature bool `json:"is_mature"`
IsAnalyzed bool `json:"is_analyzed"`
Ref *Attachment `json:"ref"`
RefID *uint `json:"ref_id"`

13
pkg/internal/models/destination.go
View File

@@ -18,10 +18,11 @@ type LocalDestination struct {
type S3Destination struct {
BaseDestination
Path string `json:"path"`
Bucket string `json:"bucket"`
Endpoint string `json:"endpoint"`
SecretID string `json:"secret_id"`
SecretKey string `json:"secret_key"`
EnableSSL bool `json:"enable_ssl"`
Path string `json:"path"`
Bucket string `json:"bucket"`
Endpoint string `json:"endpoint"`
SecretID string `json:"secret_id"`
SecretKey string `json:"secret_key"`
AccessBaseURL string `json:"access_baseurl"`
EnableSSL bool `json:"enable_ssl"`
}

25
pkg/internal/server/api/attachments_api.go
View File

@@ -47,14 +47,22 @@ func openAttachment(c *fiber.Ctx) error {
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
return c.Redirect(fmt.Sprintf(
"%s://%s.%s/%s",
protocol,
destConfigured.Bucket,
destConfigured.Endpoint,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
), fiber.StatusMovedPermanently)
if len(destConfigured.AccessBaseURL) > 0 {
return c.Redirect(fmt.Sprintf(
"%s/%s",
destConfigured.AccessBaseURL,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
), fiber.StatusMovedPermanently)
} else {
protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
return c.Redirect(fmt.Sprintf(
"%s://%s.%s/%s",
protocol,
destConfigured.Bucket,
destConfigured.Endpoint,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
), fiber.StatusMovedPermanently)
}
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
@@ -103,6 +111,7 @@ func createAttachment(c *fiber.Ctx) error {
MimeType: c.FormValue("mimetype"),
Metadata: usermeta,
IsMature: len(c.FormValue("mature")) > 0,
IsAnalyzed: false,
Destination: models.AttachmentDstTemporary,
})
if err != nil {

22
pkg/internal/services/analyzer.go
View File

@@ -35,7 +35,7 @@ func StartConsumeAnalyzeTask() {
if err := AnalyzeAttachment(task); err != nil {
log.Error().Err(err).Any("task", task).Msg("A file analyze task failed...")
} else {
log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file analyze task was completed.")
log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file analyze task was completed.")
}
}
}
@@ -51,6 +51,8 @@ func AnalyzeAttachment(file models.Attachment) error {
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
start := time.Now()
dst := filepath.Join(dest.Path, file.Uuid)
if _, err := os.Stat(dst); os.IsNotExist(err) {
return fmt.Errorf("attachment doesn't exists in temporary storage: %v", err)
@@ -85,24 +87,38 @@ func AnalyzeAttachment(file models.Attachment) error {
tx := database.C.Begin()
file.IsAnalyzed = true
linked, err := TryLinkAttachment(tx, file, file.HashCode)
if linked && err != nil {
return fmt.Errorf("unable to link file record: %v", err)
} else if !linked {
metadataCache[file.ID] = file
if err := tx.Save(&file).Error; err != nil {
tx.Rollback()
return fmt.Errorf("unable to save file record: %v", err)
}
}
tx.Commit()
log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Msg("A file analyze task was finished, starting uploading...")
start = time.Now()
// Move temporary to permanet
if !linked {
if err := ReUploadFileToPermanent(file); err != nil {
tx.Rollback()
return fmt.Errorf("unable to move file to permanet storage: %v", err)
}
}
tx.Commit()
// Recycle the temporary file
file.Destination = models.AttachmentDstTemporary
PublishDeleteFileTask(file)
// Finish
log.Info().Dur("elapsed", time.Since(start)).Uint("id", file.ID).Bool("linked", linked).Msg("A file post-analyze upload task was finished.")
return nil
}

2
pkg/internal/services/recycler.go
View File

@@ -28,7 +28,7 @@ func StartConsumeDeletionTask() {
if err := DeleteFile(task); err != nil {
log.Error().Err(err).Any("task", task).Msg("A file deletion task failed...")
} else {
log.Info().Dur("elapsed", time.Since(start)).Any("task", task).Msg("A file deletion task was completed.")
log.Info().Dur("elapsed", time.Since(start)).Uint("id", task.ID).Msg("A file deletion task was completed.")
}
}
}

9
pkg/internal/services/uploader.go
View File

@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
"github.com/gofiber/fiber/v2"
jsoniter "github.com/json-iterator/go"
@@ -39,6 +40,8 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
return fmt.Errorf("attachment isn't in temporary storage, unable to process")
}
meta.Destination = models.AttachmentDstPermanent
destMap := viper.GetStringMap("destinations.permanent")
var dest models.BaseDestination
@@ -76,6 +79,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
if err != nil {
return fmt.Errorf("unable to copy data to dest file: %v", err)
}
database.C.Save(&meta)
metadataCache[meta.ID] = meta
return nil
case models.DestinationTypeS3:
var destConfigured models.S3Destination
@@ -97,6 +103,9 @@ func ReUploadFileToPermanent(meta models.Attachment) error {
if err != nil {
return fmt.Errorf("unable to upload file to s3: %v", err)
}
database.C.Save(&meta)
metadataCache[meta.ID] = meta
return nil
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)

1
settings.toml
View File

@@ -38,4 +38,5 @@ bucket = "bucket"
endpoint = "s3.ap-east-1.amazonaws.com"
secret_id = "secret"
secret_key = "secret"
access_baseurl = "https://raw.sn.solsynth.dev"
enable_ssl = true