diff --git a/pkg/internal/database/migrator.go b/pkg/internal/database/migrator.go index edb5ebe..84b68ab 100644 --- a/pkg/internal/database/migrator.go +++ b/pkg/internal/database/migrator.go @@ -9,6 +9,7 @@ var AutoMaintainRange = []any{ &models.AttachmentPool{}, &models.Attachment{}, &models.AttachmentFragment{}, + &models.AttachmentBoost{}, &models.StickerPack{}, &models.Sticker{}, } diff --git a/pkg/internal/fs/downloader.go b/pkg/internal/fs/downloader.go new file mode 100644 index 0000000..7436892 --- /dev/null +++ b/pkg/internal/fs/downloader.go @@ -0,0 +1,52 @@ +package fs + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/spf13/viper" +) + +func DownloadFileToLocal(meta models.Attachment, dst int) (string, error) { + destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst)) + + var dest models.BaseDestination + rawDest, _ := jsoniter.Marshal(destMap) + _ = jsoniter.Unmarshal(rawDest, &dest) + + switch dest.Type { + case models.DestinationTypeLocal: + var destConfigured models.LocalDestination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + + return filepath.Join(destConfigured.Path, meta.Uuid), nil + case models.DestinationTypeS3: + var destConfigured models.S3Destination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + + client, err := minio.New(destConfigured.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(destConfigured.SecretID, destConfigured.SecretKey, ""), + Secure: destConfigured.EnableSSL, + }) + if err != nil { + return "", fmt.Errorf("unable to configure s3 client: %v", err) + } + + inDst := filepath.Join(os.TempDir(), meta.Uuid) + + err = client.FGetObject(context.Background(), destConfigured.Bucket, meta.Uuid, inDst, minio.GetObjectOptions{}) + if err != nil { + return "", fmt.Errorf("unable to upload file to s3: %v", err) + } + + return inDst, nil + default: + return "", fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) + } +} diff --git a/pkg/internal/fs/recycler.go b/pkg/internal/fs/recycler.go index c041f96..35f519a 100644 --- a/pkg/internal/fs/recycler.go +++ b/pkg/internal/fs/recycler.go @@ -104,22 +104,22 @@ func DeleteFile(meta models.Attachment) error { case models.DestinationTypeLocal: var destConfigured models.LocalDestination _ = jsoniter.Unmarshal(rawDest, &destConfigured) - return DeleteFileFromLocal(destConfigured, meta) + return DeleteFileFromLocal(destConfigured, meta.Uuid) case models.DestinationTypeS3: var destConfigured models.S3Destination _ = jsoniter.Unmarshal(rawDest, &destConfigured) - return DeleteFileFromS3(destConfigured, meta) + return DeleteFileFromS3(destConfigured, meta.Uuid) default: return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) } } -func DeleteFileFromLocal(config models.LocalDestination, meta models.Attachment) error { - fullpath := filepath.Join(config.Path, meta.Uuid) +func DeleteFileFromLocal(config models.LocalDestination, uuid string) error { + fullpath := filepath.Join(config.Path, uuid) return os.Remove(fullpath) } -func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error { +func DeleteFileFromS3(config models.S3Destination, uuid string) error { client, err := minio.New(config.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4(config.SecretID, config.SecretKey, ""), Secure: config.EnableSSL, @@ -128,7 +128,7 @@ func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error return fmt.Errorf("unable to configure s3 client: %v", err) } - err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, meta.Uuid), minio.RemoveObjectOptions{}) + err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, uuid), minio.RemoveObjectOptions{}) if err != nil { return fmt.Errorf("unable to upload file to s3: %v", err) } diff --git a/pkg/internal/models/attachments.go b/pkg/internal/models/attachments.go index 7b1e91f..194cf35 100644 --- a/pkg/internal/models/attachments.go +++ b/pkg/internal/models/attachments.go @@ -35,8 +35,6 @@ type Attachment struct { RefCount int `json:"ref_count"` Type uint `json:"type"` - FileChunks datatypes.JSONMap `json:"file_chunks"` - CleanedAt *time.Time `json:"cleaned_at"` Metadata datatypes.JSONMap `json:"metadata"` // This field is analyzer auto generated metadata @@ -62,11 +60,14 @@ type Attachment struct { Pool *AttachmentPool `json:"pool"` PoolID *uint `json:"pool_id"` + Boosts []AttachmentBoost `json:"boosts"` + AccountID uint `json:"account_id"` // Outdated fields, just for backward compatibility - IsUploaded bool `json:"is_uploaded" gorm:"-"` - IsMature bool `json:"is_mature" gorm:"-"` + FileChunks datatypes.JSONMap `json:"file_chunks" gorm:"-"` + IsUploaded bool `json:"is_uploaded" gorm:"-"` + IsMature bool `json:"is_mature" gorm:"-"` } // Data model for in progress multipart attachments diff --git a/pkg/internal/models/boost.go b/pkg/internal/models/boost.go new file mode 100644 index 0000000..5287129 --- /dev/null +++ b/pkg/internal/models/boost.go @@ -0,0 +1,24 @@ +package models + +import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda" + +const ( + BoostStatusPending = iota + BoostStatusActive + BoostStatusSuspended + BoostStatusError +) + +// AttachmentBoost is made for speed up attachment loading by copy the original attachments +// to others faster CDN or storage destinations. +type AttachmentBoost struct { + cruda.BaseModel + + Status int `json:"status"` + Destination int `json:"destination"` + + AttachmentID uint `json:"attachment_id"` + Attachment Attachment `json:"attachment"` + + AccountID uint `json:"account"` +} diff --git a/pkg/internal/server/api/boost_api.go b/pkg/internal/server/api/boost_api.go new file mode 100644 index 0000000..2a72569 --- /dev/null +++ b/pkg/internal/server/api/boost_api.go @@ -0,0 +1,88 @@ +package api + +import ( + "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/database" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/services" + "github.com/gofiber/fiber/v2" +) + +func getBoost(c *fiber.Ctx) error { + id, _ := c.ParamsInt("id", 0) + + if boost, err := services.GetBoostByID(uint(id)); err != nil { + return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + } else { + return c.JSON(boost) + } +} + +func createBoost(c *fiber.Ctx) error { + user := c.Locals("nex_user").(*sec.UserInfo) + + var data struct { + Attachment uint `json:"attachment" validate:"required"` + Destination int `json:"destination" validate:"required"` + } + + if err := exts.BindAndValidate(c, &data); err != nil { + return err + } + + var attachment models.Attachment + if err := database.C.Where("id = ?", data.Attachment).First(&attachment).Error; err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } + + if boost, err := services.CreateBoost(user, attachment, data.Destination); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else { + return c.JSON(boost) + } +} + +func updateBoost(c *fiber.Ctx) error { + user := c.Locals("nex_user").(*sec.UserInfo) + id, _ := c.ParamsInt("id", 0) + + var data struct { + Status int `json:"status" validate:"required"` + } + + if err := exts.BindAndValidate(c, &data); err != nil { + return err + } + + boost, err := services.GetBoostByID(uint(id)) + if err != nil { + return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + } else if boost.AccountID != user.ID { + return fiber.NewError(fiber.StatusNotFound, "record not created by you") + } + + if boost, err := services.UpdateBoostStatus(boost, data.Status); err != nil { + return fiber.NewError(fiber.StatusBadRequest, err.Error()) + } else { + return c.JSON(boost) + } +} + +func deleteBoost(c *fiber.Ctx) error { + user := c.Locals("nex_user").(*sec.UserInfo) + id, _ := c.ParamsInt("id", 0) + + boost, err := services.GetBoostByID(uint(id)) + if err != nil { + return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + } else if boost.AccountID != user.ID { + return fiber.NewError(fiber.StatusNotFound, "record not created by you") + } + + if err := services.DeleteBoost(boost); err != nil { + return fiber.NewError(fiber.StatusInternalServerError, err.Error()) + } else { + return c.SendStatus(fiber.StatusOK) + } +} diff --git a/pkg/internal/server/api/index.go b/pkg/internal/server/api/index.go index 69167b5..bbcd925 100644 --- a/pkg/internal/server/api/index.go +++ b/pkg/internal/server/api/index.go @@ -10,6 +10,13 @@ func MapAPIs(app *fiber.App, baseURL string) { api := app.Group(baseURL).Name("API") { + boost := api.Group("/boosts").Name("Boosts API") + { + boost.Get("/:id", getBoost) + boost.Post("/", sec.ValidatorMiddleware, createBoost) + boost.Put("/:id", sec.ValidatorMiddleware, updateBoost) + } + api.Get("/pools", listPost) api.Get("/pools/:id", getPool) api.Post("/pools", sec.ValidatorMiddleware, createPool) diff --git a/pkg/internal/services/analyzer.go b/pkg/internal/services/analyzer.go index f15b500..ef25360 100644 --- a/pkg/internal/services/analyzer.go +++ b/pkg/internal/services/analyzer.go @@ -241,7 +241,7 @@ func AnalyzeAttachment(file models.Attachment) error { if !linked { go func() { start = time.Now() - if err := ReUploadFileToPermanent(file, 1); err != nil { + if err := ReUploadFile(file, 1); err != nil { log.Warn().Any("file", file).Err(err).Msg("Unable to move file to permanet storage...") } else { // Recycle the temporary file diff --git a/pkg/internal/services/attachments.go b/pkg/internal/services/attachments.go index dd35352..fc4ac6e 100644 --- a/pkg/internal/services/attachments.go +++ b/pkg/internal/services/attachments.go @@ -35,6 +35,7 @@ func GetAttachmentByID(id uint) (models.Attachment, error) { Preload("Pool"). Preload("Thumbnail"). Preload("Compressed"). + Preload("Boosts"). First(&attachment).Error; err != nil { return attachment, err } else { @@ -64,6 +65,7 @@ func GetAttachmentByRID(rid string) (models.Attachment, error) { Preload("Pool"). Preload("Thumbnail"). Preload("Compressed"). + Preload("Boosts"). First(&attachment).Error; err != nil { return attachment, err } else { diff --git a/pkg/internal/services/boost.go b/pkg/internal/services/boost.go new file mode 100644 index 0000000..1a4d9d2 --- /dev/null +++ b/pkg/internal/services/boost.go @@ -0,0 +1,96 @@ +package services + +import ( + "fmt" + + "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/database" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/fs" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" + jsoniter "github.com/json-iterator/go" + "github.com/rs/zerolog/log" + "github.com/spf13/cast" + "github.com/spf13/viper" +) + +func GetBoostByID(id uint) (models.AttachmentBoost, error) { + var boost models.AttachmentBoost + if err := database.C. + Where("id = ?", id). + Preload("Attachment"). + First(&boost).Error; err != nil { + return boost, err + } + return boost, nil +} + +func CreateBoost(user *sec.UserInfo, source models.Attachment, destination int) (models.AttachmentBoost, error) { + boost := models.AttachmentBoost{ + Status: models.BoostStatusPending, + Destination: destination, + AttachmentID: source.ID, + Attachment: source, + AccountID: user.ID, + } + + dests := cast.ToSlice(viper.Get("destinations")) + if destination >= len(dests) { + return boost, fmt.Errorf("invalid destination: %d", destination) + } + + if err := database.C.Create(&boost).Error; err != nil { + return boost, err + } + + boost.Attachment = source + go ActivateBoost(boost) + + return boost, nil +} + +func ActivateBoost(boost models.AttachmentBoost) { + dests := cast.ToSlice(viper.Get("destinations")) + if boost.Destination >= len(dests) { + log.Warn().Any("boost", boost).Msg("Unable to activate boost, invalid destination...") + database.C.Model(&boost).Update("status", models.BoostStatusError) + return + } + + if err := ReUploadFile(boost.Attachment, boost.Destination); err != nil { + log.Warn().Any("boost", boost).Err(err).Msg("Unable to activate boost...") + database.C.Model(&boost).Update("status", models.BoostStatusError) + return + } + + log.Info().Any("boost", boost).Msg("Boost was activated successfully.") + database.C.Model(&boost).Update("status", models.BoostStatusActive) +} + +func UpdateBoostStatus(boost models.AttachmentBoost, status int) (models.AttachmentBoost, error) { + if status != models.BoostStatusActive && status != models.BoostStatusSuspended { + return boost, fmt.Errorf("invalid status: %d", status) + } + err := database.C.Save(&boost).Error + return boost, err +} + +func DeleteBoost(boost models.AttachmentBoost) error { + destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", boost.Destination)) + + var dest models.BaseDestination + rawDest, _ := jsoniter.Marshal(destMap) + _ = jsoniter.Unmarshal(rawDest, &dest) + + switch dest.Type { + case models.DestinationTypeLocal: + var destConfigured models.LocalDestination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + return fs.DeleteFileFromLocal(destConfigured, boost.Attachment.Uuid) + case models.DestinationTypeS3: + var destConfigured models.S3Destination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + return fs.DeleteFileFromS3(destConfigured, boost.Attachment.Uuid) + default: + return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) + } +} diff --git a/pkg/internal/services/uploader.go b/pkg/internal/services/uploader.go index 23da7a9..f3a27d7 100644 --- a/pkg/internal/services/uploader.go +++ b/pkg/internal/services/uploader.go @@ -9,6 +9,7 @@ import ( "path/filepath" "git.solsynth.dev/hypernet/paperclip/pkg/internal/database" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/fs" "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" "github.com/gofiber/fiber/v2" jsoniter "github.com/json-iterator/go" @@ -34,7 +35,7 @@ func UploadFileToTemporary(ctx *fiber.Ctx, file *multipart.FileHeader, meta mode } } -func ReUploadFileToPermanent(meta models.Attachment, dst int) error { +func ReUploadFile(meta models.Attachment, dst int) error { if dst == models.AttachmentDstTemporary || meta.Destination == dst { return fmt.Errorf("destnation cannot be reversed temporary or the same as the original") } @@ -42,6 +43,19 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error { return fmt.Errorf("attachment isn't in temporary storage, unable to process") } + prevDst := meta.Destination + inDst, err := fs.DownloadFileToLocal(meta, prevDst) + if err != nil { + return fmt.Errorf("unable to retrieve file content: %v", err) + } + + cleanupDst := func() { + if prevDst == models.AttachmentDstTemporary { + return + } + os.Remove(inDst) + } + meta.Destination = dst destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst)) @@ -49,16 +63,6 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error { rawDest, _ := jsoniter.Marshal(destMap) _ = jsoniter.Unmarshal(rawDest, &dest) - prevDestMap := viper.GetStringMap("destinations.0") - - // Currently, the temporary destination only supports the local. - // So we can do this. - var prevDest models.LocalDestination - prevRawDest, _ := jsoniter.Marshal(prevDestMap) - _ = jsoniter.Unmarshal(prevRawDest, &prevDest) - - inDst := filepath.Join(prevDest.Path, meta.Uuid) - switch dest.Type { case models.DestinationTypeLocal: var destConfigured models.LocalDestination @@ -83,6 +87,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error { database.C.Save(&meta) CacheAttachment(meta) + cleanupDst() return nil case models.DestinationTypeS3: var destConfigured models.S3Destination @@ -107,6 +112,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error { database.C.Save(&meta) CacheAttachment(meta) + cleanupDst() return nil default: return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)