✨ Boost CRUD API

pkg/internal/database/migrator.go

@@ -9,6 +9,7 @@ var AutoMaintainRange = []any{
 	&models.AttachmentPool{},
 	&models.Attachment{},
 	&models.AttachmentFragment{},
+	&models.AttachmentBoost{},
 	&models.StickerPack{},
 	&models.Sticker{},
 }

pkg/internal/fs/downloader.go

@@ -0,0 +1,52 @@
+package fs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/spf13/viper"
+)
+
+func DownloadFileToLocal(meta models.Attachment, dst int) (string, error) {
+	destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst))
+
+	var dest models.BaseDestination
+	rawDest, _ := jsoniter.Marshal(destMap)
+	_ = jsoniter.Unmarshal(rawDest, &dest)
+
+	switch dest.Type {
+	case models.DestinationTypeLocal:
+		var destConfigured models.LocalDestination
+		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
+
+		return filepath.Join(destConfigured.Path, meta.Uuid), nil
+	case models.DestinationTypeS3:
+		var destConfigured models.S3Destination
+		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
+
+		client, err := minio.New(destConfigured.Endpoint, &minio.Options{
+			Creds:  credentials.NewStaticV4(destConfigured.SecretID, destConfigured.SecretKey, ""),
+			Secure: destConfigured.EnableSSL,
+		})
+		if err != nil {
+			return "", fmt.Errorf("unable to configure s3 client: %v", err)
+		}
+
+		inDst := filepath.Join(os.TempDir(), meta.Uuid)
+
+		err = client.FGetObject(context.Background(), destConfigured.Bucket, meta.Uuid, inDst, minio.GetObjectOptions{})
+		if err != nil {
+			return "", fmt.Errorf("unable to upload file to s3: %v", err)
+		}
+
+		return inDst, nil
+	default:
+		return "", fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
+	}
+}

pkg/internal/fs/recycler.go

@@ -104,22 +104,22 @@ func DeleteFile(meta models.Attachment) error {
 	case models.DestinationTypeLocal:
 		var destConfigured models.LocalDestination
 		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
-		return DeleteFileFromLocal(destConfigured, meta)
+		return DeleteFileFromLocal(destConfigured, meta.Uuid)
 	case models.DestinationTypeS3:
 		var destConfigured models.S3Destination
 		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
-		return DeleteFileFromS3(destConfigured, meta)
+		return DeleteFileFromS3(destConfigured, meta.Uuid)
 	default:
 		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
 	}
 }
 
-func DeleteFileFromLocal(config models.LocalDestination, meta models.Attachment) error {
+func DeleteFileFromLocal(config models.LocalDestination, uuid string) error {
-	fullpath := filepath.Join(config.Path, meta.Uuid)
+	fullpath := filepath.Join(config.Path, uuid)
 	return os.Remove(fullpath)
 }
 
-func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error {
+func DeleteFileFromS3(config models.S3Destination, uuid string) error {
 	client, err := minio.New(config.Endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(config.SecretID, config.SecretKey, ""),
 		Secure: config.EnableSSL,
@@ -128,7 +128,7 @@ func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error
 		return fmt.Errorf("unable to configure s3 client: %v", err)
 	}
 
-	err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, meta.Uuid), minio.RemoveObjectOptions{})
+	err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, uuid), minio.RemoveObjectOptions{})
 	if err != nil {
 		return fmt.Errorf("unable to upload file to s3: %v", err)
 	}

pkg/internal/models/attachments.go

@@ -35,8 +35,6 @@ type Attachment struct {
 	RefCount    int    `json:"ref_count"`
 	Type        uint   `json:"type"`
 
-	FileChunks datatypes.JSONMap `json:"file_chunks"`
-
 	CleanedAt *time.Time `json:"cleaned_at"`
 
 	Metadata datatypes.JSONMap `json:"metadata"` // This field is analyzer auto generated metadata
@@ -62,11 +60,14 @@ type Attachment struct {
 	Pool   *AttachmentPool `json:"pool"`
 	PoolID *uint           `json:"pool_id"`
 
+	Boosts []AttachmentBoost `json:"boosts"`
+
 	AccountID uint `json:"account_id"`
 
 	// Outdated fields, just for backward compatibility
-	IsUploaded bool `json:"is_uploaded" gorm:"-"`
+	FileChunks datatypes.JSONMap `json:"file_chunks" gorm:"-"`
-	IsMature   bool `json:"is_mature" gorm:"-"`
+	IsUploaded bool              `json:"is_uploaded" gorm:"-"`
+	IsMature   bool              `json:"is_mature" gorm:"-"`
 }
 
 // Data model for in progress multipart attachments

pkg/internal/models/boost.go

@@ -0,0 +1,24 @@
+package models
+
+import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
+
+const (
+	BoostStatusPending = iota
+	BoostStatusActive
+	BoostStatusSuspended
+	BoostStatusError
+)
+
+// AttachmentBoost is made for speed up attachment loading by copy the original attachments
+// to others faster CDN or storage destinations.
+type AttachmentBoost struct {
+	cruda.BaseModel
+
+	Status      int `json:"status"`
+	Destination int `json:"destination"`
+
+	AttachmentID uint       `json:"attachment_id"`
+	Attachment   Attachment `json:"attachment"`
+
+	AccountID uint `json:"account"`
+}

pkg/internal/server/api/boost_api.go

@@ -0,0 +1,88 @@
+package api
+
+import (
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
+	"github.com/gofiber/fiber/v2"
+)
+
+func getBoost(c *fiber.Ctx) error {
+	id, _ := c.ParamsInt("id", 0)
+
+	if boost, err := services.GetBoostByID(uint(id)); err != nil {
+		return fiber.NewError(fiber.StatusInternalServerError, err.Error())
+	} else {
+		return c.JSON(boost)
+	}
+}
+
+func createBoost(c *fiber.Ctx) error {
+	user := c.Locals("nex_user").(*sec.UserInfo)
+
+	var data struct {
+		Attachment  uint `json:"attachment" validate:"required"`
+		Destination int  `json:"destination" validate:"required"`
+	}
+
+	if err := exts.BindAndValidate(c, &data); err != nil {
+		return err
+	}
+
+	var attachment models.Attachment
+	if err := database.C.Where("id = ?", data.Attachment).First(&attachment).Error; err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	}
+
+	if boost, err := services.CreateBoost(user, attachment, data.Destination); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	} else {
+		return c.JSON(boost)
+	}
+}
+
+func updateBoost(c *fiber.Ctx) error {
+	user := c.Locals("nex_user").(*sec.UserInfo)
+	id, _ := c.ParamsInt("id", 0)
+
+	var data struct {
+		Status int `json:"status" validate:"required"`
+	}
+
+	if err := exts.BindAndValidate(c, &data); err != nil {
+		return err
+	}
+
+	boost, err := services.GetBoostByID(uint(id))
+	if err != nil {
+		return fiber.NewError(fiber.StatusInternalServerError, err.Error())
+	} else if boost.AccountID != user.ID {
+		return fiber.NewError(fiber.StatusNotFound, "record not created by you")
+	}
+
+	if boost, err := services.UpdateBoostStatus(boost, data.Status); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	} else {
+		return c.JSON(boost)
+	}
+}
+
+func deleteBoost(c *fiber.Ctx) error {
+	user := c.Locals("nex_user").(*sec.UserInfo)
+	id, _ := c.ParamsInt("id", 0)
+
+	boost, err := services.GetBoostByID(uint(id))
+	if err != nil {
+		return fiber.NewError(fiber.StatusInternalServerError, err.Error())
+	} else if boost.AccountID != user.ID {
+		return fiber.NewError(fiber.StatusNotFound, "record not created by you")
+	}
+
+	if err := services.DeleteBoost(boost); err != nil {
+		return fiber.NewError(fiber.StatusInternalServerError, err.Error())
+	} else {
+		return c.SendStatus(fiber.StatusOK)
+	}
+}

pkg/internal/server/api/index.go

@@ -10,6 +10,13 @@ func MapAPIs(app *fiber.App, baseURL string) {
 
 	api := app.Group(baseURL).Name("API")
 	{
+		boost := api.Group("/boosts").Name("Boosts API")
+		{
+			boost.Get("/:id", getBoost)
+			boost.Post("/", sec.ValidatorMiddleware, createBoost)
+			boost.Put("/:id", sec.ValidatorMiddleware, updateBoost)
+		}
+
 		api.Get("/pools", listPost)
 		api.Get("/pools/:id", getPool)
 		api.Post("/pools", sec.ValidatorMiddleware, createPool)

pkg/internal/services/analyzer.go

@@ -241,7 +241,7 @@ func AnalyzeAttachment(file models.Attachment) error {
 	if !linked {
 		go func() {
 			start = time.Now()
-			if err := ReUploadFileToPermanent(file, 1); err != nil {
+			if err := ReUploadFile(file, 1); err != nil {
 				log.Warn().Any("file", file).Err(err).Msg("Unable to move file to permanet storage...")
 			} else {
 				// Recycle the temporary file

pkg/internal/services/attachments.go

@@ -35,6 +35,7 @@ func GetAttachmentByID(id uint) (models.Attachment, error) {
 		Preload("Pool").
 		Preload("Thumbnail").
 		Preload("Compressed").
+		Preload("Boosts").
 		First(&attachment).Error; err != nil {
 		return attachment, err
 	} else {
@@ -64,6 +65,7 @@ func GetAttachmentByRID(rid string) (models.Attachment, error) {
 		Preload("Pool").
 		Preload("Thumbnail").
 		Preload("Compressed").
+		Preload("Boosts").
 		First(&attachment).Error; err != nil {
 		return attachment, err
 	} else {

pkg/internal/services/boost.go

@@ -0,0 +1,96 @@
+package services
+
+import (
+	"fmt"
+
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/cast"
+	"github.com/spf13/viper"
+)
+
+func GetBoostByID(id uint) (models.AttachmentBoost, error) {
+	var boost models.AttachmentBoost
+	if err := database.C.
+		Where("id = ?", id).
+		Preload("Attachment").
+		First(&boost).Error; err != nil {
+		return boost, err
+	}
+	return boost, nil
+}
+
+func CreateBoost(user *sec.UserInfo, source models.Attachment, destination int) (models.AttachmentBoost, error) {
+	boost := models.AttachmentBoost{
+		Status:       models.BoostStatusPending,
+		Destination:  destination,
+		AttachmentID: source.ID,
+		Attachment:   source,
+		AccountID:    user.ID,
+	}
+
+	dests := cast.ToSlice(viper.Get("destinations"))
+	if destination >= len(dests) {
+		return boost, fmt.Errorf("invalid destination: %d", destination)
+	}
+
+	if err := database.C.Create(&boost).Error; err != nil {
+		return boost, err
+	}
+
+	boost.Attachment = source
+	go ActivateBoost(boost)
+
+	return boost, nil
+}
+
+func ActivateBoost(boost models.AttachmentBoost) {
+	dests := cast.ToSlice(viper.Get("destinations"))
+	if boost.Destination >= len(dests) {
+		log.Warn().Any("boost", boost).Msg("Unable to activate boost, invalid destination...")
+		database.C.Model(&boost).Update("status", models.BoostStatusError)
+		return
+	}
+
+	if err := ReUploadFile(boost.Attachment, boost.Destination); err != nil {
+		log.Warn().Any("boost", boost).Err(err).Msg("Unable to activate boost...")
+		database.C.Model(&boost).Update("status", models.BoostStatusError)
+		return
+	}
+
+	log.Info().Any("boost", boost).Msg("Boost was activated successfully.")
+	database.C.Model(&boost).Update("status", models.BoostStatusActive)
+}
+
+func UpdateBoostStatus(boost models.AttachmentBoost, status int) (models.AttachmentBoost, error) {
+	if status != models.BoostStatusActive && status != models.BoostStatusSuspended {
+		return boost, fmt.Errorf("invalid status: %d", status)
+	}
+	err := database.C.Save(&boost).Error
+	return boost, err
+}
+
+func DeleteBoost(boost models.AttachmentBoost) error {
+	destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", boost.Destination))
+
+	var dest models.BaseDestination
+	rawDest, _ := jsoniter.Marshal(destMap)
+	_ = jsoniter.Unmarshal(rawDest, &dest)
+
+	switch dest.Type {
+	case models.DestinationTypeLocal:
+		var destConfigured models.LocalDestination
+		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
+		return fs.DeleteFileFromLocal(destConfigured, boost.Attachment.Uuid)
+	case models.DestinationTypeS3:
+		var destConfigured models.S3Destination
+		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
+		return fs.DeleteFileFromS3(destConfigured, boost.Attachment.Uuid)
+	default:
+		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
+	}
+}

pkg/internal/services/uploader.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
 	"github.com/gofiber/fiber/v2"
 	jsoniter "github.com/json-iterator/go"
@@ -34,7 +35,7 @@ func UploadFileToTemporary(ctx *fiber.Ctx, file *multipart.FileHeader, meta mode
 	}
 }
 
-func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
+func ReUploadFile(meta models.Attachment, dst int) error {
 	if dst == models.AttachmentDstTemporary || meta.Destination == dst {
 		return fmt.Errorf("destnation cannot be reversed temporary or the same as the original")
 	}
@@ -42,6 +43,19 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
 		return fmt.Errorf("attachment isn't in temporary storage, unable to process")
 	}
 
+	prevDst := meta.Destination
+	inDst, err := fs.DownloadFileToLocal(meta, prevDst)
+	if err != nil {
+		return fmt.Errorf("unable to retrieve file content: %v", err)
+	}
+
+	cleanupDst := func() {
+		if prevDst == models.AttachmentDstTemporary {
+			return
+		}
+		os.Remove(inDst)
+	}
+
 	meta.Destination = dst
 	destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst))
 
@@ -49,16 +63,6 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
 	rawDest, _ := jsoniter.Marshal(destMap)
 	_ = jsoniter.Unmarshal(rawDest, &dest)
 
-	prevDestMap := viper.GetStringMap("destinations.0")
-
-	// Currently, the temporary destination only supports the local.
-	// So we can do this.
-	var prevDest models.LocalDestination
-	prevRawDest, _ := jsoniter.Marshal(prevDestMap)
-	_ = jsoniter.Unmarshal(prevRawDest, &prevDest)
-
-	inDst := filepath.Join(prevDest.Path, meta.Uuid)
-
 	switch dest.Type {
 	case models.DestinationTypeLocal:
 		var destConfigured models.LocalDestination
@@ -83,6 +87,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
 
 		database.C.Save(&meta)
 		CacheAttachment(meta)
+		cleanupDst()
 		return nil
 	case models.DestinationTypeS3:
 		var destConfigured models.S3Destination
@@ -107,6 +112,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
 
 		database.C.Save(&meta)
 		CacheAttachment(meta)
+		cleanupDst()
 		return nil
 	default:
 		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)