HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Boost CRUD API

Browse Source
This commit is contained in:
LittleSheep 2024-12-28 21:41:13 +08:00
parent edbe412f97
commit d59966a03e
11 changed files with 299 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/internal/database/migrator.go
View File

@ -9,6 +9,7 @@ var AutoMaintainRange = []any{
&models.AttachmentPool{},
&models.Attachment{},
&models.AttachmentFragment{},
&models.AttachmentBoost{},
&models.StickerPack{},
&models.Sticker{},
}

52
pkg/internal/fs/downloader.go Normal file
View File

@ -0,0 +1,52 @@
package fs
import (
"context"
"fmt"
"os"
"path/filepath"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/spf13/viper"
)
func DownloadFileToLocal(meta models.Attachment, dst int) (string, error) {
destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst))
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return filepath.Join(destConfigured.Path, meta.Uuid), nil
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
client, err := minio.New(destConfigured.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(destConfigured.SecretID, destConfigured.SecretKey, ""),
Secure: destConfigured.EnableSSL,
})
if err != nil {
return "", fmt.Errorf("unable to configure s3 client: %v", err)
}
inDst := filepath.Join(os.TempDir(), meta.Uuid)
err = client.FGetObject(context.Background(), destConfigured.Bucket, meta.Uuid, inDst, minio.GetObjectOptions{})
if err != nil {
return "", fmt.Errorf("unable to upload file to s3: %v", err)
}
return inDst, nil
default:
return "", fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}

12
pkg/internal/fs/recycler.go
View File

@ -104,22 +104,22 @@ func DeleteFile(meta models.Attachment) error {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return DeleteFileFromLocal(destConfigured, meta)
return DeleteFileFromLocal(destConfigured, meta.Uuid)
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return DeleteFileFromS3(destConfigured, meta)
return DeleteFileFromS3(destConfigured, meta.Uuid)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}
func DeleteFileFromLocal(config models.LocalDestination, meta models.Attachment) error {
fullpath := filepath.Join(config.Path, meta.Uuid)
func DeleteFileFromLocal(config models.LocalDestination, uuid string) error {
fullpath := filepath.Join(config.Path, uuid)
return os.Remove(fullpath)
}
func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error {
func DeleteFileFromS3(config models.S3Destination, uuid string) error {
client, err := minio.New(config.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(config.SecretID, config.SecretKey, ""),
Secure: config.EnableSSL,
@ -128,7 +128,7 @@ func DeleteFileFromS3(config models.S3Destination, meta models.Attachment) error
return fmt.Errorf("unable to configure s3 client: %v", err)
}
err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, meta.Uuid), minio.RemoveObjectOptions{})
err = client.RemoveObject(context.Background(), config.Bucket, filepath.Join(config.Path, uuid), minio.RemoveObjectOptions{})
if err != nil {
return fmt.Errorf("unable to upload file to s3: %v", err)
}

9
pkg/internal/models/attachments.go
View File

@ -35,8 +35,6 @@ type Attachment struct {
RefCount int `json:"ref_count"`
Type uint `json:"type"`
FileChunks datatypes.JSONMap `json:"file_chunks"`
CleanedAt *time.Time `json:"cleaned_at"`
Metadata datatypes.JSONMap `json:"metadata"` // This field is analyzer auto generated metadata
@ -62,11 +60,14 @@ type Attachment struct {
Pool *AttachmentPool `json:"pool"`
PoolID *uint `json:"pool_id"`
Boosts []AttachmentBoost `json:"boosts"`
AccountID uint `json:"account_id"`
// Outdated fields, just for backward compatibility
IsUploaded bool `json:"is_uploaded" gorm:"-"`
IsMature bool `json:"is_mature" gorm:"-"`
FileChunks datatypes.JSONMap `json:"file_chunks" gorm:"-"`
IsUploaded bool `json:"is_uploaded" gorm:"-"`
IsMature bool `json:"is_mature" gorm:"-"`
}
// Data model for in progress multipart attachments

24
pkg/internal/models/boost.go Normal file
View File

@ -0,0 +1,24 @@
package models
import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
const (
BoostStatusPending = iota
BoostStatusActive
BoostStatusSuspended
BoostStatusError
)
// AttachmentBoost is made for speed up attachment loading by copy the original attachments
// to others faster CDN or storage destinations.
type AttachmentBoost struct {
cruda.BaseModel
Status int `json:"status"`
Destination int `json:"destination"`
AttachmentID uint `json:"attachment_id"`
Attachment Attachment `json:"attachment"`
AccountID uint `json:"account"`
}

88
pkg/internal/server/api/boost_api.go Normal file
View File

@ -0,0 +1,88 @@
package api
import (
"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
"github.com/gofiber/fiber/v2"
)
func getBoost(c *fiber.Ctx) error {
id, _ := c.ParamsInt("id", 0)
if boost, err := services.GetBoostByID(uint(id)); err != nil {
return fiber.NewError(fiber.StatusInternalServerError, err.Error())
} else {
return c.JSON(boost)
}
}
func createBoost(c *fiber.Ctx) error {
user := c.Locals("nex_user").(*sec.UserInfo)
var data struct {
Attachment uint `json:"attachment" validate:"required"`
Destination int `json:"destination" validate:"required"`
}
if err := exts.BindAndValidate(c, &data); err != nil {
return err
}
var attachment models.Attachment
if err := database.C.Where("id = ?", data.Attachment).First(&attachment).Error; err != nil {
return fiber.NewError(fiber.StatusBadRequest, err.Error())
}
if boost, err := services.CreateBoost(user, attachment, data.Destination); err != nil {
return fiber.NewError(fiber.StatusBadRequest, err.Error())
} else {
return c.JSON(boost)
}
}
func updateBoost(c *fiber.Ctx) error {
user := c.Locals("nex_user").(*sec.UserInfo)
id, _ := c.ParamsInt("id", 0)
var data struct {
Status int `json:"status" validate:"required"`
}
if err := exts.BindAndValidate(c, &data); err != nil {
return err
}
boost, err := services.GetBoostByID(uint(id))
if err != nil {
return fiber.NewError(fiber.StatusInternalServerError, err.Error())
} else if boost.AccountID != user.ID {
return fiber.NewError(fiber.StatusNotFound, "record not created by you")
}
if boost, err := services.UpdateBoostStatus(boost, data.Status); err != nil {
return fiber.NewError(fiber.StatusBadRequest, err.Error())
} else {
return c.JSON(boost)
}
}
func deleteBoost(c *fiber.Ctx) error {
user := c.Locals("nex_user").(*sec.UserInfo)
id, _ := c.ParamsInt("id", 0)
boost, err := services.GetBoostByID(uint(id))
if err != nil {
return fiber.NewError(fiber.StatusInternalServerError, err.Error())
} else if boost.AccountID != user.ID {
return fiber.NewError(fiber.StatusNotFound, "record not created by you")
}
if err := services.DeleteBoost(boost); err != nil {
return fiber.NewError(fiber.StatusInternalServerError, err.Error())
} else {
return c.SendStatus(fiber.StatusOK)
}
}

7
pkg/internal/server/api/index.go
View File

@ -10,6 +10,13 @@ func MapAPIs(app *fiber.App, baseURL string) {
api := app.Group(baseURL).Name("API")
{
boost := api.Group("/boosts").Name("Boosts API")
{
boost.Get("/:id", getBoost)
boost.Post("/", sec.ValidatorMiddleware, createBoost)
boost.Put("/:id", sec.ValidatorMiddleware, updateBoost)
}
api.Get("/pools", listPost)
api.Get("/pools/:id", getPool)
api.Post("/pools", sec.ValidatorMiddleware, createPool)

2
pkg/internal/services/analyzer.go
View File

@ -241,7 +241,7 @@ func AnalyzeAttachment(file models.Attachment) error {
if !linked {
go func() {
start = time.Now()
if err := ReUploadFileToPermanent(file, 1); err != nil {
if err := ReUploadFile(file, 1); err != nil {
log.Warn().Any("file", file).Err(err).Msg("Unable to move file to permanet storage...")
} else {
// Recycle the temporary file

2
pkg/internal/services/attachments.go
View File

@ -35,6 +35,7 @@ func GetAttachmentByID(id uint) (models.Attachment, error) {
Preload("Pool").
Preload("Thumbnail").
Preload("Compressed").
Preload("Boosts").
First(&attachment).Error; err != nil {
return attachment, err
} else {
@ -64,6 +65,7 @@ func GetAttachmentByRID(rid string) (models.Attachment, error) {
Preload("Pool").
Preload("Thumbnail").
Preload("Compressed").
Preload("Boosts").
First(&attachment).Error; err != nil {
return attachment, err
} else {

96
pkg/internal/services/boost.go Normal file
View File

@ -0,0 +1,96 @@
package services
import (
"fmt"
"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
jsoniter "github.com/json-iterator/go"
"github.com/rs/zerolog/log"
"github.com/spf13/cast"
"github.com/spf13/viper"
)
func GetBoostByID(id uint) (models.AttachmentBoost, error) {
var boost models.AttachmentBoost
if err := database.C.
Where("id = ?", id).
Preload("Attachment").
First(&boost).Error; err != nil {
return boost, err
}
return boost, nil
}
func CreateBoost(user *sec.UserInfo, source models.Attachment, destination int) (models.AttachmentBoost, error) {
boost := models.AttachmentBoost{
Status: models.BoostStatusPending,
Destination: destination,
AttachmentID: source.ID,
Attachment: source,
AccountID: user.ID,
}
dests := cast.ToSlice(viper.Get("destinations"))
if destination >= len(dests) {
return boost, fmt.Errorf("invalid destination: %d", destination)
}
if err := database.C.Create(&boost).Error; err != nil {
return boost, err
}
boost.Attachment = source
go ActivateBoost(boost)
return boost, nil
}
func ActivateBoost(boost models.AttachmentBoost) {
dests := cast.ToSlice(viper.Get("destinations"))
if boost.Destination >= len(dests) {
log.Warn().Any("boost", boost).Msg("Unable to activate boost, invalid destination...")
database.C.Model(&boost).Update("status", models.BoostStatusError)
return
}
if err := ReUploadFile(boost.Attachment, boost.Destination); err != nil {
log.Warn().Any("boost", boost).Err(err).Msg("Unable to activate boost...")
database.C.Model(&boost).Update("status", models.BoostStatusError)
return
}
log.Info().Any("boost", boost).Msg("Boost was activated successfully.")
database.C.Model(&boost).Update("status", models.BoostStatusActive)
}
func UpdateBoostStatus(boost models.AttachmentBoost, status int) (models.AttachmentBoost, error) {
if status != models.BoostStatusActive && status != models.BoostStatusSuspended {
return boost, fmt.Errorf("invalid status: %d", status)
}
err := database.C.Save(&boost).Error
return boost, err
}
func DeleteBoost(boost models.AttachmentBoost) error {
destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", boost.Destination))
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return fs.DeleteFileFromLocal(destConfigured, boost.Attachment.Uuid)
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return fs.DeleteFileFromS3(destConfigured, boost.Attachment.Uuid)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}

28
pkg/internal/services/uploader.go
View File

@ -9,6 +9,7 @@ import (
"path/filepath"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
"github.com/gofiber/fiber/v2"
jsoniter "github.com/json-iterator/go"
@ -34,7 +35,7 @@ func UploadFileToTemporary(ctx *fiber.Ctx, file *multipart.FileHeader, meta mode
}
}
func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
func ReUploadFile(meta models.Attachment, dst int) error {
if dst == models.AttachmentDstTemporary || meta.Destination == dst {
return fmt.Errorf("destnation cannot be reversed temporary or the same as the original")
}
@ -42,6 +43,19 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
return fmt.Errorf("attachment isn't in temporary storage, unable to process")
}
prevDst := meta.Destination
inDst, err := fs.DownloadFileToLocal(meta, prevDst)
if err != nil {
return fmt.Errorf("unable to retrieve file content: %v", err)
}
cleanupDst := func() {
if prevDst == models.AttachmentDstTemporary {
return
}
os.Remove(inDst)
}
meta.Destination = dst
destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", dst))
@ -49,16 +63,6 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
prevDestMap := viper.GetStringMap("destinations.0")
// Currently, the temporary destination only supports the local.
// So we can do this.
var prevDest models.LocalDestination
prevRawDest, _ := jsoniter.Marshal(prevDestMap)
_ = jsoniter.Unmarshal(prevRawDest, &prevDest)
inDst := filepath.Join(prevDest.Path, meta.Uuid)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
@ -83,6 +87,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
database.C.Save(&meta)
CacheAttachment(meta)
cleanupDst()
return nil
case models.DestinationTypeS3:
var destConfigured models.S3Destination
@ -107,6 +112,7 @@ func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
database.C.Save(&meta)
CacheAttachment(meta)
cleanupDst()
return nil
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)