HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Basic queue for processing

Browse Source
This commit is contained in:
LittleSheep
2024-07-28 21:03:56 +08:00
parent 2a94bb20f8
commit 10879bef14
13 changed files with 140 additions and 635 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
pkg/internal/grpc/attachments.go
View File

@ -1,76 +0,0 @@
package grpc
import (
"context"
"fmt"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
"git.solsynth.dev/hydrogen/paperclip/pkg/proto"
"google.golang.org/protobuf/types/known/emptypb"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
jsoniter "github.com/json-iterator/go"
"github.com/samber/lo"
)
func (v *Server) GetAttachment(ctx context.Context, request *proto.AttachmentLookupRequest) (*proto.Attachment, error) {
var attachment models.Attachment
tx := database.C.Model(&models.Attachment{})
if request.Id != nil {
tx = tx.Where("id = ?", request.GetId())
}
if request.Uuid != nil {
tx = tx.Where("uuid = ?", request.GetUuid())
}
if request.Usage != nil {
tx = tx.Where("usage = ?", request.GetUsage())
}
if err := tx.First(&attachment).Error; err != nil {
return nil, err
}
rawMetadata, _ := jsoniter.Marshal(attachment.Metadata)
if attachment.AccountID == nil {
attachment.AccountID = lo.ToPtr[uint](0)
}
return &proto.Attachment{
Id: uint64(attachment.ID),
Uuid: attachment.Uuid,
Size: attachment.Size,
Name: attachment.Name,
Alt: attachment.Alternative,
Usage: attachment.Usage,
Mimetype: attachment.MimeType,
Hash: attachment.HashCode,
Destination: attachment.Destination,
Metadata: rawMetadata,
IsMature: attachment.IsMature,
AccountId: uint64(*attachment.AccountID),
}, nil
}
func (v *Server) CheckAttachmentExists(ctx context.Context, request *proto.AttachmentLookupRequest) (*emptypb.Empty, error) {
tx := database.C.Model(&models.Attachment{})
if request.Id != nil {
tx = tx.Where("id = ?", request.GetId())
}
if request.Uuid != nil {
tx = tx.Where("uuid = ?", request.GetUuid())
}
if request.Usage != nil {
tx = tx.Where("usage = ?", request.GetUsage())
}
var count int64
if err := tx.Model(&models.Attachment{}).Count(&count).Error; err != nil {
return nil, err
} else if count == 0 {
return nil, fmt.Errorf("record not found")
}
return &emptypb.Empty{}, nil
}

6
pkg/internal/grpc/server.go
View File

@ -1,16 +1,15 @@
package grpc
import (
"git.solsynth.dev/hydrogen/paperclip/pkg/proto"
"net"
"github.com/spf13/viper"
"google.golang.org/grpc"
health "google.golang.org/grpc/health/grpc_health_v1"
"google.golang.org/grpc/reflection"
"net"
)
type Server struct {
proto.UnimplementedAttachmentsServer
}
var S *grpc.Server
@ -18,7 +17,6 @@ var S *grpc.Server
func NewGRPC() {
S = grpc.NewServer()
proto.RegisterAttachmentsServer(S, &Server{})
health.RegisterHealthServer(S, &Server{})
reflection.Register(S)

27
pkg/internal/models/attachments.go
View File

@ -2,21 +2,28 @@ package models
import "gorm.io/datatypes"
type AttachmentDst = int8
const (
AttachmentDstTemporary = AttachmentDst(iota)
AttachmentDstPermanent
)
type Attachment struct {
BaseModel
Uuid string `json:"uuid"`
Size int64 `json:"size"`
Name string `json:"name"`
Alternative string `json:"alt"`
Usage string `json:"usage"`
MimeType string `json:"mimetype"`
HashCode string `json:"hash"`
Destination string `json:"destination"`
Uuid string `json:"uuid"`
Size int64 `json:"size"`
Name string `json:"name"`
Alternative string `json:"alt"`
Usage string `json:"usage"`
MimeType string `json:"mimetype"`
HashCode string `json:"hash"`
Destination AttachmentDst `json:"destination"`
Metadata datatypes.JSONMap `json:"metadata"`
IsMature bool `json:"is_mature"`
Account *Account `json:"account"`
AccountID *uint `json:"account_id"`
Account Account `json:"account"`
AccountID uint `json:"account_id"`
}

31
pkg/internal/server/api/attachments_api.go
View File

@ -25,17 +25,18 @@ func openAttachment(c *fiber.Ctx) error {
return fiber.NewError(fiber.StatusNotFound)
}
destMap := viper.GetStringMap("destinations")
dest, destOk := destMap[metadata.Destination]
if !destOk {
return fiber.NewError(fiber.StatusInternalServerError, "invalid destination: destination configuration was not found")
var destMap map[string]any
if metadata.Destination == models.AttachmentDstTemporary {
destMap = viper.GetStringMap("destinations.temporary")
} else {
destMap = viper.GetStringMap("destinations.permanent")
}
var destParsed models.BaseDestination
rawDest, _ := jsoniter.Marshal(dest)
_ = jsoniter.Unmarshal(rawDest, &destParsed)
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch destParsed.Type {
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
@ -43,7 +44,6 @@ func openAttachment(c *fiber.Ctx) error {
c.Set(fiber.HeaderContentType, metadata.MimeType)
}
return c.SendFile(filepath.Join(destConfigured.Path, metadata.Uuid), false)
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
@ -54,10 +54,9 @@ func openAttachment(c *fiber.Ctx) error {
destConfigured.Bucket,
destConfigured.Endpoint,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
))
), fiber.StatusMovedPermanently)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", destParsed.Type)
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}
@ -79,8 +78,6 @@ func createAttachment(c *fiber.Ctx) error {
}
user = lo.ToPtr(c.Locals("user").(models.Account))
destName := c.Query("destination", viper.GetString("preferred_destination"))
hash := c.FormValue("hash")
if len(hash) != 64 {
return fiber.NewError(fiber.StatusBadRequest, "please provide a sha-256 hash code, length should be 64 characters")
@ -110,7 +107,7 @@ func createAttachment(c *fiber.Ctx) error {
MimeType: c.FormValue("mimetype"),
Metadata: usermeta,
IsMature: len(c.FormValue("mature")) > 0,
Destination: destName,
Destination: models.AttachmentDstTemporary,
})
if err != nil {
tx.Rollback()
@ -118,7 +115,7 @@ func createAttachment(c *fiber.Ctx) error {
}
if !linked {
if err := services.UploadFile(destName, c, file, metadata); err != nil {
if err := services.UploadFileToTemporary(c, file, metadata); err != nil {
tx.Rollback()
return fiber.NewError(fiber.StatusBadRequest, err.Error())
}
@ -176,7 +173,7 @@ func deleteAttachment(c *fiber.Ctx) error {
attachment, err := services.GetAttachmentByID(uint(id))
if err != nil {
return fiber.NewError(fiber.StatusNotFound, err.Error())
} else if attachment.AccountID == nil || *attachment.AccountID != user.ID {
} else if attachment.AccountID != user.ID {
return fiber.NewError(fiber.StatusNotFound, "record not created by you")
}

68
pkg/internal/services/analyzer.go Normal file
View File

@ -0,0 +1,68 @@
package services
import (
"fmt"
"image"
"os"
"path/filepath"
"strings"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
jsoniter "github.com/json-iterator/go"
"github.com/spf13/viper"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
)
var fileAnalyzeQueue = make(chan models.Attachment, 256)
func PublishAnalyzeTask(file models.Attachment) {
fileAnalyzeQueue <- file
}
func AnalyzeAttachment(file models.Attachment) error {
if file.Destination != models.AttachmentDstTemporary {
return fmt.Errorf("attachment isn't in temporary storage, unable to analyze")
}
destMap := viper.GetStringMap("destinations.temporary")
var dest models.LocalDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
dst := filepath.Join(dest.Path, file.Uuid)
if _, err := os.Stat(dst); !os.IsExist(err) {
return fmt.Errorf("attachment doesn't exists in temporary storage")
}
if t := strings.SplitN(file.MimeType, "/", 2)[0]; t == "image" {
// Dealing with image
reader, err := os.Open(dst)
if err != nil {
return fmt.Errorf("unable to open file: %v", err)
}
defer reader.Close()
im, _, err := image.Decode(reader)
if err != nil {
return fmt.Errorf("unable to decode file as an image: %v", err)
}
width := im.Bounds().Dx()
height := im.Bounds().Dy()
ratio := width / height
file.Metadata = map[string]any{
"width": width,
"height": height,
"ratio": ratio,
}
}
if err := database.C.Save(&file).Error; err != nil {
return fmt.Errorf("unable to save file record: %v", err)
}
return nil
}

10
pkg/internal/services/attachments.go
View File

@ -58,19 +58,13 @@ func NewAttachmentMetadata(tx *gorm.DB, user *models.Account, file *multipart.Fi
exists.Metadata = attachment.Metadata
attachment = exists
attachment.ID = 0
if user != nil {
attachment.AccountID = &user.ID
}
attachment.AccountID = user.ID
} else {
// Upload the new file
attachment.Uuid = uuid.NewString()
attachment.Size = file.Size
attachment.Name = file.Filename
if user != nil {
attachment.AccountID = &user.ID
}
attachment.AccountID = user.ID
// If the user didn't provide file mimetype manually, we have to detect it
if len(attachment.MimeType) == 0 {

19
pkg/internal/services/recycler.go
View File

@ -14,17 +14,18 @@ import (
)
func DeleteFile(meta models.Attachment) error {
destMap := viper.GetStringMap("destinations")
dest, destOk := destMap[meta.Destination]
if !destOk {
return fmt.Errorf("invalid destination: destination configuration was not found")
var destMap map[string]any
if meta.Destination == models.AttachmentDstTemporary {
destMap = viper.GetStringMap("destinations.temporary")
} else {
destMap = viper.GetStringMap("destinations.permanent")
}
var destParsed models.BaseDestination
rawDest, _ := jsoniter.Marshal(dest)
_ = jsoniter.Unmarshal(rawDest, &destParsed)
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch destParsed.Type {
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
@ -34,7 +35,7 @@ func DeleteFile(meta models.Attachment) error {
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return DeleteFileFromS3(destConfigured, meta)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", destParsed.Type)
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}

33
pkg/internal/services/uploader.go
View File

@ -16,18 +16,31 @@ import (
"github.com/spf13/viper"
)
func UploadFile(destName string, ctx *fiber.Ctx, file *multipart.FileHeader, meta models.Attachment) error {
destMap := viper.GetStringMap("destinations")
dest, destOk := destMap[destName]
if !destOk {
return fmt.Errorf("invalid destination: destination configuration was not found")
func UploadFileToTemporary(ctx *fiber.Ctx, file *multipart.FileHeader, meta models.Attachment) error {
destMap := viper.GetStringMap("destinations.temporary")
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return UploadFileToLocal(destConfigured, ctx, file, meta)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}
var destParsed models.BaseDestination
rawDest, _ := jsoniter.Marshal(dest)
_ = jsoniter.Unmarshal(rawDest, &destParsed)
func UploadFileToPermanent(ctx *fiber.Ctx, file *multipart.FileHeader, meta models.Attachment) error {
destMap := viper.GetStringMap("destinations.permanent")
switch destParsed.Type {
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
@ -37,7 +50,7 @@ func UploadFile(destName string, ctx *fiber.Ctx, file *multipart.FileHeader, met
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
return UploadFileToS3(destConfigured, file, meta)
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", destParsed.Type)
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
}
}