From 49a8159e35860159837fb3e76c1a890ac92c1e66 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Sat, 28 Dec 2024 23:10:57 +0800 Subject: [PATCH] :recycle: Refactored opener :sparkles: Support boost in opener --- pkg/internal/models/destination.go | 7 +- pkg/internal/server/api/attachments_api.go | 67 ++------ .../{destinations.go => destinations_api.go} | 4 +- pkg/internal/services/destinations.go | 41 +++++ pkg/internal/services/opener.go | 155 ++++++++++++++++++ pkg/main.go | 1 + 6 files changed, 220 insertions(+), 55 deletions(-) rename pkg/internal/server/api/{destinations.go => destinations_api.go} (84%) create mode 100644 pkg/internal/services/destinations.go create mode 100644 pkg/internal/services/opener.go diff --git a/pkg/internal/models/destination.go b/pkg/internal/models/destination.go index 96c2524..c8ae162 100644 --- a/pkg/internal/models/destination.go +++ b/pkg/internal/models/destination.go @@ -6,9 +6,10 @@ const ( ) type BaseDestination struct { - Type string `json:"type"` - Label string `json:"label"` - LabelRegion string `json:"label_region"` + Type string `json:"type"` + Label string `json:"label"` + Region string `json:"region"` + IsBoost bool `json:"is_boost"` } type LocalDestination struct { diff --git a/pkg/internal/server/api/attachments_api.go b/pkg/internal/server/api/attachments_api.go index e11006e..a2d349d 100644 --- a/pkg/internal/server/api/attachments_api.go +++ b/pkg/internal/server/api/attachments_api.go @@ -2,8 +2,7 @@ package api import ( "fmt" - "net/url" - "path/filepath" + "strings" "git.solsynth.dev/hypernet/nexus/pkg/nex/sec" "git.solsynth.dev/hypernet/paperclip/pkg/internal/database" @@ -12,64 +11,32 @@ import ( "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" "git.solsynth.dev/hypernet/paperclip/pkg/internal/services" "github.com/gofiber/fiber/v2" - jsoniter "github.com/json-iterator/go" - "github.com/samber/lo" - "github.com/spf13/viper" ) func openAttachment(c *fiber.Ctx) error { id := c.Params("id") + region := c.Query("region") + + var err error + var url, mimetype string + if len(region) > 0 { + url, mimetype, err = services.OpenAttachmentByRID(id, region) + } else { + url, mimetype, err = services.OpenAttachmentByRID(id) + } - metadata, err := services.GetAttachmentByRID(id) if err != nil { - return fiber.NewError(fiber.StatusNotFound) + return fiber.NewError(fiber.StatusNotFound, err.Error()) } - destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", metadata.Destination)) + c.Set(fiber.HeaderContentType, mimetype) - var dest models.BaseDestination - rawDest, _ := jsoniter.Marshal(destMap) - _ = jsoniter.Unmarshal(rawDest, &dest) - - switch dest.Type { - case models.DestinationTypeLocal: - var destConfigured models.LocalDestination - _ = jsoniter.Unmarshal(rawDest, &destConfigured) - if len(destConfigured.AccessBaseURL) > 0 && !c.QueryBool("direct", false) { - // This will drop all query parameters, - // for not it's okay because the openAttachment api won't take any query parameters - return c.Redirect(fmt.Sprintf( - "%s%s?direct=true", - destConfigured.AccessBaseURL, - c.Path(), - ), fiber.StatusMovedPermanently) - } - if len(metadata.MimeType) > 0 { - c.Set(fiber.HeaderContentType, metadata.MimeType) - } - return c.SendFile(filepath.Join(destConfigured.Path, metadata.Uuid)) - case models.DestinationTypeS3: - var destConfigured models.S3Destination - _ = jsoniter.Unmarshal(rawDest, &destConfigured) - if len(destConfigured.AccessBaseURL) > 0 { - return c.Redirect(fmt.Sprintf( - "%s/%s", - destConfigured.AccessBaseURL, - url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)), - ), fiber.StatusMovedPermanently) - } else { - protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http") - return c.Redirect(fmt.Sprintf( - "%s://%s.%s/%s", - protocol, - destConfigured.Bucket, - destConfigured.Endpoint, - url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)), - ), fiber.StatusMovedPermanently) - } - default: - return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) + if strings.HasPrefix(url, "file://") { + fp := strings.Replace(url, "file://", "", 1) + return c.SendFile(fp) } + + return c.Redirect(url, fiber.StatusFound) } func getAttachmentMeta(c *fiber.Ctx) error { diff --git a/pkg/internal/server/api/destinations.go b/pkg/internal/server/api/destinations_api.go similarity index 84% rename from pkg/internal/server/api/destinations.go rename to pkg/internal/server/api/destinations_api.go index b94223e..3250b47 100644 --- a/pkg/internal/server/api/destinations.go +++ b/pkg/internal/server/api/destinations_api.go @@ -8,9 +8,9 @@ import ( ) func listDestination(c *fiber.Ctx) error { - var destinations []models.LocalDestination + var destinations []models.BaseDestination for _, value := range viper.GetStringSlice("destinations") { - var parsed models.LocalDestination + var parsed models.BaseDestination raw, _ := jsoniter.Marshal(value) _ = jsoniter.Unmarshal(raw, &parsed) destinations = append(destinations, parsed) diff --git a/pkg/internal/services/destinations.go b/pkg/internal/services/destinations.go new file mode 100644 index 0000000..ed54289 --- /dev/null +++ b/pkg/internal/services/destinations.go @@ -0,0 +1,41 @@ +package services + +import ( + "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" + jsoniter "github.com/json-iterator/go" + "github.com/rs/zerolog/log" + "github.com/spf13/viper" +) + +type destinationMapping struct { + Index int + Raw []byte +} + +var ( + destinationsByIndex = make(map[int]destinationMapping) + destinationsByRegion = make(map[string]destinationMapping) +) + +func BuildDestinationMapping() { + count := 0 + for idx, value := range viper.GetStringSlice("destinations") { + var parsed models.BaseDestination + raw, _ := jsoniter.Marshal(value) + _ = jsoniter.Unmarshal(raw, &parsed) + + mapping := destinationMapping{ + Index: idx, + Raw: raw, + } + + if len(parsed.Region) > 0 { + destinationsByIndex[idx] = mapping + destinationsByRegion[parsed.Region] = mapping + } + + count++ + } + + log.Info().Int("count", count).Msg("Destinations mapping built") +} diff --git a/pkg/internal/services/opener.go b/pkg/internal/services/opener.go new file mode 100644 index 0000000..abd0aa9 --- /dev/null +++ b/pkg/internal/services/opener.go @@ -0,0 +1,155 @@ +package services + +import ( + "context" + "encoding/json" + "fmt" + "math/rand/v2" + nurl "net/url" + "path/filepath" + "time" + + localCache "git.solsynth.dev/hypernet/paperclip/pkg/internal/cache" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/database" + "git.solsynth.dev/hypernet/paperclip/pkg/internal/models" + "github.com/eko/gocache/lib/v4/cache" + "github.com/eko/gocache/lib/v4/marshaler" + "github.com/eko/gocache/lib/v4/store" + jsoniter "github.com/json-iterator/go" + "github.com/samber/lo" +) + +type openAttachmentResult struct { + Attachment models.Attachment `json:"attachment"` + Boosts []models.AttachmentBoost `json:"boost"` +} + +func GetAttachmentOpenCacheKey(rid string) any { + return fmt.Sprintf("attachment-open#%s", rid) +} + +func OpenAttachmentByRID(rid string, region ...string) (url string, mimetype string, err error) { + cacheManager := cache.New[any](localCache.S) + marshal := marshaler.New(cacheManager) + contx := context.Background() + + var result *openAttachmentResult + if val, err := marshal.Get( + contx, + GetAttachmentOpenCacheKey(rid), + new(openAttachmentResult), + ); err == nil { + result = val.(*openAttachmentResult) + } + + if result == nil { + var attachment models.Attachment + if err = database.C.Where(models.Attachment{ + Rid: rid, + }). + Preload("Pool"). + Preload("Thumbnail"). + Preload("Compressed"). + Preload("Boosts"). + First(&attachment).Error; err != nil { + return + } + + var boosts []models.AttachmentBoost + boosts, err = ListBoostByAttachment(attachment.ID) + if err != nil { + return + } + + result = &openAttachmentResult{ + Attachment: attachment, + Boosts: boosts, + } + } + + if len(result.Attachment.MimeType) > 0 { + mimetype = result.Attachment.MimeType + } + + var dest models.BaseDestination + var rawDest []byte + + if len(region) > 0 { + if des, ok := destinationsByRegion[region[0]]; ok { + for _, boost := range result.Boosts { + if boost.Destination == des.Index { + rawDest = des.Raw + json.Unmarshal(rawDest, &dest) + } + } + } + } + if rawDest == nil { + if len(result.Boosts) > 0 { + randomIdx := rand.IntN(len(result.Boosts)) + if des, ok := destinationsByIndex[randomIdx]; ok { + rawDest = des.Raw + json.Unmarshal(rawDest, &dest) + } + } else { + if des, ok := destinationsByIndex[result.Attachment.Destination]; ok { + rawDest = des.Raw + json.Unmarshal(rawDest, &dest) + } + } + } + + if rawDest == nil { + err = fmt.Errorf("no destination found") + return + } + + switch dest.Type { + case models.DestinationTypeLocal: + var destConfigured models.LocalDestination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + url = "file://" + filepath.Join(destConfigured.Path, result.Attachment.Uuid) + return + case models.DestinationTypeS3: + var destConfigured models.S3Destination + _ = jsoniter.Unmarshal(rawDest, &destConfigured) + if len(destConfigured.AccessBaseURL) > 0 { + url = fmt.Sprintf( + "%s/%s", + destConfigured.AccessBaseURL, + nurl.QueryEscape(filepath.Join(destConfigured.Path, result.Attachment.Uuid)), + ) + } else { + protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http") + url = fmt.Sprintf( + "%s://%s.%s/%s", + protocol, + destConfigured.Bucket, + destConfigured.Endpoint, + nurl.QueryEscape(filepath.Join(destConfigured.Path, result.Attachment.Uuid)), + ) + } + return + default: + err = fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) + return + } +} + +func CacheOpenAttachment(item *openAttachmentResult) { + if item == nil { + return + } + + cacheManager := cache.New[any](localCache.S) + marshal := marshaler.New(cacheManager) + contx := context.Background() + + _ = marshal.Set( + contx, + GetAttachmentCacheKey(item.Attachment.Rid), + *item, + store.WithExpiration(60*time.Minute), + store.WithTags([]string{"attachment-open", fmt.Sprintf("user#%s", item.Attachment.Rid)}), + ) +} diff --git a/pkg/main.go b/pkg/main.go index a859823..44aaefd 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -93,6 +93,7 @@ func main() { go grpc.NewGrpc().Listen() // Post-boot actions + services.BuildDestinationMapping() services.ScanUnanalyzedFileFromDatabase() fs.RunMarkLifecycleDeletionTask()