HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Refactored opener

Browse Source 
✨ Support boost in opener
This commit is contained in:
LittleSheep 2024-12-28 23:10:57 +08:00
parent 8888f7661a
commit 49a8159e35
6 changed files with 220 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/internal/models/destination.go
View File

@ -6,9 +6,10 @@ const (
)
type BaseDestination struct {
Type string `json:"type"`
Label string `json:"label"`
LabelRegion string `json:"label_region"`
Type string `json:"type"`
Label string `json:"label"`
Region string `json:"region"`
IsBoost bool `json:"is_boost"`
}
type LocalDestination struct {

67
pkg/internal/server/api/attachments_api.go
View File

@ -2,8 +2,7 @@ package api
import (
"fmt"
"net/url"
"path/filepath"
"strings"
"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
@ -12,64 +11,32 @@ import (
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
"github.com/gofiber/fiber/v2"
jsoniter "github.com/json-iterator/go"
"github.com/samber/lo"
"github.com/spf13/viper"
)
func openAttachment(c *fiber.Ctx) error {
id := c.Params("id")
region := c.Query("region")
var err error
var url, mimetype string
if len(region) > 0 {
url, mimetype, err = services.OpenAttachmentByRID(id, region)
} else {
url, mimetype, err = services.OpenAttachmentByRID(id)
}
metadata, err := services.GetAttachmentByRID(id)
if err != nil {
return fiber.NewError(fiber.StatusNotFound)
return fiber.NewError(fiber.StatusNotFound, err.Error())
}
destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", metadata.Destination))
c.Set(fiber.HeaderContentType, mimetype)
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest)
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
if len(destConfigured.AccessBaseURL) > 0 && !c.QueryBool("direct", false) {
// This will drop all query parameters,
// for not it's okay because the openAttachment api won't take any query parameters
return c.Redirect(fmt.Sprintf(
"%s%s?direct=true",
destConfigured.AccessBaseURL,
c.Path(),
), fiber.StatusMovedPermanently)
}
if len(metadata.MimeType) > 0 {
c.Set(fiber.HeaderContentType, metadata.MimeType)
}
return c.SendFile(filepath.Join(destConfigured.Path, metadata.Uuid))
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
if len(destConfigured.AccessBaseURL) > 0 {
return c.Redirect(fmt.Sprintf(
"%s/%s",
destConfigured.AccessBaseURL,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
), fiber.StatusMovedPermanently)
} else {
protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
return c.Redirect(fmt.Sprintf(
"%s://%s.%s/%s",
protocol,
destConfigured.Bucket,
destConfigured.Endpoint,
url.QueryEscape(filepath.Join(destConfigured.Path, metadata.Uuid)),
), fiber.StatusMovedPermanently)
}
default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
if strings.HasPrefix(url, "file://") {
fp := strings.Replace(url, "file://", "", 1)
return c.SendFile(fp)
}
return c.Redirect(url, fiber.StatusFound)
}
func getAttachmentMeta(c *fiber.Ctx) error {

4
pkg/internal/server/api/destinations.go → pkg/internal/server/api/destinations_api.go
View File

@ -8,9 +8,9 @@ import (
)
func listDestination(c *fiber.Ctx) error {
var destinations []models.LocalDestination
var destinations []models.BaseDestination
for _, value := range viper.GetStringSlice("destinations") {
var parsed models.LocalDestination
var parsed models.BaseDestination
raw, _ := jsoniter.Marshal(value)
_ = jsoniter.Unmarshal(raw, &parsed)
destinations = append(destinations, parsed)

41
pkg/internal/services/destinations.go Normal file
View File

@ -0,0 +1,41 @@
package services
import (
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
jsoniter "github.com/json-iterator/go"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
)
type destinationMapping struct {
Index int
Raw []byte
}
var (
destinationsByIndex = make(map[int]destinationMapping)
destinationsByRegion = make(map[string]destinationMapping)
)
func BuildDestinationMapping() {
count := 0
for idx, value := range viper.GetStringSlice("destinations") {
var parsed models.BaseDestination
raw, _ := jsoniter.Marshal(value)
_ = jsoniter.Unmarshal(raw, &parsed)
mapping := destinationMapping{
Index: idx,
Raw: raw,
}
if len(parsed.Region) > 0 {
destinationsByIndex[idx] = mapping
destinationsByRegion[parsed.Region] = mapping
}
count++
}
log.Info().Int("count", count).Msg("Destinations mapping built")
}

155
pkg/internal/services/opener.go Normal file
View File

@ -0,0 +1,155 @@
package services
import (
"context"
"encoding/json"
"fmt"
"math/rand/v2"
nurl "net/url"
"path/filepath"
"time"
localCache "git.solsynth.dev/hypernet/paperclip/pkg/internal/cache"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
"git.solsynth.dev/hypernet/paperclip/pkg/internal/models"
"github.com/eko/gocache/lib/v4/cache"
"github.com/eko/gocache/lib/v4/marshaler"
"github.com/eko/gocache/lib/v4/store"
jsoniter "github.com/json-iterator/go"
"github.com/samber/lo"
)
type openAttachmentResult struct {
Attachment models.Attachment `json:"attachment"`
Boosts []models.AttachmentBoost `json:"boost"`
}
func GetAttachmentOpenCacheKey(rid string) any {
return fmt.Sprintf("attachment-open#%s", rid)
}
func OpenAttachmentByRID(rid string, region ...string) (url string, mimetype string, err error) {
cacheManager := cache.New[any](localCache.S)
marshal := marshaler.New(cacheManager)
contx := context.Background()
var result *openAttachmentResult
if val, err := marshal.Get(
contx,
GetAttachmentOpenCacheKey(rid),
new(openAttachmentResult),
); err == nil {
result = val.(*openAttachmentResult)
}
if result == nil {
var attachment models.Attachment
if err = database.C.Where(models.Attachment{
Rid: rid,
}).
Preload("Pool").
Preload("Thumbnail").
Preload("Compressed").
Preload("Boosts").
First(&attachment).Error; err != nil {
return
}
var boosts []models.AttachmentBoost
boosts, err = ListBoostByAttachment(attachment.ID)
if err != nil {
return
}
result = &openAttachmentResult{
Attachment: attachment,
Boosts: boosts,
}
}
if len(result.Attachment.MimeType) > 0 {
mimetype = result.Attachment.MimeType
}
var dest models.BaseDestination
var rawDest []byte
if len(region) > 0 {
if des, ok := destinationsByRegion[region[0]]; ok {
for _, boost := range result.Boosts {
if boost.Destination == des.Index {
rawDest = des.Raw
json.Unmarshal(rawDest, &dest)
}
}
}
}
if rawDest == nil {
if len(result.Boosts) > 0 {
randomIdx := rand.IntN(len(result.Boosts))
if des, ok := destinationsByIndex[randomIdx]; ok {
rawDest = des.Raw
json.Unmarshal(rawDest, &dest)
}
} else {
if des, ok := destinationsByIndex[result.Attachment.Destination]; ok {
rawDest = des.Raw
json.Unmarshal(rawDest, &dest)
}
}
}
if rawDest == nil {
err = fmt.Errorf("no destination found")
return
}
switch dest.Type {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
url = "file://" + filepath.Join(destConfigured.Path, result.Attachment.Uuid)
return
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
if len(destConfigured.AccessBaseURL) > 0 {
url = fmt.Sprintf(
"%s/%s",
destConfigured.AccessBaseURL,
nurl.QueryEscape(filepath.Join(destConfigured.Path, result.Attachment.Uuid)),
)
} else {
protocol := lo.Ternary(destConfigured.EnableSSL, "https", "http")
url = fmt.Sprintf(
"%s://%s.%s/%s",
protocol,
destConfigured.Bucket,
destConfigured.Endpoint,
nurl.QueryEscape(filepath.Join(destConfigured.Path, result.Attachment.Uuid)),
)
}
return
default:
err = fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
return
}
}
func CacheOpenAttachment(item *openAttachmentResult) {
if item == nil {
return
}
cacheManager := cache.New[any](localCache.S)
marshal := marshaler.New(cacheManager)
contx := context.Background()
_ = marshal.Set(
contx,
GetAttachmentCacheKey(item.Attachment.Rid),
*item,
store.WithExpiration(60*time.Minute),
store.WithTags([]string{"attachment-open", fmt.Sprintf("user#%s", item.Attachment.Rid)}),
)
}

1
pkg/main.go
View File

@ -93,6 +93,7 @@ func main() {
go grpc.NewGrpc().Listen()
// Post-boot actions
services.BuildDestinationMapping()
services.ScanUnanalyzedFileFromDatabase()
fs.RunMarkLifecycleDeletionTask()