✨ Referenced external attachment

.gitignore

@@ -3,3 +3,4 @@
 /keys
 
 .DS_Store
+.idea

pkg/filekit/models/attachments.go

@@ -14,7 +14,8 @@ import (
 )
 
 const (
-	AttachmentDstTemporary = 0 // The destination 0 is a reserved config for pre-upload processing
+	AttachmentDstExternal  = -1 // The destination marked the file did not store inside our service
+	AttachmentDstTemporary = 0  // Destination 0 is a reserved config for pre-upload processing
 )
 
 const (
@@ -63,6 +64,7 @@ type Attachment struct {
 
 	Ref    *Attachment `json:"ref"`
 	RefID  *uint       `json:"ref_id"`
+	RefURL *string     `json:"ref_url"` // External URL for the attachment
 
 	Pool   *AttachmentPool `json:"pool"`
 	PoolID *uint           `json:"pool_id"`
@@ -71,15 +73,13 @@ type Attachment struct {
 
 	AccountID uint           `json:"account_id"`
 	Account   models.Account `gorm:"-" json:"account"`
-
-	// Outdated fields, just for backward compatibility
-	FileChunks datatypes.JSONMap `json:"file_chunks" gorm:"-"`
-	IsUploaded bool              `json:"is_uploaded" gorm:"-"`
-	IsMature   bool              `json:"is_mature" gorm:"-"`
 }
 
 func (v *Attachment) AfterUpdate(tx *gorm.DB) error {
-	cachekit.FKey(cachekit.DAAttachment, v.Rid)
+	_ = cachekit.Delete(
+		gap.Ca,
+		cachekit.FKey(cachekit.DAAttachment, v.Rid),
+	)
 
 	return nil
 }
@@ -114,7 +114,7 @@ type AttachmentFragment struct {
 }
 
 func (v *AttachmentFragment) AfterUpdate(tx *gorm.DB) error {
-	cachekit.Delete(
+	_ = cachekit.Delete(
 		gap.Ca,
 		cachekit.FKey("attachment-fragment", v.Rid),
 	)
@@ -122,7 +122,7 @@ func (v *AttachmentFragment) AfterUpdate(tx *gorm.DB) error {
 	return nil
 }
 
-func (v AttachmentFragment) ToAttachment() Attachment {
+func (v *AttachmentFragment) ToAttachment() Attachment {
 	return Attachment{
 		Rid:         v.Rid,
 		Uuid:        v.Uuid,

pkg/internal/fs/downloader.go

@@ -6,8 +6,8 @@ import (
 	"os"
 	"path/filepath"
 
-	pkg "git.solsynth.dev/hypernet/paperclip/pkg/internal"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	pkg "git.solsynth.dev/hypernet/paperclip/pkg/internal"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7"
 	"github.com/spf13/viper"

pkg/internal/fs/merger.go

@@ -6,8 +6,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"github.com/spf13/viper"
 )
 

pkg/internal/server/api/boost_api.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
 	"github.com/gofiber/fiber/v2"

pkg/internal/server/api/index.go

@@ -37,6 +37,7 @@ func MapAPIs(app *fiber.App, baseURL string) {
 			attachments.Get("/:id/meta", getAttachmentMeta)
 			attachments.Get("/:id", openAttachment)
 			attachments.Post("/", sec.ValidatorMiddleware, createAttachmentDirectly)
+			attachments.Post("/referenced", sec.ValidatorMiddleware, createAttachmentWithURL)
 			attachments.Put("/:id", sec.ValidatorMiddleware, updateAttachmentMeta)
 			attachments.Delete("/:id", sec.ValidatorMiddleware, deleteAttachment)
 		}

pkg/internal/server/api/sticker_packs_api.go

@@ -2,9 +2,9 @@ package api
 
 import (
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/gap"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
 	"git.solsynth.dev/hypernet/passport/pkg/authkit"

pkg/internal/server/api/up_direct_api.go

@@ -2,10 +2,11 @@ package api
 
 import (
 	"fmt"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
 
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
 	"github.com/gofiber/fiber/v2"
 	jsoniter "github.com/json-iterator/go"
@@ -81,3 +82,50 @@ func createAttachmentDirectly(c *fiber.Ctx) error {
 
 	return c.JSON(metadata)
 }
+
+func createAttachmentWithURL(c *fiber.Ctx) error {
+	user := c.Locals("nex_user").(*sec.UserInfo)
+
+	poolAlias := c.FormValue("pool")
+	aliasingMap := viper.GetStringMapString("pools.aliases")
+	if val, ok := aliasingMap[poolAlias]; ok {
+		poolAlias = val
+	}
+
+	var data struct {
+		URL         string         `json:"url"`
+		Metadata    map[string]any `json:"metadata"`
+		Mimetype    string         `json:"mimetype"`
+		Name        string         `json:"filename"`
+		Alternative string         `json:"alt"`
+	}
+	if err := exts.BindAndValidate(c, &data); err != nil {
+		return err
+	}
+
+	if !user.HasPermNode("CreateReferencedAttachments", true) {
+		return fiber.NewError(fiber.StatusForbidden, "you are not permitted to create attachments with URL")
+	}
+
+	pool, err := services.GetAttachmentPoolByAlias(poolAlias)
+	if err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("unable to get attachment pool info: %v", err))
+	}
+
+	attachment := models.Attachment{
+		Name:        data.Name,
+		Alternative: data.Alternative,
+		MimeType:    c.FormValue("mimetype"),
+		Usermeta:    data.Metadata,
+		IsAnalyzed:  true,
+		Destination: models.AttachmentDstExternal,
+		Pool:        &pool,
+		PoolID:      &pool.ID,
+	}
+
+	if attachment, err = services.NewRefURLAttachment(database.C, user, attachment); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	}
+
+	return c.JSON(attachment)
+}

pkg/internal/server/api/up_multipart_api.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/server/exts"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/services"
 	"github.com/gofiber/fiber/v2"

pkg/internal/services/analyzer.go

@@ -16,9 +16,9 @@ import (
 	"github.com/barasher/go-exiftool"
 	"github.com/samber/lo"
 
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/k0kubun/go-ansi"
 	"github.com/kettek/apng"

pkg/internal/services/attachments.go

@@ -158,6 +158,25 @@ func NewAttachmentMetadata(tx *gorm.DB, user *sec.UserInfo, file *multipart.File
 	return attachment, nil
 }
 
+func NewRefURLAttachment(tx *gorm.DB, user *sec.UserInfo, attachment models.Attachment) (models.Attachment, error) {
+	if attachment.RefURL == nil {
+		return attachment, fmt.Errorf("attachment doesn't have a ref url")
+	}
+
+	attachment.Uuid = uuid.NewString()
+	attachment.Rid = RandString(16)
+	attachment.Size = 0
+	attachment.Destination = models.AttachmentDstExternal
+	attachment.Type = models.AttachmentTypeNormal
+	attachment.AccountID = user.ID
+
+	if err := tx.Save(&attachment).Error; err != nil {
+		return attachment, fmt.Errorf("failed to save attachment record: %v", err)
+	}
+
+	return attachment, nil
+}
+
 func TryLinkAttachment(tx *gorm.DB, og models.Attachment, hash string) (bool, error) {
 	prev, err := GetAttachmentByHash(hash)
 	if err != nil {

pkg/internal/services/boost.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cast"

pkg/internal/services/cleaner.go

@@ -3,8 +3,8 @@ package services
 import (
 	"time"
 
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 
 	"github.com/rs/zerolog/log"
 )

pkg/internal/services/fragments.go

@@ -12,9 +12,9 @@ import (
 
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/cachekit"
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/gap"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"
 	jsoniter "github.com/json-iterator/go"

pkg/internal/services/opener.go

@@ -66,6 +66,12 @@ func OpenAttachmentByRID(rid string, preferredSize int, region ...string) (url s
 		mimetype = result.Attachment.MimeType
 	}
 
+	if result.Attachment.RefURL != nil {
+		url = *result.Attachment.RefURL
+		filesize = 0
+		return
+	}
+
 	filesize = result.Attachment.Size
 
 	var dest models.BaseDestination

pkg/internal/services/payment.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"time"
 
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/gap"
 	wproto "git.solsynth.dev/hypernet/wallet/pkg/proto"

pkg/internal/services/pools.go

@@ -1,8 +1,8 @@
 package services
 
 import (
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 )
 
 func ListAttachmentPool() ([]models.AttachmentPool, error) {

pkg/internal/services/related.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 )
 
 func SetAttachmentAsThumbnail(item models.Attachment) (models.Attachment, error) {

pkg/internal/services/sticker_packs.go

@@ -2,8 +2,8 @@ package services
 
 import (
 	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"gorm.io/gorm"
 )
 

pkg/internal/services/stickers.go

@@ -6,8 +6,8 @@ import (
 
 	"gorm.io/gorm"
 
-	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
+	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"github.com/spf13/viper"
 )
 

pkg/internal/services/uploader.go

@@ -8,9 +8,9 @@ import (
 	"os"
 	"path/filepath"
 
+	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hypernet/paperclip/pkg/internal/fs"
-	"git.solsynth.dev/hypernet/paperclip/pkg/filekit/models"
 	"github.com/gofiber/fiber/v2"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7"