Attachment has pool

This commit is contained in:
LittleSheep 2024-08-18 14:09:52 +08:00
parent dd0f7399a6
commit a82fb3a49c
11 changed files with 102 additions and 32 deletions

View File

@ -4,17 +4,18 @@
<option name="autoReloadType" value="ALL" /> <option name="autoReloadType" value="ALL" />
</component> </component>
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: List attachment original filter"> <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Attachment pool basis">
<change afterPath="$PROJECT_DIR$/pkg/internal/models/pools.go" afterDir="false" /> <change afterPath="$PROJECT_DIR$/pkg/internal/services/random_id.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/pkg/internal/server/api/pools_api.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/pkg/internal/services/pools.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/database/migrator.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/database/migrator.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/go.mod" beforeDir="false" afterPath="$PROJECT_DIR$/go.mod" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/accounts.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/accounts.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/go.sum" beforeDir="false" afterPath="$PROJECT_DIR$/go.sum" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/metadata.go" beforeDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/models/pools.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/pools.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/index.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/index.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/pools.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/pools.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -127,7 +128,8 @@
<MESSAGE value=":recycle: Moved onto dealer" /> <MESSAGE value=":recycle: Moved onto dealer" />
<MESSAGE value=":sparkles: Self reference detection" /> <MESSAGE value=":sparkles: Self reference detection" />
<MESSAGE value=":sparkles: List attachment original filter" /> <MESSAGE value=":sparkles: List attachment original filter" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: List attachment original filter" /> <MESSAGE value=":sparkles: Attachment pool basis" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Attachment pool basis" />
</component> </component>
<component name="VgoProject"> <component name="VgoProject">
<settings-migrated>true</settings-migrated> <settings-migrated>true</settings-migrated>

6
go.mod
View File

@ -8,12 +8,15 @@ require (
github.com/gofiber/fiber/v2 v2.52.4 github.com/gofiber/fiber/v2 v2.52.4
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/json-iterator/go v1.1.12 github.com/json-iterator/go v1.1.12
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
github.com/minio/minio-go/v7 v7.0.70 github.com/minio/minio-go/v7 v7.0.70
github.com/robfig/cron/v3 v3.0.1 github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.31.0 github.com/rs/zerolog v1.31.0
github.com/samber/lo v1.39.0 github.com/samber/lo v1.39.0
github.com/schollz/progressbar/v3 v3.14.4
github.com/spf13/viper v1.18.2 github.com/spf13/viper v1.18.2
google.golang.org/grpc v1.64.0 google.golang.org/grpc v1.64.0
gopkg.in/vansante/go-ffprobe.v2 v2.2.0
gorm.io/datatypes v1.2.0 gorm.io/datatypes v1.2.0
gorm.io/driver/postgres v1.5.4 gorm.io/driver/postgres v1.5.4
gorm.io/gorm v1.25.6 gorm.io/gorm v1.25.6
@ -48,7 +51,6 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect github.com/jinzhu/now v1.1.5 // indirect
github.com/jpillora/backoff v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/leodido/go-urn v1.2.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect
@ -70,7 +72,6 @@ require (
github.com/rs/xid v1.5.0 // indirect github.com/rs/xid v1.5.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/schollz/progressbar/v3 v3.14.4 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cast v1.6.0 // indirect
@ -91,7 +92,6 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
google.golang.org/protobuf v1.34.2 // indirect google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/vansante/go-ffprobe.v2 v2.2.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/mysql v1.5.2 // indirect gorm.io/driver/mysql v1.5.2 // indirect
) )

2
go.sum
View File

@ -362,8 +362,6 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

View File

@ -15,11 +15,14 @@ const (
type Attachment struct { type Attachment struct {
BaseModel BaseModel
Uuid string `json:"uuid"` // Random ID is for accessing (appear in URL)
Rid string `json:"rid" gorm:"uniqueIndex"`
// Unique ID is for storing (appear in local file name or object name)
Uuid string `json:"uuid"`
Size int64 `json:"size"` Size int64 `json:"size"`
Name string `json:"name"` Name string `json:"name"`
Alternative string `json:"alt"` Alternative string `json:"alt"`
Usage string `json:"usage"`
MimeType string `json:"mimetype"` MimeType string `json:"mimetype"`
HashCode string `json:"hash"` HashCode string `json:"hash"`
Destination AttachmentDst `json:"destination"` Destination AttachmentDst `json:"destination"`

View File

@ -17,6 +17,8 @@ type AttachmentPool struct {
} }
type AttachmentPoolConfig struct { type AttachmentPoolConfig struct {
ExistLifecycle *int `json:"exist_lifecycle"` MaxFileSize *int64 `json:"max_file_size"`
IsPublicAccessible bool `json:"is_public_accessible"` ExistLifecycle *int64 `json:"exist_lifecycle"`
AllowCrossPoolIngress bool `json:"allow_cross_pool_ingress"`
AllowCrossPoolEgress bool `json:"allow_cross_pool_egress"`
} }

View File

@ -85,9 +85,18 @@ func createAttachment(c *fiber.Ctx) error {
} }
user := c.Locals("user").(models.Account) user := c.Locals("user").(models.Account)
usage := c.FormValue("usage") poolAlias := c.FormValue("pool")
if !lo.Contains(viper.GetStringSlice("accepts_usage"), usage) { if len(poolAlias) == 0 {
return fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("disallowed usage: %s", usage)) poolAlias = c.FormValue("usage")
}
aliasingMap := viper.GetStringMapString("pools.aliases")
if val, ok := aliasingMap[poolAlias]; ok {
poolAlias = val
}
pool, err := services.GetAttachmentPoolByAlias(poolAlias)
if err != nil {
return fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("unable to get attachment pool info: %v", err))
} }
file, err := c.FormFile("file") file, err := c.FormFile("file")
@ -97,6 +106,8 @@ func createAttachment(c *fiber.Ctx) error {
if err = gap.H.EnsureGrantedPerm(c, "CreateAttachments", file.Size); err != nil { if err = gap.H.EnsureGrantedPerm(c, "CreateAttachments", file.Size); err != nil {
return err return err
} else if pool.Config.Data().MaxFileSize != nil && file.Size > *pool.Config.Data().MaxFileSize {
return fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("attachment pool %s doesn't allow file larger than %d", pool.Alias, *pool.Config.Data().MaxFileSize))
} }
usermeta := make(map[string]any) usermeta := make(map[string]any)
@ -105,13 +116,14 @@ func createAttachment(c *fiber.Ctx) error {
tx := database.C.Begin() tx := database.C.Begin()
metadata, err := services.NewAttachmentMetadata(tx, user, file, models.Attachment{ metadata, err := services.NewAttachmentMetadata(tx, user, file, models.Attachment{
Usage: usage,
Alternative: c.FormValue("alt"), Alternative: c.FormValue("alt"),
MimeType: c.FormValue("mimetype"), MimeType: c.FormValue("mimetype"),
Metadata: usermeta, Metadata: usermeta,
IsMature: len(c.FormValue("mature")) > 0, IsMature: len(c.FormValue("mature")) > 0,
IsAnalyzed: false, IsAnalyzed: false,
Destination: models.AttachmentDstTemporary, Destination: models.AttachmentDstTemporary,
Pool: &pool,
PoolID: &pool.ID,
}) })
if err != nil { if err != nil {
tx.Rollback() tx.Rollback()
@ -126,6 +138,7 @@ func createAttachment(c *fiber.Ctx) error {
tx.Commit() tx.Commit()
metadata.Account = user metadata.Account = user
metadata.Pool = &pool
services.PublishAnalyzeTask(metadata) services.PublishAnalyzeTask(metadata)
return c.JSON(metadata) return c.JSON(metadata)
@ -141,7 +154,6 @@ func updateAttachmentMeta(c *fiber.Ctx) error {
var data struct { var data struct {
Alternative string `json:"alt"` Alternative string `json:"alt"`
Usage string `json:"usage"`
IsMature bool `json:"is_mature"` IsMature bool `json:"is_mature"`
} }
@ -155,7 +167,6 @@ func updateAttachmentMeta(c *fiber.Ctx) error {
} }
attachment.Alternative = data.Alternative attachment.Alternative = data.Alternative
attachment.Usage = data.Usage
attachment.IsMature = data.IsMature attachment.IsMature = data.IsMature
if attachment, err := services.UpdateAttachment(attachment); err != nil { if attachment, err := services.UpdateAttachment(attachment); err != nil {

View File

@ -99,7 +99,7 @@ func AnalyzeAttachment(file models.Attachment) error {
var start time.Time var start time.Time
// Do analyze job // Do analyze jobs
if !file.IsAnalyzed || len(file.HashCode) == 0 { if !file.IsAnalyzed || len(file.HashCode) == 0 {
destMap := viper.GetStringMap("destinations.temporary") destMap := viper.GetStringMap("destinations.temporary")

View File

@ -28,7 +28,7 @@ func GetAttachmentByID(id uint) (models.Attachment, error) {
var attachment models.Attachment var attachment models.Attachment
if err := database.C.Where(models.Attachment{ if err := database.C.Where(models.Attachment{
BaseModel: models.BaseModel{ID: id}, BaseModel: models.BaseModel{ID: id},
}).Preload("Account").First(&attachment).Error; err != nil { }).Preload("Pool").Preload("Account").First(&attachment).Error; err != nil {
return attachment, err return attachment, err
} else { } else {
MaintainAttachmentCache() MaintainAttachmentCache()
@ -42,7 +42,7 @@ func GetAttachmentByHash(hash string) (models.Attachment, error) {
var attachment models.Attachment var attachment models.Attachment
if err := database.C.Where(models.Attachment{ if err := database.C.Where(models.Attachment{
HashCode: hash, HashCode: hash,
}).First(&attachment).Error; err != nil { }).Preload("Pool").First(&attachment).Error; err != nil {
return attachment, err return attachment, err
} }
return attachment, nil return attachment, nil
@ -60,6 +60,7 @@ func CacheAttachment(id uint, item models.Attachment) {
} }
func NewAttachmentMetadata(tx *gorm.DB, user models.Account, file *multipart.FileHeader, attachment models.Attachment) (models.Attachment, error) { func NewAttachmentMetadata(tx *gorm.DB, user models.Account, file *multipart.FileHeader, attachment models.Attachment) (models.Attachment, error) {
attachment.Rid = RandString(16)
attachment.Uuid = uuid.NewString() attachment.Uuid = uuid.NewString()
attachment.Size = file.Size attachment.Size = file.Size
attachment.Name = file.Filename attachment.Name = file.Filename
@ -104,6 +105,13 @@ func TryLinkAttachment(tx *gorm.DB, og models.Attachment, hash string) (bool, er
return false, err return false, err
} }
if prev.PoolID != nil && og.PoolID != nil && prev.PoolID != og.PoolID {
if !prev.Pool.Config.Data().AllowCrossPoolEgress || !og.Pool.Config.Data().AllowCrossPoolIngress {
// Pool config doesn't allow reference
return false, nil
}
}
prev.RefCount++ prev.RefCount++
og.RefID = &prev.ID og.RefID = &prev.ID
og.Uuid = prev.Uuid og.Uuid = prev.Uuid

View File

@ -21,6 +21,14 @@ func GetAttachmentPool(id uint) (models.AttachmentPool, error) {
return pool, nil return pool, nil
} }
func GetAttachmentPoolByAlias(alias string) (models.AttachmentPool, error) {
var pool models.AttachmentPool
if err := database.C.Where("alias = ?", alias).First(&pool).Error; err != nil {
return pool, err
}
return pool, nil
}
func GetAttachmentPoolWithUser(id uint, userId uint) (models.AttachmentPool, error) { func GetAttachmentPoolWithUser(id uint, userId uint) (models.AttachmentPool, error) {
var pool models.AttachmentPool var pool models.AttachmentPool
if err := database.C.Where("id = ? AND account_id = ?", id, userId).First(&pool).Error; err != nil { if err := database.C.Where("id = ? AND account_id = ?", id, userId).First(&pool).Error; err != nil {

View File

@ -0,0 +1,31 @@
package services
import (
"math/rand"
"strings"
)
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterIdxBits = 6
letterIdxMask = 1<<letterIdxBits - 1
letterIdxMax = 63 / letterIdxBits
)
func RandString(length int) string {
builder := strings.Builder{}
builder.Grow(length)
for idx, cache, remain := length-1, rand.Int63(), letterIdxMax; idx >= 0; {
if remain == 0 {
cache, remain = rand.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
builder.WriteByte(letterBytes[idx])
idx--
}
cache >>= letterIdxBits
remain--
}
return builder.String()
}

View File

@ -5,11 +5,11 @@ grpc_bind = "0.0.0.0:7443"
domain = "usercontent.solsynth.dev" domain = "usercontent.solsynth.dev"
accepts_usage = [ accepts_usage = [
"p.avatar", "p.avatar",
"p.banner", "p.banner",
"i.attachment", "i.attachment",
"m.attachment", "m.attachment",
"sticker", "sticker",
] ]
[workers] [workers]
@ -33,6 +33,13 @@ refresh_token_duration = 2592000
dsn = "host=localhost user=postgres password=password dbname=hy_paperclip port=5432 sslmode=disable" dsn = "host=localhost user=postgres password=password dbname=hy_paperclip port=5432 sslmode=disable"
prefix = "paperclip_" prefix = "paperclip_"
[pools.aliases]
"p.avatar" = "avatar"
"p.banner" = "avatar"
"i.attachment" = "interactive"
"m.attachment" = "messaging"
"sticker" = "sticker"
[destinations.temporary] [destinations.temporary]
type = "local" type = "local"
path = "uploads" path = "uploads"