HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ 支援分片上传 #3

Merged
LittleSheep merged 3 commits from feature/multipart-upload into master 2024-08-20 14:57:06 +00:00
Conversation 1 Commits 3 Files Changed 14 +21 -24
6 changed files with 21 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 37c47f9839 - Show all commits

19
.idea/workspace.xml
View File

@ -4,19 +4,12 @@
<option name="autoReloadType" value="ALL" />
</component>
<component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Un-public indexable &amp; select by pools">
<change afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_direct_api.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" />
<change afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Multipart file upload">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/index.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/index.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/random_id.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/random_id.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/main.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
@ -128,7 +121,6 @@
</option>
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value=":truck: Update url mapping" />
<MESSAGE value=":bug: Fix uuid duplicate when link exists" />
<MESSAGE value="&#10;:sparkles: Add health check" />
<MESSAGE value=":arrow_up: Upgrade Passport and use Hyper SDK" />
@ -153,7 +145,8 @@
<MESSAGE value=":boom: Use attachment rid instead of primary key when create" />
<MESSAGE value=":bug: Fix crash on maintain cache" />
<MESSAGE value=":sparkles: Un-public indexable &amp; select by pools" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Un-public indexable &amp; select by pools" />
<MESSAGE value=":sparkles: Multipart file upload" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Multipart file upload" />
</component>
<component name="VgoProject">
<settings-migrated>true</settings-migrated>

1
pkg/internal/models/attachments.go
View File

@ -25,7 +25,6 @@ type Attachment struct {
Alternative string `json:"alt"`
MimeType string `json:"mimetype"`
HashCode string `json:"hash"`
UserHash *string `json:"user_hash"`
Destination AttachmentDst `json:"destination"`
RefCount int `json:"ref_count"`

7
pkg/internal/server/api/up_multipart_api.go
View File

@ -19,7 +19,6 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
var data struct {
Pool string `json:"pool" validate:"required"`
Size int64 `json:"size" validate:"required"`
Hash string `json:"hash" validate:"required"`
Alternative string `json:"alt"`
MimeType string `json:"mimetype"`
Metadata map[string]any `json:"metadata"`
@ -43,7 +42,6 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
}
metadata, err := services.NewAttachmentPlaceholder(database.C, user, models.Attachment{
UserHash: &data.Hash,
Alternative: data.Alternative,
MimeType: data.MimeType,
Metadata: data.Metadata,
@ -57,7 +55,10 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
return fiber.NewError(fiber.StatusBadRequest, err.Error())
}
return c.JSON(metadata)
return c.JSON(fiber.Map{
"chunk_size": viper.GetInt64("performance.file_chunk_size"),
"meta": metadata,
})
}
func uploadAttachmentMultipart(c *fiber.Ctx) error {

14
pkg/internal/services/analyzer.go
View File

@ -99,6 +99,14 @@ func AnalyzeAttachment(file models.Attachment) error {
var start time.Time
if len(file.HashCode) == 0 {
if hash, err := HashAttachment(file); err != nil {
return err
} else {
file.HashCode = hash
}
}
// Do analyze jobs
if !file.IsAnalyzed || len(file.HashCode) == 0 {
destMap := viper.GetStringMap("destinations.temporary")
@ -158,12 +166,6 @@ func AnalyzeAttachment(file models.Attachment) error {
"color_space": stream.ColorSpace,
}
}
if hash, err := HashAttachment(file); err != nil {
return err
} else {
file.HashCode = hash
}
}
tx := database.C.Begin()

2
pkg/internal/services/merger.go
View File

@ -44,5 +44,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
meta.IsUploaded = true
database.C.Save(&meta)
PublishAnalyzeTask(meta)
return meta, nil
}

2
settings.toml
View File

@ -30,7 +30,7 @@ access_token_duration = 300
refresh_token_duration = 2592000
[performance]
file_chunk_size = 5242880
file_chunk_size = 26214400
[database]
dsn = "host=localhost user=postgres password=password dbname=hy_paperclip port=5432 sslmode=disable"