HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Fix concurrent upload multipart cause incomplete

Browse Source
This commit is contained in:
LittleSheep 2024-08-22 01:30:14 +08:00
parent 10eb0ba3bc
commit 644af9e7b2
4 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.idea/workspace.xml generated
View File

@ -4,9 +4,10 @@
<option name="autoReloadType" value="ALL" /> <option name="autoReloadType" value="ALL" />
</component> </component>
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Fix direct upload will save as non-uploaded"> <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Fix uploader still using old cache api">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/attachments.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
@ -97,8 +98,8 @@
<component name="SharedIndexes"> <component name="SharedIndexes">
<attachedChunks> <attachedChunks>
<set> <set>
<option value="bundled-gosdk-5df93f7ad4aa-dfc284eb1eb8-org.jetbrains.plugins.go.sharedIndexes.bundled-GO-242.20224.306" /> <option value="bundled-gosdk-5df93f7ad4aa-dfc284eb1eb8-org.jetbrains.plugins.go.sharedIndexes.bundled-GO-242.20224.424" />
<option value="bundled-js-predefined-d6986cc7102b-410509235cf1-JavaScript-GO-242.20224.306" /> <option value="bundled-js-predefined-d6986cc7102b-410509235cf1-JavaScript-GO-242.20224.424" />
</set> </set>
</attachedChunks> </attachedChunks>
</component> </component>
@ -118,7 +119,6 @@
</option> </option>
</component> </component>
<component name="VcsManagerConfiguration"> <component name="VcsManagerConfiguration">
<MESSAGE value=":ambulance: Fix getting user panic" />
<MESSAGE value=":zap: Add cache into metadata fetching" /> <MESSAGE value=":zap: Add cache into metadata fetching" />
<MESSAGE value=":recycle: Moved onto dealer" /> <MESSAGE value=":recycle: Moved onto dealer" />
<MESSAGE value=":sparkles: Self reference detection" /> <MESSAGE value=":sparkles: Self reference detection" />
@ -143,7 +143,8 @@
<MESSAGE value=":bug: Fix merged file isn't add to cache" /> <MESSAGE value=":bug: Fix merged file isn't add to cache" />
<MESSAGE value=":bug: Fix analyzer didn't cache attachment" /> <MESSAGE value=":bug: Fix analyzer didn't cache attachment" />
<MESSAGE value=":bug: Fix direct upload will save as non-uploaded" /> <MESSAGE value=":bug: Fix direct upload will save as non-uploaded" />
<option name="LAST_COMMIT_MESSAGE" value=":bug: Fix direct upload will save as non-uploaded" /> <MESSAGE value=":bug: Fix uploader still using old cache api" />
<option name="LAST_COMMIT_MESSAGE" value=":bug: Fix uploader still using old cache api" />
</component> </component>
<component name="VgoProject"> <component name="VgoProject">
<settings-migrated>true</settings-migrated> <settings-migrated>true</settings-migrated>

4
pkg/internal/services/merger.go
View File

@ -27,7 +27,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
// Merge files // Merge files
for _, chunk := range arrange { for _, chunk := range arrange {
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk)) chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, chunk))
chunkFile, err := os.Open(chunkPath) chunkFile, err := os.Open(chunkPath)
if err != nil { if err != nil {
return meta, err return meta, err
@ -52,7 +52,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
// Clean up // Clean up
for _, chunk := range arrange { for _, chunk := range arrange {
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk)) chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, chunk))
_ = os.Remove(chunkPath) _ = os.Remove(chunkPath)
} }

2
pkg/internal/services/recycler.go
View File

@ -104,7 +104,7 @@ func DeleteFile(meta models.Attachment) error {
_ = jsoniter.Unmarshal(rawDest, &dest) _ = jsoniter.Unmarshal(rawDest, &dest)
for cid := range meta.FileChunks { for cid := range meta.FileChunks {
path := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid)) path := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
_ = os.Remove(path) _ = os.Remove(path)
} }

9
pkg/internal/services/uploader.go
View File

@ -46,7 +46,12 @@ func UploadChunkToTemporary(ctx *fiber.Ctx, cid string, file *multipart.FileHead
case models.DestinationTypeLocal: case models.DestinationTypeLocal:
var destConfigured models.LocalDestination var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured) _ = jsoniter.Unmarshal(rawDest, &destConfigured)
return ctx.SaveFile(file, filepath.Join(destConfigured.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid))) tempPath := filepath.Join(destConfigured.Path, fmt.Sprintf("%s.part%s.partial", meta.Uuid, cid))
destPath := filepath.Join(destConfigured.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
if err := ctx.SaveFile(file, tempPath); err != nil {
return err
}
return os.Rename(tempPath, destPath)
default: default:
return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type) return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
} }
@ -59,7 +64,7 @@ func CheckChunkExistsInTemporary(meta models.Attachment, cid string) bool {
rawDest, _ := jsoniter.Marshal(destMap) rawDest, _ := jsoniter.Marshal(destMap)
_ = jsoniter.Unmarshal(rawDest, &dest) _ = jsoniter.Unmarshal(rawDest, &dest)
path := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid)) path := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
return false return false
} else { } else {