🐛 Fix concurrent upload multipart cause incomplete
This commit is contained in:
		
							
								
								
									
										13
									
								
								.idea/workspace.xml
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										13
									
								
								.idea/workspace.xml
									
									
									
										generated
									
									
									
								
							@@ -4,9 +4,10 @@
 | 
				
			|||||||
    <option name="autoReloadType" value="ALL" />
 | 
					    <option name="autoReloadType" value="ALL" />
 | 
				
			||||||
  </component>
 | 
					  </component>
 | 
				
			||||||
  <component name="ChangeListManager">
 | 
					  <component name="ChangeListManager">
 | 
				
			||||||
    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Fix direct upload will save as non-uploaded">
 | 
					    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Fix uploader still using old cache api">
 | 
				
			||||||
      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
 | 
					      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
 | 
				
			||||||
      <change beforePath="$PROJECT_DIR$/pkg/internal/services/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/attachments.go" afterDir="false" />
 | 
					      <change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
 | 
				
			||||||
 | 
					      <change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
 | 
				
			||||||
      <change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" />
 | 
					      <change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" />
 | 
				
			||||||
    </list>
 | 
					    </list>
 | 
				
			||||||
    <option name="SHOW_DIALOG" value="false" />
 | 
					    <option name="SHOW_DIALOG" value="false" />
 | 
				
			||||||
@@ -97,8 +98,8 @@
 | 
				
			|||||||
  <component name="SharedIndexes">
 | 
					  <component name="SharedIndexes">
 | 
				
			||||||
    <attachedChunks>
 | 
					    <attachedChunks>
 | 
				
			||||||
      <set>
 | 
					      <set>
 | 
				
			||||||
        <option value="bundled-gosdk-5df93f7ad4aa-dfc284eb1eb8-org.jetbrains.plugins.go.sharedIndexes.bundled-GO-242.20224.306" />
 | 
					        <option value="bundled-gosdk-5df93f7ad4aa-dfc284eb1eb8-org.jetbrains.plugins.go.sharedIndexes.bundled-GO-242.20224.424" />
 | 
				
			||||||
        <option value="bundled-js-predefined-d6986cc7102b-410509235cf1-JavaScript-GO-242.20224.306" />
 | 
					        <option value="bundled-js-predefined-d6986cc7102b-410509235cf1-JavaScript-GO-242.20224.424" />
 | 
				
			||||||
      </set>
 | 
					      </set>
 | 
				
			||||||
    </attachedChunks>
 | 
					    </attachedChunks>
 | 
				
			||||||
  </component>
 | 
					  </component>
 | 
				
			||||||
@@ -118,7 +119,6 @@
 | 
				
			|||||||
    </option>
 | 
					    </option>
 | 
				
			||||||
  </component>
 | 
					  </component>
 | 
				
			||||||
  <component name="VcsManagerConfiguration">
 | 
					  <component name="VcsManagerConfiguration">
 | 
				
			||||||
    <MESSAGE value=":ambulance: Fix getting user panic" />
 | 
					 | 
				
			||||||
    <MESSAGE value=":zap: Add cache into metadata fetching" />
 | 
					    <MESSAGE value=":zap: Add cache into metadata fetching" />
 | 
				
			||||||
    <MESSAGE value=":recycle: Moved onto dealer" />
 | 
					    <MESSAGE value=":recycle: Moved onto dealer" />
 | 
				
			||||||
    <MESSAGE value=":sparkles: Self reference detection" />
 | 
					    <MESSAGE value=":sparkles: Self reference detection" />
 | 
				
			||||||
@@ -143,7 +143,8 @@
 | 
				
			|||||||
    <MESSAGE value=":bug: Fix merged file isn't add to cache" />
 | 
					    <MESSAGE value=":bug: Fix merged file isn't add to cache" />
 | 
				
			||||||
    <MESSAGE value=":bug: Fix analyzer didn't cache attachment" />
 | 
					    <MESSAGE value=":bug: Fix analyzer didn't cache attachment" />
 | 
				
			||||||
    <MESSAGE value=":bug: Fix direct upload will save as non-uploaded" />
 | 
					    <MESSAGE value=":bug: Fix direct upload will save as non-uploaded" />
 | 
				
			||||||
    <option name="LAST_COMMIT_MESSAGE" value=":bug: Fix direct upload will save as non-uploaded" />
 | 
					    <MESSAGE value=":bug: Fix uploader still using old cache api" />
 | 
				
			||||||
 | 
					    <option name="LAST_COMMIT_MESSAGE" value=":bug: Fix uploader still using old cache api" />
 | 
				
			||||||
  </component>
 | 
					  </component>
 | 
				
			||||||
  <component name="VgoProject">
 | 
					  <component name="VgoProject">
 | 
				
			||||||
    <settings-migrated>true</settings-migrated>
 | 
					    <settings-migrated>true</settings-migrated>
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -27,7 +27,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
	// Merge files
 | 
						// Merge files
 | 
				
			||||||
	for _, chunk := range arrange {
 | 
						for _, chunk := range arrange {
 | 
				
			||||||
		chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
 | 
							chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, chunk))
 | 
				
			||||||
		chunkFile, err := os.Open(chunkPath)
 | 
							chunkFile, err := os.Open(chunkPath)
 | 
				
			||||||
		if err != nil {
 | 
							if err != nil {
 | 
				
			||||||
			return meta, err
 | 
								return meta, err
 | 
				
			||||||
@@ -52,7 +52,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
	// Clean up
 | 
						// Clean up
 | 
				
			||||||
	for _, chunk := range arrange {
 | 
						for _, chunk := range arrange {
 | 
				
			||||||
		chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
 | 
							chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, chunk))
 | 
				
			||||||
		_ = os.Remove(chunkPath)
 | 
							_ = os.Remove(chunkPath)
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -104,7 +104,7 @@ func DeleteFile(meta models.Attachment) error {
 | 
				
			|||||||
		_ = jsoniter.Unmarshal(rawDest, &dest)
 | 
							_ = jsoniter.Unmarshal(rawDest, &dest)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		for cid := range meta.FileChunks {
 | 
							for cid := range meta.FileChunks {
 | 
				
			||||||
			path := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid))
 | 
								path := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
 | 
				
			||||||
			_ = os.Remove(path)
 | 
								_ = os.Remove(path)
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 
 | 
				
			|||||||
@@ -46,7 +46,12 @@ func UploadChunkToTemporary(ctx *fiber.Ctx, cid string, file *multipart.FileHead
 | 
				
			|||||||
	case models.DestinationTypeLocal:
 | 
						case models.DestinationTypeLocal:
 | 
				
			||||||
		var destConfigured models.LocalDestination
 | 
							var destConfigured models.LocalDestination
 | 
				
			||||||
		_ = jsoniter.Unmarshal(rawDest, &destConfigured)
 | 
							_ = jsoniter.Unmarshal(rawDest, &destConfigured)
 | 
				
			||||||
		return ctx.SaveFile(file, filepath.Join(destConfigured.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid)))
 | 
							tempPath := filepath.Join(destConfigured.Path, fmt.Sprintf("%s.part%s.partial", meta.Uuid, cid))
 | 
				
			||||||
 | 
							destPath := filepath.Join(destConfigured.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
 | 
				
			||||||
 | 
							if err := ctx.SaveFile(file, tempPath); err != nil {
 | 
				
			||||||
 | 
								return err
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							return os.Rename(tempPath, destPath)
 | 
				
			||||||
	default:
 | 
						default:
 | 
				
			||||||
		return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
 | 
							return fmt.Errorf("invalid destination: unsupported protocol %s", dest.Type)
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
@@ -59,7 +64,7 @@ func CheckChunkExistsInTemporary(meta models.Attachment, cid string) bool {
 | 
				
			|||||||
	rawDest, _ := jsoniter.Marshal(destMap)
 | 
						rawDest, _ := jsoniter.Marshal(destMap)
 | 
				
			||||||
	_ = jsoniter.Unmarshal(rawDest, &dest)
 | 
						_ = jsoniter.Unmarshal(rawDest, &dest)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	path := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, cid))
 | 
						path := filepath.Join(dest.Path, fmt.Sprintf("%s.part%s", meta.Uuid, cid))
 | 
				
			||||||
	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
 | 
						if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
 | 
				
			||||||
		return false
 | 
							return false
 | 
				
			||||||
	} else {
 | 
						} else {
 | 
				
			||||||
 
 | 
				
			|||||||
		Reference in New Issue
	
	Block a user