🐛 Bug fixes and optimization

.idea/workspace.xml

@@ -4,13 +4,12 @@
     <option name="autoReloadType" value="ALL" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Multipart file upload">
+    <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Make it more implementable">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -45,33 +44,33 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;DefaultGoTemplateProperty&quot;: &quot;Go File&quot;,
-    &quot;Go Build.Backend.executor&quot;: &quot;Run&quot;,
-    &quot;Go 构建.Backend.executor&quot;: &quot;Run&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.go.formatter.settings.were.checked&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.go.migrated.go.modules.settings&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.go.modules.automatic.dependencies.download&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.go.modules.go.list.on.any.changes.was.set&quot;: &quot;true&quot;,
-    &quot;git-widget-placeholder&quot;: &quot;feature/multipart-upload&quot;,
-    &quot;go.import.settings.migrated&quot;: &quot;true&quot;,
-    &quot;go.sdk.automatically.set&quot;: &quot;true&quot;,
-    &quot;last_opened_file_path&quot;: &quot;/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc&quot;,
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
-    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
-    &quot;run.code.analysis.last.selected.profile&quot;: &quot;pProject Default&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;,
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "DefaultGoTemplateProperty": "Go File",
+    "Go Build.Backend.executor": "Run",
+    "Go 构建.Backend.executor": "Run",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "RunOnceActivity.go.formatter.settings.were.checked": "true",
+    "RunOnceActivity.go.migrated.go.modules.settings": "true",
+    "RunOnceActivity.go.modules.automatic.dependencies.download": "true",
+    "RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
+    "git-widget-placeholder": "feature/multipart-upload",
+    "go.import.settings.migrated": "true",
+    "go.sdk.automatically.set": "true",
+    "last_opened_file_path": "/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc",
+    "node.js.detected.package.eslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "nodejs_package_manager_path": "npm",
+    "run.code.analysis.last.selected.profile": "pProject Default",
+    "settings.editor.selected.configurable": "preferences.pluginManager",
+    "vue.rearranger.settings.migration": "true"
   },
-  &quot;keyToStringList&quot;: {
-    &quot;DatabaseDriversLRU&quot;: [
-      &quot;postgresql&quot;
+  "keyToStringList": {
+    "DatabaseDriversLRU": [
+      "postgresql"
     ]
   }
-}</component>
+}]]></component>
   <component name="RecentsManager">
     <key name="CopyFile.RECENT_KEYS">
       <recent name="$PROJECT_DIR$/pkg/internal/grpc" />
@@ -121,7 +120,6 @@
     </option>
   </component>
   <component name="VcsManagerConfiguration">
-    <MESSAGE value=":bug: Fix uuid duplicate when link exists" />
     <MESSAGE value="&#10;:sparkles: Add health check" />
     <MESSAGE value=":arrow_up: Upgrade Passport and use Hyper SDK" />
     <MESSAGE value=":arrow_up: Upgrade Passport to fix bug" />
@@ -146,7 +144,8 @@
     <MESSAGE value=":bug: Fix crash on maintain cache" />
     <MESSAGE value=":sparkles: Un-public indexable &amp; select by pools" />
     <MESSAGE value=":sparkles: Multipart file upload" />
-    <option name="LAST_COMMIT_MESSAGE" value=":sparkles: Multipart file upload" />
+    <MESSAGE value=":sparkles: Make it more implementable" />
+    <option name="LAST_COMMIT_MESSAGE" value=":sparkles: Make it more implementable" />
   </component>
   <component name="VgoProject">
     <settings-migrated>true</settings-migrated>

pkg/internal/server/api/up_multipart_api.go

@@ -1,10 +1,12 @@
 package api
 
 import (
+	"encoding/json"
 	"fmt"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/gap"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
+	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/server/exts"
 	"git.solsynth.dev/hydrogen/paperclip/pkg/internal/services"
 	"github.com/gofiber/fiber/v2"
 	"github.com/spf13/viper"
@@ -19,12 +21,17 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
 	var data struct {
 		Pool        string         `json:"pool" validate:"required"`
 		Size        int64          `json:"size" validate:"required"`
+		FileName    string         `json:"name" validate:"required"`
 		Alternative string         `json:"alt"`
 		MimeType    string         `json:"mimetype"`
 		Metadata    map[string]any `json:"metadata"`
 		IsMature    bool           `json:"is_mature"`
 	}
 
+	if err := exts.BindAndValidate(c, &data); err != nil {
+		return err
+	}
+
 	aliasingMap := viper.GetStringMapString("pools.aliases")
 	if val, ok := aliasingMap[data.Pool]; ok {
 		data.Pool = val
@@ -42,6 +49,8 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
 	}
 
 	metadata, err := services.NewAttachmentPlaceholder(database.C, user, models.Attachment{
+		Name:        data.FileName,
+		Size:        data.Size,
 		Alternative: data.Alternative,
 		MimeType:    data.MimeType,
 		Metadata:    data.Metadata,
@@ -57,6 +66,7 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
 
 	return c.JSON(fiber.Map{
 		"chunk_size":  viper.GetInt64("performance.file_chunk_size"),
+		"chunk_count": len(metadata.FileChunks),
 		"meta":        metadata,
 	})
 }
@@ -100,8 +110,9 @@ func uploadAttachmentMultipart(c *fiber.Ctx) error {
 		if !services.CheckChunkExistsInTemporary(meta, cid) {
 			isAllUploaded = false
 			break
-		} else if val, ok := idx.(int); ok {
-			chunkArrange[val] = cid
+		} else if val, ok := idx.(json.Number); ok {
+			data, _ := val.Int64()
+			chunkArrange[data] = cid
 		}
 	}
 

pkg/internal/services/analyzer.go

@@ -53,7 +53,10 @@ func ScanUnanalyzedFileFromDatabase() {
 	}
 
 	var attachments []models.Attachment
-	if err := database.C.Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).Find(&attachments).Error; err != nil {
+	if err := database.C.
+		Where("is_uploaded = ?", true).
+		Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).
+		Find(&attachments).Error; err != nil {
 		log.Error().Err(err).Msg("Scan unanalyzed files from database failed...")
 		return
 	}
@@ -93,7 +96,9 @@ func ScanUnanalyzedFileFromDatabase() {
 }
 
 func AnalyzeAttachment(file models.Attachment) error {
-	if file.Destination != models.AttachmentDstTemporary {
+	if !file.IsUploaded {
+		return fmt.Errorf("file isn't finish multipart upload")
+	} else if file.Destination != models.AttachmentDstTemporary {
 		return fmt.Errorf("attachment isn't in temporary storage, unable to analyze")
 	}
 

pkg/internal/services/merger.go

@@ -25,6 +25,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
 	}
 	defer destFile.Close()
 
+	// Merge files
 	for _, chunk := range arrange {
 		chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
 		chunkFile, err := os.Open(chunkPath)
@@ -41,10 +42,17 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
 		_ = chunkFile.Close()
 	}
 
+	// Do post-upload tasks
 	meta.IsUploaded = true
 	database.C.Save(&meta)
 
 	PublishAnalyzeTask(meta)
 
+	// Clean up
+	for _, chunk := range arrange {
+		chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
+		_ = os.Remove(chunkPath)
+	}
+
 	return meta, nil
 }

pkg/internal/services/recycler.go

@@ -64,7 +64,7 @@ func RunMarkLifecycleDeletionTask() {
 }
 
 func RunMarkMultipartDeletionTask() {
-	lifecycle := time.Now().Add(-24 * time.Hour)
+	lifecycle := time.Now().Add(-60 * time.Minute)
 	tx := database.C.
 		Where("created_at < ?", lifecycle).
 		Where("is_uploaded = ?", false).