🐛 Bug fixes and optimization
This commit is contained in:
parent
37c47f9839
commit
7a8fa116d3
@ -4,13 +4,12 @@
|
||||
<option name="autoReloadType" value="ALL" />
|
||||
</component>
|
||||
<component name="ChangeListManager">
|
||||
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Multipart file upload">
|
||||
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Make it more implementable">
|
||||
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
|
||||
<change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
|
||||
</list>
|
||||
<option name="SHOW_DIALOG" value="false" />
|
||||
<option name="HIGHLIGHT_CONFLICTS" value="true" />
|
||||
@ -45,33 +44,33 @@
|
||||
<option name="hideEmptyMiddlePackages" value="true" />
|
||||
<option name="showLibraryContents" value="true" />
|
||||
</component>
|
||||
<component name="PropertiesComponent">{
|
||||
"keyToString": {
|
||||
"DefaultGoTemplateProperty": "Go File",
|
||||
"Go Build.Backend.executor": "Run",
|
||||
"Go 构建.Backend.executor": "Run",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"RunOnceActivity.go.formatter.settings.were.checked": "true",
|
||||
"RunOnceActivity.go.migrated.go.modules.settings": "true",
|
||||
"RunOnceActivity.go.modules.automatic.dependencies.download": "true",
|
||||
"RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
|
||||
"git-widget-placeholder": "feature/multipart-upload",
|
||||
"go.import.settings.migrated": "true",
|
||||
"go.sdk.automatically.set": "true",
|
||||
"last_opened_file_path": "/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc",
|
||||
"node.js.detected.package.eslint": "true",
|
||||
"node.js.selected.package.eslint": "(autodetect)",
|
||||
"nodejs_package_manager_path": "npm",
|
||||
"run.code.analysis.last.selected.profile": "pProject Default",
|
||||
"settings.editor.selected.configurable": "preferences.pluginManager",
|
||||
"vue.rearranger.settings.migration": "true"
|
||||
<component name="PropertiesComponent"><![CDATA[{
|
||||
"keyToString": {
|
||||
"DefaultGoTemplateProperty": "Go File",
|
||||
"Go Build.Backend.executor": "Run",
|
||||
"Go 构建.Backend.executor": "Run",
|
||||
"RunOnceActivity.ShowReadmeOnStart": "true",
|
||||
"RunOnceActivity.go.formatter.settings.were.checked": "true",
|
||||
"RunOnceActivity.go.migrated.go.modules.settings": "true",
|
||||
"RunOnceActivity.go.modules.automatic.dependencies.download": "true",
|
||||
"RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
|
||||
"git-widget-placeholder": "feature/multipart-upload",
|
||||
"go.import.settings.migrated": "true",
|
||||
"go.sdk.automatically.set": "true",
|
||||
"last_opened_file_path": "/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc",
|
||||
"node.js.detected.package.eslint": "true",
|
||||
"node.js.selected.package.eslint": "(autodetect)",
|
||||
"nodejs_package_manager_path": "npm",
|
||||
"run.code.analysis.last.selected.profile": "pProject Default",
|
||||
"settings.editor.selected.configurable": "preferences.pluginManager",
|
||||
"vue.rearranger.settings.migration": "true"
|
||||
},
|
||||
"keyToStringList": {
|
||||
"DatabaseDriversLRU": [
|
||||
"postgresql"
|
||||
"keyToStringList": {
|
||||
"DatabaseDriversLRU": [
|
||||
"postgresql"
|
||||
]
|
||||
}
|
||||
}</component>
|
||||
}]]></component>
|
||||
<component name="RecentsManager">
|
||||
<key name="CopyFile.RECENT_KEYS">
|
||||
<recent name="$PROJECT_DIR$/pkg/internal/grpc" />
|
||||
@ -121,7 +120,6 @@
|
||||
</option>
|
||||
</component>
|
||||
<component name="VcsManagerConfiguration">
|
||||
<MESSAGE value=":bug: Fix uuid duplicate when link exists" />
|
||||
<MESSAGE value=" :sparkles: Add health check" />
|
||||
<MESSAGE value=":arrow_up: Upgrade Passport and use Hyper SDK" />
|
||||
<MESSAGE value=":arrow_up: Upgrade Passport to fix bug" />
|
||||
@ -146,7 +144,8 @@
|
||||
<MESSAGE value=":bug: Fix crash on maintain cache" />
|
||||
<MESSAGE value=":sparkles: Un-public indexable & select by pools" />
|
||||
<MESSAGE value=":sparkles: Multipart file upload" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Multipart file upload" />
|
||||
<MESSAGE value=":sparkles: Make it more implementable" />
|
||||
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Make it more implementable" />
|
||||
</component>
|
||||
<component name="VgoProject">
|
||||
<settings-migrated>true</settings-migrated>
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
|
||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/gap"
|
||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
|
||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/server/exts"
|
||||
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/services"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/spf13/viper"
|
||||
@ -19,12 +21,17 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
|
||||
var data struct {
|
||||
Pool string `json:"pool" validate:"required"`
|
||||
Size int64 `json:"size" validate:"required"`
|
||||
FileName string `json:"name" validate:"required"`
|
||||
Alternative string `json:"alt"`
|
||||
MimeType string `json:"mimetype"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
IsMature bool `json:"is_mature"`
|
||||
}
|
||||
|
||||
if err := exts.BindAndValidate(c, &data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
aliasingMap := viper.GetStringMapString("pools.aliases")
|
||||
if val, ok := aliasingMap[data.Pool]; ok {
|
||||
data.Pool = val
|
||||
@ -42,6 +49,8 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
|
||||
}
|
||||
|
||||
metadata, err := services.NewAttachmentPlaceholder(database.C, user, models.Attachment{
|
||||
Name: data.FileName,
|
||||
Size: data.Size,
|
||||
Alternative: data.Alternative,
|
||||
MimeType: data.MimeType,
|
||||
Metadata: data.Metadata,
|
||||
@ -57,6 +66,7 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"chunk_size": viper.GetInt64("performance.file_chunk_size"),
|
||||
"chunk_count": len(metadata.FileChunks),
|
||||
"meta": metadata,
|
||||
})
|
||||
}
|
||||
@ -100,8 +110,9 @@ func uploadAttachmentMultipart(c *fiber.Ctx) error {
|
||||
if !services.CheckChunkExistsInTemporary(meta, cid) {
|
||||
isAllUploaded = false
|
||||
break
|
||||
} else if val, ok := idx.(int); ok {
|
||||
chunkArrange[val] = cid
|
||||
} else if val, ok := idx.(json.Number); ok {
|
||||
data, _ := val.Int64()
|
||||
chunkArrange[data] = cid
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -53,7 +53,10 @@ func ScanUnanalyzedFileFromDatabase() {
|
||||
}
|
||||
|
||||
var attachments []models.Attachment
|
||||
if err := database.C.Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).Find(&attachments).Error; err != nil {
|
||||
if err := database.C.
|
||||
Where("is_uploaded = ?", true).
|
||||
Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).
|
||||
Find(&attachments).Error; err != nil {
|
||||
log.Error().Err(err).Msg("Scan unanalyzed files from database failed...")
|
||||
return
|
||||
}
|
||||
@ -93,7 +96,9 @@ func ScanUnanalyzedFileFromDatabase() {
|
||||
}
|
||||
|
||||
func AnalyzeAttachment(file models.Attachment) error {
|
||||
if file.Destination != models.AttachmentDstTemporary {
|
||||
if !file.IsUploaded {
|
||||
return fmt.Errorf("file isn't finish multipart upload")
|
||||
} else if file.Destination != models.AttachmentDstTemporary {
|
||||
return fmt.Errorf("attachment isn't in temporary storage, unable to analyze")
|
||||
}
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
// Merge files
|
||||
for _, chunk := range arrange {
|
||||
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
|
||||
chunkFile, err := os.Open(chunkPath)
|
||||
@ -41,10 +42,17 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
|
||||
_ = chunkFile.Close()
|
||||
}
|
||||
|
||||
// Do post-upload tasks
|
||||
meta.IsUploaded = true
|
||||
database.C.Save(&meta)
|
||||
|
||||
PublishAnalyzeTask(meta)
|
||||
|
||||
// Clean up
|
||||
for _, chunk := range arrange {
|
||||
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
|
||||
_ = os.Remove(chunkPath)
|
||||
}
|
||||
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
@ -64,7 +64,7 @@ func RunMarkLifecycleDeletionTask() {
|
||||
}
|
||||
|
||||
func RunMarkMultipartDeletionTask() {
|
||||
lifecycle := time.Now().Add(-24 * time.Hour)
|
||||
lifecycle := time.Now().Add(-60 * time.Minute)
|
||||
tx := database.C.
|
||||
Where("created_at < ?", lifecycle).
|
||||
Where("is_uploaded = ?", false).
|
||||
|
||||
Loading…
Reference in New Issue
Block a user