HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Bug fixes and optimization

Browse Source
This commit is contained in:
LittleSheep 2024-08-20 22:55:58 +08:00
parent 37c47f9839
commit 7a8fa116d3
5 changed files with 59 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
.idea/workspace.xml generated
View File

@ -4,13 +4,12 @@
<option name="autoReloadType" value="ALL" /> <option name="autoReloadType" value="ALL" />
</component> </component>
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Multipart file upload"> <list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":sparkles: Make it more implementable">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/up_multipart_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/merger.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/merger.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pkg/internal/services/recycler.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/recycler.go" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -45,33 +44,33 @@
<option name="hideEmptyMiddlePackages" value="true" /> <option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" /> <option name="showLibraryContents" value="true" />
</component> </component>
<component name="PropertiesComponent">{ <component name="PropertiesComponent"><![CDATA[{
&quot;keyToString&quot;: { "keyToString": {
&quot;DefaultGoTemplateProperty&quot;: &quot;Go File&quot;, "DefaultGoTemplateProperty": "Go File",
&quot;Go Build.Backend.executor&quot;: &quot;Run&quot;, "Go Build.Backend.executor": "Run",
&quot;Go 构建.Backend.executor&quot;: &quot;Run&quot;, "Go 构建.Backend.executor": "Run",
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;, "RunOnceActivity.ShowReadmeOnStart": "true",
&quot;RunOnceActivity.go.formatter.settings.were.checked&quot;: &quot;true&quot;, "RunOnceActivity.go.formatter.settings.were.checked": "true",
&quot;RunOnceActivity.go.migrated.go.modules.settings&quot;: &quot;true&quot;, "RunOnceActivity.go.migrated.go.modules.settings": "true",
&quot;RunOnceActivity.go.modules.automatic.dependencies.download&quot;: &quot;true&quot;, "RunOnceActivity.go.modules.automatic.dependencies.download": "true",
&quot;RunOnceActivity.go.modules.go.list.on.any.changes.was.set&quot;: &quot;true&quot;, "RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
&quot;git-widget-placeholder&quot;: &quot;feature/multipart-upload&quot;, "git-widget-placeholder": "feature/multipart-upload",
&quot;go.import.settings.migrated&quot;: &quot;true&quot;, "go.import.settings.migrated": "true",
&quot;go.sdk.automatically.set&quot;: &quot;true&quot;, "go.sdk.automatically.set": "true",
&quot;last_opened_file_path&quot;: &quot;/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc&quot;, "last_opened_file_path": "/Users/littlesheep/Documents/Projects/Hydrogen/Paperclip/pkg/internal/grpc",
&quot;node.js.detected.package.eslint&quot;: &quot;true&quot;, "node.js.detected.package.eslint": "true",
&quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;, "node.js.selected.package.eslint": "(autodetect)",
&quot;nodejs_package_manager_path&quot;: &quot;npm&quot;, "nodejs_package_manager_path": "npm",
&quot;run.code.analysis.last.selected.profile&quot;: &quot;pProject Default&quot;, "run.code.analysis.last.selected.profile": "pProject Default",
&quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;, "settings.editor.selected.configurable": "preferences.pluginManager",
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot; "vue.rearranger.settings.migration": "true"
}, },
&quot;keyToStringList&quot;: { "keyToStringList": {
&quot;DatabaseDriversLRU&quot;: [ "DatabaseDriversLRU": [
&quot;postgresql&quot; "postgresql"
] ]
} }
}</component> }]]></component>
<component name="RecentsManager"> <component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS"> <key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$/pkg/internal/grpc" /> <recent name="$PROJECT_DIR$/pkg/internal/grpc" />
@ -121,7 +120,6 @@
</option> </option>
</component> </component>
<component name="VcsManagerConfiguration"> <component name="VcsManagerConfiguration">
<MESSAGE value=":bug: Fix uuid duplicate when link exists" />
<MESSAGE value="&#10;:sparkles: Add health check" /> <MESSAGE value="&#10;:sparkles: Add health check" />
<MESSAGE value=":arrow_up: Upgrade Passport and use Hyper SDK" /> <MESSAGE value=":arrow_up: Upgrade Passport and use Hyper SDK" />
<MESSAGE value=":arrow_up: Upgrade Passport to fix bug" /> <MESSAGE value=":arrow_up: Upgrade Passport to fix bug" />
@ -146,7 +144,8 @@
<MESSAGE value=":bug: Fix crash on maintain cache" /> <MESSAGE value=":bug: Fix crash on maintain cache" />
<MESSAGE value=":sparkles: Un-public indexable &amp; select by pools" /> <MESSAGE value=":sparkles: Un-public indexable &amp; select by pools" />
<MESSAGE value=":sparkles: Multipart file upload" /> <MESSAGE value=":sparkles: Multipart file upload" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Multipart file upload" /> <MESSAGE value=":sparkles: Make it more implementable" />
<option name="LAST_COMMIT_MESSAGE" value=":sparkles: Make it more implementable" />
</component> </component>
<component name="VgoProject"> <component name="VgoProject">
<settings-migrated>true</settings-migrated> <settings-migrated>true</settings-migrated>

15
pkg/internal/server/api/up_multipart_api.go
View File

@ -1,10 +1,12 @@
package api package api
import ( import (
"encoding/json"
"fmt" "fmt"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/database" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/database"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/gap" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/gap"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/models" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/models"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/server/exts"
"git.solsynth.dev/hydrogen/paperclip/pkg/internal/services" "git.solsynth.dev/hydrogen/paperclip/pkg/internal/services"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
"github.com/spf13/viper" "github.com/spf13/viper"
@ -19,12 +21,17 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
var data struct { var data struct {
Pool string `json:"pool" validate:"required"` Pool string `json:"pool" validate:"required"`
Size int64 `json:"size" validate:"required"` Size int64 `json:"size" validate:"required"`
FileName string `json:"name" validate:"required"`
Alternative string `json:"alt"` Alternative string `json:"alt"`
MimeType string `json:"mimetype"` MimeType string `json:"mimetype"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
IsMature bool `json:"is_mature"` IsMature bool `json:"is_mature"`
} }
if err := exts.BindAndValidate(c, &data); err != nil {
return err
}
aliasingMap := viper.GetStringMapString("pools.aliases") aliasingMap := viper.GetStringMapString("pools.aliases")
if val, ok := aliasingMap[data.Pool]; ok { if val, ok := aliasingMap[data.Pool]; ok {
data.Pool = val data.Pool = val
@ -42,6 +49,8 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
} }
metadata, err := services.NewAttachmentPlaceholder(database.C, user, models.Attachment{ metadata, err := services.NewAttachmentPlaceholder(database.C, user, models.Attachment{
Name: data.FileName,
Size: data.Size,
Alternative: data.Alternative, Alternative: data.Alternative,
MimeType: data.MimeType, MimeType: data.MimeType,
Metadata: data.Metadata, Metadata: data.Metadata,
@ -57,6 +66,7 @@ func createAttachmentMultipartPlaceholder(c *fiber.Ctx) error {
return c.JSON(fiber.Map{ return c.JSON(fiber.Map{
"chunk_size": viper.GetInt64("performance.file_chunk_size"), "chunk_size": viper.GetInt64("performance.file_chunk_size"),
"chunk_count": len(metadata.FileChunks),
"meta": metadata, "meta": metadata,
}) })
} }
@ -100,8 +110,9 @@ func uploadAttachmentMultipart(c *fiber.Ctx) error {
if !services.CheckChunkExistsInTemporary(meta, cid) { if !services.CheckChunkExistsInTemporary(meta, cid) {
isAllUploaded = false isAllUploaded = false
break break
} else if val, ok := idx.(int); ok { } else if val, ok := idx.(json.Number); ok {
chunkArrange[val] = cid data, _ := val.Int64()
chunkArrange[data] = cid
} }
} }

9
pkg/internal/services/analyzer.go
View File

@ -53,7 +53,10 @@ func ScanUnanalyzedFileFromDatabase() {
} }
var attachments []models.Attachment var attachments []models.Attachment
if err := database.C.Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).Find(&attachments).Error; err != nil { if err := database.C.
Where("is_uploaded = ?", true).
Where("destination = ? OR is_analyzed = ?", models.AttachmentDstTemporary, false).
Find(&attachments).Error; err != nil {
log.Error().Err(err).Msg("Scan unanalyzed files from database failed...") log.Error().Err(err).Msg("Scan unanalyzed files from database failed...")
return return
} }
@ -93,7 +96,9 @@ func ScanUnanalyzedFileFromDatabase() {
} }
func AnalyzeAttachment(file models.Attachment) error { func AnalyzeAttachment(file models.Attachment) error {
if file.Destination != models.AttachmentDstTemporary { if !file.IsUploaded {
return fmt.Errorf("file isn't finish multipart upload")
} else if file.Destination != models.AttachmentDstTemporary {
return fmt.Errorf("attachment isn't in temporary storage, unable to analyze") return fmt.Errorf("attachment isn't in temporary storage, unable to analyze")
} }

8
pkg/internal/services/merger.go
View File

@ -25,6 +25,7 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
} }
defer destFile.Close() defer destFile.Close()
// Merge files
for _, chunk := range arrange { for _, chunk := range arrange {
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk)) chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
chunkFile, err := os.Open(chunkPath) chunkFile, err := os.Open(chunkPath)
@ -41,10 +42,17 @@ func MergeFileChunks(meta models.Attachment, arrange []string) (models.Attachmen
_ = chunkFile.Close() _ = chunkFile.Close()
} }
// Do post-upload tasks
meta.IsUploaded = true meta.IsUploaded = true
database.C.Save(&meta) database.C.Save(&meta)
PublishAnalyzeTask(meta) PublishAnalyzeTask(meta)
// Clean up
for _, chunk := range arrange {
chunkPath := filepath.Join(dest.Path, fmt.Sprintf("%s.%s", meta.Uuid, chunk))
_ = os.Remove(chunkPath)
}
return meta, nil return meta, nil
} }

2
pkg/internal/services/recycler.go
View File

@ -64,7 +64,7 @@ func RunMarkLifecycleDeletionTask() {
} }
func RunMarkMultipartDeletionTask() { func RunMarkMultipartDeletionTask() {
lifecycle := time.Now().Add(-24 * time.Hour) lifecycle := time.Now().Add(-60 * time.Minute)
tx := database.C. tx := database.C.
Where("created_at < ?", lifecycle). Where("created_at < ?", lifecycle).
Where("is_uploaded = ?", false). Where("is_uploaded = ?", false).