HyperNet/Paperclip
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐛 Provide a way to bypass nexus and serving content to prevent nexus reverse proxy issue

Browse Source
This commit is contained in:
LittleSheep 2024-11-06 23:01:29 +08:00
parent 2f99ac7b6e
commit c41f8b77ce
9 changed files with 6460 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.idea/dataSources.local.xml generated
View File

@ -10,8 +10,16 @@
<user-name>postgres</user-name>
<schema-mapping>
<introspection-scope>
<node kind="database" qname="@">
<node kind="schema" qname="@" />
<node negative="1">
<node kind="database" qname="sn_paperclip">
<node kind="schema">
<name qname="@" />
<name qname="public" />
</node>
</node>
<node kind="database" qname="sn_passport">
<node kind="schema" qname="@" />
</node>
</node>
</introspection-scope>
</schema-mapping>

6978
.idea/dataSources/bac83d8a-c619-4680-a07f-6674b93fbfea.xml generated
View File

File diff suppressed because it is too large Load Diff

51
.idea/workspace.xml generated
View File

@ -4,9 +4,16 @@
<option name="autoReloadType" value="ALL" />
</component>
<component name="ChangeListManager">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Bug fixes">
<list default="true" id="18dd0d68-b4b8-40db-9734-9119b5c848bd" name="更改" comment=":bug: Fix hash bug">
<change beforePath="$PROJECT_DIR$/.idea/dataSources.local.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/dataSources.local.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/dataSources/bac83d8a-c619-4680-a07f-6674b93fbfea.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/dataSources/bac83d8a-c619-4680-a07f-6674b93fbfea.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/attachments.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/attachments.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/models/destination.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/models/destination.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/server/api/attachments_api.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/analyzer.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pkg/internal/services/uploader.go" beforeDir="false" afterPath="$PROJECT_DIR$/pkg/internal/services/uploader.go" afterDir="false" />
<change beforePath="$PROJECT_DIR$/settings.toml" beforeDir="false" afterPath="$PROJECT_DIR$/settings.toml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -44,7 +51,7 @@
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"DefaultGoTemplateProperty": "Go File",
"Go Build.Backend.executor": "Run",
"Go Build.Backend.executor": "Debug",
"Go 构建.Backend.executor": "Run",
"RunOnceActivity.ShowReadmeOnStart": "true",
"RunOnceActivity.go.formatter.settings.were.checked": "true",
@ -118,7 +125,6 @@
</option>
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value=":bug: Fix doesn't get has lifecycle settings buckets correctly" />
<MESSAGE value=":zap: Fix the RandString method cause the lag" />
<MESSAGE value=":bug: Fix mark clean required issue" />
<MESSAGE value=":sparkles: Support use rid to get file" />
@ -143,9 +149,46 @@
<MESSAGE value=":recycle: Migrated to nexus" />
<MESSAGE value=":wastebasket: Remove account" />
<MESSAGE value=":bug: Bug fixes" />
<option name="LAST_COMMIT_MESSAGE" value=":bug: Bug fixes" />
<MESSAGE value=":bug: Fix hash bug" />
<option name="LAST_COMMIT_MESSAGE" value=":bug: Fix hash bug" />
</component>
<component name="VgoProject">
<settings-migrated>true</settings-migrated>
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
<breakpoints>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241102174036-194d27090415/pkg/nex/sec/perms.go</url>
<line>9</line>
<option name="timeStamp" value="1" />
</line-breakpoint>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241102174036-194d27090415/pkg/nex/sec/perms.go</url>
<line>11</line>
<option name="timeStamp" value="2" />
</line-breakpoint>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241103161217-e1207a8deb28/pkg/nex/sec/perms.go</url>
<line>11</line>
<option name="timeStamp" value="3" />
</line-breakpoint>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241103161217-e1207a8deb28/pkg/nex/sec/perms.go</url>
<line>9</line>
<option name="timeStamp" value="5" />
</line-breakpoint>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241103165336-3e68cea0d783/pkg/nex/sec/perms.go</url>
<line>12</line>
<option name="timeStamp" value="6" />
</line-breakpoint>
<line-breakpoint enabled="true" type="DlvLineBreakpoint">
<url>file://$USER_HOME$/Applications/GoLang/pkg/mod/git.solsynth.dev/hypernet/nexus@v0.0.0-20241103165336-3e68cea0d783/pkg/nex/sec/perms.go</url>
<line>10</line>
<option name="timeStamp" value="7" />
</line-breakpoint>
</breakpoints>
</breakpoint-manager>
</component>
</project>

19
pkg/internal/models/attachments.go
View File

@ -7,11 +7,8 @@ import (
"gorm.io/datatypes"
)
type AttachmentDst = int8
const (
AttachmentDstTemporary = AttachmentDst(iota)
AttachmentDstPermanent
AttachmentDstTemporary = 0 // The destination 0 is a reserved config for pre-upload processing
)
type Attachment struct {
@ -22,13 +19,13 @@ type Attachment struct {
// Unique ID is for storing (appear in local file name or object name)
Uuid string `json:"uuid"`
Size int64 `json:"size"`
Name string `json:"name"`
Alternative string `json:"alt"`
MimeType string `json:"mimetype"`
HashCode string `json:"hash"`
Destination AttachmentDst `json:"destination"`
RefCount int `json:"ref_count"`
Size int64 `json:"size"`
Name string `json:"name"`
Alternative string `json:"alt"`
MimeType string `json:"mimetype"`
HashCode string `json:"hash"`
Destination int `json:"destination"`
RefCount int `json:"ref_count"`
FileChunks datatypes.JSONMap `json:"file_chunks"`

3
pkg/internal/models/destination.go
View File

@ -12,7 +12,8 @@ type BaseDestination struct {
type LocalDestination struct {
BaseDestination
Path string `json:"path"`
Path string `json:"path"`
AccessBaseURL string `json:"access_baseurl"`
}
type S3Destination struct {

18
pkg/internal/server/api/attachments_api.go
View File

@ -26,12 +26,7 @@ func openAttachment(c *fiber.Ctx) error {
return fiber.NewError(fiber.StatusNotFound, "file is in uploading progress, please wait until all chunk uploaded")
}
var destMap map[string]any
if metadata.Destination == models.AttachmentDstTemporary {
destMap = viper.GetStringMap("destinations.temporary")
} else {
destMap = viper.GetStringMap("destinations.permanent")
}
destMap := viper.GetStringMap(fmt.Sprintf("destinations.%d", metadata.Destination))
var dest models.BaseDestination
rawDest, _ := jsoniter.Marshal(destMap)
@ -41,10 +36,19 @@ func openAttachment(c *fiber.Ctx) error {
case models.DestinationTypeLocal:
var destConfigured models.LocalDestination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)
if len(destConfigured.AccessBaseURL) > 0 && !c.QueryBool("direct", false) {
// This will drop all query parameters,
// for not it's okay because the openAttachment api won't take any query parameters
return c.Redirect(fmt.Sprintf(
"%s%s?direct=true",
destConfigured.AccessBaseURL,
c.Path(),
), fiber.StatusMovedPermanently)
}
if len(metadata.MimeType) > 0 {
c.Set(fiber.HeaderContentType, metadata.MimeType)
}
return c.SendFile(filepath.Join(destConfigured.Path, metadata.Uuid), false)
return c.SendFile(filepath.Join(destConfigured.Path, metadata.Uuid))
case models.DestinationTypeS3:
var destConfigured models.S3Destination
_ = jsoniter.Unmarshal(rawDest, &destConfigured)

2
pkg/internal/services/analyzer.go
View File

@ -241,7 +241,7 @@ func AnalyzeAttachment(file models.Attachment) error {
// Move temporary to permanent
if !linked {
if err := ReUploadFileToPermanent(file); err != nil {
if err := ReUploadFileToPermanent(file, 1); err != nil {
return fmt.Errorf("unable to move file to permanet storage: %v", err)
}
}

7
pkg/internal/services/uploader.go
View File

@ -94,12 +94,15 @@ func CheckChunkExistsInTemporary(meta models.Attachment, cid string) bool {
}
}
func ReUploadFileToPermanent(meta models.Attachment) error {
func ReUploadFileToPermanent(meta models.Attachment, dst int) error {
if dst == models.AttachmentDstTemporary || meta.Destination != dst {
return fmt.Errorf("destnation cannot be reversed temporary or the same as the original")
}
if meta.Destination != models.AttachmentDstTemporary {
return fmt.Errorf("attachment isn't in temporary storage, unable to process")
}
meta.Destination = models.AttachmentDstPermanent
meta.Destination = 0
destMap := viper.GetStringMap("destinations.permanent")

6
settings.toml
View File

@ -16,13 +16,13 @@ print_routes = false
[performance]
file_chunk_size = 26214400
[destinations.temporary]
[[destinations]]
type = "local"
path = "uploads"
[destinations.permanent]
[[destinations]]
type = "local"
path = "uploads/permanent"
access_baseurl = "http://localhost:8004"
[security]
internal_public_key = "keys/internal_public_key.pem"