From 4b1953c8e75c22748bdf7db935dd0153decff601 Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Wed, 12 Mar 2025 00:09:26 +0800 Subject: [PATCH] :sparkles: Fetching other fediverse timeline --- pkg/internal/database/migrator.go | 2 + pkg/internal/models/fediverse_posts.go | 25 ++++++++ pkg/internal/services/bsky/fetch.go | 39 +++++++++++++ pkg/internal/services/fediverse.go | 72 +++++++++++++++++++++++ pkg/internal/services/mastodon/fetch.go | 76 +++++++++++++++++++++++++ pkg/internal/services/misskey/fetch.go | 49 ++++++++++++++++ pkg/main.go | 1 + 7 files changed, 264 insertions(+) create mode 100644 pkg/internal/models/fediverse_posts.go create mode 100644 pkg/internal/services/bsky/fetch.go create mode 100644 pkg/internal/services/fediverse.go create mode 100644 pkg/internal/services/mastodon/fetch.go create mode 100644 pkg/internal/services/misskey/fetch.go diff --git a/pkg/internal/database/migrator.go b/pkg/internal/database/migrator.go index 1b795c6..21e4369 100644 --- a/pkg/internal/database/migrator.go +++ b/pkg/internal/database/migrator.go @@ -16,6 +16,8 @@ var AutoMaintainRange = []any{ &models.PollAnswer{}, &models.PostFlag{}, &models.PostView{}, + &models.FediversePost{}, + &models.FediverseUser{}, } func RunMigration(source *gorm.DB) error { diff --git a/pkg/internal/models/fediverse_posts.go b/pkg/internal/models/fediverse_posts.go new file mode 100644 index 0000000..d459fa1 --- /dev/null +++ b/pkg/internal/models/fediverse_posts.go @@ -0,0 +1,25 @@ +package models + +import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda" + +type FediversePost struct { + cruda.BaseModel + + Identifier string `json:"identifier" gorm:"uniqueIndex"` + Origin string `json:"origin"` + Content string `json:"content"` + Language string `json:"language"` + Images []string `json:"images"` + + User FediverseUser `json:"user"` + UserID uint `json:"user_id"` +} + +type FediverseUser struct { + cruda.BaseModel + + Identifier string `json:"identifier" gorm:"uniqueIndex"` + Origin string `json:"origin"` + Name string `json:"name"` + Nick string `json:"nick"` +} diff --git a/pkg/internal/services/bsky/fetch.go b/pkg/internal/services/bsky/fetch.go new file mode 100644 index 0000000..c49d6b4 --- /dev/null +++ b/pkg/internal/services/bsky/fetch.go @@ -0,0 +1,39 @@ +package bsky + +import ( + "encoding/json" + "fmt" + "net/http" +) + +type BlueskyPost struct { + URI string `json:"uri"` + CID string `json:"cid"` + Record struct { + Text string `json:"text"` + CreatedAt string `json:"createdAt"` + } `json:"record"` + Author struct { + Handle string `json:"handle"` + DisplayName string `json:"displayName"` + } `json:"author"` +} + +func FetchBlueskyPublicFeed(feedURI string, limit int) ([]BlueskyPost, error) { + url := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.feed.getFeed?feed=%s&limit=%d", feedURI, limit) + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch Bluesky posts: %v", err) + } + defer resp.Body.Close() + + var response struct { + Feed []BlueskyPost `json:"feed"` + } + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to parse Bluesky JSON: %v", err) + } + + return response.Feed, nil +} diff --git a/pkg/internal/services/fediverse.go b/pkg/internal/services/fediverse.go new file mode 100644 index 0000000..416016e --- /dev/null +++ b/pkg/internal/services/fediverse.go @@ -0,0 +1,72 @@ +package services + +import ( + "fmt" + + "git.solsynth.dev/hypernet/interactive/pkg/internal/database" + "git.solsynth.dev/hypernet/interactive/pkg/internal/models" + "git.solsynth.dev/hypernet/interactive/pkg/internal/services/mastodon" + "github.com/rs/zerolog/log" + "github.com/samber/lo" + "github.com/spf13/viper" + "gorm.io/gorm/clause" +) + +type FromFediversePost interface { + ToFediversePost() models.FediversePost +} + +type FediverseFriendConfig struct { + ID string `json:"id"` + URL string `json:"url"` + Type string `json:"type"` + BatchSize int `json:"batch_size"` +} + +var fediverseFriends []FediverseFriendConfig + +func ReadFriendConfig() { + if err := viper.UnmarshalKey("fediverse.friends", &fediverseFriends); err != nil { + log.Error().Err(err).Msg("Failed to loading fediverse friend config...") + } + log.Info().Int("count", len(fediverseFriends)).Msg("Loaded fediverse friend config!") +} + +func FetchFediversePost(cfg FediverseFriendConfig) ([]models.FediversePost, error) { + var posts []models.FediversePost + switch cfg.Type { + case "mastodon": + data, err := mastodon.FetchTimeline(cfg.URL, cfg.BatchSize) + if err != nil { + return nil, err + } + posts = lo.Map(data, func(item mastodon.MastodonPost, _ int) models.FediversePost { + return item.ToFediversePost() + }) + default: + return nil, fmt.Errorf("unsupported fediverse service: %s", cfg.Type) + } + return posts, nil +} + +func FetchFediverseTimedTask() { + if len(fediverseFriends) == 0 { + return + } + + log.Debug().Msg("Starting fetching fediverse friends timeline...") + + var totalPosts []models.FediversePost + for _, friend := range fediverseFriends { + log.Info().Str("id", friend.ID).Str("url", friend.URL).Msg("Fetching fediverse friend timeline...") + posts, err := FetchFediversePost(friend) + if err != nil { + log.Error().Err(err).Str("id", friend.ID).Str("url", friend.URL).Msg("Failed to fetch fediverse friend timelime...") + } + totalPosts = append(totalPosts, posts...) + } + + if err := database.C.Clauses(clause.OnConflict{DoNothing: true}).Create(&totalPosts).Error; err != nil { + log.Error().Err(err).Msg("Failed to save timeline posts...") + } +} diff --git a/pkg/internal/services/mastodon/fetch.go b/pkg/internal/services/mastodon/fetch.go new file mode 100644 index 0000000..5618d1d --- /dev/null +++ b/pkg/internal/services/mastodon/fetch.go @@ -0,0 +1,76 @@ +package mastodon + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "git.solsynth.dev/hypernet/interactive/pkg/internal/models" + "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda" + "github.com/samber/lo" +) + +type MastodomAttachment struct { + URL string `json:"url"` + Preview string `json:"preview"` +} + +type MastodonPost struct { + ID string `json:"id"` + Content string `json:"content"` + URL string `json:"url"` + Account struct { + Acct string `json:"acct"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + Avatar string `json:"avatar"` + AccountURL string `json:"url"` + } `json:"account"` + Language string `json:"language"` + MediaAttachments []MastodomAttachment `json:"media_attachments"` + CreatedAt time.Time `json:"created_at"` + Server string `json:"-"` +} + +func (v MastodonPost) ToFediversePost() models.FediversePost { + return models.FediversePost{ + BaseModel: cruda.BaseModel{ + CreatedAt: v.CreatedAt, + }, + Identifier: v.ID, + Origin: v.Server, + Content: v.Content, + Language: v.Language, + Images: lo.Map(v.MediaAttachments, func(item MastodomAttachment, _ int) string { + return item.URL + }), User: models.FediverseUser{ + Identifier: v.Account.Acct, + Name: v.Account.Username, + Nick: v.Account.DisplayName, + Origin: v.Server, + }, + } +} + +func FetchTimeline(server string, limit int) ([]MastodonPost, error) { + url := fmt.Sprintf("%s/api/v1/timelines/public?limit=%d", server, limit) + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch public timeline: %v", err) + } + defer resp.Body.Close() + + var posts []MastodonPost + if err := json.NewDecoder(resp.Body).Decode(&posts); err != nil { + return nil, fmt.Errorf("failed to parse timeline JSON: %v", err) + } + + for idx := range posts { + posts[idx].Server = strings.Replace(strings.Replace(server, "https://", "", 1), "http://", "", 1) + } + + return posts, nil +} diff --git a/pkg/internal/services/misskey/fetch.go b/pkg/internal/services/misskey/fetch.go new file mode 100644 index 0000000..e224ec8 --- /dev/null +++ b/pkg/internal/services/misskey/fetch.go @@ -0,0 +1,49 @@ +package services + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" +) + +type MisskeyPost struct { + ID string `json:"id"` + Text string `json:"text"` + User struct { + Username string `json:"username"` + DisplayName string `json:"name"` + AvatarURL string `json:"avatarUrl"` + } `json:"user"` + CreatedAt string `json:"createdAt"` +} + +func FetchTimeline(server, token string, limit int) ([]MisskeyPost, error) { + url := fmt.Sprintf("%s/api/notes/global-timeline", server) + + payload := map[string]interface{}{ + "limit": limit, + "i": token, + } + body, _ := json.Marshal(payload) + + req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch Misskey posts: %v", err) + } + defer resp.Body.Close() + + var posts []MisskeyPost + if err := json.NewDecoder(resp.Body).Decode(&posts); err != nil { + return nil, fmt.Errorf("failed to parse Misskey JSON: %v", err) + } + + return posts, nil +} diff --git a/pkg/main.go b/pkg/main.go index aac6a80..714dc18 100644 --- a/pkg/main.go +++ b/pkg/main.go @@ -68,6 +68,7 @@ func main() { // Configure timed tasks quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger))) quartz.AddFunc("@every 5m", services.FlushPostViews) + quartz.AddFunc("@every 5m", services.FetchFediverseTimedTask) quartz.Start() // Initialize cache