✨ Fetching other fediverse timeline

pkg/internal/database/migrator.go

@@ -16,6 +16,8 @@ var AutoMaintainRange = []any{
 	&models.PollAnswer{},
 	&models.PostFlag{},
 	&models.PostView{},
+	&models.FediversePost{},
+	&models.FediverseUser{},
 }
 
 func RunMigration(source *gorm.DB) error {

pkg/internal/models/fediverse_posts.go

@@ -0,0 +1,25 @@
+package models
+
+import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
+
+type FediversePost struct {
+	cruda.BaseModel
+
+	Identifier string   `json:"identifier" gorm:"uniqueIndex"`
+	Origin     string   `json:"origin"`
+	Content    string   `json:"content"`
+	Language   string   `json:"language"`
+	Images     []string `json:"images"`
+
+	User   FediverseUser `json:"user"`
+	UserID uint          `json:"user_id"`
+}
+
+type FediverseUser struct {
+	cruda.BaseModel
+
+	Identifier string `json:"identifier" gorm:"uniqueIndex"`
+	Origin     string `json:"origin"`
+	Name       string `json:"name"`
+	Nick       string `json:"nick"`
+}

pkg/internal/services/bsky/fetch.go

@@ -0,0 +1,39 @@
+package bsky
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type BlueskyPost struct {
+	URI    string `json:"uri"`
+	CID    string `json:"cid"`
+	Record struct {
+		Text      string `json:"text"`
+		CreatedAt string `json:"createdAt"`
+	} `json:"record"`
+	Author struct {
+		Handle      string `json:"handle"`
+		DisplayName string `json:"displayName"`
+	} `json:"author"`
+}
+
+func FetchBlueskyPublicFeed(feedURI string, limit int) ([]BlueskyPost, error) {
+	url := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.feed.getFeed?feed=%s&limit=%d", feedURI, limit)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch Bluesky posts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var response struct {
+		Feed []BlueskyPost `json:"feed"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return nil, fmt.Errorf("failed to parse Bluesky JSON: %v", err)
+	}
+
+	return response.Feed, nil
+}

pkg/internal/services/fediverse.go

@@ -0,0 +1,72 @@
+package services
+
+import (
+	"fmt"
+
+	"git.solsynth.dev/hypernet/interactive/pkg/internal/database"
+	"git.solsynth.dev/hypernet/interactive/pkg/internal/models"
+	"git.solsynth.dev/hypernet/interactive/pkg/internal/services/mastodon"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"github.com/spf13/viper"
+	"gorm.io/gorm/clause"
+)
+
+type FromFediversePost interface {
+	ToFediversePost() models.FediversePost
+}
+
+type FediverseFriendConfig struct {
+	ID        string `json:"id"`
+	URL       string `json:"url"`
+	Type      string `json:"type"`
+	BatchSize int    `json:"batch_size"`
+}
+
+var fediverseFriends []FediverseFriendConfig
+
+func ReadFriendConfig() {
+	if err := viper.UnmarshalKey("fediverse.friends", &fediverseFriends); err != nil {
+		log.Error().Err(err).Msg("Failed to loading fediverse friend config...")
+	}
+	log.Info().Int("count", len(fediverseFriends)).Msg("Loaded fediverse friend config!")
+}
+
+func FetchFediversePost(cfg FediverseFriendConfig) ([]models.FediversePost, error) {
+	var posts []models.FediversePost
+	switch cfg.Type {
+	case "mastodon":
+		data, err := mastodon.FetchTimeline(cfg.URL, cfg.BatchSize)
+		if err != nil {
+			return nil, err
+		}
+		posts = lo.Map(data, func(item mastodon.MastodonPost, _ int) models.FediversePost {
+			return item.ToFediversePost()
+		})
+	default:
+		return nil, fmt.Errorf("unsupported fediverse service: %s", cfg.Type)
+	}
+	return posts, nil
+}
+
+func FetchFediverseTimedTask() {
+	if len(fediverseFriends) == 0 {
+		return
+	}
+
+	log.Debug().Msg("Starting fetching fediverse friends timeline...")
+
+	var totalPosts []models.FediversePost
+	for _, friend := range fediverseFriends {
+		log.Info().Str("id", friend.ID).Str("url", friend.URL).Msg("Fetching fediverse friend timeline...")
+		posts, err := FetchFediversePost(friend)
+		if err != nil {
+			log.Error().Err(err).Str("id", friend.ID).Str("url", friend.URL).Msg("Failed to fetch fediverse friend timelime...")
+		}
+		totalPosts = append(totalPosts, posts...)
+	}
+
+	if err := database.C.Clauses(clause.OnConflict{DoNothing: true}).Create(&totalPosts).Error; err != nil {
+		log.Error().Err(err).Msg("Failed to save timeline posts...")
+	}
+}

pkg/internal/services/mastodon/fetch.go

@@ -0,0 +1,76 @@
+package mastodon
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.solsynth.dev/hypernet/interactive/pkg/internal/models"
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
+	"github.com/samber/lo"
+)
+
+type MastodomAttachment struct {
+	URL     string `json:"url"`
+	Preview string `json:"preview"`
+}
+
+type MastodonPost struct {
+	ID      string `json:"id"`
+	Content string `json:"content"`
+	URL     string `json:"url"`
+	Account struct {
+		Acct        string `json:"acct"`
+		Username    string `json:"username"`
+		DisplayName string `json:"display_name"`
+		Avatar      string `json:"avatar"`
+		AccountURL  string `json:"url"`
+	} `json:"account"`
+	Language         string               `json:"language"`
+	MediaAttachments []MastodomAttachment `json:"media_attachments"`
+	CreatedAt        time.Time            `json:"created_at"`
+	Server           string               `json:"-"`
+}
+
+func (v MastodonPost) ToFediversePost() models.FediversePost {
+	return models.FediversePost{
+		BaseModel: cruda.BaseModel{
+			CreatedAt: v.CreatedAt,
+		},
+		Identifier: v.ID,
+		Origin:     v.Server,
+		Content:    v.Content,
+		Language:   v.Language,
+		Images: lo.Map(v.MediaAttachments, func(item MastodomAttachment, _ int) string {
+			return item.URL
+		}), User: models.FediverseUser{
+			Identifier: v.Account.Acct,
+			Name:       v.Account.Username,
+			Nick:       v.Account.DisplayName,
+			Origin:     v.Server,
+		},
+	}
+}
+
+func FetchTimeline(server string, limit int) ([]MastodonPost, error) {
+	url := fmt.Sprintf("%s/api/v1/timelines/public?limit=%d", server, limit)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch public timeline: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var posts []MastodonPost
+	if err := json.NewDecoder(resp.Body).Decode(&posts); err != nil {
+		return nil, fmt.Errorf("failed to parse timeline JSON: %v", err)
+	}
+
+	for idx := range posts {
+		posts[idx].Server = strings.Replace(strings.Replace(server, "https://", "", 1), "http://", "", 1)
+	}
+
+	return posts, nil
+}

pkg/internal/services/misskey/fetch.go

@@ -0,0 +1,49 @@
+package services
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type MisskeyPost struct {
+	ID   string `json:"id"`
+	Text string `json:"text"`
+	User struct {
+		Username    string `json:"username"`
+		DisplayName string `json:"name"`
+		AvatarURL   string `json:"avatarUrl"`
+	} `json:"user"`
+	CreatedAt string `json:"createdAt"`
+}
+
+func FetchTimeline(server, token string, limit int) ([]MisskeyPost, error) {
+	url := fmt.Sprintf("%s/api/notes/global-timeline", server)
+
+	payload := map[string]interface{}{
+		"limit": limit,
+		"i":     token,
+	}
+	body, _ := json.Marshal(payload)
+
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch Misskey posts: %v", err)
+	}
+	defer resp.Body.Close()
+
+	var posts []MisskeyPost
+	if err := json.NewDecoder(resp.Body).Decode(&posts); err != nil {
+		return nil, fmt.Errorf("failed to parse Misskey JSON: %v", err)
+	}
+
+	return posts, nil
+}

pkg/main.go

@@ -68,6 +68,7 @@ func main() {
 	// Configure timed tasks
 	quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
 	quartz.AddFunc("@every 5m", services.FlushPostViews)
+	quartz.AddFunc("@every 5m", services.FetchFediverseTimedTask)
 	quartz.Start()
 
 	// Initialize cache