HyperNet/Interactive
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

♻️ Upgrade to new feed reader

Browse Source
This commit is contained in:
LittleSheep 2025-04-05 01:12:03 +08:00
parent 3c3cbd9c29
commit 349d768d22
9 changed files with 18 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
/keys
.DS_Store
.idea

2
pkg/internal/database/migrator.go
View File

@ -16,8 +16,6 @@ var AutoMaintainRange = []any{
&models.PollAnswer{},
&models.PostFlag{},
&models.PostView{},
&models.FediverseUser{},
&models.FediversePost{},
}
func RunMigration(source *gorm.DB) error {

29
pkg/internal/models/fediverse_posts.go
View File

@ -1,29 +0,0 @@
package models
import (
"git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
"gorm.io/datatypes"
)
type FediversePost struct {
cruda.BaseModel
Identifier string `json:"identifier" gorm:"uniqueIndex"`
Origin string `json:"origin"`
Content string `json:"content"`
Language string `json:"language"`
Images datatypes.JSONSlice[string] `json:"images"`
User FediverseUser `json:"user"`
UserID uint `json:"user_id"`
}
type FediverseUser struct {
cruda.BaseModel
Identifier string `json:"identifier" gorm:"uniqueIndex"`
Origin string `json:"origin"`
Avatar string `json:"avatar"`
Name string `json:"name"`
Nick string `json:"nick"`
}

39
pkg/internal/services/bsky/fetch.go
View File

@ -1,39 +0,0 @@
package bsky
import (
"encoding/json"
"fmt"
"net/http"
)
type BlueskyPost struct {
URI string `json:"uri"`
CID string `json:"cid"`
Record struct {
Text string `json:"text"`
CreatedAt string `json:"createdAt"`
} `json:"record"`
Author struct {
Handle string `json:"handle"`
DisplayName string `json:"displayName"`
} `json:"author"`
}
func FetchBlueskyPublicFeed(feedURI string, limit int) ([]BlueskyPost, error) {
url := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.feed.getFeed?feed=%s&limit=%d", feedURI, limit)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch Bluesky posts: %v", err)
}
defer resp.Body.Close()
var response struct {
Feed []BlueskyPost `json:"feed"`
}
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to parse Bluesky JSON: %v", err)
}
return response.Feed, nil
}

116
pkg/internal/services/fediverse.go
View File

@ -1,116 +0,0 @@
package services
import (
"fmt"
"git.solsynth.dev/hypernet/interactive/pkg/internal/database"
"git.solsynth.dev/hypernet/interactive/pkg/internal/models"
"git.solsynth.dev/hypernet/interactive/pkg/internal/services/mastodon"
"github.com/rs/zerolog/log"
"github.com/samber/lo"
"github.com/spf13/viper"
"gorm.io/gorm/clause"
)
type FromFediversePost interface {
ToFediversePost() models.FediversePost
}
type FediverseFriendConfig struct {
ID string `json:"id"`
URL string `json:"url"`
Type string `json:"type"`
Trending bool `json:"trending"`
}
var fediverseFriends []FediverseFriendConfig
func ReadFriendConfig() {
if err := viper.UnmarshalKey("fediverse.friends", &fediverseFriends); err != nil {
log.Error().Err(err).Msg("Failed to loading fediverse friend config...")
}
log.Info().Int("count", len(fediverseFriends)).Msg("Loaded fediverse friend config!")
}
func FetchFediversePost(cfg FediverseFriendConfig) ([]models.FediversePost, error) {
switch cfg.Type {
case "mastodon":
data, err := mastodon.FetchTimeline(cfg.URL, 50, cfg.Trending)
if err != nil {
return nil, err
}
posts := lo.Map(data, func(item mastodon.MastodonPost, _ int) models.FediversePost {
return item.ToFediversePost()
})
return posts, nil
default:
// TODO Other platform fetching is still under development
// DO NOT USE THEM
return nil, fmt.Errorf("unsupported fediverse service: %s", cfg.Type)
}
}
func FetchFediverseTimedTask() {
if len(fediverseFriends) == 0 {
return
}
log.Debug().Msg("Starting fetching fediverse friends timeline...")
var totalPosts []models.FediversePost
var totalUsers []models.FediverseUser
userMap := make(map[string]models.FediverseUser)
for _, friend := range fediverseFriends {
log.Info().Str("id", friend.ID).Str("url", friend.URL).Msg("Fetching fediverse friend timeline...")
posts, err := FetchFediversePost(friend)
if err != nil {
log.Error().Err(err).Str("id", friend.ID).Str("url", friend.URL).Msg("Failed to fetch fediverse friend timeline...")
continue
}
log.Info().Str("id", friend.ID).Str("url", friend.URL).Int("count", len(posts)).Msg("Fetched fediverse friend timeline...")
for _, post := range posts {
if _, exists := userMap[post.User.Identifier]; !exists {
userMap[post.User.Identifier] = post.User
}
}
totalPosts = append(totalPosts, posts...)
}
for _, user := range userMap {
totalUsers = append(totalUsers, user)
}
if len(totalUsers) > 0 {
if err := database.C.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "identifier"}},
DoUpdates: clause.AssignmentColumns([]string{"name", "nick", "avatar"}),
}).Create(&totalUsers).Error; err != nil {
log.Error().Err(err).Msg("Failed to save fediverse users...")
}
for _, user := range totalUsers {
userMap[user.Identifier] = user
}
}
for i := range totalPosts {
if user, exists := userMap[totalPosts[i].User.Identifier]; exists {
totalPosts[i].UserID = user.ID
totalPosts[i].User = user
} else {
log.Warn().Str("user_identifier", totalPosts[i].User.Identifier).Msg("User ID not found for post, skipping")
totalPosts = append(totalPosts[:i], totalPosts[i+1:]...) // Remove invalid post
i-- // Adjust index after removal
}
}
if len(totalPosts) > 0 {
if err := database.C.Clauses(clause.OnConflict{DoNothing: true}).Create(&totalPosts).Error; err != nil {
log.Error().Err(err).Msg("Failed to save timeline posts...")
}
}
}

105
pkg/internal/services/mastodon/fetch.go
View File

@ -1,105 +0,0 @@
package mastodon
import (
"fmt"
"io"
"net/http"
"strings"
"time"
"git.solsynth.dev/hypernet/interactive/pkg/internal/models"
"git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
"github.com/gofiber/fiber/v2"
jsoniter "github.com/json-iterator/go"
"github.com/rs/zerolog/log"
"github.com/samber/lo"
)
type MastodomAttachment struct {
URL string `json:"url"`
Preview string `json:"preview"`
}
type MastodonPost struct {
ID string `json:"id"`
Content string `json:"content"`
URL string `json:"url"`
Sensitive bool `json:"sensitive"`
Account struct {
Acct string `json:"acct"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
Avatar string `json:"avatar"`
AccountURL string `json:"url"`
} `json:"account"`
Language string `json:"language"`
MediaAttachments []MastodomAttachment `json:"media_attachments"`
CreatedAt time.Time `json:"created_at"`
Server string `json:"-"`
}
func (v MastodonPost) ToFediversePost() models.FediversePost {
return models.FediversePost{
BaseModel: cruda.BaseModel{
CreatedAt: v.CreatedAt,
},
Identifier: fmt.Sprintf("%s@%s", v.ID, v.Server),
Origin: v.Server,
Content: v.Content,
Language: v.Language,
Images: lo.Map(v.MediaAttachments, func(item MastodomAttachment, _ int) string {
return item.URL
}), User: models.FediverseUser{
Identifier: v.Account.Acct,
Name: v.Account.Username,
Nick: v.Account.DisplayName,
Avatar: v.Account.Avatar,
Origin: v.Server,
},
}
}
func FetchTimeline(server string, limit int, useTrend bool) ([]MastodonPost, error) {
url := fmt.Sprintf(
lo.Ternary(
useTrend,
"%s/api/v1/trends/statuses?limit=%d",
"%s/api/v1/timelines/public?local=true&limit=%d",
),
server,
limit,
)
log.Debug().Str("url", url).Msg("Fetching mastodon timeline...")
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("failed to fetch public timeline: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %v", err)
}
if resp.StatusCode != fiber.StatusOK {
return nil, fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, body)
}
log.Debug().Str("url", url).Msg("Fetched mastodon timeline...")
var posts []MastodonPost
if err := jsoniter.Unmarshal(body, &posts); err != nil {
return nil, fmt.Errorf("failed to parse timeline JSON: %v", err)
}
posts = lo.Filter(posts, func(item MastodonPost, index int) bool {
return !item.Sensitive
})
for idx := range posts {
posts[idx].Server = strings.Replace(strings.Replace(server, "https://", "", 1), "http://", "", 1)
}
return posts, nil
}

49
pkg/internal/services/misskey/fetch.go
View File

@ -1,49 +0,0 @@
package services
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
type MisskeyPost struct {
ID string `json:"id"`
Text string `json:"text"`
User struct {
Username string `json:"username"`
DisplayName string `json:"name"`
AvatarURL string `json:"avatarUrl"`
} `json:"user"`
CreatedAt string `json:"createdAt"`
}
func FetchTimeline(server, token string, limit int) ([]MisskeyPost, error) {
url := fmt.Sprintf("%s/api/notes/global-timeline", server)
payload := map[string]interface{}{
"limit": limit,
"i": token,
}
body, _ := json.Marshal(payload)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch Misskey posts: %v", err)
}
defer resp.Body.Close()
var posts []MisskeyPost
if err := json.NewDecoder(resp.Body).Decode(&posts); err != nil {
return nil, fmt.Errorf("failed to parse Misskey JSON: %v", err)
}
return posts, nil
}

63
pkg/internal/services/queries/feed.go
View File

@ -34,8 +34,7 @@ func GetFeed(c *fiber.Ctx, limit int, user *uint, cursor *time.Time) ([]FeedEntr
// Planing the feed
limitF := float64(limit)
interCount := int(math.Ceil(limitF * 0.5))
fediCount := int(math.Ceil(limitF * 0.25))
newsCount := int(math.Ceil(limitF * 0.25))
readerCount := int(math.Ceil(limitF * 0.5))
// Internal posts
interTx, err := services.UniversalPostFilter(c, database.C)
@ -45,32 +44,21 @@ func GetFeed(c *fiber.Ctx, limit int, user *uint, cursor *time.Time) ([]FeedEntr
if cursor != nil {
interTx = interTx.Where("published_at < ?", *cursor)
}
interPosts, err := ListPostForFeed(interTx, interCount, user, c.Get("X-API-Version", "1"))
posts, err := ListPostForFeed(interTx, interCount, user, c.Get("X-API-Version", "1"))
if err != nil {
return nil, fmt.Errorf("failed to load interactive posts: %v", err)
}
feed = append(feed, interPosts...)
// Fediverse posts
fediTx := database.C
if cursor != nil {
fediTx = fediTx.Where("created_at < ?", *cursor)
}
fediPosts, err := ListFediversePostForFeed(fediTx, fediCount)
if err != nil {
return feed, fmt.Errorf("failed to load fediverse posts: %v", err)
}
feed = append(feed, fediPosts...)
feed = append(feed, posts...)
sort.Slice(feed, func(i, j int) bool {
return feed[i].CreatedAt.After(feed[j].CreatedAt)
})
// News today - from Reader
if news, err := ListNewsForFeed(newsCount, cursor); err != nil {
if news, err := ListNewsForFeed(readerCount, cursor); err != nil {
log.Error().Err(err).Msg("Failed to load news in getting feed...")
} else {
feed = append(feed, news)
feed = append(feed, news...)
}
return feed, nil
@ -100,27 +88,10 @@ func ListPostForFeed(tx *gorm.DB, limit int, user *uint, api string) ([]FeedEntr
return entries, nil
}
func ListFediversePostForFeed(tx *gorm.DB, limit int) ([]FeedEntry, error) {
var posts []models.FediversePost
if err := tx.
Preload("User").Limit(limit).
Find(&posts).Error; err != nil {
return nil, err
}
entries := lo.Map(posts, func(post models.FediversePost, _ int) FeedEntry {
return FeedEntry{
Type: "fediverse.post",
Data: post,
CreatedAt: post.CreatedAt,
}
})
return entries, nil
}
func ListNewsForFeed(limit int, cursor *time.Time) (FeedEntry, error) {
func ListNewsForFeed(limit int, cursor *time.Time) ([]FeedEntry, error) {
conn, err := gap.Nx.GetClientGrpcConn("re")
if err != nil {
return FeedEntry{}, fmt.Errorf("failed to get grpc connection with reader: %v", err)
return nil, fmt.Errorf("failed to get grpc connection with reader: %v", err)
}
client := proto.NewFeedServiceClient(conn)
request := &proto.GetFeedRequest{
@ -131,16 +102,16 @@ func ListNewsForFeed(limit int, cursor *time.Time) (FeedEntry, error) {
}
resp, err := client.GetFeed(context.Background(), request)
if err != nil {
return FeedEntry{}, fmt.Errorf("failed to get feed from reader: %v", err)
return nil, fmt.Errorf("failed to get feed from reader: %v", err)
}
var createdAt time.Time
return FeedEntry{
Type: "reader.news",
CreatedAt: createdAt,
Data: lo.Map(resp.Items, func(item *proto.FeedItem, _ int) map[string]any {
cta := time.UnixMilli(int64(item.CreatedAt))
createdAt = lo.Ternary(createdAt.Before(cta), cta, createdAt)
return nex.DecodeMap(item.Content)
}),
}, nil
return lo.Map(resp.Items, func(item *proto.FeedItem, _ int) FeedEntry {
cta := time.UnixMilli(int64(item.CreatedAt))
createdAt = lo.Ternary(createdAt.Before(cta), cta, createdAt)
return FeedEntry{
Type: item.Type,
Data: nex.DecodeMap(item.Content),
CreatedAt: cta,
}
}), nil
}

2
pkg/main.go
View File

@ -43,7 +43,6 @@ func main() {
if err := viper.ReadInConfig(); err != nil {
log.Panic().Err(err).Msg("An error occurred when loading settings.")
}
services.ReadFriendConfig()
// Connect to nexus
if err := gap.InitializeToNexus(); err != nil {
@ -68,7 +67,6 @@ func main() {
// Configure timed tasks
quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
quartz.AddFunc("@every 5m", services.FlushPostViews)
quartz.AddFunc("@every 5m", services.FetchFediverseTimedTask)
quartz.Start()
// App