🎉 Initial Commit

pkg/internal/cache/store.go

@@ -0,0 +1,24 @@
+package cache
+
+import (
+	"github.com/dgraph-io/ristretto"
+	"github.com/eko/gocache/lib/v4/store"
+	ristrettoCache "github.com/eko/gocache/store/ristretto/v4"
+)
+
+var S store.StoreInterface
+
+func NewStore() error {
+	ristretto, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters: 1000,
+		MaxCost:     100,
+		BufferItems: 64,
+	})
+	if err != nil {
+		return err
+	}
+
+	S = ristrettoCache.NewRistretto(ristretto)
+
+	return nil
+}

pkg/internal/database/migrator.go

@@ -0,0 +1,20 @@
+package database
+
+import (
+	"git.solsynth.dev/hypernet/reader/pkg/internal/models"
+	"gorm.io/gorm"
+)
+
+var AutoMaintainRange = []any{
+	&models.LinkMeta{},
+}
+
+func RunMigration(source *gorm.DB) error {
+	if err := source.AutoMigrate(
+		AutoMaintainRange...,
+	); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/internal/database/source.go

@@ -0,0 +1,27 @@
+package database
+
+import (
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/gap"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"github.com/spf13/viper"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var C *gorm.DB
+
+func NewGorm() error {
+	var err error
+
+	dsn, err := cruda.NewCrudaConn(gap.Nx).AllocDatabase("reader")
+	C, err = gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.New(&log.Logger, logger.Config{
+		Colorful:                  true,
+		IgnoreRecordNotFoundError: true,
+		LogLevel:                  lo.Ternary(viper.GetBool("debug.database"), logger.Info, logger.Silent),
+	})})
+
+	return err
+}

pkg/internal/gap/server.go

@@ -0,0 +1,43 @@
+package gap
+
+import (
+	"fmt"
+	"git.solsynth.dev/hypernet/nexus/pkg/nex"
+	"git.solsynth.dev/hypernet/nexus/pkg/proto"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+var Nx *nex.Conn
+
+func InitializeToNexus() error {
+	grpcBind := strings.SplitN(viper.GetString("grpc_bind"), ":", 2)
+	httpBind := strings.SplitN(viper.GetString("bind"), ":", 2)
+
+	outboundIp, _ := nex.GetOutboundIP()
+
+	grpcOutbound := fmt.Sprintf("%s:%s", outboundIp, grpcBind[1])
+	httpOutbound := fmt.Sprintf("%s:%s", outboundIp, httpBind[1])
+
+	var err error
+	Nx, err = nex.NewNexusConn(viper.GetString("nexus_addr"), &proto.ServiceInfo{
+		Id:       viper.GetString("id"),
+		Type:     "uc",
+		Label:    "Reader",
+		GrpcAddr: grpcOutbound,
+		HttpAddr: lo.ToPtr("http://" + httpOutbound + "/api"),
+	})
+	if err == nil {
+		go func() {
+			err := Nx.RunRegistering()
+			if err != nil {
+				log.Error().Err(err).Msg("An error occurred while registering service...")
+			}
+		}()
+	}
+
+	return err
+}

pkg/internal/grpc/health.go

@@ -0,0 +1,26 @@
+package grpc
+
+import (
+	"context"
+	health "google.golang.org/grpc/health/grpc_health_v1"
+	"time"
+)
+
+func (v *Server) Check(ctx context.Context, request *health.HealthCheckRequest) (*health.HealthCheckResponse, error) {
+	return &health.HealthCheckResponse{
+		Status: health.HealthCheckResponse_SERVING,
+	}, nil
+}
+
+func (v *Server) Watch(request *health.HealthCheckRequest, server health.Health_WatchServer) error {
+	for {
+		if server.Send(&health.HealthCheckResponse{
+			Status: health.HealthCheckResponse_SERVING,
+		}) != nil {
+			break
+		}
+		time.Sleep(1000 * time.Millisecond)
+	}
+
+	return nil
+}

pkg/internal/grpc/server.go

@@ -0,0 +1,40 @@
+package grpc
+
+import (
+	"net"
+
+	"git.solsynth.dev/hypernet/nexus/pkg/proto"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+	health "google.golang.org/grpc/health/grpc_health_v1"
+	"google.golang.org/grpc/reflection"
+)
+
+type Server struct {
+	proto.UnimplementedDirectoryServiceServer
+	health.UnimplementedHealthServer
+
+	srv *grpc.Server
+}
+
+func NewGrpc() *Server {
+	server := &Server{
+		srv: grpc.NewServer(),
+	}
+
+	proto.RegisterDirectoryServiceServer(server.srv, server)
+	health.RegisterHealthServer(server.srv, server)
+
+	reflection.Register(server.srv)
+
+	return server
+}
+
+func (v *Server) Listen() error {
+	listener, err := net.Listen("tcp", viper.GetString("grpc_bind"))
+	if err != nil {
+		return err
+	}
+
+	return v.srv.Serve(listener)
+}

pkg/internal/grpc/services.go

@@ -0,0 +1,42 @@
+package grpc
+
+import (
+	"context"
+	"git.solsynth.dev/hypernet/nexus/pkg/nex"
+	"strconv"
+
+	"git.solsynth.dev/hypernet/nexus/pkg/proto"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/database"
+)
+
+func (v *Server) BroadcastEvent(ctx context.Context, in *proto.EventInfo) (*proto.EventResponse, error) {
+	switch in.GetEvent() {
+	case "deletion":
+		data := nex.DecodeMap(in.GetData())
+		resType, ok := data["type"].(string)
+		if !ok {
+			break
+		}
+		switch resType {
+		case "account":
+			id, ok := data["id"].(string)
+			if !ok {
+				break
+			}
+			numericId, err := strconv.Atoi(id)
+			if err != nil {
+				break
+			}
+			tx := database.C.Begin()
+			for _, model := range database.AutoMaintainRange {
+				switch model.(type) {
+				default:
+					tx.Delete(model, "account_id = ?", numericId)
+				}
+			}
+			tx.Commit()
+		}
+	}
+
+	return &proto.EventResponse{}, nil
+}

pkg/internal/meta.go

@@ -0,0 +1,5 @@
+package pkg
+
+const (
+	AppVersion = "1.0.0"
+)

pkg/internal/models/link.go

@@ -0,0 +1,18 @@
+package models
+
+import "git.solsynth.dev/hypernet/nexus/pkg/nex/cruda"
+
+type LinkMeta struct {
+	cruda.BaseModel
+
+	Entry       string  `json:"entry_id" gorm:"uniqueIndex"`
+	Icon        string  `json:"icon"`
+	URL         string  `json:"url"`
+	Title       *string `json:"title"`
+	Image       *string `json:"image"`
+	Video       *string `json:"video"`
+	Audio       *string `json:"audio"`
+	Description *string `json:"description"`
+	SiteName    *string `json:"site_name"`
+	Type        *string `json:"type"`
+}

pkg/internal/server/api/expand_link_api.go

@@ -0,0 +1,33 @@
+package api
+
+import (
+	"encoding/base64"
+	"sync"
+
+	"git.solsynth.dev/hypernet/reader/pkg/internal/services"
+	"github.com/gofiber/fiber/v2"
+)
+
+var expandInProgress sync.Map
+
+func getLinkMeta(c *fiber.Ctx) error {
+	targetEncoded := c.Params("*1")
+	targetRaw, _ := base64.StdEncoding.DecodeString(targetEncoded)
+
+	if ch, loaded := expandInProgress.LoadOrStore(targetEncoded, make(chan struct{})); loaded {
+		// If the request is already in progress, wait for it to complete
+		<-ch.(chan struct{})
+	} else {
+		// If this is the first request, process it and signal others
+		defer func() {
+			close(ch.(chan struct{}))
+			expandInProgress.Delete(targetEncoded)
+		}()
+	}
+
+	if meta, err := services.ScrapLink(string(targetRaw)); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	} else {
+		return c.JSON(meta)
+	}
+}

pkg/internal/server/api/index.go

@@ -0,0 +1,12 @@
+package api
+
+import (
+	"github.com/gofiber/fiber/v2"
+)
+
+func MapAPIs(app *fiber.App, baseURL string) {
+	api := app.Group(baseURL).Name("API")
+	{
+		api.Get("/link/*", getLinkMeta)
+	}
+}

pkg/internal/server/exts/utils.go

@@ -0,0 +1,18 @@
+package exts
+
+import (
+	"github.com/go-playground/validator/v10"
+	"github.com/gofiber/fiber/v2"
+)
+
+var validation = validator.New(validator.WithRequiredStructEnabled())
+
+func BindAndValidate(c *fiber.Ctx, out any) error {
+	if err := c.BodyParser(out); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	} else if err := validation.Struct(out); err != nil {
+		return fiber.NewError(fiber.StatusBadRequest, err.Error())
+	}
+
+	return nil
+}

pkg/internal/server/server.go

@@ -0,0 +1,71 @@
+package server
+
+import (
+	"strings"
+
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+
+	"git.solsynth.dev/hypernet/reader/pkg/internal/server/api"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/cors"
+	"github.com/gofiber/fiber/v2/middleware/idempotency"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	jsoniter "github.com/json-iterator/go"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+)
+
+var IReader *sec.InternalTokenReader
+
+type App struct {
+	app *fiber.App
+}
+
+func NewServer() *App {
+	app := fiber.New(fiber.Config{
+		DisableStartupMessage: true,
+		EnableIPValidation:    true,
+		ServerHeader:          "Hypernet.Reader",
+		AppName:               "Hypernet.Reader",
+		ProxyHeader:           fiber.HeaderXForwardedFor,
+		JSONEncoder:           jsoniter.ConfigCompatibleWithStandardLibrary.Marshal,
+		JSONDecoder:           jsoniter.ConfigCompatibleWithStandardLibrary.Unmarshal,
+		BodyLimit:             512 * 1024 * 1024 * 1024, // 512 TiB
+		EnablePrintRoutes:     viper.GetBool("debug.print_routes"),
+	})
+
+	app.Use(idempotency.New())
+	app.Use(cors.New(cors.Config{
+		AllowCredentials: true,
+		AllowMethods: strings.Join([]string{
+			fiber.MethodGet,
+			fiber.MethodPost,
+			fiber.MethodHead,
+			fiber.MethodOptions,
+			fiber.MethodPut,
+			fiber.MethodDelete,
+			fiber.MethodPatch,
+		}, ","),
+		AllowOriginsFunc: func(origin string) bool {
+			return true
+		},
+	}))
+
+	app.Use(logger.New(logger.Config{
+		Format: "${status} | ${latency} | ${method} ${path}\n",
+		Output: log.Logger,
+	}))
+
+	app.Use(sec.ContextMiddleware(IReader))
+
+	api.MapAPIs(app, "/api")
+
+	return &App{app}
+}
+
+func (v *App) Listen() {
+	if err := v.app.Listen(viper.GetString("bind")); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when starting server...")
+	}
+}

pkg/internal/services/cleaner.go

@@ -0,0 +1,24 @@
+package services
+
+import (
+	database2 "git.solsynth.dev/hypernet/reader/pkg/internal/database"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+func DoAutoDatabaseCleanup() {
+	deadline := time.Now().Add(60 * time.Minute)
+	log.Debug().Time("deadline", deadline).Msg("Now cleaning up entire database...")
+
+	var count int64
+	for _, model := range database2.AutoMaintainRange {
+		tx := database2.C.Unscoped().Delete(model, "deleted_at >= ?", deadline)
+		if tx.Error != nil {
+			log.Error().Err(tx.Error).Msg("An error occurred when running auth context cleanup...")
+		}
+		count += tx.RowsAffected
+	}
+
+	log.Debug().Int64("affected", count).Msg("Clean up entire database accomplished.")
+}

pkg/internal/services/scraper.go

@@ -0,0 +1,116 @@
+package services
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"net"
+	"net/http"
+	"time"
+
+	"git.solsynth.dev/hypernet/reader/pkg/internal/database"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/models"
+	"github.com/gocolly/colly"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+	"github.com/spf13/viper"
+)
+
+func GetLinkMetaFromCache(target string) (models.LinkMeta, error) {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	var meta models.LinkMeta
+	if err := database.C.Where("entry = ?", entry).First(&meta).Error; err != nil {
+		return meta, err
+	}
+	return meta, nil
+}
+
+func SaveLinkMetaToCache(target string, meta models.LinkMeta) error {
+	hash := md5.Sum([]byte(target))
+	entry := hex.EncodeToString(hash[:])
+	meta.Entry = entry
+	return database.C.Save(&meta).Error
+}
+
+func ScrapLink(target string) (*models.LinkMeta, error) {
+	if cache, err := GetLinkMetaFromCache(target); err == nil {
+		log.Debug().Str("url", target).Msg("Expanding link... hit cache")
+		return &cache, nil
+	}
+
+	c := colly.NewCollector(
+		colly.UserAgent(viper.GetString("scraper.user-agent")),
+		colly.MaxDepth(3),
+	)
+
+	c.WithTransport(&http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   60 * time.Second,
+			KeepAlive: 360 * time.Second,
+		}).DialContext,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	})
+
+	meta := &models.LinkMeta{
+		URL: target,
+	}
+
+	c.OnHTML("title", func(e *colly.HTMLElement) {
+		meta.Title = &e.Text
+	})
+	c.OnHTML("meta[name]", func(e *colly.HTMLElement) {
+		switch e.Attr("name") {
+		case "description":
+			meta.Description = lo.ToPtr(e.Attr("content"))
+		}
+	})
+	c.OnHTML("meta[property]", func(e *colly.HTMLElement) {
+		switch e.Attr("property") {
+		case "og:title":
+			meta.Title = lo.ToPtr(e.Attr("content"))
+		case "og:description":
+			meta.Description = lo.ToPtr(e.Attr("content"))
+		case "og:image":
+			meta.Image = lo.ToPtr(e.Attr("content"))
+		case "og:video":
+			meta.Video = lo.ToPtr(e.Attr("content"))
+		case "og:audio":
+			meta.Audio = lo.ToPtr(e.Attr("content"))
+		case "og:site_name":
+			meta.SiteName = lo.ToPtr(e.Attr("content"))
+		case "og:type":
+			meta.Type = lo.ToPtr(e.Attr("content"))
+		}
+	})
+	c.OnHTML("link[rel]", func(e *colly.HTMLElement) {
+		if e.Attr("rel") == "icon" {
+			meta.Icon = e.Request.AbsoluteURL(e.Attr("href"))
+		}
+	})
+
+	c.OnRequest(func(r *colly.Request) {
+		log.Debug().Str("url", target).Msg("Expanding link... requesting")
+	})
+	c.RedirectHandler = func(req *http.Request, via []*http.Request) error {
+		log.Debug().Str("url", req.URL.String()).Msg("Expanding link... redirecting")
+		return nil
+	}
+
+	c.OnResponse(func(r *colly.Response) {
+		log.Debug().Str("url", target).Msg("Expanding link... analyzing")
+	})
+	c.OnError(func(r *colly.Response, err error) {
+		log.Warn().Err(err).Str("url", target).Msg("Expanding link... failed")
+	})
+
+	c.OnScraped(func(r *colly.Response) {
+		_ = SaveLinkMetaToCache(target, *meta)
+		log.Debug().Str("url", target).Msg("Expanding link... finished")
+	})
+
+	return meta, c.Visit(target)
+}

pkg/main.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"fmt"
+	"git.solsynth.dev/hypernet/nexus/pkg/nex/sec"
+	pkg "git.solsynth.dev/hypernet/reader/pkg/internal"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/gap"
+	"github.com/fatih/color"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"git.solsynth.dev/hypernet/reader/pkg/internal/cache"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/database"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/grpc"
+
+	"git.solsynth.dev/hypernet/reader/pkg/internal/server"
+	"git.solsynth.dev/hypernet/reader/pkg/internal/services"
+	"github.com/robfig/cron/v3"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+)
+
+func init() {
+	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout})
+}
+
+func main() {
+	// Booting screen
+	fmt.Println(color.YellowString(" ____                           _ _\n|  _ \\ __ _ _ __   ___ _ __ ___| (_)_ __\n| |_) / _` | '_ \\ / _ \\ '__/ __| | | '_ \\\n|  __/ (_| | |_) |  __/ | | (__| | | |_) |\n|_|   \\__,_| .__/ \\___|_|  \\___|_|_| .__/\n           |_|                     |_|"))
+	fmt.Printf("%s v%s\n", color.New(color.FgHiYellow).Add(color.Bold).Sprintf("Hypernet.Reader"), pkg.AppVersion)
+	fmt.Printf("The upload service in Hypernet\n")
+	color.HiBlack("=====================================================\n")
+
+	// Configure settings
+	viper.AddConfigPath(".")
+	viper.AddConfigPath("..")
+	viper.SetConfigName("settings")
+	viper.SetConfigType("toml")
+
+	// Load settings
+	if err := viper.ReadInConfig(); err != nil {
+		log.Panic().Err(err).Msg("An error occurred when loading settings.")
+	}
+
+	// Connect to nexus
+	if err := gap.InitializeToNexus(); err != nil {
+		log.Error().Err(err).Msg("An error occurred when registering service to nexus...")
+	}
+
+	// Load keypair
+	if reader, err := sec.NewInternalTokenReader(viper.GetString("security.internal_public_key")); err != nil {
+		log.Error().Err(err).Msg("An error occurred when reading internal public key for jwt. Authentication related features will be disabled.")
+	} else {
+		server.IReader = reader
+		log.Info().Msg("Internal jwt public key loaded.")
+	}
+
+	// Connect to database
+	if err := database.NewGorm(); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when connect to database.")
+	} else if err := database.RunMigration(database.C); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when running database auto migration.")
+	}
+
+	// Initialize cache
+	if err := cache.NewStore(); err != nil {
+		log.Fatal().Err(err).Msg("An error occurred when initializing cache.")
+	}
+
+	// Set up some workers
+	for idx := 0; idx < viper.GetInt("workers.files_deletion"); idx++ {
+		go services.StartConsumeDeletionTask()
+	}
+	for idx := 0; idx < viper.GetInt("workers.files_analyze"); idx++ {
+		go services.StartConsumeAnalyzeTask()
+	}
+
+	// Configure timed tasks
+	quartz := cron.New(cron.WithLogger(cron.VerbosePrintfLogger(&log.Logger)))
+	quartz.AddFunc("@every 60m", services.DoAutoDatabaseCleanup)
+	quartz.AddFunc("@every 60m", services.RunMarkLifecycleDeletionTask)
+	quartz.AddFunc("@every 60m", services.RunMarkMultipartDeletionTask)
+	quartz.AddFunc("@midnight", services.RunScheduleDeletionTask)
+	quartz.Start()
+
+	// Server
+	go server.NewServer().Listen()
+
+	// Grpc Server
+	go grpc.NewGrpc().Listen()
+
+	// Post-boot actions
+	services.ScanUnanalyzedFileFromDatabase()
+	services.RunMarkLifecycleDeletionTask()
+
+	// Messages
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+	<-quit
+
+	quartz.Stop()
+}