HyperNet/Reader
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

👔 Scrap the website according the feed whatever it provide content or not

Browse Source
This commit is contained in:
LittleSheep 2025-01-26 00:41:33 +08:00
parent 01b0cbce3e
commit fbbe72bd54
1 changed files with 6 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/internal/services/reader.go
View File

@ -145,18 +145,13 @@ func newsSourceReadFeed(src models.NewsSource) ([]models.NewsArticle, error) {
if item.Image != nil { if item.Image != nil {
parent.Thumbnail = item.Image.URL parent.Thumbnail = item.Image.URL
} }
if len(item.Content) > 0 {
// Good website, provide content, skip scraping of it article, err := ScrapNews(item.Link, parent)
parent.Content = item.Content if err != nil {
result = append(result, pgConvert(parent)) log.Warn().Err(err).Str("url", item.Link).Msg("Failed to scrap a news article...")
} else { continue
article, err := ScrapNews(item.Link, parent)
if err != nil {
log.Warn().Err(err).Str("url", item.Link).Msg("Failed to scrap a news article...")
continue
}
result = append(result, pgConvert(*article))
} }
result = append(result, pgConvert(*article))
log.Debug().Str("url", item.Link).Msg("Scraped a news article...") log.Debug().Str("url", item.Link).Msg("Scraped a news article...")
} }