sneak/orangesite
1
0
Fork 0
Code Issues 5 Pull Requests Releases Wiki Activity

bring repo into policy compliance; vendor assets; Gitea CI

Browse Source 
Vendor the front-end assets and drop the third-party CDN dependencies
(BootstrapCDN is being sunset): the bootstrap 4.0.0 css/js, jquery
3.2.1 slim, and popper 1.12.9 now live under static/ and are served
from the app, byte-for-byte identical to the previous SRI-pinned files.

Migrate CI from Drone to a Gitea Actions workflow that runs
docker build . on push, with the checkout action pinned by SHA.

Bring the repo up to standard:

- add REPO_POLICIES.md, .editorconfig, .dockerignore, .golangci.yml,
  and a comprehensive root-anchored .gitignore
- rewrite the Makefile with the required test/lint/fmt/fmt-check/check/
  docker/hooks targets (golangci-lint, 30s test timeout, verbose rerun
  on failure, check modifies nothing)
- rewrite the Dockerfile as a hash-pinned multistage build: a lint
  stage (golangci-lint), a glibc build+test stage (the legacy sqlite
  driver needs cgo+glibc), and a debian-slim runtime carrying the
  binary, templates, and static assets
- add real tests for the hn package
- bring the code into golangci-lint (default: all) compliance: fix the
  malformed gorm struct tags, check previously-ignored errors, dispatch
  the zerolog error event, avoid a uint->Duration overflow, split long
  functions, and add doc comments — all behaviour-preserving
- expand the README with the required sections
This commit is contained in:
Jeffrey Paul
2026-07-26 23:55:21 +07:00
parent 000f5bbaa6
commit 3ab9637246
23 changed files with 1009 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
hn/fetcher.go
View File

@@ -6,36 +6,195 @@ import (
"time"
"github.com/jinzhu/gorm"
// sqlite3 dialect registered with gorm via import side effects.
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/peterhellberg/hn"
"github.com/rs/zerolog"
)
const (
// frontPageSize is the number of stories shown on the HN front page.
frontPageSize = 30
// defaultFetchInterval is how often the front page is scraped.
defaultFetchInterval = 60 * time.Second
// httpClientTimeout bounds each request to the HN API.
httpClientTimeout = 5 * time.Second
)
// Fetcher periodically scrapes the HN front page and records front-page
// tenure in the database.
type Fetcher struct {
nextFetch time.Time
fetchInterval time.Duration
db *gorm.DB
hn *hn.Client
log *zerolog.Logger
}
// NewFetcher builds a Fetcher that scrapes the HN front page once a minute.
func NewFetcher(db *gorm.DB) *Fetcher {
f := new(Fetcher)
f.db = db
f.fetchIntervalSecs = 60
f.fetchInterval = defaultFetchInterval
f.hn = hn.NewClient(&http.Client{
Timeout: time.Duration(5 * time.Second),
Timeout: httpClientTimeout,
})
return f
}
type Fetcher struct {
nextFetch time.Time
fetchIntervalSecs uint
db *gorm.DB
hn *hn.Client
log *zerolog.Logger
}
// AddLogger attaches a logger to the fetcher.
func (f *Fetcher) AddLogger(l *zerolog.Logger) {
f.log = l
}
func (f *Fetcher) run() {
// StoreFrontPage scrapes the current HN front page, records new and changed
// stories, and marks stories that have left the front page.
func (f *Fetcher) StoreFrontPage() error {
ids, err := f.hn.TopStories()
if err != nil {
return err
}
t := time.Now()
for i, id := range ids[:frontPageSize] {
item, err := f.hn.Item(id)
if err != nil {
return err
}
// check to see if the item was on the frontpage already or not
var c int
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
if c == 0 {
f.recordNewStory(id, i, t, item)
} else {
f.updateExistingStory(id, i, item)
}
}
return f.markDepartedStories(ids, t)
}
// recordNewStory records a story appearing on the front page for the first
// time.
func (f *Fetcher) recordNewStory(id, rank int, t time.Time, item *hn.Item) {
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
Disappeared: time.Time{},
HighestRank: uint(rank + 1),
Rank: uint(rank + 1),
Title: item.Title,
Score: uint(item.Score),
URL: item.URL,
}
f.db.Create(&r)
f.log.Info().
Uint("hnid", uint(id)).
Uint("rank", uint(rank+1)).
Str("title", item.Title).
Int("score", item.Score).
Str("url", item.URL).
Msg("HN new story on frontpage")
}
// updateExistingStory updates the rank, record rank, and score of a story
// still on (or returned to) the front page.
func (f *Fetcher) updateExistingStory(id, rank int, item *hn.Item) {
var old HNFrontPage
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
if old.Rank != uint(rank+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrank", old.Rank).
Uint("newrank", uint(rank+1)).
Int("score", item.Score).
Str("title", item.Title).
Str("url", item.URL).
Msg("HN story rank changed, recording new rank")
old.Rank = uint(rank + 1)
old.Score = uint(item.Score)
}
if old.HighestRank > uint(rank+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrecord", old.HighestRank).
Uint("newrecord", uint(rank+1)).
Msg("recording new record high rank for story")
old.HighestRank = uint(rank + 1)
}
if old.Score != uint(item.Score) {
old.Score = uint(item.Score)
}
// in any case it's here now
old.Disappeared = time.Time{}
f.db.Save(&old)
}
// markDepartedStories marks any active front-page rows whose story no longer
// appears in the current scrape as departed at time t.
func (f *Fetcher) markDepartedStories(ids []int, t time.Time) error {
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
if err != nil {
f.log.Error().Err(err).Msg("querying active frontpage items")
return err
}
var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
err = f.db.ScanRows(fpitems, &item)
if err != nil {
f.log.Error().Err(err).Msg("scanning frontpage row")
continue
}
exitedFrontPage := true
for _, xd := range ids[:frontPageSize] {
if item.HNID == uint(xd) {
exitedFrontPage = false
}
}
if exitedFrontPage {
toupdate = append(toupdate, item.HNID)
dur := t.Sub(item.Appeared).String()
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
// close result before we do the update
_ = fpitems.Close()
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
return nil
}
// run migrates the schema and then scrapes the front page forever on the
// configured interval.
func (f *Fetcher) run() {
if os.Getenv("DEBUG") != "" {
f.db.LogMode(true)
}
@@ -45,157 +204,18 @@ func (f *Fetcher) run() {
f.db.AutoMigrate(&HNFrontPage{})
for {
f.log.Info().
Msg("fetching top stories from HN")
f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second)
f.log.Info().Msg("fetching top stories from HN")
f.nextFetch = time.Now().Add(f.fetchInterval)
err := f.StoreFrontPage()
if err != nil {
panic(err)
}
until := time.Until(f.nextFetch)
countdown := time.NewTimer(until)
f.log.Info().Msgf("waiting %s until next fetch", until)
<-countdown.C
}
}
func (f *Fetcher) StoreFrontPage() error {
// FIXME set fetchid
//r, err := f.db.Table("hn_story_rank").Select("MAX(FetchID)").Rows()
//pp.Print(r)
//Select("max(FetchID)").Find(&HNStoryRank)
ids, err := f.hn.TopStories()
t := time.Now()
if err != nil {
return err
}
// 30 items on HN frontpage.
for i, id := range ids[:30] {
item, err := f.hn.Item(id)
if err != nil {
return (err)
}
/*
s := HNStoryRank{
HNID: uint(id),
Rank: uint(i + 1),
URL: item.URL,
Title: item.Title,
Score: item.Score,
FetchedAt: t,
}
*/
//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
// FIXME this will grow unbounded and make the file too big if
// I don't clean this up or otherwise limit the data in here
// disabled for now
//f.db.Create(&s)
//FIXME check to see if the same HNID was already on the frontpage
//or not so we don't spam the db
// check to see if the item was on the frontpage already or not
var c int
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
if c == 0 {
// first appearance on frontpage
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
Disappeared: time.Time{},
HighestRank: uint(i + 1),
Rank: uint(i + 1),
Title: item.Title,
Score: uint(item.Score),
URL: item.URL,
}
f.db.Create(&r)
f.log.Info().
Uint("hnid", uint(id)).
Uint("rank", uint(i+1)).
Str("title", item.Title).
Int("score", item.Score).
Str("url", item.URL).
Msg("HN new story on frontpage")
} else {
// it's still here, (or back)
var old HNFrontPage
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
if old.Rank != uint(i+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrank", old.Rank).
Uint("newrank", uint(i+1)).
Int("score", item.Score).
Str("title", item.Title).
Str("url", item.URL).
Msg("HN story rank changed, recording new rank")
old.Rank = uint(i + 1)
old.Score = uint(item.Score)
}
if old.HighestRank > uint(i+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrecord", old.HighestRank).
Uint("newrecord", uint(i+1)).
Msg("recording new record high rank for story")
old.HighestRank = uint(i + 1)
}
if old.Score != uint(item.Score) {
old.Score = uint(item.Score)
}
// in any case it's here now
old.Disappeared = time.Time{}
f.db.Save(&old)
}
}
// FIXME iterate over frontpage items still active in DB and note any
// that are no longer on the scrape
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
if err != nil {
f.log.Error().
Err(err)
}
var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
f.db.ScanRows(fpitems, &item)
//pp.Print(item)
exitedFrontPage := true
for _, xd := range ids[:30] {
if item.HNID == uint(xd) {
exitedFrontPage = false
}
}
if exitedFrontPage {
toupdate = append(toupdate, item.HNID)
//item.Disappeared = t
dur := t.Sub(item.Appeared).String()
//f.db.Save(&item)
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
fpitems.Close() // close result before we do the update
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
return nil
}