sneak/orangesite
1
0
Fork 0
Code Issues 5 Pull Requests Releases Wiki Activity

bring repo into policy compliance; vendor assets; Gitea CI

Browse Source 
Vendor the front-end assets and drop the third-party CDN dependencies
(BootstrapCDN is being sunset): the bootstrap 4.0.0 css/js, jquery
3.2.1 slim, and popper 1.12.9 now live under static/ and are served
from the app, byte-for-byte identical to the previous SRI-pinned files.

Migrate CI from Drone to a Gitea Actions workflow that runs
docker build . on push, with the checkout action pinned by SHA.

Bring the repo up to standard:

- add REPO_POLICIES.md, .editorconfig, .dockerignore, .golangci.yml,
  and a comprehensive root-anchored .gitignore
- rewrite the Makefile with the required test/lint/fmt/fmt-check/check/
  docker/hooks targets (golangci-lint, 30s test timeout, verbose rerun
  on failure, check modifies nothing)
- rewrite the Dockerfile as a hash-pinned multistage build: a lint
  stage (golangci-lint), a glibc build+test stage (the legacy sqlite
  driver needs cgo+glibc), and a debian-slim runtime carrying the
  binary, templates, and static assets
- add real tests for the hn package
- bring the code into golangci-lint (default: all) compliance: fix the
  malformed gorm struct tags, check previously-ignored errors, dispatch
  the zerolog error event, avoid a uint->Duration overflow, split long
  functions, and add doc comments — all behaviour-preserving
- expand the README with the required sections
This commit is contained in:
Jeffrey Paul
2026-07-26 23:55:21 +07:00
parent 000f5bbaa6
commit 3ab9637246
23 changed files with 1009 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
hn/db.go
View File

@@ -1,3 +1,5 @@
// Package hn implements the orangesite Hacker News front-page scraper, its
// gorm-backed storage schema, and the HTTP handlers that render it.
package hn
import (
@@ -8,10 +10,17 @@ import (
// this schema is quite redundant, i know
// HNFrontPage records a story's tenure on the front page: when it appeared,
// when it disappeared, and the best rank it reached. The HN-prefixed name is
// retained deliberately: gorm derives the table name ("hn_front_pages") from
// it, so renaming would orphan the existing production database.
//
//nolint:revive // see above: renaming changes the gorm table name
type HNFrontPage struct {
gorm.Model
InternalID uint64 `gorm:"primary_key;auto_increment:true`
HNID uint // HN integer id
InternalID uint64
HNID uint // HN integer id
Appeared time.Time
Disappeared time.Time
HighestRank uint // frontpage index
@@ -21,9 +30,15 @@ type HNFrontPage struct {
URL string // duh
}
// HNStoryRank is a point-in-time snapshot of a single story's rank. The
// HN-prefixed name is retained deliberately for the same gorm table-name
// reason as HNFrontPage.
//
//nolint:revive // renaming changes the gorm table name
type HNStoryRank struct {
gorm.Model
InternalStoryID uint64 `gorm:"primary_key;auto_increment:true`
InternalStoryID uint64
HNID uint // HN integer id
Title string // submission title
URL string // duh
@@ -32,9 +47,11 @@ type HNStoryRank struct {
FetchedAt time.Time // identical within fetchid
}
// FrontPageCache is a cached view of a story's front-page lifetime.
type FrontPageCache struct {
gorm.Model
CacheID uint64 `gorm:"primary_key;auto_increment:true`
CacheID uint64
HNID uint
HighestRankReached uint
URL string

330
hn/fetcher.go
View File

@@ -6,36 +6,195 @@ import (
"time"
"github.com/jinzhu/gorm"
// sqlite3 dialect registered with gorm via import side effects.
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/peterhellberg/hn"
"github.com/rs/zerolog"
)
const (
// frontPageSize is the number of stories shown on the HN front page.
frontPageSize = 30
// defaultFetchInterval is how often the front page is scraped.
defaultFetchInterval = 60 * time.Second
// httpClientTimeout bounds each request to the HN API.
httpClientTimeout = 5 * time.Second
)
// Fetcher periodically scrapes the HN front page and records front-page
// tenure in the database.
type Fetcher struct {
nextFetch time.Time
fetchInterval time.Duration
db *gorm.DB
hn *hn.Client
log *zerolog.Logger
}
// NewFetcher builds a Fetcher that scrapes the HN front page once a minute.
func NewFetcher(db *gorm.DB) *Fetcher {
f := new(Fetcher)
f.db = db
f.fetchIntervalSecs = 60
f.fetchInterval = defaultFetchInterval
f.hn = hn.NewClient(&http.Client{
Timeout: time.Duration(5 * time.Second),
Timeout: httpClientTimeout,
})
return f
}
type Fetcher struct {
nextFetch time.Time
fetchIntervalSecs uint
db *gorm.DB
hn *hn.Client
log *zerolog.Logger
}
// AddLogger attaches a logger to the fetcher.
func (f *Fetcher) AddLogger(l *zerolog.Logger) {
f.log = l
}
func (f *Fetcher) run() {
// StoreFrontPage scrapes the current HN front page, records new and changed
// stories, and marks stories that have left the front page.
func (f *Fetcher) StoreFrontPage() error {
ids, err := f.hn.TopStories()
if err != nil {
return err
}
t := time.Now()
for i, id := range ids[:frontPageSize] {
item, err := f.hn.Item(id)
if err != nil {
return err
}
// check to see if the item was on the frontpage already or not
var c int
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
if c == 0 {
f.recordNewStory(id, i, t, item)
} else {
f.updateExistingStory(id, i, item)
}
}
return f.markDepartedStories(ids, t)
}
// recordNewStory records a story appearing on the front page for the first
// time.
func (f *Fetcher) recordNewStory(id, rank int, t time.Time, item *hn.Item) {
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
Disappeared: time.Time{},
HighestRank: uint(rank + 1),
Rank: uint(rank + 1),
Title: item.Title,
Score: uint(item.Score),
URL: item.URL,
}
f.db.Create(&r)
f.log.Info().
Uint("hnid", uint(id)).
Uint("rank", uint(rank+1)).
Str("title", item.Title).
Int("score", item.Score).
Str("url", item.URL).
Msg("HN new story on frontpage")
}
// updateExistingStory updates the rank, record rank, and score of a story
// still on (or returned to) the front page.
func (f *Fetcher) updateExistingStory(id, rank int, item *hn.Item) {
var old HNFrontPage
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
if old.Rank != uint(rank+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrank", old.Rank).
Uint("newrank", uint(rank+1)).
Int("score", item.Score).
Str("title", item.Title).
Str("url", item.URL).
Msg("HN story rank changed, recording new rank")
old.Rank = uint(rank + 1)
old.Score = uint(item.Score)
}
if old.HighestRank > uint(rank+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrecord", old.HighestRank).
Uint("newrecord", uint(rank+1)).
Msg("recording new record high rank for story")
old.HighestRank = uint(rank + 1)
}
if old.Score != uint(item.Score) {
old.Score = uint(item.Score)
}
// in any case it's here now
old.Disappeared = time.Time{}
f.db.Save(&old)
}
// markDepartedStories marks any active front-page rows whose story no longer
// appears in the current scrape as departed at time t.
func (f *Fetcher) markDepartedStories(ids []int, t time.Time) error {
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
if err != nil {
f.log.Error().Err(err).Msg("querying active frontpage items")
return err
}
var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
err = f.db.ScanRows(fpitems, &item)
if err != nil {
f.log.Error().Err(err).Msg("scanning frontpage row")
continue
}
exitedFrontPage := true
for _, xd := range ids[:frontPageSize] {
if item.HNID == uint(xd) {
exitedFrontPage = false
}
}
if exitedFrontPage {
toupdate = append(toupdate, item.HNID)
dur := t.Sub(item.Appeared).String()
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
// close result before we do the update
_ = fpitems.Close()
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
return nil
}
// run migrates the schema and then scrapes the front page forever on the
// configured interval.
func (f *Fetcher) run() {
if os.Getenv("DEBUG") != "" {
f.db.LogMode(true)
}
@@ -45,157 +204,18 @@ func (f *Fetcher) run() {
f.db.AutoMigrate(&HNFrontPage{})
for {
f.log.Info().
Msg("fetching top stories from HN")
f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second)
f.log.Info().Msg("fetching top stories from HN")
f.nextFetch = time.Now().Add(f.fetchInterval)
err := f.StoreFrontPage()
if err != nil {
panic(err)
}
until := time.Until(f.nextFetch)
countdown := time.NewTimer(until)
f.log.Info().Msgf("waiting %s until next fetch", until)
<-countdown.C
}
}
func (f *Fetcher) StoreFrontPage() error {
// FIXME set fetchid
//r, err := f.db.Table("hn_story_rank").Select("MAX(FetchID)").Rows()
//pp.Print(r)
//Select("max(FetchID)").Find(&HNStoryRank)
ids, err := f.hn.TopStories()
t := time.Now()
if err != nil {
return err
}
// 30 items on HN frontpage.
for i, id := range ids[:30] {
item, err := f.hn.Item(id)
if err != nil {
return (err)
}
/*
s := HNStoryRank{
HNID: uint(id),
Rank: uint(i + 1),
URL: item.URL,
Title: item.Title,
Score: item.Score,
FetchedAt: t,
}
*/
//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
// FIXME this will grow unbounded and make the file too big if
// I don't clean this up or otherwise limit the data in here
// disabled for now
//f.db.Create(&s)
//FIXME check to see if the same HNID was already on the frontpage
//or not so we don't spam the db
// check to see if the item was on the frontpage already or not
var c int
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
if c == 0 {
// first appearance on frontpage
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
Disappeared: time.Time{},
HighestRank: uint(i + 1),
Rank: uint(i + 1),
Title: item.Title,
Score: uint(item.Score),
URL: item.URL,
}
f.db.Create(&r)
f.log.Info().
Uint("hnid", uint(id)).
Uint("rank", uint(i+1)).
Str("title", item.Title).
Int("score", item.Score).
Str("url", item.URL).
Msg("HN new story on frontpage")
} else {
// it's still here, (or back)
var old HNFrontPage
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
if old.Rank != uint(i+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrank", old.Rank).
Uint("newrank", uint(i+1)).
Int("score", item.Score).
Str("title", item.Title).
Str("url", item.URL).
Msg("HN story rank changed, recording new rank")
old.Rank = uint(i + 1)
old.Score = uint(item.Score)
}
if old.HighestRank > uint(i+1) {
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrecord", old.HighestRank).
Uint("newrecord", uint(i+1)).
Msg("recording new record high rank for story")
old.HighestRank = uint(i + 1)
}
if old.Score != uint(item.Score) {
old.Score = uint(item.Score)
}
// in any case it's here now
old.Disappeared = time.Time{}
f.db.Save(&old)
}
}
// FIXME iterate over frontpage items still active in DB and note any
// that are no longer on the scrape
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
if err != nil {
f.log.Error().
Err(err)
}
var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
f.db.ScanRows(fpitems, &item)
//pp.Print(item)
exitedFrontPage := true
for _, xd := range ids[:30] {
if item.HNID == uint(xd) {
exitedFrontPage = false
}
}
if exitedFrontPage {
toupdate = append(toupdate, item.HNID)
//item.Disappeared = t
dur := t.Sub(item.Appeared).String()
//f.db.Save(&item)
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
fpitems.Close() // close result before we do the update
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
return nil
}

78
hn/handlers.go
View File

@@ -10,38 +10,58 @@ import (
"github.com/labstack/echo"
)
// RequestHandlerSet holds the dependencies shared by the HTTP handlers.
type RequestHandlerSet struct {
db *gorm.DB
version string
}
// NewRequestHandlerSet builds a RequestHandlerSet for the given version and
// database handle.
func NewRequestHandlerSet(version string, db *gorm.DB) *RequestHandlerSet {
rhs := new(RequestHandlerSet)
rhs.db = db
rhs.version = version
return rhs
}
func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
// exitRow is a story that has left the front page within the last 24h.
type exitRow struct {
Duration string
DurationSecs uint
URL string
Title string
HighestRank uint
HNID uint
Score uint
TimeGone string
TimeGoneSecs uint
}
// currentRow is a story currently on the front page.
type currentRow struct {
Duration string
DurationSecs uint
URL string
Title string
Score uint
HighestRank uint
HNID uint
Rank uint
}
// exitedRows returns the stories that left the front page in the last 24h,
// most recently departed first.
func (r *RequestHandlerSet) exitedRows() []exitRow {
last24h := time.Now().Add(time.Second * 86400 * -1)
var fpi []HNFrontPage
r.db.Where("disappeared is not ? and disappeared > ?", time.Time{}, last24h).Order("disappeared desc").Find(&fpi)
type fprow struct {
Duration string
DurationSecs uint
URL string
Title string
HighestRank uint
HNID uint
Score uint
TimeGone string
TimeGoneSecs uint
}
var fprows []fprow
rows := make([]exitRow, 0, len(fpi))
for _, item := range fpi {
fprows = append(fprows, fprow{
rows = append(rows, exitRow{
Duration: u.TimeDiffHuman(item.Disappeared, item.Appeared),
DurationSecs: u.TimeDiffAbsSeconds(item.Disappeared, item.Appeared),
URL: item.URL,
@@ -54,23 +74,17 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
})
}
type rowtwo struct {
Duration string
DurationSecs uint
URL string
Title string
Score uint
HighestRank uint
HNID uint
Rank uint
}
var currentfp []rowtwo
return rows
}
// currentRows returns the stories currently on the front page, best rank first.
func (r *RequestHandlerSet) currentRows() []currentRow {
var cur []HNFrontPage
r.db.Where("disappeared is ?", time.Time{}).Order("rank asc").Find(&cur)
rows := make([]currentRow, 0, len(cur))
for _, item := range cur {
currentfp = append(currentfp, rowtwo{
rows = append(rows, currentRow{
Duration: u.TimeDiffHuman(time.Now(), item.Appeared),
DurationSecs: u.TimeDiffAbsSeconds(time.Now(), item.Appeared),
URL: item.URL,
@@ -82,12 +96,17 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
})
}
return rows
}
func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano),
"exits": fprows,
"current": currentfp,
"exits": r.exitedRows(),
"current": r.currentRows(),
"gitrev": r.version,
}
return c.Render(http.StatusOK, "index.html", tc)
}
@@ -96,5 +115,6 @@ func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
"time": time.Now().UTC().Format(time.RFC3339Nano),
"gitrev": r.version,
}
return c.Render(http.StatusOK, "about.html", tc)
}

73
hn/hn_test.go Normal file
View File

@@ -0,0 +1,73 @@
package hn_test
import (
"testing"
"time"
"git.eeqj.de/sneak/orangesite/hn"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
func TestNewFetcher(t *testing.T) {
t.Parallel()
f := hn.NewFetcher(nil)
if f == nil {
t.Fatal("NewFetcher returned nil")
}
}
func TestNewRequestHandlerSet(t *testing.T) {
t.Parallel()
rhs := hn.NewRequestHandlerSet("v1.2.3", nil)
if rhs == nil {
t.Fatal("NewRequestHandlerSet returned nil")
}
}
// TestHNFrontPageRoundTrip exercises the gorm schema against an in-memory
// sqlite database: migrate, insert, and read back a front-page row.
func TestHNFrontPageRoundTrip(t *testing.T) {
t.Parallel()
db, err := gorm.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open in-memory db: %v", err)
}
defer func() { _ = db.Close() }()
err = db.AutoMigrate(&hn.HNFrontPage{}).Error
if err != nil {
t.Fatalf("automigrate: %v", err)
}
want := hn.HNFrontPage{
HNID: 42,
Appeared: time.Now(),
Disappeared: time.Time{},
HighestRank: 1,
Rank: 3,
Title: "hello world",
Score: 100,
URL: "https://example.com",
}
err = db.Create(&want).Error
if err != nil {
t.Fatalf("create: %v", err)
}
var got hn.HNFrontPage
err = db.Model(&hn.HNFrontPage{}).Where("hn_id = ?", 42).First(&got).Error
if err != nil {
t.Fatalf("query: %v", err)
}
if got.Title != want.Title || got.Score != want.Score || got.URL != want.URL {
t.Errorf("round-trip mismatch: got %+v, want title=%q score=%d url=%q",
got, want.Title, want.Score, want.URL)
}
}

30
hn/server.go
View File

@@ -5,6 +5,7 @@ import (
"time"
"github.com/jinzhu/gorm"
// sqlite3 dialect registered with gorm via import side effects.
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/labstack/echo"
"github.com/labstack/echo/middleware"
@@ -17,6 +18,8 @@ import (
"github.com/rs/zerolog/log"
)
// App holds the running server's dependencies: the echo instance, the
// logger, the database handle, the startup time, and the background fetcher.
type App struct {
version string
buildarch string
@@ -27,6 +30,8 @@ type App struct {
fetcher *Fetcher
}
// RunServer boots the application, starts the background fetcher, and blocks
// serving HTTP until the server exits, returning a process exit code.
func RunServer(version string, buildarch string) int {
a := new(App)
a.version = version
@@ -34,7 +39,7 @@ func RunServer(version string, buildarch string) int {
a.startup = time.Now()
a.init()
defer a.db.Close()
defer func() { _ = a.db.Close() }()
a.fetcher = NewFetcher(a.db)
a.fetcher.AddLogger(a.log)
@@ -48,6 +53,7 @@ func (a *App) init() {
// setup logging
l := log.With().Caller().Logger()
log.Logger = l
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
if tty {
out := zerolog.NewConsoleWriter(
@@ -58,11 +64,14 @@ func (a *App) init() {
)
log.Logger = log.Output(out)
}
// always log in UTC
zerolog.TimestampFunc = func() time.Time {
return time.Now().UTC()
}
zerolog.SetGlobalLevel(zerolog.InfoLevel)
if os.Getenv("DEBUG") != "" {
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
@@ -71,18 +80,18 @@ func (a *App) init() {
a.identify()
// open db
// FIXME make configurable path
DATABASE_PATH := "/data/storage.sqlite"
// database path defaults to the container data volume, overridable
// with the DATABASE_PATH environment variable
dbPath := "/data/storage.sqlite"
if os.Getenv("DATABASE_PATH") != "" {
DATABASE_PATH = os.Getenv("DATABASE_PATH")
dbPath = os.Getenv("DATABASE_PATH")
}
db, err := gorm.Open("sqlite3", DATABASE_PATH)
db, err := gorm.Open("sqlite3", dbPath)
if err != nil {
panic("failed to open database: " + err.Error())
}
a.db = db
}
@@ -94,7 +103,6 @@ func (a *App) identify() {
}
func (a *App) runForever() int {
// Echo instance
a.e = echo.New()
@@ -120,16 +128,18 @@ func (a *App) runForever() int {
if err != nil {
a.e.Logger.Fatal(err)
}
a.e.Renderer = r
rhs := NewRequestHandlerSet(a.version, a.db)
// Routes
a.e.Static("/static", "static")
a.e.GET("/", rhs.indexHandler)
a.e.GET("/about", rhs.aboutHandler)
// Start server
// Start server (blocks; Fatal exits the process on error)
a.e.Logger.Fatal(a.e.Start(":8080"))
return 0 //FIXME setup graceful shutdown
return 0
}