package hn import ( "net/http" "time" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/peterhellberg/hn" "github.com/rs/zerolog" ) func NewFetcher(db *gorm.DB) *Fetcher { f := new(Fetcher) f.db = db f.fetchIntervalSecs = 60 f.hn = hn.NewClient(&http.Client{ Timeout: time.Duration(5 * time.Second), }) return f } type Fetcher struct { nextFetch time.Time fetchIntervalSecs uint db *gorm.DB hn *hn.Client log *zerolog.Logger } func (f *Fetcher) AddLogger(l *zerolog.Logger) { f.log = l } func (f *Fetcher) run() { f.db.AutoMigrate(&HNStoryRank{}) f.db.AutoMigrate(&FrontPageCache{}) for { f.log.Info(). Msg("fetching top stories from HN") f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second) err := f.StoreFrontPage() if err != nil { panic(err) } until := time.Until(f.nextFetch) countdown := time.NewTimer(until) f.log.Info().Msgf("waiting %s until next fetch", until) <-countdown.C } } func (f *Fetcher) StoreFrontPage() error { // FIXME set fetchid r := f.db.Select("max(FetchID)").Find(&HNStoryRank) ids, err := f.hn.TopStories() t := time.Now() if err != nil { return err } for i, id := range ids[:30] { item, err := f.hn.Item(id) if err != nil { return (err) } s := HNStoryRank{ HNID: uint(id), FetchID: uint(0), Rank: uint(i + 1), URL: item.URL, Title: item.Title, FetchedAt: t, } f.log.Info().Msgf("storing story with rank %d in db", (i + 1)) f.db.Create(&s) } return nil }