progress

hn/fetcher.go

@@ -0,0 +1,83 @@
+package hn
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/peterhellberg/hn"
+	"github.com/rs/zerolog"
+)
+
+func NewFetcher(db *gorm.DB) *Fetcher {
+	f := new(Fetcher)
+	f.db = db
+	f.fetchIntervalSecs = 60
+	f.hn = hn.NewClient(&http.Client{
+		Timeout: time.Duration(5 * time.Second),
+	})
+	return f
+}
+
+type Fetcher struct {
+	nextFetch         time.Time
+	fetchIntervalSecs uint
+	db                *gorm.DB
+	hn                *hn.Client
+	log               *zerolog.Logger
+}
+
+func (f *Fetcher) AddLogger(l *zerolog.Logger) {
+	f.log = l
+}
+
+func (f *Fetcher) run() {
+	f.db.AutoMigrate(&HNStoryRank{})
+	f.db.AutoMigrate(&FrontPageCache{})
+
+	for {
+		f.log.Info().
+			Msg("fetching top stories from HN")
+		f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second)
+		err := f.StoreFrontPage()
+		if err != nil {
+			panic(err)
+		}
+		until := time.Until(f.nextFetch)
+		countdown := time.NewTimer(until)
+		f.log.Info().Msgf("waiting %s until next fetch", until)
+		<-countdown.C
+	}
+}
+
+func (f *Fetcher) StoreFrontPage() error {
+
+	// FIXME set fetchid
+	r := f.db.Select("max(FetchID)").Find(&HNStoryRank)
+
+	ids, err := f.hn.TopStories()
+	t := time.Now()
+	if err != nil {
+		return err
+	}
+
+	for i, id := range ids[:30] {
+
+		item, err := f.hn.Item(id)
+		if err != nil {
+			return (err)
+		}
+		s := HNStoryRank{
+			HNID:      uint(id),
+			FetchID:   uint(0),
+			Rank:      uint(i + 1),
+			URL:       item.URL,
+			Title:     item.Title,
+			FetchedAt: t,
+		}
+		f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
+		f.db.Create(&s)
+	}
+	return nil
+}