progress

hn/db.go

@@ -0,0 +1,29 @@
+package hn
+
+import (
+	"time"
+
+	"github.com/jinzhu/gorm"
+)
+
+type HNStoryRank struct {
+	gorm.Model
+	InternalStoryID uint64    `gorm:"primary_key;auto_increment:true`
+	HNID            uint      // HN integer id
+	Title           string    // submission title
+	URL             string    // duh
+	FetchID         uint      // integer identifying fetch batch
+	Rank            uint      // frontpage index
+	FetchedAt       time.Time // identical within fetchid
+}
+
+type FrontPageCache struct {
+	gorm.Model
+	CacheID            uint64 `gorm:"primary_key;auto_increment:true`
+	HNID               uint
+	HighestRankReached uint
+	URL                string
+	Title              string
+	Appeared           time.Time
+	Disappeared        time.Time
+}

hn/fetcher.go

@@ -0,0 +1,83 @@
+package hn
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	"github.com/peterhellberg/hn"
+	"github.com/rs/zerolog"
+)
+
+func NewFetcher(db *gorm.DB) *Fetcher {
+	f := new(Fetcher)
+	f.db = db
+	f.fetchIntervalSecs = 60
+	f.hn = hn.NewClient(&http.Client{
+		Timeout: time.Duration(5 * time.Second),
+	})
+	return f
+}
+
+type Fetcher struct {
+	nextFetch         time.Time
+	fetchIntervalSecs uint
+	db                *gorm.DB
+	hn                *hn.Client
+	log               *zerolog.Logger
+}
+
+func (f *Fetcher) AddLogger(l *zerolog.Logger) {
+	f.log = l
+}
+
+func (f *Fetcher) run() {
+	f.db.AutoMigrate(&HNStoryRank{})
+	f.db.AutoMigrate(&FrontPageCache{})
+
+	for {
+		f.log.Info().
+			Msg("fetching top stories from HN")
+		f.nextFetch = time.Now().Add(time.Duration(f.fetchIntervalSecs) * time.Second)
+		err := f.StoreFrontPage()
+		if err != nil {
+			panic(err)
+		}
+		until := time.Until(f.nextFetch)
+		countdown := time.NewTimer(until)
+		f.log.Info().Msgf("waiting %s until next fetch", until)
+		<-countdown.C
+	}
+}
+
+func (f *Fetcher) StoreFrontPage() error {
+
+	// FIXME set fetchid
+	r := f.db.Select("max(FetchID)").Find(&HNStoryRank)
+
+	ids, err := f.hn.TopStories()
+	t := time.Now()
+	if err != nil {
+		return err
+	}
+
+	for i, id := range ids[:30] {
+
+		item, err := f.hn.Item(id)
+		if err != nil {
+			return (err)
+		}
+		s := HNStoryRank{
+			HNID:      uint(id),
+			FetchID:   uint(0),
+			Rank:      uint(i + 1),
+			URL:       item.URL,
+			Title:     item.Title,
+			FetchedAt: t,
+		}
+		f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
+		f.db.Create(&s)
+	}
+	return nil
+}

hn/handlers.go

@@ -2,10 +2,22 @@ package hn
 
 import (
 	"net/http"
+	"time"
 
+	"github.com/flosch/pongo2"
 	"github.com/labstack/echo"
 )
 
 func indexHandler(c echo.Context) error {
-	return c.Render(http.StatusOK, "index", nil)
+	tc := pongo2.Context{
+		"time": time.Now().UTC().Format(time.RFC3339Nano),
+	}
+	return c.Render(http.StatusOK, "index.html", tc)
+}
+
+func aboutHandler(c echo.Context) error {
+	tc := pongo2.Context{
+		"time": time.Now().UTC().Format(time.RFC3339Nano),
+	}
+	return c.Render(http.StatusOK, "about.html", tc)
 }

hn/server.go

@@ -8,19 +8,23 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/labstack/echo"
 	"github.com/labstack/echo/middleware"
-	"github.com/labstack/gommon/log"
+	gl "github.com/labstack/gommon/log"
+	"github.com/mattn/go-isatty"
+	ep2 "github.com/mayowa/echo-pongo2"
 	"github.com/ziflex/lecho/v2"
-)
 
-// required for orm
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+)
 
 type App struct {
 	version   string
 	buildarch string
 	e         *echo.Echo
-	logger    *string
+	log       *zerolog.Logger
 	db        *gorm.DB
 	startup   time.Time
+	fetcher   *Fetcher
 }
 
 func RunServer(version string, buildarch string) int {
@@ -28,18 +32,69 @@ func RunServer(version string, buildarch string) int {
 	a.version = version
 	a.buildarch = buildarch
 	a.startup = time.Now()
-	a.runForever()
-	return 0
+
+	a.init()
+	defer a.db.Close()
+
+	a.fetcher = NewFetcher(a.db)
+	a.fetcher.AddLogger(a.log)
+
+	go a.fetcher.run()
+
+	return a.runForever()
 }
 
-func (a *App) runForever() {
+func (a *App) init() {
+	// setup logging
+	l := log.With().Caller().Logger()
+	log.Logger = l
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if tty {
+		out := zerolog.NewConsoleWriter(
+			func(w *zerolog.ConsoleWriter) {
+				// Customize time format
+				w.TimeFormat = time.RFC3339
+			},
+		)
+		log.Logger = log.Output(out)
+	}
+	// always log in UTC
+	zerolog.TimestampFunc = func() time.Time {
+		return time.Now().UTC()
+	}
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	if os.Getenv("DEBUG") != "" {
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+
+	a.log = &log.Logger
+
+	a.identify()
+
+	// open db
+	// FIXME make configurable path
+	db, err := gorm.Open("sqlite3", "storage.db")
+	if err != nil {
+		panic("failed to connect database")
+	}
+	a.db = db
+}
+
+func (a *App) identify() {
+	log.Info().
+		Str("version", a.version).
+		Str("buildarch", a.buildarch).
+		Msg("starting")
+}
+
+func (a *App) runForever() int {
 
 	// Echo instance
 	a.e = echo.New()
 
-	lev := log.INFO
+	lev := gl.INFO
 	if os.Getenv("DEBUG") != "" {
-		lev = log.DEBUG
+		lev = gl.DEBUG
 	}
 
 	logger := lecho.New(
@@ -55,11 +110,17 @@ func (a *App) runForever() {
 	a.e.Use(middleware.Logger())
 	a.e.Use(middleware.Recover())
 
-	a.e.Renderer = NewTemplate("./view/")
-
+	r, err := ep2.NewRenderer("view")
+	if err != nil {
+		a.e.Logger.Fatal(err)
+	}
+	a.e.Renderer = r
 	// Routes
 	a.e.GET("/", indexHandler)
+	a.e.GET("/about", aboutHandler)
 
 	// Start server
 	a.e.Logger.Fatal(a.e.Start(":8080"))
+
+	return 0 //FIXME setup graceful shutdown
 }

hn/templates.go

@@ -1,40 +0,0 @@
-package hn
-
-import (
-	"errors"
-	"html/template"
-	"io"
-
-	"github.com/labstack/echo"
-)
-
-// Define the template registry struct
-type TemplateRegistry struct {
-	templates map[string]*template.Template
-}
-
-// Implement e.Renderer interface
-func (t *TemplateRegistry) Render(w io.Writer, name string, data interface{}, c echo.Context) error {
-	tmpl, ok := t.templates[name]
-	if !ok {
-		err := errors.New("Template not found -> " + name)
-		return err
-	}
-	return tmpl.ExecuteTemplate(w, "base.html", data)
-}
-
-func NewTemplate(templatesDir string) *TemplateRegistry {
-	//ext := ".html"
-
-	ins := TemplateRegistry{
-		templates: map[string]*template.Template{},
-	}
-
-	//layout := templatesDir + "base" + ext
-
-	templates := make(map[string]*template.Template)
-	templates["index"] = template.Must(template.ParseFiles("_pages/index.html", "_layouts/base.html"))
-	//templates["about.html"] = template.Must(template.ParseFiles("view/about.html", "view/base.html"))
-
-	return &ins
-}