From f8d5b6c614be33041c5581a9c53876f7daa916a6 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 24 Mar 2020 20:55:03 -0700 Subject: [PATCH] alpha version --- hn/fetcher.go | 48 ++++++++++++++++++++++++----------- hn/handlers.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++--- hn/server.go | 7 ++++-- view/base.html | 5 ++++ view/index.html | 57 +++++++++++++++++++++++++++++++++++++++-- view/page.html | 2 +- 6 files changed, 164 insertions(+), 22 deletions(-) diff --git a/hn/fetcher.go b/hn/fetcher.go index 379bee4..df069fc 100644 --- a/hn/fetcher.go +++ b/hn/fetcher.go @@ -1,16 +1,19 @@ package hn import ( - "fmt" "net/http" + "os" "time" "github.com/jinzhu/gorm" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/peterhellberg/hn" + "github.com/rs/zerolog" ) +const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00" + func NewFetcher(db *gorm.DB) *Fetcher { f := new(Fetcher) f.db = db @@ -34,8 +37,14 @@ func (f *Fetcher) AddLogger(l *zerolog.Logger) { } func (f *Fetcher) run() { + + if os.Getenv("DEBUG") != "" { + f.db.LogMode(true) + } + f.db.AutoMigrate(&HNStoryRank{}) f.db.AutoMigrate(&FrontPageCache{}) + f.db.AutoMigrate(&HNFrontPage{}) for { f.log.Info(). @@ -80,18 +89,21 @@ func (f *Fetcher) StoreFrontPage() error { Title: item.Title, FetchedAt: t, } - f.log.Info().Msgf("storing story with rank %d in db", (i + 1)) + //f.log.Debug().Msgf("storing story with rank %d in db", (i + 1)) + // FIXME this will grow unbounded and make the file too big if + // I don't clean this up or otherwise limit the data in here f.db.Create(&s) // check to see if the item was on the frontpage already or not var c int - f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).Count(&c) + f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c) if c == 0 { // first appearance on frontpage r := HNFrontPage{ HNID: uint(id), Appeared: t, HighestRank: uint(i + 1), + Rank: uint(i + 1), Title: item.Title, URL: item.URL, } @@ -105,7 +117,7 @@ func (f *Fetcher) StoreFrontPage() error { } else { // it's still here, compare its ranking var old HNFrontPage - f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).First(&old) + f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old) // FIXME update highestrank if new is lower needSave := false if old.Rank != uint(i+1) { @@ -121,12 +133,12 @@ func (f *Fetcher) StoreFrontPage() error { } if old.HighestRank > uint(i+1) { - old.HighestRank = uint(i + 1) f.log.Info(). Uint("hnid", uint(id)). - Uint("oldrank", old.Rank). - Uint("newrank", uint(i+1)). + Uint("oldrecord", old.HighestRank). + Uint("newrecord", uint(i+1)). Msg("recording new record high rank for story") + old.HighestRank = uint(i + 1) needSave = true } if needSave { @@ -138,12 +150,16 @@ func (f *Fetcher) StoreFrontPage() error { // FIXME iterate over frontpage items still active in DB and note any // that are no longer on the scrape - fpitems, err := f.db.Model(&HNFrontPage{}).Where("Disappeared is NULL").Rows() - defer fpitems.Close() + fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows() + if err != nil { + f.log.Error(). + Err(err) + } + var toupdate []uint for fpitems.Next() { var item HNFrontPage f.db.ScanRows(fpitems, &item) - fmt.Println(item) + //pp.Print(item) exitedFrontPage := true for _, xd := range ids[:30] { if item.HNID == uint(xd) { @@ -151,18 +167,22 @@ func (f *Fetcher) StoreFrontPage() error { } } if exitedFrontPage { - item.Disappeared = t - dur := item.Disappeared.Sub(item.Appeared) - f.db.Save(&item) + toupdate = append(toupdate, item.HNID) + //item.Disappeared = t + dur := t.Sub(item.Appeared).String() + //f.db.Save(&item) f.log.Info(). Uint("hnid", item.HNID). Uint("HighestRank", item.HighestRank). Str("title", item.Title). - Dur("fpduration", dur). + Str("time_on_frontpage", dur). Str("url", item.URL). Msg("HN story exited frontpage") } } + fpitems.Close() // close tx before we do the update + f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t) + return nil } diff --git a/hn/handlers.go b/hn/handlers.go index a7b70f6..1388106 100644 --- a/hn/handlers.go +++ b/hn/handlers.go @@ -5,17 +5,78 @@ import ( "time" "github.com/flosch/pongo2" + "github.com/jinzhu/gorm" "github.com/labstack/echo" ) -func indexHandler(c echo.Context) error { +type RequestHandlerSet struct { + db *gorm.DB +} + +func NewRequestHandlerSet(db *gorm.DB) *RequestHandlerSet { + rhs := new(RequestHandlerSet) + rhs.db = db + return rhs +} + +func (r *RequestHandlerSet) indexHandler(c echo.Context) error { + var fpi []HNFrontPage + r.db.Where("disappeared is not ?", SQLITE_NULL_DATETIME).Order("disappeared desc").Find(&fpi) + + type fprow struct { + Duration string + URL string + Title string + HighestRank uint + HNID uint + TimeGone string + } + var fprows []fprow + + for _, item := range fpi { + fprows = append(fprows, fprow{ + Duration: item.Disappeared.Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(), + URL: item.URL, + HNID: item.HNID, + Title: item.Title, + HighestRank: item.HighestRank, + TimeGone: time.Now().Round(time.Minute).Sub(item.Disappeared.Round(time.Minute)).String(), + }) + } + + type rowtwo struct { + Duration string + URL string + Title string + HighestRank uint + HNID uint + Rank uint + } + var currentfp []rowtwo + + var cur []HNFrontPage + r.db.Where("disappeared is ?", SQLITE_NULL_DATETIME).Order("rank asc").Find(&cur) + + for _, item := range cur { + currentfp = append(currentfp, rowtwo{ + Duration: time.Now().Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(), + URL: item.URL, + HNID: item.HNID, + Title: item.Title, + HighestRank: item.HighestRank, + Rank: item.Rank, + }) + } + tc := pongo2.Context{ - "time": time.Now().UTC().Format(time.RFC3339Nano), + "time": time.Now().UTC().Format(time.RFC3339Nano), + "exits": fprows, + "current": currentfp, } return c.Render(http.StatusOK, "index.html", tc) } -func aboutHandler(c echo.Context) error { +func (r *RequestHandlerSet) aboutHandler(c echo.Context) error { tc := pongo2.Context{ "time": time.Now().UTC().Format(time.RFC3339Nano), } diff --git a/hn/server.go b/hn/server.go index a9bfe0f..91c85e3 100644 --- a/hn/server.go +++ b/hn/server.go @@ -115,9 +115,12 @@ func (a *App) runForever() int { a.e.Logger.Fatal(err) } a.e.Renderer = r + + rhs := NewRequestHandlerSet(a.db) + // Routes - a.e.GET("/", indexHandler) - a.e.GET("/about", aboutHandler) + a.e.GET("/", rhs.indexHandler) + a.e.GET("/about", rhs.aboutHandler) // Start server a.e.Logger.Fatal(a.e.Start(":8080")) diff --git a/view/base.html b/view/base.html index f1ead83..153f6b3 100644 --- a/view/base.html +++ b/view/base.html @@ -16,6 +16,11 @@ body { background: #f6f6ef; } + +#pagebody { + margin-top: 4em; +} + diff --git a/view/index.html b/view/index.html index 16099e9..ae52af5 100644 --- a/view/index.html +++ b/view/index.html @@ -2,8 +2,61 @@ {% block content %}
-

Index Page

-

{{ time }}

+ +

Links Exiting The Front Page

+ + + + + + + + + + +{% for exit in exits %} + + + + + + + +{% endfor %} + +
Hang TimeTitleHighest RankTime Since Wipeout
{{exit.Duration}} + {{exit.Title}} (comments){{exit.HighestRank}}{{exit.TimeGone}}
+ + + +

Current Top30

+ + + + + + + + + +{% for i in current %} + + + + + +{% endfor %} + +
Hang TimeTitleHighest Rank
{{i.Duration}} + {{i.Title}} (comments){{i.HighestRank}}
+ + + +{{ time }}
{% endblock %} diff --git a/view/page.html b/view/page.html index c0e91e2..5a33e9f 100644 --- a/view/page.html +++ b/view/page.html @@ -3,7 +3,7 @@ {% block body %} {% include "navbar.html" %} -
+
{% block content %}