bugfixes, i think

hn/fetcher.go

@@ -12,8 +12,6 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00"
-
 func NewFetcher(db *gorm.DB) *Fetcher {
 	f := new(Fetcher)
 	f.db = db
@@ -99,14 +97,18 @@ func (f *Fetcher) StoreFrontPage() error {
 		// disabled for now
 		//f.db.Create(&s)
 
+		//FIXME check to see if the same HNID was already on the frontpage
+		//or not so we don't spam the db
+
 		// check to see if the item was on the frontpage already or not
 		var c int
-		f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c)
+		f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c)
 		if c == 0 {
 			// first appearance on frontpage
 			r := HNFrontPage{
 				HNID:        uint(id),
 				Appeared:    t,
+				Disappeared: time.Time{},
 				HighestRank: uint(i + 1),
 				Rank:        uint(i + 1),
 				Title:       item.Title,
@@ -122,11 +124,10 @@ func (f *Fetcher) StoreFrontPage() error {
 				Str("url", item.URL).
 				Msg("HN new story on frontpage")
 		} else {
-			// it's still here, compare its ranking
+			// it's still here, (or back)
 			var old HNFrontPage
-			f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old)
-			// FIXME update highestrank if new is lower
-			needSave := false
+			f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old)
+
 			if old.Rank != uint(i+1) {
 				f.log.Info().
 					Uint("hnid", uint(id)).
@@ -138,7 +139,6 @@ func (f *Fetcher) StoreFrontPage() error {
 					Msg("HN story rank changed, recording new rank")
 				old.Rank = uint(i + 1)
 				old.Score = uint(item.Score)
-				needSave = true
 			}
 
 			if old.HighestRank > uint(i+1) {
@@ -148,24 +148,22 @@ func (f *Fetcher) StoreFrontPage() error {
 					Uint("newrecord", uint(i+1)).
 					Msg("recording new record high rank for story")
 				old.HighestRank = uint(i + 1)
-				needSave = true
 			}
 
 			if old.Score != uint(item.Score) {
 				old.Score = uint(item.Score)
-				needSave = true
 			}
 
-			if needSave {
-				f.db.Save(&old)
-			}
+			// in any case it's here now
+			old.Disappeared = time.Time{}
+			f.db.Save(&old)
 		}
 
 	}
 
 	// FIXME iterate over frontpage items still active in DB and note any
 	// that are no longer on the scrape
-	fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows()
+	fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows()
 	if err != nil {
 		f.log.Error().
 			Err(err)
@@ -196,8 +194,8 @@ func (f *Fetcher) StoreFrontPage() error {
 
 		}
 	}
-	fpitems.Close() // close tx before we do the update
-	f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t)
+	fpitems.Close() // close result before we do the update
+	f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t)
 
 	return nil
 }

hn/handlers.go

@@ -10,18 +10,20 @@ import (
 )
 
 type RequestHandlerSet struct {
-	db *gorm.DB
+	db      *gorm.DB
+	version string
 }
 
-func NewRequestHandlerSet(db *gorm.DB) *RequestHandlerSet {
+func NewRequestHandlerSet(version string, db *gorm.DB) *RequestHandlerSet {
 	rhs := new(RequestHandlerSet)
 	rhs.db = db
+	rhs.version = version
 	return rhs
 }
 
 func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
 	var fpi []HNFrontPage
-	r.db.Where("disappeared is not ?", SQLITE_NULL_DATETIME).Order("disappeared desc").Find(&fpi)
+	r.db.Where("disappeared is not ?", time.Time{}).Order("disappeared desc").Find(&fpi)
 
 	type fprow struct {
 		Duration     string
@@ -63,7 +65,7 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
 	var currentfp []rowtwo
 
 	var cur []HNFrontPage
-	r.db.Where("disappeared is ?", SQLITE_NULL_DATETIME).Order("rank asc").Find(&cur)
+	r.db.Where("disappeared is ?", time.Time{}).Order("rank asc").Find(&cur)
 
 	for _, item := range cur {
 		currentfp = append(currentfp, rowtwo{
@@ -82,13 +84,15 @@ func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
 		"time":    time.Now().UTC().Format(time.RFC3339Nano),
 		"exits":   fprows,
 		"current": currentfp,
+		"gitrev":  r.version,
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }
 
 func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
 	tc := pongo2.Context{
-		"time": time.Now().UTC().Format(time.RFC3339Nano),
+		"time":   time.Now().UTC().Format(time.RFC3339Nano),
+		"gitrev": r.version,
 	}
 	return c.Render(http.StatusOK, "about.html", tc)
 }

hn/server.go

@@ -122,7 +122,7 @@ func (a *App) runForever() int {
 	}
 	a.e.Renderer = r
 
-	rhs := NewRequestHandlerSet(a.db)
+	rhs := NewRequestHandlerSet(a.version, a.db)
 
 	// Routes
 	a.e.GET("/", rhs.indexHandler)