alpha version

hn/fetcher.go

@@ -1,16 +1,19 @@
 package hn
 
 import (
-	"fmt"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/peterhellberg/hn"
+
 	"github.com/rs/zerolog"
 )
 
+const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00"
+
 func NewFetcher(db *gorm.DB) *Fetcher {
 	f := new(Fetcher)
 	f.db = db
@@ -34,8 +37,14 @@ func (f *Fetcher) AddLogger(l *zerolog.Logger) {
 }
 
 func (f *Fetcher) run() {
+
+	if os.Getenv("DEBUG") != "" {
+		f.db.LogMode(true)
+	}
+
 	f.db.AutoMigrate(&HNStoryRank{})
 	f.db.AutoMigrate(&FrontPageCache{})
+	f.db.AutoMigrate(&HNFrontPage{})
 
 	for {
 		f.log.Info().
@@ -80,18 +89,21 @@ func (f *Fetcher) StoreFrontPage() error {
 			Title:     item.Title,
 			FetchedAt: t,
 		}
-		f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
+		//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
+		// FIXME this will grow unbounded and make the file too big if
+		// I don't clean this up or otherwise limit the data in here
 		f.db.Create(&s)
 
 		// check to see if the item was on the frontpage already or not
 		var c int
-		f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).Count(&c)
+		f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c)
 		if c == 0 {
 			// first appearance on frontpage
 			r := HNFrontPage{
 				HNID:        uint(id),
 				Appeared:    t,
 				HighestRank: uint(i + 1),
+				Rank:        uint(i + 1),
 				Title:       item.Title,
 				URL:         item.URL,
 			}
@@ -105,7 +117,7 @@ func (f *Fetcher) StoreFrontPage() error {
 		} else {
 			// it's still here, compare its ranking
 			var old HNFrontPage
-			f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).First(&old)
+			f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old)
 			// FIXME update highestrank if new is lower
 			needSave := false
 			if old.Rank != uint(i+1) {
@@ -121,12 +133,12 @@ func (f *Fetcher) StoreFrontPage() error {
 			}
 
 			if old.HighestRank > uint(i+1) {
-				old.HighestRank = uint(i + 1)
 				f.log.Info().
 					Uint("hnid", uint(id)).
-					Uint("oldrank", old.Rank).
-					Uint("newrank", uint(i+1)).
+					Uint("oldrecord", old.HighestRank).
+					Uint("newrecord", uint(i+1)).
 					Msg("recording new record high rank for story")
+				old.HighestRank = uint(i + 1)
 				needSave = true
 			}
 			if needSave {
@@ -138,12 +150,16 @@ func (f *Fetcher) StoreFrontPage() error {
 
 	// FIXME iterate over frontpage items still active in DB and note any
 	// that are no longer on the scrape
-	fpitems, err := f.db.Model(&HNFrontPage{}).Where("Disappeared is NULL").Rows()
-	defer fpitems.Close()
+	fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows()
+	if err != nil {
+		f.log.Error().
+			Err(err)
+	}
+	var toupdate []uint
 	for fpitems.Next() {
 		var item HNFrontPage
 		f.db.ScanRows(fpitems, &item)
-		fmt.Println(item)
+		//pp.Print(item)
 		exitedFrontPage := true
 		for _, xd := range ids[:30] {
 			if item.HNID == uint(xd) {
@@ -151,18 +167,22 @@ func (f *Fetcher) StoreFrontPage() error {
 			}
 		}
 		if exitedFrontPage {
-			item.Disappeared = t
-			dur := item.Disappeared.Sub(item.Appeared)
-			f.db.Save(&item)
+			toupdate = append(toupdate, item.HNID)
+			//item.Disappeared = t
+			dur := t.Sub(item.Appeared).String()
+			//f.db.Save(&item)
 			f.log.Info().
 				Uint("hnid", item.HNID).
 				Uint("HighestRank", item.HighestRank).
 				Str("title", item.Title).
-				Dur("fpduration", dur).
+				Str("time_on_frontpage", dur).
 				Str("url", item.URL).
 				Msg("HN story exited frontpage")
 
 		}
 	}
+	fpitems.Close() // close tx before we do the update
+	f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t)
+
 	return nil
 }