alpha version

hn/fetcher.go

@@ -1,16 +1,19 @@
 package hn
 
 import (
-	"fmt"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/peterhellberg/hn"
+
 	"github.com/rs/zerolog"
 )
 
+const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00"
+
 func NewFetcher(db *gorm.DB) *Fetcher {
 	f := new(Fetcher)
 	f.db = db
@@ -34,8 +37,14 @@ func (f *Fetcher) AddLogger(l *zerolog.Logger) {
 }
 
 func (f *Fetcher) run() {
+
+	if os.Getenv("DEBUG") != "" {
+		f.db.LogMode(true)
+	}
+
 	f.db.AutoMigrate(&HNStoryRank{})
 	f.db.AutoMigrate(&FrontPageCache{})
+	f.db.AutoMigrate(&HNFrontPage{})
 
 	for {
 		f.log.Info().
@@ -80,18 +89,21 @@ func (f *Fetcher) StoreFrontPage() error {
 			Title:     item.Title,
 			FetchedAt: t,
 		}
-		f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
+		//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
+		// FIXME this will grow unbounded and make the file too big if
+		// I don't clean this up or otherwise limit the data in here
 		f.db.Create(&s)
 
 		// check to see if the item was on the frontpage already or not
 		var c int
-		f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).Count(&c)
+		f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c)
 		if c == 0 {
 			// first appearance on frontpage
 			r := HNFrontPage{
 				HNID:        uint(id),
 				Appeared:    t,
 				HighestRank: uint(i + 1),
+				Rank:        uint(i + 1),
 				Title:       item.Title,
 				URL:         item.URL,
 			}
@@ -105,7 +117,7 @@ func (f *Fetcher) StoreFrontPage() error {
 		} else {
 			// it's still here, compare its ranking
 			var old HNFrontPage
-			f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).First(&old)
+			f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old)
 			// FIXME update highestrank if new is lower
 			needSave := false
 			if old.Rank != uint(i+1) {
@@ -121,12 +133,12 @@ func (f *Fetcher) StoreFrontPage() error {
 			}
 
 			if old.HighestRank > uint(i+1) {
-				old.HighestRank = uint(i + 1)
 				f.log.Info().
 					Uint("hnid", uint(id)).
-					Uint("oldrank", old.Rank).
+					Uint("oldrecord", old.HighestRank).
-					Uint("newrank", uint(i+1)).
+					Uint("newrecord", uint(i+1)).
 					Msg("recording new record high rank for story")
+				old.HighestRank = uint(i + 1)
 				needSave = true
 			}
 			if needSave {
@@ -138,12 +150,16 @@ func (f *Fetcher) StoreFrontPage() error {
 
 	// FIXME iterate over frontpage items still active in DB and note any
 	// that are no longer on the scrape
-	fpitems, err := f.db.Model(&HNFrontPage{}).Where("Disappeared is NULL").Rows()
+	fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows()
-	defer fpitems.Close()
+	if err != nil {
+		f.log.Error().
+			Err(err)
+	}
+	var toupdate []uint
 	for fpitems.Next() {
 		var item HNFrontPage
 		f.db.ScanRows(fpitems, &item)
-		fmt.Println(item)
+		//pp.Print(item)
 		exitedFrontPage := true
 		for _, xd := range ids[:30] {
 			if item.HNID == uint(xd) {
@@ -151,18 +167,22 @@ func (f *Fetcher) StoreFrontPage() error {
 			}
 		}
 		if exitedFrontPage {
-			item.Disappeared = t
+			toupdate = append(toupdate, item.HNID)
-			dur := item.Disappeared.Sub(item.Appeared)
+			//item.Disappeared = t
-			f.db.Save(&item)
+			dur := t.Sub(item.Appeared).String()
+			//f.db.Save(&item)
 			f.log.Info().
 				Uint("hnid", item.HNID).
 				Uint("HighestRank", item.HighestRank).
 				Str("title", item.Title).
-				Dur("fpduration", dur).
+				Str("time_on_frontpage", dur).
 				Str("url", item.URL).
 				Msg("HN story exited frontpage")
 
 		}
 	}
+	fpitems.Close() // close tx before we do the update
+	f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t)
+
 	return nil
 }

hn/handlers.go

@@ -5,17 +5,78 @@ import (
 	"time"
 
 	"github.com/flosch/pongo2"
+	"github.com/jinzhu/gorm"
 	"github.com/labstack/echo"
 )
 
-func indexHandler(c echo.Context) error {
+type RequestHandlerSet struct {
+	db *gorm.DB
+}
+
+func NewRequestHandlerSet(db *gorm.DB) *RequestHandlerSet {
+	rhs := new(RequestHandlerSet)
+	rhs.db = db
+	return rhs
+}
+
+func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
+	var fpi []HNFrontPage
+	r.db.Where("disappeared is not ?", SQLITE_NULL_DATETIME).Order("disappeared desc").Find(&fpi)
+
+	type fprow struct {
+		Duration    string
+		URL         string
+		Title       string
+		HighestRank uint
+		HNID        uint
+		TimeGone    string
+	}
+	var fprows []fprow
+
+	for _, item := range fpi {
+		fprows = append(fprows, fprow{
+			Duration:    item.Disappeared.Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
+			URL:         item.URL,
+			HNID:        item.HNID,
+			Title:       item.Title,
+			HighestRank: item.HighestRank,
+			TimeGone:    time.Now().Round(time.Minute).Sub(item.Disappeared.Round(time.Minute)).String(),
+		})
+	}
+
+	type rowtwo struct {
+		Duration    string
+		URL         string
+		Title       string
+		HighestRank uint
+		HNID        uint
+		Rank        uint
+	}
+	var currentfp []rowtwo
+
+	var cur []HNFrontPage
+	r.db.Where("disappeared is ?", SQLITE_NULL_DATETIME).Order("rank asc").Find(&cur)
+
+	for _, item := range cur {
+		currentfp = append(currentfp, rowtwo{
+			Duration:    time.Now().Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
+			URL:         item.URL,
+			HNID:        item.HNID,
+			Title:       item.Title,
+			HighestRank: item.HighestRank,
+			Rank:        item.Rank,
+		})
+	}
+
 	tc := pongo2.Context{
-		"time": time.Now().UTC().Format(time.RFC3339Nano),
+		"time":    time.Now().UTC().Format(time.RFC3339Nano),
+		"exits":   fprows,
+		"current": currentfp,
 	}
 	return c.Render(http.StatusOK, "index.html", tc)
 }
 
-func aboutHandler(c echo.Context) error {
+func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
 	tc := pongo2.Context{
 		"time": time.Now().UTC().Format(time.RFC3339Nano),
 	}

hn/server.go

@@ -115,9 +115,12 @@ func (a *App) runForever() int {
 		a.e.Logger.Fatal(err)
 	}
 	a.e.Renderer = r
+
+	rhs := NewRequestHandlerSet(a.db)
+
 	// Routes
-	a.e.GET("/", indexHandler)
+	a.e.GET("/", rhs.indexHandler)
-	a.e.GET("/about", aboutHandler)
+	a.e.GET("/about", rhs.aboutHandler)
 
 	// Start server
 	a.e.Logger.Fatal(a.e.Start(":8080"))

view/base.html

@@ -16,6 +16,11 @@
 body {
     background: #f6f6ef;
 }
+
+#pagebody {
+    margin-top: 4em;
+}
+
 </style>
 
 </head>

view/index.html

@@ -2,8 +2,61 @@
 
 {% block content %}
 <div class="col-lg-12">
-<h1>Index Page</h1>
 
-<h2>{{ time }}</h2>
+
+<h2>Links Exiting The Front Page</h2>
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">Hang Time</th>
+  <th scope="col">Title</th>
+  <th scope="col">Highest Rank</th>
+  <th scope="col">Time Since Wipeout</th>
+</tr>
+</thead>
+<tbody>
+{% for exit in exits %}
+<!--
+    {{ exit|stringformat:'%#v'}}
+-->
+
+    <tr>
+      <td scope="row">{{exit.Duration}}</th>
+      <td><a href="{{exit.URL}}">{{exit.Title}}</a> <small>(<a
+             href="https://news.ycombinator.com/item?id={{exit.HNID}}">comments</a>)</small></td>
+      <td>{{exit.HighestRank}}</td>
+      <td>{{exit.TimeGone}}</td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
+
+
+
+<h2>Current Top30</h2>
+<table class="table table-striped table-hover">
+  <thead class="thead-dark">
+<tr>
+  <th scope="col">Hang Time</th>
+  <th scope="col">Title</th>
+  <th scope="col">Highest Rank</th>
+</tr>
+</thead>
+<tbody>
+{% for i in current %}
+
+    <tr>
+      <td scope="row">{{i.Duration}}</th>
+      <td><a href="{{i.URL}}">{{i.Title}}</a> <small>(<a
+             href="https://news.ycombinator.com/item?id={{i.HNID}}">comments</a>)</small></td>
+      <td>{{i.HighestRank}}</td>
+    </tr>
+{% endfor %}
+ </tbody>
+</table>
+
+
+
+<small>{{ time }}</small>
 </div>
 {% endblock %}

view/page.html

@@ -3,7 +3,7 @@
 {% block body %}
     {% include "navbar.html" %}
     <!-- Page Content -->
-    <div class="container">
+    <div class="container" id="pagebody">
         <div class="row">
             {% block content %}
             <div class="col-lg-12 text-center">