diff --git a/hn/fetcher.go b/hn/fetcher.go
index 379bee4..df069fc 100644
--- a/hn/fetcher.go
+++ b/hn/fetcher.go
@@ -1,16 +1,19 @@
package hn
import (
- "fmt"
"net/http"
+ "os"
"time"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/peterhellberg/hn"
+
"github.com/rs/zerolog"
)
+const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00"
+
func NewFetcher(db *gorm.DB) *Fetcher {
f := new(Fetcher)
f.db = db
@@ -34,8 +37,14 @@ func (f *Fetcher) AddLogger(l *zerolog.Logger) {
}
func (f *Fetcher) run() {
+
+ if os.Getenv("DEBUG") != "" {
+ f.db.LogMode(true)
+ }
+
f.db.AutoMigrate(&HNStoryRank{})
f.db.AutoMigrate(&FrontPageCache{})
+ f.db.AutoMigrate(&HNFrontPage{})
for {
f.log.Info().
@@ -80,18 +89,21 @@ func (f *Fetcher) StoreFrontPage() error {
Title: item.Title,
FetchedAt: t,
}
- f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
+ //f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
+ // FIXME this will grow unbounded and make the file too big if
+ // I don't clean this up or otherwise limit the data in here
f.db.Create(&s)
// check to see if the item was on the frontpage already or not
var c int
- f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).Count(&c)
+ f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c)
if c == 0 {
// first appearance on frontpage
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
HighestRank: uint(i + 1),
+ Rank: uint(i + 1),
Title: item.Title,
URL: item.URL,
}
@@ -105,7 +117,7 @@ func (f *Fetcher) StoreFrontPage() error {
} else {
// it's still here, compare its ranking
var old HNFrontPage
- f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).First(&old)
+ f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old)
// FIXME update highestrank if new is lower
needSave := false
if old.Rank != uint(i+1) {
@@ -121,12 +133,12 @@ func (f *Fetcher) StoreFrontPage() error {
}
if old.HighestRank > uint(i+1) {
- old.HighestRank = uint(i + 1)
f.log.Info().
Uint("hnid", uint(id)).
- Uint("oldrank", old.Rank).
- Uint("newrank", uint(i+1)).
+ Uint("oldrecord", old.HighestRank).
+ Uint("newrecord", uint(i+1)).
Msg("recording new record high rank for story")
+ old.HighestRank = uint(i + 1)
needSave = true
}
if needSave {
@@ -138,12 +150,16 @@ func (f *Fetcher) StoreFrontPage() error {
// FIXME iterate over frontpage items still active in DB and note any
// that are no longer on the scrape
- fpitems, err := f.db.Model(&HNFrontPage{}).Where("Disappeared is NULL").Rows()
- defer fpitems.Close()
+ fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows()
+ if err != nil {
+ f.log.Error().
+ Err(err)
+ }
+ var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
f.db.ScanRows(fpitems, &item)
- fmt.Println(item)
+ //pp.Print(item)
exitedFrontPage := true
for _, xd := range ids[:30] {
if item.HNID == uint(xd) {
@@ -151,18 +167,22 @@ func (f *Fetcher) StoreFrontPage() error {
}
}
if exitedFrontPage {
- item.Disappeared = t
- dur := item.Disappeared.Sub(item.Appeared)
- f.db.Save(&item)
+ toupdate = append(toupdate, item.HNID)
+ //item.Disappeared = t
+ dur := t.Sub(item.Appeared).String()
+ //f.db.Save(&item)
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
- Dur("fpduration", dur).
+ Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
+ fpitems.Close() // close tx before we do the update
+ f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t)
+
return nil
}
diff --git a/hn/handlers.go b/hn/handlers.go
index a7b70f6..1388106 100644
--- a/hn/handlers.go
+++ b/hn/handlers.go
@@ -5,17 +5,78 @@ import (
"time"
"github.com/flosch/pongo2"
+ "github.com/jinzhu/gorm"
"github.com/labstack/echo"
)
-func indexHandler(c echo.Context) error {
+type RequestHandlerSet struct {
+ db *gorm.DB
+}
+
+func NewRequestHandlerSet(db *gorm.DB) *RequestHandlerSet {
+ rhs := new(RequestHandlerSet)
+ rhs.db = db
+ return rhs
+}
+
+func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
+ var fpi []HNFrontPage
+ r.db.Where("disappeared is not ?", SQLITE_NULL_DATETIME).Order("disappeared desc").Find(&fpi)
+
+ type fprow struct {
+ Duration string
+ URL string
+ Title string
+ HighestRank uint
+ HNID uint
+ TimeGone string
+ }
+ var fprows []fprow
+
+ for _, item := range fpi {
+ fprows = append(fprows, fprow{
+ Duration: item.Disappeared.Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
+ URL: item.URL,
+ HNID: item.HNID,
+ Title: item.Title,
+ HighestRank: item.HighestRank,
+ TimeGone: time.Now().Round(time.Minute).Sub(item.Disappeared.Round(time.Minute)).String(),
+ })
+ }
+
+ type rowtwo struct {
+ Duration string
+ URL string
+ Title string
+ HighestRank uint
+ HNID uint
+ Rank uint
+ }
+ var currentfp []rowtwo
+
+ var cur []HNFrontPage
+ r.db.Where("disappeared is ?", SQLITE_NULL_DATETIME).Order("rank asc").Find(&cur)
+
+ for _, item := range cur {
+ currentfp = append(currentfp, rowtwo{
+ Duration: time.Now().Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
+ URL: item.URL,
+ HNID: item.HNID,
+ Title: item.Title,
+ HighestRank: item.HighestRank,
+ Rank: item.Rank,
+ })
+ }
+
tc := pongo2.Context{
- "time": time.Now().UTC().Format(time.RFC3339Nano),
+ "time": time.Now().UTC().Format(time.RFC3339Nano),
+ "exits": fprows,
+ "current": currentfp,
}
return c.Render(http.StatusOK, "index.html", tc)
}
-func aboutHandler(c echo.Context) error {
+func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano),
}
diff --git a/hn/server.go b/hn/server.go
index a9bfe0f..91c85e3 100644
--- a/hn/server.go
+++ b/hn/server.go
@@ -115,9 +115,12 @@ func (a *App) runForever() int {
a.e.Logger.Fatal(err)
}
a.e.Renderer = r
+
+ rhs := NewRequestHandlerSet(a.db)
+
// Routes
- a.e.GET("/", indexHandler)
- a.e.GET("/about", aboutHandler)
+ a.e.GET("/", rhs.indexHandler)
+ a.e.GET("/about", rhs.aboutHandler)
// Start server
a.e.Logger.Fatal(a.e.Start(":8080"))
diff --git a/view/base.html b/view/base.html
index f1ead83..153f6b3 100644
--- a/view/base.html
+++ b/view/base.html
@@ -16,6 +16,11 @@
body {
background: #f6f6ef;
}
+
+#pagebody {
+ margin-top: 4em;
+}
+
diff --git a/view/index.html b/view/index.html
index 16099e9..ae52af5 100644
--- a/view/index.html
+++ b/view/index.html
@@ -2,8 +2,61 @@
{% block content %}
-
Index Page
-
{{ time }}
+
+
Links Exiting The Front Page
+
+
+
+ Hang Time |
+ Title |
+ Highest Rank |
+ Time Since Wipeout |
+
+
+
+{% for exit in exits %}
+
+
+
+ {{exit.Duration}}
+ | {{exit.Title}} (comments) |
+ {{exit.HighestRank}} |
+ {{exit.TimeGone}} |
+
+{% endfor %}
+
+
+
+
+
+
Current Top30
+
+
+
+ Hang Time |
+ Title |
+ Highest Rank |
+
+
+
+{% for i in current %}
+
+
+ {{i.Duration}}
+ | {{i.Title}} (comments) |
+ {{i.HighestRank}} |
+
+{% endfor %}
+
+
+
+
+
+
{{ time }}
{% endblock %}
diff --git a/view/page.html b/view/page.html
index c0e91e2..5a33e9f 100644
--- a/view/page.html
+++ b/view/page.html
@@ -3,7 +3,7 @@
{% block body %}
{% include "navbar.html" %}
-