sneak/orangesite
1
0
Fork 0
Code Issues 5 Pull Requests Releases Wiki Activity

alpha version
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Jeffrey Paul 2020-03-24 20:55:03 -07:00
parent 7e2704ce7d
commit f8d5b6c614
6 changed files with 164 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
hn/fetcher.go
View File

@ -1,16 +1,19 @@
package hn
import (
"fmt"
"net/http"
"os"
"time"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/peterhellberg/hn"
"github.com/rs/zerolog"
)
const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00"
func NewFetcher(db *gorm.DB) *Fetcher {
f := new(Fetcher)
f.db = db
@ -34,8 +37,14 @@ func (f *Fetcher) AddLogger(l *zerolog.Logger) {
}
func (f *Fetcher) run() {
if os.Getenv("DEBUG") != "" {
f.db.LogMode(true)
}
f.db.AutoMigrate(&HNStoryRank{})
f.db.AutoMigrate(&FrontPageCache{})
f.db.AutoMigrate(&HNFrontPage{})
for {
f.log.Info().
@ -80,18 +89,21 @@ func (f *Fetcher) StoreFrontPage() error {
Title: item.Title,
FetchedAt: t,
}
f.log.Info().Msgf("storing story with rank %d in db", (i + 1))
//f.log.Debug().Msgf("storing story with rank %d in db", (i + 1))
// FIXME this will grow unbounded and make the file too big if
// I don't clean this up or otherwise limit the data in here
f.db.Create(&s)
// check to see if the item was on the frontpage already or not
var c int
f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).Count(&c)
f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c)
if c == 0 {
// first appearance on frontpage
r := HNFrontPage{
HNID: uint(id),
Appeared: t,
HighestRank: uint(i + 1),
Rank: uint(i + 1),
Title: item.Title,
URL: item.URL,
}
@ -105,7 +117,7 @@ func (f *Fetcher) StoreFrontPage() error {
} else {
// it's still here, compare its ranking
var old HNFrontPage
f.db.Model(&HNFrontPage{}).Where("HNID = ? and Disappeared is NULL", id).First(&old)
f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old)
// FIXME update highestrank if new is lower
needSave := false
if old.Rank != uint(i+1) {
@ -121,12 +133,12 @@ func (f *Fetcher) StoreFrontPage() error {
}
if old.HighestRank > uint(i+1) {
old.HighestRank = uint(i + 1)
f.log.Info().
Uint("hnid", uint(id)).
Uint("oldrank", old.Rank).
Uint("newrank", uint(i+1)).
Uint("oldrecord", old.HighestRank).
Uint("newrecord", uint(i+1)).
Msg("recording new record high rank for story")
old.HighestRank = uint(i + 1)
needSave = true
}
if needSave {
@ -138,12 +150,16 @@ func (f *Fetcher) StoreFrontPage() error {
// FIXME iterate over frontpage items still active in DB and note any
// that are no longer on the scrape
fpitems, err := f.db.Model(&HNFrontPage{}).Where("Disappeared is NULL").Rows()
defer fpitems.Close()
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows()
if err != nil {
f.log.Error().
Err(err)
}
var toupdate []uint
for fpitems.Next() {
var item HNFrontPage
f.db.ScanRows(fpitems, &item)
fmt.Println(item)
//pp.Print(item)
exitedFrontPage := true
for _, xd := range ids[:30] {
if item.HNID == uint(xd) {
@ -151,18 +167,22 @@ func (f *Fetcher) StoreFrontPage() error {
}
}
if exitedFrontPage {
item.Disappeared = t
dur := item.Disappeared.Sub(item.Appeared)
f.db.Save(&item)
toupdate = append(toupdate, item.HNID)
//item.Disappeared = t
dur := t.Sub(item.Appeared).String()
//f.db.Save(&item)
f.log.Info().
Uint("hnid", item.HNID).
Uint("HighestRank", item.HighestRank).
Str("title", item.Title).
Dur("fpduration", dur).
Str("time_on_frontpage", dur).
Str("url", item.URL).
Msg("HN story exited frontpage")
}
}
fpitems.Close() // close tx before we do the update
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t)
return nil
}

67
hn/handlers.go
View File

@ -5,17 +5,78 @@ import (
"time"
"github.com/flosch/pongo2"
"github.com/jinzhu/gorm"
"github.com/labstack/echo"
)
func indexHandler(c echo.Context) error {
type RequestHandlerSet struct {
db *gorm.DB
}
func NewRequestHandlerSet(db *gorm.DB) *RequestHandlerSet {
rhs := new(RequestHandlerSet)
rhs.db = db
return rhs
}
func (r *RequestHandlerSet) indexHandler(c echo.Context) error {
var fpi []HNFrontPage
r.db.Where("disappeared is not ?", SQLITE_NULL_DATETIME).Order("disappeared desc").Find(&fpi)
type fprow struct {
Duration string
URL string
Title string
HighestRank uint
HNID uint
TimeGone string
}
var fprows []fprow
for _, item := range fpi {
fprows = append(fprows, fprow{
Duration: item.Disappeared.Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
URL: item.URL,
HNID: item.HNID,
Title: item.Title,
HighestRank: item.HighestRank,
TimeGone: time.Now().Round(time.Minute).Sub(item.Disappeared.Round(time.Minute)).String(),
})
}
type rowtwo struct {
Duration string
URL string
Title string
HighestRank uint
HNID uint
Rank uint
}
var currentfp []rowtwo
var cur []HNFrontPage
r.db.Where("disappeared is ?", SQLITE_NULL_DATETIME).Order("rank asc").Find(&cur)
for _, item := range cur {
currentfp = append(currentfp, rowtwo{
Duration: time.Now().Round(time.Minute).Sub(item.Appeared.Round(time.Minute)).String(),
URL: item.URL,
HNID: item.HNID,
Title: item.Title,
HighestRank: item.HighestRank,
Rank: item.Rank,
})
}
tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano),
"time": time.Now().UTC().Format(time.RFC3339Nano),
"exits": fprows,
"current": currentfp,
}
return c.Render(http.StatusOK, "index.html", tc)
}
func aboutHandler(c echo.Context) error {
func (r *RequestHandlerSet) aboutHandler(c echo.Context) error {
tc := pongo2.Context{
"time": time.Now().UTC().Format(time.RFC3339Nano),
}

7
hn/server.go
View File

@ -115,9 +115,12 @@ func (a *App) runForever() int {
a.e.Logger.Fatal(err)
}
a.e.Renderer = r
rhs := NewRequestHandlerSet(a.db)
// Routes
a.e.GET("/", indexHandler)
a.e.GET("/about", aboutHandler)
a.e.GET("/", rhs.indexHandler)
a.e.GET("/about", rhs.aboutHandler)
// Start server
a.e.Logger.Fatal(a.e.Start(":8080"))

5
view/base.html
View File

@ -16,6 +16,11 @@
body {
background: #f6f6ef;
}
#pagebody {
margin-top: 4em;
}
</style>
</head>

57
view/index.html
View File

@ -2,8 +2,61 @@
{% block content %}
<div class="col-lg-12">
<h1>Index Page</h1>
<h2>{{ time }}</h2>
<h2>Links Exiting The Front Page</h2>
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Hang Time</th>
<th scope="col">Title</th>
<th scope="col">Highest Rank</th>
<th scope="col">Time Since Wipeout</th>
</tr>
</thead>
<tbody>
{% for exit in exits %}
<!--
{{ exit|stringformat:'%#v'}}
-->
<tr>
<td scope="row">{{exit.Duration}}</th>
<td><a href="{{exit.URL}}">{{exit.Title}}</a> <small>(<a
href="https://news.ycombinator.com/item?id={{exit.HNID}}">comments</a>)</small></td>
<td>{{exit.HighestRank}}</td>
<td>{{exit.TimeGone}}</td>
</tr>
{% endfor %}
</tbody>
</table>
<h2>Current Top30</h2>
<table class="table table-striped table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">Hang Time</th>
<th scope="col">Title</th>
<th scope="col">Highest Rank</th>
</tr>
</thead>
<tbody>
{% for i in current %}
<tr>
<td scope="row">{{i.Duration}}</th>
<td><a href="{{i.URL}}">{{i.Title}}</a> <small>(<a
href="https://news.ycombinator.com/item?id={{i.HNID}}">comments</a>)</small></td>
<td>{{i.HighestRank}}</td>
</tr>
{% endfor %}
</tbody>
</table>
<small>{{ time }}</small>
</div>
{% endblock %}

2
view/page.html
View File

@ -3,7 +3,7 @@
{% block body %}
{% include "navbar.html" %}
<!-- Page Content -->
<div class="container">
<div class="container" id="pagebody">
<div class="row">
{% block content %}
<div class="col-lg-12 text-center">