|
|
|
@ -12,8 +12,6 @@ import ( |
|
|
|
|
"github.com/rs/zerolog" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
const SQLITE_NULL_DATETIME = "0001-01-01 00:00:00+00:00" |
|
|
|
|
|
|
|
|
|
func NewFetcher(db *gorm.DB) *Fetcher { |
|
|
|
|
f := new(Fetcher) |
|
|
|
|
f.db = db |
|
|
|
@ -99,14 +97,18 @@ func (f *Fetcher) StoreFrontPage() error { |
|
|
|
|
// disabled for now
|
|
|
|
|
//f.db.Create(&s)
|
|
|
|
|
|
|
|
|
|
//FIXME check to see if the same HNID was already on the frontpage
|
|
|
|
|
//or not so we don't spam the db
|
|
|
|
|
|
|
|
|
|
// check to see if the item was on the frontpage already or not
|
|
|
|
|
var c int |
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).Count(&c) |
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).Count(&c) |
|
|
|
|
if c == 0 { |
|
|
|
|
// first appearance on frontpage
|
|
|
|
|
r := HNFrontPage{ |
|
|
|
|
HNID: uint(id), |
|
|
|
|
Appeared: t, |
|
|
|
|
Disappeared: time.Time{}, |
|
|
|
|
HighestRank: uint(i + 1), |
|
|
|
|
Rank: uint(i + 1), |
|
|
|
|
Title: item.Title, |
|
|
|
@ -122,11 +124,10 @@ func (f *Fetcher) StoreFrontPage() error { |
|
|
|
|
Str("url", item.URL). |
|
|
|
|
Msg("HN new story on frontpage") |
|
|
|
|
} else { |
|
|
|
|
// it's still here, compare its ranking
|
|
|
|
|
// it's still here, (or back)
|
|
|
|
|
var old HNFrontPage |
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("hn_id = ? and disappeared is ?", id, SQLITE_NULL_DATETIME).First(&old) |
|
|
|
|
// FIXME update highestrank if new is lower
|
|
|
|
|
needSave := false |
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("hn_id = ?", id).First(&old) |
|
|
|
|
|
|
|
|
|
if old.Rank != uint(i+1) { |
|
|
|
|
f.log.Info(). |
|
|
|
|
Uint("hnid", uint(id)). |
|
|
|
@ -138,7 +139,6 @@ func (f *Fetcher) StoreFrontPage() error { |
|
|
|
|
Msg("HN story rank changed, recording new rank") |
|
|
|
|
old.Rank = uint(i + 1) |
|
|
|
|
old.Score = uint(item.Score) |
|
|
|
|
needSave = true |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if old.HighestRank > uint(i+1) { |
|
|
|
@ -148,24 +148,22 @@ func (f *Fetcher) StoreFrontPage() error { |
|
|
|
|
Uint("newrecord", uint(i+1)). |
|
|
|
|
Msg("recording new record high rank for story") |
|
|
|
|
old.HighestRank = uint(i + 1) |
|
|
|
|
needSave = true |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if old.Score != uint(item.Score) { |
|
|
|
|
old.Score = uint(item.Score) |
|
|
|
|
needSave = true |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if needSave { |
|
|
|
|
f.db.Save(&old) |
|
|
|
|
} |
|
|
|
|
// in any case it's here now
|
|
|
|
|
old.Disappeared = time.Time{} |
|
|
|
|
f.db.Save(&old) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// FIXME iterate over frontpage items still active in DB and note any
|
|
|
|
|
// that are no longer on the scrape
|
|
|
|
|
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", SQLITE_NULL_DATETIME).Rows() |
|
|
|
|
fpitems, err := f.db.Model(&HNFrontPage{}).Where("disappeared is ?", time.Time{}).Rows() |
|
|
|
|
if err != nil { |
|
|
|
|
f.log.Error(). |
|
|
|
|
Err(err) |
|
|
|
@ -196,8 +194,8 @@ func (f *Fetcher) StoreFrontPage() error { |
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
fpitems.Close() // close tx before we do the update
|
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", SQLITE_NULL_DATETIME, toupdate).Update("Disappeared", t) |
|
|
|
|
fpitems.Close() // close result before we do the update
|
|
|
|
|
f.db.Model(&HNFrontPage{}).Where("disappeared is ? and hn_id in (?)", time.Time{}, toupdate).Update("Disappeared", t) |
|
|
|
|
|
|
|
|
|
return nil |
|
|
|
|
} |
|
|
|
|