sneak
/
historyposter
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

getting close to working, switching to work on server for a sec
continuous-integration/drone/push Build is failing Details

This commit is contained in:
Jeffrey Paul 2020-09-21 20:20:24 -07:00
parent da09de0293
commit a52878297f
6 changed files with 365 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -39,6 +39,9 @@ lint:
debug: build
GOTRACEBACK=all HISTORYPOSTER_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
debugger:
cd cmd/historyposter && dlv debug main.go
run: build
./$(FN)

3
go.mod
View File

@ -3,9 +3,10 @@ module git.eeqj.de/sneak/historyposter
go 1.15
require (
git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570
github.com/k0kubun/pp v3.0.1+incompatible
github.com/mattn/go-isatty v0.0.12
github.com/pelletier/go-toml v1.8.1 // indirect
github.com/mattn/go-sqlite3 v1.14.3
github.com/rs/zerolog v1.20.0
github.com/spf13/viper v1.7.1
)

9
go.sum
View File

@ -11,6 +11,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570 h1:Zn8Wgv8xjJZbeqaTEygI2oJskWOv0rTA7dSWDnUhC7g=
git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@ -68,6 +70,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4 h1:60gBOooTSmNtrqNaRvrDbi8VAne0REaek2agjnITKSw=
github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@ -111,6 +115,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@ -126,9 +132,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

159
hp/detector.go
View File

@ -1,11 +1,68 @@
package process
package hp
import (
"context"
"database/sql"
"io/ioutil"
"os"
"path/filepath"
"time"
"git.eeqj.de/sneak/goutil"
"github.com/k0kubun/pp"
_ "github.com/mattn/go-sqlite3"
"github.com/rs/zerolog/log"
)
func findHistoryFiles() []string {
func (hp *HistoryPoster) postUrls(ctx context.Context, cancel context.CancelFunc) {
log.Info().Msg("finding history files")
files, err := findHistoryFiles()
if err != nil {
hp.shutdown(err.Error(), -1)
}
for _, v := range files {
hl, err := dumpHistoryFromChromeHistoryFile(v)
if err != nil {
log.Error().
Err(err).
Msg("unable to read history from file")
hp.shutdown(err.Error(), -1)
}
for _, hitem := range hl {
hp.processUrlFromHistory(hitem)
}
}
}
func (hp *HistoryPoster) processUrlFromHistory(hi historyItem) {
log.Debug().
Str("url", hi.url).
Msg("got url to process")
if hp.store.UrlAlreadySeen(hi.url) {
return
}
log.Debug().
Str("url", hi.url).
Msg("url is new, must be posted")
err := hp.postUrl(hi)
if err != nil {
log.Error().
Err(err).
Msg("url could not be posted :(")
} else {
hp.store.MarkUrlSeen(hi.url)
}
}
func (hp *HistoryPoster) postUrl(hi historyItem) error {
// FIXME
//panic("unimplemented")
return nil
}
func findHistoryFiles() ([]string, error) {
//FIXME make this support safari one day
home := os.Getenv("HOME")
chromeDir := home + "/Library/Application Support/Google/Chrome"
@ -14,10 +71,104 @@ func findHistoryFiles() []string {
output = append(output, defaultProfileDir+"/History")
otherProfiles, err := filepath.Glob(chromeDir + "/Profile *")
if err != nil {
return output
return nil, err
}
for _, v := range otherProfiles {
output = append(output, v+"/History")
}
return output
// FIXME check to see if these files actually exist or not
return output, nil
}
type historyItem struct {
last_visit_time time.Time
url string
title string
visit_count int
}
func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) {
tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter")
if err != nil {
return nil, err
}
log.Debug().
Str("tempdir", tempdir).
Msg("created tempdir")
dbfn := tempdir + "/History"
goutil.CopyFile(path, dbfn)
log.Debug().
Str("dbfn", dbfn).
Msg("copied history file")
defer func() {
os.RemoveAll(tempdir)
log.Debug().
Str("tempdir", tempdir).
Msg("removed tempdir")
}()
db, err := sql.Open("sqlite3", dbfn)
if err != nil {
return nil, err
}
log.Debug().
Str("dbfn", dbfn).
Msg("history file opened")
defer func() {
db.Close()
log.Debug().
Str("filename", dbfn).
Msg("closed history file")
}()
query := `
SELECT
last_visit_time,
url,
title,
visit_count
FROM
urls
ORDER BY
last_visit_time DESC
`
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer rows.Close()
output := make([]historyItem, 0)
for rows.Next() {
//log.Debug().Msg("processing row")
var last_visit_time int64
var url string
var title string
var visit_count int
err := rows.Scan(&last_visit_time, &url, &title, &visit_count)
if err != nil {
log.Debug().Err(err).Msg("row error")
return nil, err
}
t := goutil.TimeFromWebKit(last_visit_time).UTC()
hi := historyItem{
last_visit_time: t,
url: url,
title: title,
visit_count: visit_count,
}
output = append(output, hi)
}
pp.Print(output)
return output, nil
}

137
hp/historyposter.go
View File

@ -1,26 +1,43 @@
package process
package hp
import (
"context"
"io"
"os"
"os/signal"
"path/filepath"
"runtime"
"syscall"
"time"
"github.com/k0kubun/pp"
"git.eeqj.de/sneak/goutil"
"git.eeqj.de/sneak/historyposter/store"
"github.com/mattn/go-isatty"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
)
// HistoryPoster is the main app framework object
type HistoryPoster struct {
version string
buildarch string
startup time.Time
appCtx context.Context
shutdownFunc context.CancelFunc
newUrlChan chan string
exitCode int
store *store.Store
logfh *os.File
}
// CLIEntry is the main entrypoint
func CLIEntry(version, buildarch string) int {
hp := new(HistoryPoster)
hp.version = version
hp.buildarch = buildarch
hp.startup = time.Now()
hp.exitCode = 0
hp.newUrlChan = make(chan string)
c := make(chan os.Signal)
@ -30,6 +47,13 @@ func CLIEntry(version, buildarch string) int {
hp.configure()
hp.setupLogging()
store, err := store.NewStore()
if err != nil {
hp.shutdown("cannot create state file: "+err.Error(), -1)
return hp.exitCode
}
hp.store = store
hp.appCtx, hp.shutdownFunc = context.WithCancel(context.Background())
go func() {
// this sits and waits for an interrupt to be received
@ -40,26 +64,16 @@ func CLIEntry(version, buildarch string) int {
return hp.runForever(hp.appCtx)
}
// HistoryPoster is the main app framework object
type HistoryPoster struct {
version string
buildarch string
startup time.Time
appCtx context.Context
shutdownFunc context.CancelFunc
newUrlChan chan string
}
func (hp *HistoryPoster) configure() {
viper.SetConfigName("historyposter")
viper.SetConfigType("yaml")
viper.AddConfigPath("/etc/historyposter") // path to look for the config file in
viper.AddConfigPath("$HOME/.config/historyposter") // call multiple times to add many search paths
viper.SetEnvPrefix("HISTORYPOSTER")
viper.AutomaticEnv()
viper.SetDefault("Debug", false)
viper.SetDefault("Logfile", os.Getenv("HOME")+"/Library/Logs/historyposter/historyposter.log")
if err := viper.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
@ -68,47 +82,92 @@ func (hp *HistoryPoster) configure() {
// Config file was found but another error was produced
log.Panic().
Err(err).
Msg("cannot read config file")
Msg("config file malformed")
}
}
if viper.GetBool("debug") {
pp.Print(viper.AllSettings())
}
//if viper.GetBool("debug") {
// pp.Print(viper.AllSettings())
//}
}
func (hp *HistoryPoster) runForever(ctx context.Context) int {
log.Info().Msg("this is where i do stuff")
log.Info().Msg("entering main loop")
_ = findHistoryFiles()
interval := 60 * time.Second
timeout := 10 * time.Second
ticker := time.NewTicker(interval)
go func() {
// do it once right now, without an insta-tick
go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
// then go do it repeatedly:
for {
select {
case <-ticker.C:
go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
case <-ctx.Done():
ticker.Stop()
return
}
}
}()
<-ctx.Done()
log.Info().Msgf("shutting down")
return 0
hp.cleanup()
log.Info().Msgf("exiting")
hp.logfh.Close()
return hp.exitCode
}
func (hp *HistoryPoster) cleanup() {
log.Info().Msgf("begin cleanup")
hp.store.Close()
}
func (hp *HistoryPoster) setupLogging() {
log.Logger = log.With().Caller().Logger()
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
if tty {
out := zerolog.NewConsoleWriter(
func(w *zerolog.ConsoleWriter) {
// Customize time format
w.TimeFormat = time.RFC3339
},
)
log.Logger = log.Output(out)
}
// always log in UTC
zerolog.TimestampFunc = func() time.Time {
return time.Now().UTC()
}
log.Logger = log.With().Caller().Logger()
zerolog.SetGlobalLevel(zerolog.InfoLevel)
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
consoleWriter := zerolog.NewConsoleWriter(
func(w *zerolog.ConsoleWriter) {
// Customize time format
w.TimeFormat = time.RFC3339
},
)
var writers []io.Writer
if tty {
writers = append(writers, consoleWriter)
}
logfile := viper.GetString("Logfile")
logfileDir := filepath.Dir(logfile)
err := goutil.Mkdirp(logfileDir)
if err != nil {
log.Error().Err(err).Msg("unable to create log dir")
}
hp.logfh, err = os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
panic("unable to open logfile: " + err.Error())
}
writers = append(writers, hp.logfh)
multi := zerolog.MultiLevelWriter(writers...)
logger := zerolog.New(multi).With().Timestamp().Logger().With().Caller().Logger()
log.Logger = logger
// FIXME get caller back in there zerolog.New(multi).Caller().Logger()
if viper.GetBool("debug") {
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
@ -123,3 +182,9 @@ func (hp *HistoryPoster) identify() {
Str("os", runtime.GOOS).
Msg("starting")
}
func (hp *HistoryPoster) shutdown(reason string, exitcode int) {
log.Info().Msgf("shutting down: %s", reason)
hp.exitCode = exitcode
hp.shutdownFunc()
}

97
store/store.go Normal file
View File

@ -0,0 +1,97 @@
package store
import (
"database/sql"
"fmt"
"os"
"git.eeqj.de/sneak/goutil"
"github.com/rs/zerolog/log"
)
const tablename string = "postedUrls"
type Store struct {
dbdir string
dbfn string
db *sql.DB
}
func NewStore() (*Store, error) {
s := new(Store)
s.dbdir = os.Getenv("HOME") + "/Library/Application Support/historyposter"
err := goutil.Mkdirp(s.dbdir)
if err != nil {
log.Error().
Err(err).
Str("dir", s.dbdir).
Msg("unable to create directory")
return nil, err
}
s.dbfn = s.dbdir + "/postedurlstore.db"
log.Info().Msg("opening store db")
db, err := sql.Open("sqlite3", s.dbfn)
if err != nil {
log.Error().Err(err)
return nil, err
}
s.db = db
if s.CreateTables() != nil {
log.Error().
Err(err).
Msg("unable to create tables")
return nil, err
}
return s, nil
}
func (s *Store) CreateTables() error {
q := fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url VARCHAR(255) NOT NULL,
posted DATE NULL
);
`, tablename)
_, err := s.db.Exec(q)
return err
}
func (s *Store) Close() {
log.Info().Msg("closing store db")
s.db.Close()
}
func (s *Store) MarkUrlSeen(url string) {
q := fmt.Sprintf(`INSERT into %s (url, posted) VALUES (?, date('now'));`, tablename)
_, err := s.db.Exec(q, url)
if err != nil {
log.Error().
Str("url", url).
Err(err).Msg("unable to insert url into db")
}
log.Debug().
Str("url", url).
Msg("url added to db")
}
func (s *Store) UrlAlreadySeen(url string) bool {
q := fmt.Sprintf(`select id from %s where url = ?;`, tablename)
row := s.db.QueryRow(q, url)
var id int
err := row.Scan(&id)
if err == sql.ErrNoRows {
return false
}
if err != nil {
log.Error().Err(err).Msg("sql error looking up url")
return false
}
if id > 0 {
return true
}
log.Fatal().Msg("shouldn't happen")
return false
}