getting close to working, switching to work on server for a sec

Makefile

@@ -39,6 +39,9 @@ lint:
 debug: build
 	GOTRACEBACK=all HISTORYPOSTER_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log
 
+debugger:
+	cd cmd/historyposter && dlv debug main.go
+
 run: build
 	./$(FN)
 

go.mod

@@ -3,9 +3,10 @@ module git.eeqj.de/sneak/historyposter
 go 1.15
 
 require (
+	git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570
 	github.com/k0kubun/pp v3.0.1+incompatible
 	github.com/mattn/go-isatty v0.0.12
-	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/mattn/go-sqlite3 v1.14.3
 	github.com/rs/zerolog v1.20.0
 	github.com/spf13/viper v1.7.1
 )

go.sum

@@ -11,6 +11,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570 h1:Zn8Wgv8xjJZbeqaTEygI2oJskWOv0rTA7dSWDnUhC7g=
+git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -68,6 +70,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4 h1:60gBOooTSmNtrqNaRvrDbi8VAne0REaek2agjnITKSw=
+github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -111,6 +115,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA=
+github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -126,9 +132,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
-github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

hp/detector.go

@@ -1,11 +1,68 @@
-package process
+package hp
 
 import (
+	"context"
+	"database/sql"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
+
+	"git.eeqj.de/sneak/goutil"
+	"github.com/k0kubun/pp"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/rs/zerolog/log"
 )
 
-func findHistoryFiles() []string {
+func (hp *HistoryPoster) postUrls(ctx context.Context, cancel context.CancelFunc) {
+	log.Info().Msg("finding history files")
+	files, err := findHistoryFiles()
+	if err != nil {
+		hp.shutdown(err.Error(), -1)
+	}
+
+	for _, v := range files {
+		hl, err := dumpHistoryFromChromeHistoryFile(v)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("unable to read history from file")
+			hp.shutdown(err.Error(), -1)
+		}
+		for _, hitem := range hl {
+			hp.processUrlFromHistory(hitem)
+		}
+	}
+
+}
+
+func (hp *HistoryPoster) processUrlFromHistory(hi historyItem) {
+	log.Debug().
+		Str("url", hi.url).
+		Msg("got url to process")
+	if hp.store.UrlAlreadySeen(hi.url) {
+		return
+	}
+	log.Debug().
+		Str("url", hi.url).
+		Msg("url is new, must be posted")
+	err := hp.postUrl(hi)
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("url could not be posted :(")
+	} else {
+		hp.store.MarkUrlSeen(hi.url)
+	}
+}
+
+func (hp *HistoryPoster) postUrl(hi historyItem) error {
+	// FIXME
+	//panic("unimplemented")
+	return nil
+}
+
+func findHistoryFiles() ([]string, error) {
 	//FIXME make this support safari one day
 	home := os.Getenv("HOME")
 	chromeDir := home + "/Library/Application Support/Google/Chrome"
@@ -14,10 +71,104 @@ func findHistoryFiles() []string {
 	output = append(output, defaultProfileDir+"/History")
 	otherProfiles, err := filepath.Glob(chromeDir + "/Profile *")
 	if err != nil {
-		return output
+		return nil, err
 	}
 	for _, v := range otherProfiles {
 		output = append(output, v+"/History")
 	}
-	return output
+	// FIXME check to see if these files actually exist or not
+	return output, nil
+}
+
+type historyItem struct {
+	last_visit_time time.Time
+	url             string
+	title           string
+	visit_count     int
+}
+
+func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) {
+	tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter")
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug().
+		Str("tempdir", tempdir).
+		Msg("created tempdir")
+
+	dbfn := tempdir + "/History"
+	goutil.CopyFile(path, dbfn)
+	log.Debug().
+		Str("dbfn", dbfn).
+		Msg("copied history file")
+
+	defer func() {
+		os.RemoveAll(tempdir)
+		log.Debug().
+			Str("tempdir", tempdir).
+			Msg("removed tempdir")
+	}()
+
+	db, err := sql.Open("sqlite3", dbfn)
+
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug().
+		Str("dbfn", dbfn).
+		Msg("history file opened")
+
+	defer func() {
+		db.Close()
+		log.Debug().
+			Str("filename", dbfn).
+			Msg("closed history file")
+	}()
+
+	query := `
+	    SELECT
+		last_visit_time,
+		url,
+	        title,
+	        visit_count
+	    FROM
+		urls
+	    ORDER BY
+		last_visit_time DESC
+	`
+
+	rows, err := db.Query(query)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	output := make([]historyItem, 0)
+	for rows.Next() {
+		//log.Debug().Msg("processing row")
+		var last_visit_time int64
+		var url string
+		var title string
+		var visit_count int
+		err := rows.Scan(&last_visit_time, &url, &title, &visit_count)
+		if err != nil {
+			log.Debug().Err(err).Msg("row error")
+			return nil, err
+		}
+		t := goutil.TimeFromWebKit(last_visit_time).UTC()
+		hi := historyItem{
+			last_visit_time: t,
+			url:             url,
+			title:           title,
+			visit_count:     visit_count,
+		}
+		output = append(output, hi)
+	}
+
+	pp.Print(output)
+	return output, nil
 }

hp/historyposter.go

@@ -1,26 +1,43 @@
-package process
+package hp
 
 import (
 	"context"
+	"io"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"syscall"
 	"time"
 
-	"github.com/k0kubun/pp"
+	"git.eeqj.de/sneak/goutil"
+	"git.eeqj.de/sneak/historyposter/store"
 	"github.com/mattn/go-isatty"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )
 
+// HistoryPoster is the main app framework object
+type HistoryPoster struct {
+	version      string
+	buildarch    string
+	startup      time.Time
+	appCtx       context.Context
+	shutdownFunc context.CancelFunc
+	newUrlChan   chan string
+	exitCode     int
+	store        *store.Store
+	logfh        *os.File
+}
+
 // CLIEntry is the main entrypoint
 func CLIEntry(version, buildarch string) int {
 	hp := new(HistoryPoster)
 	hp.version = version
 	hp.buildarch = buildarch
 	hp.startup = time.Now()
+	hp.exitCode = 0
 	hp.newUrlChan = make(chan string)
 
 	c := make(chan os.Signal)
@@ -30,6 +47,13 @@ func CLIEntry(version, buildarch string) int {
 	hp.configure()
 	hp.setupLogging()
 
+	store, err := store.NewStore()
+	if err != nil {
+		hp.shutdown("cannot create state file: "+err.Error(), -1)
+		return hp.exitCode
+	}
+	hp.store = store
+
 	hp.appCtx, hp.shutdownFunc = context.WithCancel(context.Background())
 	go func() {
 		// this sits and waits for an interrupt to be received
@@ -40,26 +64,16 @@ func CLIEntry(version, buildarch string) int {
 	return hp.runForever(hp.appCtx)
 }
 
-// HistoryPoster is the main app framework object
-type HistoryPoster struct {
-	version      string
-	buildarch    string
-	startup      time.Time
-	appCtx       context.Context
-	shutdownFunc context.CancelFunc
-	newUrlChan   chan string
-}
-
 func (hp *HistoryPoster) configure() {
 	viper.SetConfigName("historyposter")
 	viper.SetConfigType("yaml")
 	viper.AddConfigPath("/etc/historyposter")          // path to look for the config file in
 	viper.AddConfigPath("$HOME/.config/historyposter") // call multiple times to add many search paths
-
 	viper.SetEnvPrefix("HISTORYPOSTER")
 	viper.AutomaticEnv()
 
 	viper.SetDefault("Debug", false)
+	viper.SetDefault("Logfile", os.Getenv("HOME")+"/Library/Logs/historyposter/historyposter.log")
 
 	if err := viper.ReadInConfig(); err != nil {
 		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
@@ -68,47 +82,92 @@ func (hp *HistoryPoster) configure() {
 			// Config file was found but another error was produced
 			log.Panic().
 				Err(err).
-				Msg("cannot read config file")
+				Msg("config file malformed")
 		}
 	}
 
-	if viper.GetBool("debug") {
+	//if viper.GetBool("debug") {
-		pp.Print(viper.AllSettings())
+	//	pp.Print(viper.AllSettings())
-	}
+	//}
 }
 
 func (hp *HistoryPoster) runForever(ctx context.Context) int {
 
-	log.Info().Msg("this is where i do stuff")
+	log.Info().Msg("entering main loop")
 
-	_ = findHistoryFiles()
+	interval := 60 * time.Second
+	timeout := 10 * time.Second
+
+	ticker := time.NewTicker(interval)
+	go func() {
+		// do it once right now, without an insta-tick
+		go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
+
+		// then go do it repeatedly:
+		for {
+			select {
+			case <-ticker.C:
+				go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
+			case <-ctx.Done():
+				ticker.Stop()
+				return
+			}
+		}
+	}()
 
 	<-ctx.Done()
-	log.Info().Msgf("shutting down")
+	hp.cleanup()
-	return 0
+	log.Info().Msgf("exiting")
+	hp.logfh.Close()
+	return hp.exitCode
 }
 
+func (hp *HistoryPoster) cleanup() {
+	log.Info().Msgf("begin cleanup")
+	hp.store.Close()
+}
 func (hp *HistoryPoster) setupLogging() {
+	// always log in UTC
+	zerolog.TimestampFunc = func() time.Time {
+		return time.Now().UTC()
+	}
 	log.Logger = log.With().Caller().Logger()
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 
 	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
 
-	if tty {
+	consoleWriter := zerolog.NewConsoleWriter(
-		out := zerolog.NewConsoleWriter(
 		func(w *zerolog.ConsoleWriter) {
 			// Customize time format
 			w.TimeFormat = time.RFC3339
 		},
 	)
-		log.Logger = log.Output(out)
+
+	var writers []io.Writer
+	if tty {
+		writers = append(writers, consoleWriter)
 	}
 
-	// always log in UTC
+	logfile := viper.GetString("Logfile")
-	zerolog.TimestampFunc = func() time.Time {
+	logfileDir := filepath.Dir(logfile)
-		return time.Now().UTC()
+	err := goutil.Mkdirp(logfileDir)
+	if err != nil {
+		log.Error().Err(err).Msg("unable to create log dir")
 	}
 
-	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	hp.logfh, err = os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		panic("unable to open logfile: " + err.Error())
+	}
+
+	writers = append(writers, hp.logfh)
+
+	multi := zerolog.MultiLevelWriter(writers...)
+	logger := zerolog.New(multi).With().Timestamp().Logger().With().Caller().Logger()
+
+	log.Logger = logger
+	// FIXME get caller back in there zerolog.New(multi).Caller().Logger()
+
 	if viper.GetBool("debug") {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
@@ -123,3 +182,9 @@ func (hp *HistoryPoster) identify() {
 		Str("os", runtime.GOOS).
 		Msg("starting")
 }
+
+func (hp *HistoryPoster) shutdown(reason string, exitcode int) {
+	log.Info().Msgf("shutting down: %s", reason)
+	hp.exitCode = exitcode
+	hp.shutdownFunc()
+}

store/store.go

@@ -0,0 +1,97 @@
+package store
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+
+	"git.eeqj.de/sneak/goutil"
+	"github.com/rs/zerolog/log"
+)
+
+const tablename string = "postedUrls"
+
+type Store struct {
+	dbdir string
+	dbfn  string
+	db    *sql.DB
+}
+
+func NewStore() (*Store, error) {
+	s := new(Store)
+	s.dbdir = os.Getenv("HOME") + "/Library/Application Support/historyposter"
+	err := goutil.Mkdirp(s.dbdir)
+	if err != nil {
+		log.Error().
+			Err(err).
+			Str("dir", s.dbdir).
+			Msg("unable to create directory")
+		return nil, err
+	}
+	s.dbfn = s.dbdir + "/postedurlstore.db"
+	log.Info().Msg("opening store db")
+	db, err := sql.Open("sqlite3", s.dbfn)
+	if err != nil {
+		log.Error().Err(err)
+		return nil, err
+	}
+	s.db = db
+	if s.CreateTables() != nil {
+		log.Error().
+			Err(err).
+			Msg("unable to create tables")
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *Store) CreateTables() error {
+	q := fmt.Sprintf(`
+	CREATE TABLE IF NOT EXISTS %s (
+	    id INTEGER PRIMARY KEY AUTOINCREMENT,
+	    url VARCHAR(255) NOT NULL,
+	    posted DATE NULL
+	);
+    `, tablename)
+	_, err := s.db.Exec(q)
+	return err
+}
+
+func (s *Store) Close() {
+	log.Info().Msg("closing store db")
+	s.db.Close()
+}
+
+func (s *Store) MarkUrlSeen(url string) {
+	q := fmt.Sprintf(`INSERT into %s (url, posted) VALUES (?, date('now'));`, tablename)
+	_, err := s.db.Exec(q, url)
+	if err != nil {
+		log.Error().
+			Str("url", url).
+			Err(err).Msg("unable to insert url into db")
+	}
+	log.Debug().
+		Str("url", url).
+		Msg("url added to db")
+}
+
+func (s *Store) UrlAlreadySeen(url string) bool {
+	q := fmt.Sprintf(`select id from %s where url = ?;`, tablename)
+
+	row := s.db.QueryRow(q, url)
+
+	var id int
+	err := row.Scan(&id)
+	if err == sql.ErrNoRows {
+		return false
+	}
+	if err != nil {
+		log.Error().Err(err).Msg("sql error looking up url")
+		return false
+	}
+	if id > 0 {
+		return true
+	}
+	log.Fatal().Msg("shouldn't happen")
+	return false
+}