getting close to working, switching to work on server for a sec

2020-09-21 20:20:24 -07:00
parent da09de0293
commit a52878297f
6 changed files with 365 additions and 43 deletions
--- a/hp/detector.go
+++ b/hp/detector.go
@@ -1,11 +1,68 @@
-package process
+package hp

 import (
+	"context"
+	"database/sql"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"time"
+
+	"git.eeqj.de/sneak/goutil"
+	"github.com/k0kubun/pp"
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/rs/zerolog/log"
 )

-func findHistoryFiles() []string {
+func (hp *HistoryPoster) postUrls(ctx context.Context, cancel context.CancelFunc) {
+	log.Info().Msg("finding history files")
+	files, err := findHistoryFiles()
+	if err != nil {
+		hp.shutdown(err.Error(), -1)
+	}
+
+	for _, v := range files {
+		hl, err := dumpHistoryFromChromeHistoryFile(v)
+		if err != nil {
+			log.Error().
+				Err(err).
+				Msg("unable to read history from file")
+			hp.shutdown(err.Error(), -1)
+		}
+		for _, hitem := range hl {
+			hp.processUrlFromHistory(hitem)
+		}
+	}
+
+}
+
+func (hp *HistoryPoster) processUrlFromHistory(hi historyItem) {
+	log.Debug().
+		Str("url", hi.url).
+		Msg("got url to process")
+	if hp.store.UrlAlreadySeen(hi.url) {
+		return
+	}
+	log.Debug().
+		Str("url", hi.url).
+		Msg("url is new, must be posted")
+	err := hp.postUrl(hi)
+	if err != nil {
+		log.Error().
+			Err(err).
+			Msg("url could not be posted :(")
+	} else {
+		hp.store.MarkUrlSeen(hi.url)
+	}
+}
+
+func (hp *HistoryPoster) postUrl(hi historyItem) error {
+	// FIXME
+	//panic("unimplemented")
+	return nil
+}
+
+func findHistoryFiles() ([]string, error) {
 	//FIXME make this support safari one day
 	home := os.Getenv("HOME")
 	chromeDir := home + "/Library/Application Support/Google/Chrome"
@@ -14,10 +71,104 @@ func findHistoryFiles() []string {
 	output = append(output, defaultProfileDir+"/History")
 	otherProfiles, err := filepath.Glob(chromeDir + "/Profile *")
 	if err != nil {
-		return output
+		return nil, err
 	}
 	for _, v := range otherProfiles {
 		output = append(output, v+"/History")
 	}
-	return output
+	// FIXME check to see if these files actually exist or not
+	return output, nil
+}
+
+type historyItem struct {
+	last_visit_time time.Time
+	url             string
+	title           string
+	visit_count     int
+}
+
+func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) {
+	tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter")
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug().
+		Str("tempdir", tempdir).
+		Msg("created tempdir")
+
+	dbfn := tempdir + "/History"
+	goutil.CopyFile(path, dbfn)
+	log.Debug().
+		Str("dbfn", dbfn).
+		Msg("copied history file")
+
+	defer func() {
+		os.RemoveAll(tempdir)
+		log.Debug().
+			Str("tempdir", tempdir).
+			Msg("removed tempdir")
+	}()
+
+	db, err := sql.Open("sqlite3", dbfn)
+
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debug().
+		Str("dbfn", dbfn).
+		Msg("history file opened")
+
+	defer func() {
+		db.Close()
+		log.Debug().
+			Str("filename", dbfn).
+			Msg("closed history file")
+	}()
+
+	query := `
+	    SELECT
+		last_visit_time,
+		url,
+	        title,
+	        visit_count
+	    FROM
+		urls
+	    ORDER BY
+		last_visit_time DESC
+	`
+
+	rows, err := db.Query(query)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	output := make([]historyItem, 0)
+	for rows.Next() {
+		//log.Debug().Msg("processing row")
+		var last_visit_time int64
+		var url string
+		var title string
+		var visit_count int
+		err := rows.Scan(&last_visit_time, &url, &title, &visit_count)
+		if err != nil {
+			log.Debug().Err(err).Msg("row error")
+			return nil, err
+		}
+		t := goutil.TimeFromWebKit(last_visit_time).UTC()
+		hi := historyItem{
+			last_visit_time: t,
+			url:             url,
+			title:           title,
+			visit_count:     visit_count,
+		}
+		output = append(output, hi)
+	}
+
+	pp.Print(output)
+	return output, nil
 }
--- a/hp/historyposter.go
+++ b/hp/historyposter.go
@@ -1,26 +1,43 @@
-package process
+package hp

 import (
 	"context"
+	"io"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"runtime"
 	"syscall"
 	"time"

-	"github.com/k0kubun/pp"
+	"git.eeqj.de/sneak/goutil"
+	"git.eeqj.de/sneak/historyposter/store"
 	"github.com/mattn/go-isatty"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )

+// HistoryPoster is the main app framework object
+type HistoryPoster struct {
+	version      string
+	buildarch    string
+	startup      time.Time
+	appCtx       context.Context
+	shutdownFunc context.CancelFunc
+	newUrlChan   chan string
+	exitCode     int
+	store        *store.Store
+	logfh        *os.File
+}
+
 // CLIEntry is the main entrypoint
 func CLIEntry(version, buildarch string) int {
 	hp := new(HistoryPoster)
 	hp.version = version
 	hp.buildarch = buildarch
 	hp.startup = time.Now()
+	hp.exitCode = 0
 	hp.newUrlChan = make(chan string)

 	c := make(chan os.Signal)
@@ -30,6 +47,13 @@ func CLIEntry(version, buildarch string) int {
 	hp.configure()
 	hp.setupLogging()

+	store, err := store.NewStore()
+	if err != nil {
+		hp.shutdown("cannot create state file: "+err.Error(), -1)
+		return hp.exitCode
+	}
+	hp.store = store
+
 	hp.appCtx, hp.shutdownFunc = context.WithCancel(context.Background())
 	go func() {
 		// this sits and waits for an interrupt to be received
@@ -40,26 +64,16 @@ func CLIEntry(version, buildarch string) int {
 	return hp.runForever(hp.appCtx)
 }

-// HistoryPoster is the main app framework object
-type HistoryPoster struct {
-	version      string
-	buildarch    string
-	startup      time.Time
-	appCtx       context.Context
-	shutdownFunc context.CancelFunc
-	newUrlChan   chan string
-}
-
 func (hp *HistoryPoster) configure() {
 	viper.SetConfigName("historyposter")
 	viper.SetConfigType("yaml")
 	viper.AddConfigPath("/etc/historyposter")          // path to look for the config file in
 	viper.AddConfigPath("$HOME/.config/historyposter") // call multiple times to add many search paths
-
 	viper.SetEnvPrefix("HISTORYPOSTER")
 	viper.AutomaticEnv()

 	viper.SetDefault("Debug", false)
+	viper.SetDefault("Logfile", os.Getenv("HOME")+"/Library/Logs/historyposter/historyposter.log")

 	if err := viper.ReadInConfig(); err != nil {
 		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
@@ -68,47 +82,92 @@ func (hp *HistoryPoster) configure() {
 			// Config file was found but another error was produced
 			log.Panic().
 				Err(err).
-				Msg("cannot read config file")
+				Msg("config file malformed")
 		}
 	}

-	if viper.GetBool("debug") {
-		pp.Print(viper.AllSettings())
-	}
+	//if viper.GetBool("debug") {
+	//	pp.Print(viper.AllSettings())
+	//}
 }

 func (hp *HistoryPoster) runForever(ctx context.Context) int {

-	log.Info().Msg("this is where i do stuff")
+	log.Info().Msg("entering main loop")

-	_ = findHistoryFiles()
+	interval := 60 * time.Second
+	timeout := 10 * time.Second
+
+	ticker := time.NewTicker(interval)
+	go func() {
+		// do it once right now, without an insta-tick
+		go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
+
+		// then go do it repeatedly:
+		for {
+			select {
+			case <-ticker.C:
+				go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }()
+			case <-ctx.Done():
+				ticker.Stop()
+				return
+			}
+		}
+	}()

 	<-ctx.Done()
-	log.Info().Msgf("shutting down")
-	return 0
+	hp.cleanup()
+	log.Info().Msgf("exiting")
+	hp.logfh.Close()
+	return hp.exitCode
 }

+func (hp *HistoryPoster) cleanup() {
+	log.Info().Msgf("begin cleanup")
+	hp.store.Close()
+}
 func (hp *HistoryPoster) setupLogging() {
-	log.Logger = log.With().Caller().Logger()
-
-	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
-
-	if tty {
-		out := zerolog.NewConsoleWriter(
-			func(w *zerolog.ConsoleWriter) {
-				// Customize time format
-				w.TimeFormat = time.RFC3339
-			},
-		)
-		log.Logger = log.Output(out)
-	}
-
 	// always log in UTC
 	zerolog.TimestampFunc = func() time.Time {
 		return time.Now().UTC()
 	}
-
+	log.Logger = log.With().Caller().Logger()
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+
+	consoleWriter := zerolog.NewConsoleWriter(
+		func(w *zerolog.ConsoleWriter) {
+			// Customize time format
+			w.TimeFormat = time.RFC3339
+		},
+	)
+
+	var writers []io.Writer
+	if tty {
+		writers = append(writers, consoleWriter)
+	}
+
+	logfile := viper.GetString("Logfile")
+	logfileDir := filepath.Dir(logfile)
+	err := goutil.Mkdirp(logfileDir)
+	if err != nil {
+		log.Error().Err(err).Msg("unable to create log dir")
+	}
+
+	hp.logfh, err = os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		panic("unable to open logfile: " + err.Error())
+	}
+
+	writers = append(writers, hp.logfh)
+
+	multi := zerolog.MultiLevelWriter(writers...)
+	logger := zerolog.New(multi).With().Timestamp().Logger().With().Caller().Logger()
+
+	log.Logger = logger
+	// FIXME get caller back in there zerolog.New(multi).Caller().Logger()
+
 	if viper.GetBool("debug") {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
@@ -123,3 +182,9 @@ func (hp *HistoryPoster) identify() {
 		Str("os", runtime.GOOS).
 		Msg("starting")
 }
+
+func (hp *HistoryPoster) shutdown(reason string, exitcode int) {
+	log.Info().Msgf("shutting down: %s", reason)
+	hp.exitCode = exitcode
+	hp.shutdownFunc()
+}