diff --git a/Makefile b/Makefile index a21fe13..8ef7be2 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ lint: debug: build GOTRACEBACK=all HISTORYPOSTER_DEBUG=1 ./$(FN) 2>&1 | tee -a debug.log +debugger: + cd cmd/historyposter && dlv debug main.go + run: build ./$(FN) diff --git a/go.mod b/go.mod index b334b56..0fb8c79 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,10 @@ module git.eeqj.de/sneak/historyposter go 1.15 require ( + git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570 github.com/k0kubun/pp v3.0.1+incompatible github.com/mattn/go-isatty v0.0.12 - github.com/pelletier/go-toml v1.8.1 // indirect + github.com/mattn/go-sqlite3 v1.14.3 github.com/rs/zerolog v1.20.0 github.com/spf13/viper v1.7.1 ) diff --git a/go.sum b/go.sum index 48d463b..48d1dbd 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570 h1:Zn8Wgv8xjJZbeqaTEygI2oJskWOv0rTA7dSWDnUhC7g= +git.eeqj.de/sneak/goutil v0.0.0-20200922001804-e36581f20570/go.mod h1:eczIi5zp8IZnFLQbMF0Xufw6to+UMCbOxA4M4Hp7ORw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -68,6 +70,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4 h1:60gBOooTSmNtrqNaRvrDbi8VAne0REaek2agjnITKSw= +github.com/hako/durafmt v0.0.0-20191009132224-3f39dc1ed9f4/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -111,6 +115,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-sqlite3 v1.14.3 h1:j7a/xn1U6TKA/PHHxqZuzh64CdtRc7rU9M+AvkOl5bA= +github.com/mattn/go-sqlite3 v1.14.3/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -126,9 +132,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/hp/detector.go b/hp/detector.go index 2e0c4b6..a7482b1 100644 --- a/hp/detector.go +++ b/hp/detector.go @@ -1,11 +1,68 @@ -package process +package hp import ( + "context" + "database/sql" + "io/ioutil" "os" "path/filepath" + "time" + + "git.eeqj.de/sneak/goutil" + "github.com/k0kubun/pp" + _ "github.com/mattn/go-sqlite3" + "github.com/rs/zerolog/log" ) -func findHistoryFiles() []string { +func (hp *HistoryPoster) postUrls(ctx context.Context, cancel context.CancelFunc) { + log.Info().Msg("finding history files") + files, err := findHistoryFiles() + if err != nil { + hp.shutdown(err.Error(), -1) + } + + for _, v := range files { + hl, err := dumpHistoryFromChromeHistoryFile(v) + if err != nil { + log.Error(). + Err(err). + Msg("unable to read history from file") + hp.shutdown(err.Error(), -1) + } + for _, hitem := range hl { + hp.processUrlFromHistory(hitem) + } + } + +} + +func (hp *HistoryPoster) processUrlFromHistory(hi historyItem) { + log.Debug(). + Str("url", hi.url). + Msg("got url to process") + if hp.store.UrlAlreadySeen(hi.url) { + return + } + log.Debug(). + Str("url", hi.url). + Msg("url is new, must be posted") + err := hp.postUrl(hi) + if err != nil { + log.Error(). + Err(err). + Msg("url could not be posted :(") + } else { + hp.store.MarkUrlSeen(hi.url) + } +} + +func (hp *HistoryPoster) postUrl(hi historyItem) error { + // FIXME + //panic("unimplemented") + return nil +} + +func findHistoryFiles() ([]string, error) { //FIXME make this support safari one day home := os.Getenv("HOME") chromeDir := home + "/Library/Application Support/Google/Chrome" @@ -14,10 +71,104 @@ func findHistoryFiles() []string { output = append(output, defaultProfileDir+"/History") otherProfiles, err := filepath.Glob(chromeDir + "/Profile *") if err != nil { - return output + return nil, err } for _, v := range otherProfiles { output = append(output, v+"/History") } - return output + // FIXME check to see if these files actually exist or not + return output, nil +} + +type historyItem struct { + last_visit_time time.Time + url string + title string + visit_count int +} + +func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) { + tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter") + if err != nil { + return nil, err + } + + log.Debug(). + Str("tempdir", tempdir). + Msg("created tempdir") + + dbfn := tempdir + "/History" + goutil.CopyFile(path, dbfn) + log.Debug(). + Str("dbfn", dbfn). + Msg("copied history file") + + defer func() { + os.RemoveAll(tempdir) + log.Debug(). + Str("tempdir", tempdir). + Msg("removed tempdir") + }() + + db, err := sql.Open("sqlite3", dbfn) + + if err != nil { + return nil, err + } + + log.Debug(). + Str("dbfn", dbfn). + Msg("history file opened") + + defer func() { + db.Close() + log.Debug(). + Str("filename", dbfn). + Msg("closed history file") + }() + + query := ` + SELECT + last_visit_time, + url, + title, + visit_count + FROM + urls + ORDER BY + last_visit_time DESC + ` + + rows, err := db.Query(query) + + if err != nil { + return nil, err + } + + defer rows.Close() + + output := make([]historyItem, 0) + for rows.Next() { + //log.Debug().Msg("processing row") + var last_visit_time int64 + var url string + var title string + var visit_count int + err := rows.Scan(&last_visit_time, &url, &title, &visit_count) + if err != nil { + log.Debug().Err(err).Msg("row error") + return nil, err + } + t := goutil.TimeFromWebKit(last_visit_time).UTC() + hi := historyItem{ + last_visit_time: t, + url: url, + title: title, + visit_count: visit_count, + } + output = append(output, hi) + } + + pp.Print(output) + return output, nil } diff --git a/hp/historyposter.go b/hp/historyposter.go index 2f2fae6..f525049 100644 --- a/hp/historyposter.go +++ b/hp/historyposter.go @@ -1,26 +1,43 @@ -package process +package hp import ( "context" + "io" "os" "os/signal" + "path/filepath" "runtime" "syscall" "time" - "github.com/k0kubun/pp" + "git.eeqj.de/sneak/goutil" + "git.eeqj.de/sneak/historyposter/store" "github.com/mattn/go-isatty" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/spf13/viper" ) +// HistoryPoster is the main app framework object +type HistoryPoster struct { + version string + buildarch string + startup time.Time + appCtx context.Context + shutdownFunc context.CancelFunc + newUrlChan chan string + exitCode int + store *store.Store + logfh *os.File +} + // CLIEntry is the main entrypoint func CLIEntry(version, buildarch string) int { hp := new(HistoryPoster) hp.version = version hp.buildarch = buildarch hp.startup = time.Now() + hp.exitCode = 0 hp.newUrlChan = make(chan string) c := make(chan os.Signal) @@ -30,6 +47,13 @@ func CLIEntry(version, buildarch string) int { hp.configure() hp.setupLogging() + store, err := store.NewStore() + if err != nil { + hp.shutdown("cannot create state file: "+err.Error(), -1) + return hp.exitCode + } + hp.store = store + hp.appCtx, hp.shutdownFunc = context.WithCancel(context.Background()) go func() { // this sits and waits for an interrupt to be received @@ -40,26 +64,16 @@ func CLIEntry(version, buildarch string) int { return hp.runForever(hp.appCtx) } -// HistoryPoster is the main app framework object -type HistoryPoster struct { - version string - buildarch string - startup time.Time - appCtx context.Context - shutdownFunc context.CancelFunc - newUrlChan chan string -} - func (hp *HistoryPoster) configure() { viper.SetConfigName("historyposter") viper.SetConfigType("yaml") viper.AddConfigPath("/etc/historyposter") // path to look for the config file in viper.AddConfigPath("$HOME/.config/historyposter") // call multiple times to add many search paths - viper.SetEnvPrefix("HISTORYPOSTER") viper.AutomaticEnv() viper.SetDefault("Debug", false) + viper.SetDefault("Logfile", os.Getenv("HOME")+"/Library/Logs/historyposter/historyposter.log") if err := viper.ReadInConfig(); err != nil { if _, ok := err.(viper.ConfigFileNotFoundError); ok { @@ -68,47 +82,92 @@ func (hp *HistoryPoster) configure() { // Config file was found but another error was produced log.Panic(). Err(err). - Msg("cannot read config file") + Msg("config file malformed") } } - if viper.GetBool("debug") { - pp.Print(viper.AllSettings()) - } + //if viper.GetBool("debug") { + // pp.Print(viper.AllSettings()) + //} } func (hp *HistoryPoster) runForever(ctx context.Context) int { - log.Info().Msg("this is where i do stuff") + log.Info().Msg("entering main loop") - _ = findHistoryFiles() + interval := 60 * time.Second + timeout := 10 * time.Second + + ticker := time.NewTicker(interval) + go func() { + // do it once right now, without an insta-tick + go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }() + + // then go do it repeatedly: + for { + select { + case <-ticker.C: + go func() { hp.postUrls(context.WithTimeout(ctx, timeout)) }() + case <-ctx.Done(): + ticker.Stop() + return + } + } + }() <-ctx.Done() - log.Info().Msgf("shutting down") - return 0 + hp.cleanup() + log.Info().Msgf("exiting") + hp.logfh.Close() + return hp.exitCode } +func (hp *HistoryPoster) cleanup() { + log.Info().Msgf("begin cleanup") + hp.store.Close() +} func (hp *HistoryPoster) setupLogging() { - log.Logger = log.With().Caller().Logger() - - tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd()) - - if tty { - out := zerolog.NewConsoleWriter( - func(w *zerolog.ConsoleWriter) { - // Customize time format - w.TimeFormat = time.RFC3339 - }, - ) - log.Logger = log.Output(out) - } - // always log in UTC zerolog.TimestampFunc = func() time.Time { return time.Now().UTC() } - + log.Logger = log.With().Caller().Logger() zerolog.SetGlobalLevel(zerolog.InfoLevel) + + tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd()) + + consoleWriter := zerolog.NewConsoleWriter( + func(w *zerolog.ConsoleWriter) { + // Customize time format + w.TimeFormat = time.RFC3339 + }, + ) + + var writers []io.Writer + if tty { + writers = append(writers, consoleWriter) + } + + logfile := viper.GetString("Logfile") + logfileDir := filepath.Dir(logfile) + err := goutil.Mkdirp(logfileDir) + if err != nil { + log.Error().Err(err).Msg("unable to create log dir") + } + + hp.logfh, err = os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + panic("unable to open logfile: " + err.Error()) + } + + writers = append(writers, hp.logfh) + + multi := zerolog.MultiLevelWriter(writers...) + logger := zerolog.New(multi).With().Timestamp().Logger().With().Caller().Logger() + + log.Logger = logger + // FIXME get caller back in there zerolog.New(multi).Caller().Logger() + if viper.GetBool("debug") { zerolog.SetGlobalLevel(zerolog.DebugLevel) } @@ -123,3 +182,9 @@ func (hp *HistoryPoster) identify() { Str("os", runtime.GOOS). Msg("starting") } + +func (hp *HistoryPoster) shutdown(reason string, exitcode int) { + log.Info().Msgf("shutting down: %s", reason) + hp.exitCode = exitcode + hp.shutdownFunc() +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..86645c6 --- /dev/null +++ b/store/store.go @@ -0,0 +1,97 @@ +package store + +import ( + "database/sql" + "fmt" + "os" + + "git.eeqj.de/sneak/goutil" + "github.com/rs/zerolog/log" +) + +const tablename string = "postedUrls" + +type Store struct { + dbdir string + dbfn string + db *sql.DB +} + +func NewStore() (*Store, error) { + s := new(Store) + s.dbdir = os.Getenv("HOME") + "/Library/Application Support/historyposter" + err := goutil.Mkdirp(s.dbdir) + if err != nil { + log.Error(). + Err(err). + Str("dir", s.dbdir). + Msg("unable to create directory") + return nil, err + } + s.dbfn = s.dbdir + "/postedurlstore.db" + log.Info().Msg("opening store db") + db, err := sql.Open("sqlite3", s.dbfn) + if err != nil { + log.Error().Err(err) + return nil, err + } + s.db = db + if s.CreateTables() != nil { + log.Error(). + Err(err). + Msg("unable to create tables") + return nil, err + } + return s, nil +} + +func (s *Store) CreateTables() error { + q := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url VARCHAR(255) NOT NULL, + posted DATE NULL + ); + `, tablename) + _, err := s.db.Exec(q) + return err +} + +func (s *Store) Close() { + log.Info().Msg("closing store db") + s.db.Close() +} + +func (s *Store) MarkUrlSeen(url string) { + q := fmt.Sprintf(`INSERT into %s (url, posted) VALUES (?, date('now'));`, tablename) + _, err := s.db.Exec(q, url) + if err != nil { + log.Error(). + Str("url", url). + Err(err).Msg("unable to insert url into db") + } + log.Debug(). + Str("url", url). + Msg("url added to db") +} + +func (s *Store) UrlAlreadySeen(url string) bool { + q := fmt.Sprintf(`select id from %s where url = ?;`, tablename) + + row := s.db.QueryRow(q, url) + + var id int + err := row.Scan(&id) + if err == sql.ErrNoRows { + return false + } + if err != nil { + log.Error().Err(err).Msg("sql error looking up url") + return false + } + if id > 0 { + return true + } + log.Fatal().Msg("shouldn't happen") + return false +}