package historyposter import ( "bytes" "context" "database/sql" "encoding/json" "errors" "io/ioutil" "net/http" "os" "path/filepath" "strings" "time" "git.eeqj.de/sneak/goutil" "git.eeqj.de/sneak/mothership/apitypes" // db driver: _ "github.com/mattn/go-sqlite3" "github.com/rs/zerolog/log" "github.com/spf13/viper" ) const jsonContentType = "application/json; charset=utf-8" func (hp *HistoryPoster) postURLs(ctx context.Context, cancel context.CancelFunc) { log.Info().Msg("finding history files") files, err := findHistoryFiles() if err != nil { hp.shutdown(err.Error(), -1) } for _, v := range files { hl, err := dumpHistoryFromChromeHistoryFile(v) if err != nil { log.Error(). Err(err). Msg("unable to read history from file") hp.shutdown(err.Error(), -1) } for _, hitem := range hl { hp.processURLFromHistory(hitem) } } } func (hp *HistoryPoster) processURLFromHistory(hi historyItem) { log.Debug(). Str("url", hi.URL). Msg("got url to process") if hp.store.URLAlreadySeen(hi.URL) { return } log.Debug(). Str("url", hi.URL). Msg("url is new, must be posted") err := hp.postURL(hi) if err != nil { log.Error(). Err(err). Msg("url could not be posted :(") } else { hp.store.MarkURLSeen(hi.URL) } } func (hp *HistoryPoster) postURL(hi historyItem) error { reqStruct := &apitypes.MothershipHistoryRequest{ PSK: viper.GetString("PSK"), Visit: apitypes.MothershipHistoryItem{ LastVisitTime: hi.lastVisitTime.Format(time.RFC3339Nano), URL: hi.URL, }, } url := viper.GetString("APIURL") if !strings.HasPrefix(url, "https://") { log.Error().Str("url", url). Msg("HISTORYPOSTER_APIURL must begin with https://") return errors.New("bad API url, must use TLS") } reqBody := new(bytes.Buffer) err := json.NewEncoder(reqBody).Encode(reqStruct) if err != nil { log.Error().Err(err).Msg("unable to encode message to mothership") return err } req, err := http.NewRequestWithContext(hp.appCtx, "POST", url, reqBody) req.Header.Set("Content-type", jsonContentType) if err != nil { log.Error().Err(err).Msg("unable to construct request") return err } var httpClient = &http.Client{ Timeout: time.Second * 10, } res, err := httpClient.Do(req) if err != nil { log.Error().Err(err).Msg("unable to POST url to mothership") return err } defer res.Body.Close() if res.StatusCode != http.StatusOK { log.Error(). Int("statuscode", res.StatusCode). Msg("unable to POST url to mothership") return err } var apiresp apitypes.MothershipHistoryResponse err = json.NewDecoder(res.Body).Decode(&apiresp) if err != nil { log.Error().Err(err).Msg("unable to decode mothership response") return err } if apiresp.Result != "ok" { log.Error().Msg("mothership response non-ok") return errors.New("mothership response non-ok") } log.Info().Str("url", hi.URL).Msg("url sent to mothership") return nil } func findHistoryFiles() ([]string, error) { // FIXME make this support safari one day home := os.Getenv("HOME") chromeDir := home + "/Library/Application Support/Google/Chrome" defaultProfileDir := chromeDir + "/Default" output := make([]string, 0) output = append(output, defaultProfileDir+"/History") otherProfiles, err := filepath.Glob(chromeDir + "/Profile *") if err != nil { return nil, err } for _, v := range otherProfiles { output = append(output, v+"/History") } // FIXME check to see if these files actually exist or not return output, nil } type historyItem struct { lastVisitTime time.Time URL string title string visitCount int } func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) { tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter") if err != nil { return nil, err } log.Debug(). Str("tempdir", tempdir). Msg("created tempdir") dbfn := tempdir + "/History" err = goutil.CopyFile(path, dbfn) if err != nil { log.Error(). Str("source", path). Str("target", dbfn). Err(err). Msg("unable to copy history file") return nil, err } log.Debug(). Str("dbfn", dbfn). Msg("copied history file") defer func() { os.RemoveAll(tempdir) log.Debug(). Str("tempdir", tempdir). Msg("removed tempdir") }() db, err := sql.Open("sqlite3", dbfn) if err != nil { return nil, err } log.Debug(). Str("dbfn", dbfn). Msg("history file opened") defer func() { db.Close() log.Debug(). Str("filename", dbfn). Msg("closed history file") }() query := ` SELECT last_visit_time, url, title, visit_count FROM urls ORDER BY last_visit_time DESC ` rows, err := db.Query(query) if err != nil { return nil, err } if rows.Err() != nil { return nil, rows.Err() } defer rows.Close() output := make([]historyItem, 0) for rows.Next() { // log.Debug().Msg("processing row") var lastVisitTime int64 var url string var title string var visitCount int err := rows.Scan(&lastVisitTime, &url, &title, &visitCount) if err != nil { log.Debug().Err(err).Msg("row error") return nil, err } t := goutil.TimeFromWebKit(lastVisitTime).UTC() hi := historyItem{ lastVisitTime: t, URL: url, title: title, visitCount: visitCount, } output = append(output, hi) } // pp.Print(output) return output, nil }