You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

258 lines
5.3 KiB

package historyposter
import (
"bytes"
"context"
"database/sql"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"git.eeqj.de/sneak/goutil"
"git.eeqj.de/sneak/mothership/apitypes"
// db driver:
_ "github.com/mattn/go-sqlite3"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
)
const jsonContentType = "application/json; charset=utf-8"
func (hp *HistoryPoster) postURLs(ctx context.Context, cancel context.CancelFunc) {
log.Info().Msg("finding history files")
files, err := findHistoryFiles()
if err != nil {
hp.shutdown(err.Error(), -1)
}
for _, v := range files {
hl, err := dumpHistoryFromChromeHistoryFile(v)
if err != nil {
log.Error().
Err(err).
Msg("unable to read history from file")
hp.shutdown(err.Error(), -1)
}
for _, hitem := range hl {
hp.processURLFromHistory(hitem)
}
}
}
func (hp *HistoryPoster) processURLFromHistory(hi historyItem) {
log.Debug().
Str("url", hi.URL).
Msg("got url to process")
if hp.store.URLAlreadySeen(hi.URL) {
return
}
log.Debug().
Str("url", hi.URL).
Msg("url is new, must be posted")
err := hp.postURL(hi)
if err != nil {
log.Error().
Err(err).
Msg("url could not be posted :(")
} else {
hp.store.MarkURLSeen(hi.URL)
}
}
func (hp *HistoryPoster) postURL(hi historyItem) error {
reqStruct := &apitypes.MothershipHistoryRequest{
PSK: viper.GetString("PSK"),
Visit: apitypes.MothershipHistoryItem{
LastVisitTime: hi.lastVisitTime.Format(time.RFC3339Nano),
URL: hi.URL,
},
}
url := viper.GetString("APIURL")
if !strings.HasPrefix(url, "https://") {
log.Error().Str("url", url).
Msg("HISTORYPOSTER_APIURL must begin with https://")
return errors.New("bad API url, must use TLS")
}
reqBody := new(bytes.Buffer)
err := json.NewEncoder(reqBody).Encode(reqStruct)
if err != nil {
log.Error().Err(err).Msg("unable to encode message to mothership")
return err
}
req, err := http.NewRequestWithContext(hp.appCtx, "POST", url, reqBody)
req.Header.Set("Content-type", jsonContentType)
if err != nil {
log.Error().Err(err).Msg("unable to construct request")
return err
}
var httpClient = &http.Client{
Timeout: time.Second * 10,
}
res, err := httpClient.Do(req)
if err != nil {
log.Error().Err(err).Msg("unable to POST url to mothership")
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
log.Error().
Int("statuscode", res.StatusCode).
Msg("unable to POST url to mothership")
return err
}
var apiresp apitypes.MothershipHistoryResponse
err = json.NewDecoder(res.Body).Decode(&apiresp)
if err != nil {
log.Error().Err(err).Msg("unable to decode mothership response")
return err
}
if apiresp.Result != "ok" {
log.Error().Msg("mothership response non-ok")
return errors.New("mothership response non-ok")
}
log.Info().Str("url", hi.URL).Msg("url sent to mothership")
return nil
}
func findHistoryFiles() ([]string, error) {
// FIXME make this support safari one day
home := os.Getenv("HOME")
chromeDir := home + "/Library/Application Support/Google/Chrome"
defaultProfileDir := chromeDir + "/Default"
output := make([]string, 0)
output = append(output, defaultProfileDir+"/History")
otherProfiles, err := filepath.Glob(chromeDir + "/Profile *")
if err != nil {
return nil, err
}
for _, v := range otherProfiles {
output = append(output, v+"/History")
}
// FIXME check to see if these files actually exist or not
return output, nil
}
type historyItem struct {
lastVisitTime time.Time
URL string
title string
visitCount int
}
func dumpHistoryFromChromeHistoryFile(path string) ([]historyItem, error) {
tempdir, err := ioutil.TempDir(os.Getenv("TMPDIR"), "historyposter")
if err != nil {
return nil, err
}
log.Debug().
Str("tempdir", tempdir).
Msg("created tempdir")
dbfn := tempdir + "/History"
err = goutil.CopyFile(path, dbfn)
if err != nil {
log.Error().
Str("source", path).
Str("target", dbfn).
Err(err).
Msg("unable to copy history file")
return nil, err
}
log.Debug().
Str("dbfn", dbfn).
Msg("copied history file")
defer func() {
os.RemoveAll(tempdir)
log.Debug().
Str("tempdir", tempdir).
Msg("removed tempdir")
}()
db, err := sql.Open("sqlite3", dbfn)
if err != nil {
return nil, err
}
log.Debug().
Str("dbfn", dbfn).
Msg("history file opened")
defer func() {
db.Close()
log.Debug().
Str("filename", dbfn).
Msg("closed history file")
}()
query := `
SELECT
last_visit_time,
url,
title,
visit_count
FROM
urls
ORDER BY
last_visit_time DESC
`
rows, err := db.Query(query)
if err != nil {
return nil, err
}
if rows.Err() != nil {
return nil, rows.Err()
}
defer rows.Close()
output := make([]historyItem, 0)
for rows.Next() {
// log.Debug().Msg("processing row")
var lastVisitTime int64
var url string
var title string
var visitCount int
err := rows.Scan(&lastVisitTime, &url, &title, &visitCount)
if err != nil {
log.Debug().Err(err).Msg("row error")
return nil, err
}
t := goutil.TimeFromWebKit(lastVisitTime).UTC()
hi := historyItem{
lastVisitTime: t,
URL: url,
title: title,
visitCount: visitCount,
}
output = append(output, hi)
}
// pp.Print(output)
return output, nil
}