major overhaul, including:

* builds with echo now instead of gin
* beginning of web UI
* factor out util functions

database/manager.go

@@ -1,19 +1,21 @@
 package database
 
 import (
-	"errors"
-	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/jinzhu/gorm"
 
+	u "git.eeqj.de/sneak/goutil"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
 )
 
 type Manager struct {
-	db *gorm.DB
+	db                            *gorm.DB
+	cachelock                     sync.Mutex
+	recentlyInsertedTootHashCache *map[string]bool
 }
 
 func New() *Manager {
@@ -24,31 +26,21 @@ func New() *Manager {
 
 func (m *Manager) init() {
 	m.open()
+	// breaks stuff, do not use:
+	//m.db.SingularTable(true)
 	m.db.LogMode(false)
 	if viper.GetBool("Debug") {
 		m.db.LogMode(true)
 	}
-}
 
-func mkdirp(p string) error {
-	src, err := os.Stat(p)
-	if os.IsNotExist(err) {
-		errDir := os.MkdirAll(p, 0755)
-		if errDir != nil {
-			return err
-		}
-		return nil
-	}
-	if src.Mode().IsRegular() {
-		return errors.New("file already exists at path")
-	}
-	return nil
+	h := make(map[string]bool)
+	m.recentlyInsertedTootHashCache = &h
 }
 
 func (m *Manager) open() {
 	log.Info().Msg("opening database")
 	dirname := filepath.Dir(viper.GetString("DbStorageLocation"))
-	err := mkdirp(dirname)
+	err := u.Mkdirp(dirname)
 	if err != nil {
 		log.Panic().
 			Err(err).

database/model.go

@@ -22,7 +22,6 @@ type StoredToot struct {
 	TextContent   []byte
 	URL           string
 	Hostname      string
-	Fetched       time.Time
 }
 
 type APInstance struct {

database/storageconnector.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"html"
 	"strings"
-	"time"
 
 	"git.eeqj.de/sneak/feta/toot"
 
@@ -13,8 +12,32 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 )
 
+func (m *Manager) TootInsertHashCacheSize() uint {
+	m.cachelock.Lock()
+	defer m.cachelock.Unlock()
+	return uint(len(*m.recentlyInsertedTootHashCache))
+}
+
 func (m *Manager) TootExists(t *toot.Toot) bool {
 	var try StoredToot
+
+	// first, nuke cache if it gets too big to prevent
+	// unlimited memory usage
+	m.cachelock.Lock()
+	if len(*m.recentlyInsertedTootHashCache) > 1000000 {
+		emptycache := make(map[string]bool)
+		m.recentlyInsertedTootHashCache = &emptycache
+	}
+	m.cachelock.Unlock()
+
+	m.cachelock.Lock()
+	// check map
+	if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
+		m.cachelock.Unlock()
+		return true
+	}
+	m.cachelock.Unlock()
+
 	if m.db.Where("Hash = ?", t.GetHash()).First(&try).RecordNotFound() {
 		return false
 	} else {
@@ -23,12 +46,11 @@ func (m *Manager) TootExists(t *toot.Toot) bool {
 }
 
 func (m *Manager) StoreToot(t *toot.Toot) error {
-
 	nt := new(StoredToot)
 	nt.UUID = uuid.New()
 	nt.ServerCreated = t.Parsed.CreatedAt
 	nt.Original = t.Original
-	// FIXME normalize this, check for @ and append hostname if none
+	// FIXME add better validation to the parsed stuff here
 	nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost))
 	nt.URL = t.Parsed.URL
 	nt.Content = []byte(t.Parsed.Content)
@@ -37,8 +59,17 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
 	nt.TextContent = []byte(html.UnescapeString(hstg.StripTags(t.Parsed.Content)))
 	nt.Hostname = strings.ToLower(t.FromHost)
 	nt.Hash = t.GetHash()
-	nt.Fetched = time.Now()
+
+	//TODO: detect hashtags and insert hashtag records
+	//TODO: detect URLs and insert URL records
+
 	r := m.db.Create(&nt)
+
+	// put it in the cache to avoid relying on db for dedupe:
+	m.cachelock.Lock()
+	(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
+	m.cachelock.Unlock()
+
 	//panic(fmt.Sprintf("%+v", t))
 	return r.Error
 }