add LRU caching

database/manager.go

@@ -3,21 +3,24 @@ package database
 import (
 	"path/filepath"
 	"sync"
-
 	"github.com/jinzhu/gorm"
-
 	u "git.eeqj.de/sneak/goutil"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
+	"github.com/golang/groupcache/lru"
 )
 
+const cacheEntries = 1000000
+
+// Manager coordinates database and cache reads and writes
 type Manager struct {
 	db                            *gorm.DB
 	cachelock                     sync.Mutex
-	recentlyInsertedTootHashCache *map[string]bool
+	recentlyInsertedTootHashCache *lru.Cache
 }
 
+// New creates new Manager
 func New() *Manager {
 	m := new(Manager)
 	m.init()
@@ -32,9 +35,7 @@ func (m *Manager) init() {
 	if viper.GetBool("Debug") {
 		m.db.LogMode(true)
 	}
-
+	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
-	h := make(map[string]bool)
-	m.recentlyInsertedTootHashCache = &h
 }
 
 func (m *Manager) open() {

database/storageconnector.go

@@ -4,9 +4,7 @@ import (
 	"fmt"
 	"html"
 	"strings"
-
 	"git.eeqj.de/sneak/feta/toot"
-
 	"github.com/google/uuid"
 	hstg "github.com/grokify/html-strip-tags-go"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
@@ -15,24 +13,15 @@ import (
 func (m *Manager) TootInsertHashCacheSize() uint {
 	m.cachelock.Lock()
 	defer m.cachelock.Unlock()
-	return uint(len(*m.recentlyInsertedTootHashCache))
+	return uint(m.recentlyInsertedTootHashCache.Len())
 }
 
 func (m *Manager) TootExists(t *toot.Toot) bool {
 	var try StoredToot
 
-	// first, nuke cache if it gets too big to prevent
+	// check cache
-	// unlimited memory usage
 	m.cachelock.Lock()
-	if len(*m.recentlyInsertedTootHashCache) > 1000000 {
+	if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
-		emptycache := make(map[string]bool)
-		m.recentlyInsertedTootHashCache = &emptycache
-	}
-	m.cachelock.Unlock()
-
-	m.cachelock.Lock()
-	// check map
-	if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
 		m.cachelock.Unlock()
 		return true
 	}
@@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
 
 	// put it in the cache to avoid relying on db for dedupe:
 	m.cachelock.Lock()
-	(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
+	m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
 	m.cachelock.Unlock()
 
 	//panic(fmt.Sprintf("%+v", t))