diff --git a/database/manager.go b/database/manager.go index 8765e98..385dfeb 100644 --- a/database/manager.go +++ b/database/manager.go @@ -3,21 +3,24 @@ package database import ( "path/filepath" "sync" - "github.com/jinzhu/gorm" - u "git.eeqj.de/sneak/goutil" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/rs/zerolog/log" "github.com/spf13/viper" + "github.com/golang/groupcache/lru" ) +const cacheEntries = 1000000 + +// Manager coordinates database and cache reads and writes type Manager struct { db *gorm.DB cachelock sync.Mutex - recentlyInsertedTootHashCache *map[string]bool + recentlyInsertedTootHashCache *lru.Cache } +// New creates new Manager func New() *Manager { m := new(Manager) m.init() @@ -32,9 +35,7 @@ func (m *Manager) init() { if viper.GetBool("Debug") { m.db.LogMode(true) } - - h := make(map[string]bool) - m.recentlyInsertedTootHashCache = &h + m.recentlyInsertedTootHashCache = lru.New(cacheEntries) } func (m *Manager) open() { diff --git a/database/storageconnector.go b/database/storageconnector.go index 2475d7e..600c079 100644 --- a/database/storageconnector.go +++ b/database/storageconnector.go @@ -4,9 +4,7 @@ import ( "fmt" "html" "strings" - "git.eeqj.de/sneak/feta/toot" - "github.com/google/uuid" hstg "github.com/grokify/html-strip-tags-go" _ "github.com/jinzhu/gorm/dialects/sqlite" @@ -15,24 +13,15 @@ import ( func (m *Manager) TootInsertHashCacheSize() uint { m.cachelock.Lock() defer m.cachelock.Unlock() - return uint(len(*m.recentlyInsertedTootHashCache)) + return uint(m.recentlyInsertedTootHashCache.Len()) } func (m *Manager) TootExists(t *toot.Toot) bool { var try StoredToot - // first, nuke cache if it gets too big to prevent - // unlimited memory usage + // check cache m.cachelock.Lock() - if len(*m.recentlyInsertedTootHashCache) > 1000000 { - emptycache := make(map[string]bool) - m.recentlyInsertedTootHashCache = &emptycache - } - m.cachelock.Unlock() - - m.cachelock.Lock() - // check map - if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok { + if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok { m.cachelock.Unlock() return true } @@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error { // put it in the cache to avoid relying on db for dedupe: m.cachelock.Lock() - (*m.recentlyInsertedTootHashCache)[nt.Hash] = true + m.recentlyInsertedTootHashCache.Add(nt.Hash, true) m.cachelock.Unlock() //panic(fmt.Sprintf("%+v", t))