add LRU caching

master
Jeffrey Paul 4 years ago
parent 70faf517f3
commit 45848dbbdf
  1. 13
      database/manager.go
  2. 19
      database/storageconnector.go

@ -3,21 +3,24 @@ package database
import (
"path/filepath"
"sync"
"github.com/jinzhu/gorm"
u "git.eeqj.de/sneak/goutil"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"github.com/golang/groupcache/lru"
)
const cacheEntries = 1000000
// Manager coordinates database and cache reads and writes
type Manager struct {
db *gorm.DB
cachelock sync.Mutex
recentlyInsertedTootHashCache *map[string]bool
recentlyInsertedTootHashCache *lru.Cache
}
// New creates new Manager
func New() *Manager {
m := new(Manager)
m.init()
@ -32,9 +35,7 @@ func (m *Manager) init() {
if viper.GetBool("Debug") {
m.db.LogMode(true)
}
h := make(map[string]bool)
m.recentlyInsertedTootHashCache = &h
m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
}
func (m *Manager) open() {

@ -4,9 +4,7 @@ import (
"fmt"
"html"
"strings"
"git.eeqj.de/sneak/feta/toot"
"github.com/google/uuid"
hstg "github.com/grokify/html-strip-tags-go"
_ "github.com/jinzhu/gorm/dialects/sqlite"
@ -15,24 +13,15 @@ import (
func (m *Manager) TootInsertHashCacheSize() uint {
m.cachelock.Lock()
defer m.cachelock.Unlock()
return uint(len(*m.recentlyInsertedTootHashCache))
return uint(m.recentlyInsertedTootHashCache.Len())
}
func (m *Manager) TootExists(t *toot.Toot) bool {
var try StoredToot
// first, nuke cache if it gets too big to prevent
// unlimited memory usage
m.cachelock.Lock()
if len(*m.recentlyInsertedTootHashCache) > 1000000 {
emptycache := make(map[string]bool)
m.recentlyInsertedTootHashCache = &emptycache
}
m.cachelock.Unlock()
// check cache
m.cachelock.Lock()
// check map
if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
if _, ok := m.recentlyInsertedTootHashCache.Get(t.GetHash()); ok {
m.cachelock.Unlock()
return true
}
@ -67,7 +56,7 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
// put it in the cache to avoid relying on db for dedupe:
m.cachelock.Lock()
(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
m.recentlyInsertedTootHashCache.Add(nt.Hash, true)
m.cachelock.Unlock()
//panic(fmt.Sprintf("%+v", t))

Loading…
Cancel
Save