package database import ( "fmt" "html" "strings" "git.eeqj.de/sneak/feta/toot" "github.com/google/uuid" hstg "github.com/grokify/html-strip-tags-go" _ "github.com/jinzhu/gorm/dialects/sqlite" ) func (m *Manager) TootInsertHashCacheSize() uint { m.cachelock.Lock() defer m.cachelock.Unlock() return uint(len(*m.recentlyInsertedTootHashCache)) } func (m *Manager) TootExists(t *toot.Toot) bool { var try StoredToot // first, nuke cache if it gets too big to prevent // unlimited memory usage m.cachelock.Lock() if len(*m.recentlyInsertedTootHashCache) > 1000000 { emptycache := make(map[string]bool) m.recentlyInsertedTootHashCache = &emptycache } m.cachelock.Unlock() m.cachelock.Lock() // check map if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok { m.cachelock.Unlock() return true } m.cachelock.Unlock() if m.db.Where("Hash = ?", t.GetHash()).First(&try).RecordNotFound() { return false } else { return true } } func (m *Manager) StoreToot(t *toot.Toot) error { nt := new(StoredToot) nt.UUID = uuid.New() nt.ServerCreated = t.Parsed.CreatedAt nt.Original = t.Original // FIXME add better validation to the parsed stuff here nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost)) nt.URL = t.Parsed.URL nt.Content = []byte(t.Parsed.Content) // FIXME replace tags with spaces, don't just strip them, otherwise text // gets messed up. nt.TextContent = []byte(html.UnescapeString(hstg.StripTags(t.Parsed.Content))) nt.Hostname = strings.ToLower(t.FromHost) nt.Hash = t.GetHash() //TODO: detect hashtags and insert hashtag records //TODO: detect URLs and insert URL records r := m.db.Create(&nt) // put it in the cache to avoid relying on db for dedupe: m.cachelock.Lock() (*m.recentlyInsertedTootHashCache)[nt.Hash] = true m.cachelock.Unlock() //panic(fmt.Sprintf("%+v", t)) return r.Error } func (m *Manager) StoreToots(tc []*toot.Toot) error { for _, item := range tc { err := m.StoreToot(item) if err != nil { return err } } return nil }