sneak
06df947186
Some checks failed
continuous-integration/drone/push Build is failing
* builds with echo now instead of gin * beginning of web UI * factor out util functions
86 lines
2.0 KiB
Go
86 lines
2.0 KiB
Go
package database
|
|
|
|
import (
|
|
"fmt"
|
|
"html"
|
|
"strings"
|
|
|
|
"git.eeqj.de/sneak/feta/toot"
|
|
|
|
"github.com/google/uuid"
|
|
hstg "github.com/grokify/html-strip-tags-go"
|
|
_ "github.com/jinzhu/gorm/dialects/sqlite"
|
|
)
|
|
|
|
func (m *Manager) TootInsertHashCacheSize() uint {
|
|
m.cachelock.Lock()
|
|
defer m.cachelock.Unlock()
|
|
return uint(len(*m.recentlyInsertedTootHashCache))
|
|
}
|
|
|
|
func (m *Manager) TootExists(t *toot.Toot) bool {
|
|
var try StoredToot
|
|
|
|
// first, nuke cache if it gets too big to prevent
|
|
// unlimited memory usage
|
|
m.cachelock.Lock()
|
|
if len(*m.recentlyInsertedTootHashCache) > 1000000 {
|
|
emptycache := make(map[string]bool)
|
|
m.recentlyInsertedTootHashCache = &emptycache
|
|
}
|
|
m.cachelock.Unlock()
|
|
|
|
m.cachelock.Lock()
|
|
// check map
|
|
if _, ok := (*m.recentlyInsertedTootHashCache)[t.GetHash()]; ok {
|
|
m.cachelock.Unlock()
|
|
return true
|
|
}
|
|
m.cachelock.Unlock()
|
|
|
|
if m.db.Where("Hash = ?", t.GetHash()).First(&try).RecordNotFound() {
|
|
return false
|
|
} else {
|
|
return true
|
|
}
|
|
}
|
|
|
|
func (m *Manager) StoreToot(t *toot.Toot) error {
|
|
nt := new(StoredToot)
|
|
nt.UUID = uuid.New()
|
|
nt.ServerCreated = t.Parsed.CreatedAt
|
|
nt.Original = t.Original
|
|
// FIXME add better validation to the parsed stuff here
|
|
nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost))
|
|
nt.URL = t.Parsed.URL
|
|
nt.Content = []byte(t.Parsed.Content)
|
|
// FIXME replace tags with spaces, don't just strip them, otherwise text
|
|
// gets messed up.
|
|
nt.TextContent = []byte(html.UnescapeString(hstg.StripTags(t.Parsed.Content)))
|
|
nt.Hostname = strings.ToLower(t.FromHost)
|
|
nt.Hash = t.GetHash()
|
|
|
|
//TODO: detect hashtags and insert hashtag records
|
|
//TODO: detect URLs and insert URL records
|
|
|
|
r := m.db.Create(&nt)
|
|
|
|
// put it in the cache to avoid relying on db for dedupe:
|
|
m.cachelock.Lock()
|
|
(*m.recentlyInsertedTootHashCache)[nt.Hash] = true
|
|
m.cachelock.Unlock()
|
|
|
|
//panic(fmt.Sprintf("%+v", t))
|
|
return r.Error
|
|
}
|
|
|
|
func (m *Manager) StoreToots(tc []*toot.Toot) error {
|
|
for _, item := range tc {
|
|
err := m.StoreToot(item)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|