parent
255554db97
commit
d2bd99801d
@ -1,33 +0,0 @@ |
||||
package feta |
||||
|
||||
import "time" |
||||
import "github.com/rs/zerolog/log" |
||||
|
||||
type tootIngester struct { |
||||
inbound chan *toot |
||||
recentlySeen []*seenTootMemo |
||||
} |
||||
|
||||
type tootHash string |
||||
|
||||
type seenTootMemo struct { |
||||
lastSeen time.Time |
||||
tootHash tootHash |
||||
} |
||||
|
||||
func newTootIngester() *tootIngester { |
||||
ti := new(tootIngester) |
||||
ti.inbound = make(chan *toot, 1) |
||||
return ti |
||||
} |
||||
|
||||
func (ti *tootIngester) getDeliveryChannel() chan *toot { |
||||
return ti.inbound |
||||
} |
||||
|
||||
func (ti *tootIngester) ingest() { |
||||
log.Info().Msg("tootIngester starting") |
||||
for { |
||||
time.Sleep(1 * time.Second) // FIXME do something
|
||||
} |
||||
} |
@ -0,0 +1,34 @@ |
||||
package ingester |
||||
|
||||
import "time" |
||||
import "github.com/rs/zerolog/log" |
||||
import "github.com/sneak/feta/toot" |
||||
import "github.com/sneak/feta/storage" |
||||
|
||||
type TootIngester struct { |
||||
inbound chan *toot.Toot |
||||
recentlySeen []*seenTootMemo |
||||
storageBackend *storage.TootStorageBackend |
||||
} |
||||
|
||||
type seenTootMemo struct { |
||||
lastSeen time.Time |
||||
tootHash toot.TootHash |
||||
} |
||||
|
||||
func NewTootIngester() *TootIngester { |
||||
ti := new(TootIngester) |
||||
ti.inbound = make(chan *toot.Toot, 1) |
||||
return ti |
||||
} |
||||
|
||||
func (ti *TootIngester) GetDeliveryChannel() chan *toot.Toot { |
||||
return ti.inbound |
||||
} |
||||
|
||||
func (ti *TootIngester) Ingest() { |
||||
log.Info().Msg("TootIngester starting") |
||||
for { |
||||
time.Sleep(1 * time.Second) // FIXME do something
|
||||
} |
||||
} |
@ -0,0 +1,10 @@ |
||||
package jsonapis |
||||
|
||||
import "fmt" |
||||
import "encoding/json" |
||||
|
||||
func (atl *apTootList) String() string { |
||||
return fmt.Sprintf("%+v", atl) |
||||
} |
||||
|
||||
type apTootList []json.RawMessage |
@ -0,0 +1,16 @@ |
||||
package seeds |
||||
|
||||
var SeedInstances = [...]string{ |
||||
"splat.soy", |
||||
"veenus.art", |
||||
"iscute.moe", |
||||
"order.life", |
||||
"princess.cat", |
||||
"blobturtle.club", |
||||
"busshi.moe", |
||||
"thewired.xyz", |
||||
"wetfish.space", |
||||
"underscore.world", |
||||
"fedi.valkyrie.world", |
||||
"gnosis.systems", |
||||
} |
@ -0,0 +1,88 @@ |
||||
package storage |
||||
|
||||
import "errors" |
||||
import "io/ioutil" |
||||
import "os" |
||||
import "strings" |
||||
import "sync" |
||||
|
||||
import "github.com/sneak/feta/toot" |
||||
|
||||
type TootStorageBackend interface { |
||||
TootExists(t toot.Toot) bool |
||||
StoreToot(t toot.Toot) error |
||||
StoreToots(tc []*toot.Toot) error |
||||
} |
||||
|
||||
type TootFSStorage struct { |
||||
root string |
||||
} |
||||
|
||||
func NewTootFSStorage(root string) *TootFSStorage { |
||||
ts := new(TootFSStorage) |
||||
ts.root = root |
||||
return ts |
||||
} |
||||
|
||||
func (ts *TootFSStorage) StoreToots(tc []*toot.Toot) error { |
||||
var returnErrors []string |
||||
for _, item := range tc { |
||||
err := ts.StoreToot(*item) |
||||
if err != nil { |
||||
returnErrors = append(returnErrors, err.Error()) |
||||
continue |
||||
} |
||||
} |
||||
if len(returnErrors) == 0 { |
||||
return nil |
||||
} |
||||
return errors.New(strings.Join(returnErrors, "; ")) |
||||
} |
||||
|
||||
func (ts *TootFSStorage) TootExists(t toot.Toot) bool { |
||||
path := t.DiskStoragePath() |
||||
full := ts.root + "/" + path |
||||
_, err := os.Stat(full) |
||||
if os.IsNotExist(err) { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func (ts *TootFSStorage) StoreToot(t toot.Toot) error { |
||||
path := t.DiskStoragePath() |
||||
full := ts.root + "/" + path |
||||
return ioutil.WriteFile(full, t.Original, 0644) |
||||
} |
||||
|
||||
type TootMemoryStorage struct { |
||||
sync.Mutex |
||||
toots map[toot.TootHash]toot.Toot |
||||
//maxSize uint // FIXME support eviction
|
||||
} |
||||
|
||||
func NewTootMemoryStorage() *TootMemoryStorage { |
||||
ts := new(TootMemoryStorage) |
||||
ts.toots = make(map[toot.TootHash]toot.Toot) |
||||
return ts |
||||
} |
||||
|
||||
func (ts *TootMemoryStorage) StoreToot(t toot.Toot) { |
||||
th := t.Hash |
||||
if ts.TootExists(th) { |
||||
return |
||||
} |
||||
ts.Lock() |
||||
defer ts.Unlock() |
||||
ts.toots[th] = t |
||||
return |
||||
} |
||||
|
||||
func (ts *TootMemoryStorage) TootExists(th toot.TootHash) bool { |
||||
ts.Lock() |
||||
defer ts.Unlock() |
||||
if _, ok := ts.toots[th]; ok { //this syntax is so gross
|
||||
return true |
||||
} |
||||
return false |
||||
} |
@ -1,37 +0,0 @@ |
||||
package feta |
||||
|
||||
import "encoding/json" |
||||
|
||||
type toot struct { |
||||
original *json.RawMessage |
||||
parsed *tootFromAPI |
||||
} |
||||
|
||||
func newToots(input []*json.RawMessage) []*toot { |
||||
l := make([]*toot, 0) |
||||
for x := range input { |
||||
t := newToot(x) |
||||
if t != nil { |
||||
l = append(l, t) |
||||
} |
||||
} |
||||
return l |
||||
} |
||||
|
||||
func newToot(input *json.RawMessage) *toot { |
||||
t := new(toot) |
||||
t.original = input |
||||
t.parsed = new(tootFromAPI) |
||||
err = json.Unmarshal(*input, t.parsed) |
||||
if err != nil { |
||||
t.parsed = nil |
||||
} |
||||
return t |
||||
} |
||||
|
||||
func (t *toot) identityHashInput() string { |
||||
// id + datestamp + acct + content
|
||||
} |
||||
|
||||
func (t *toot) hash() tootHash { |
||||
} |
@ -0,0 +1,107 @@ |
||||
package toot |
||||
|
||||
import "fmt" |
||||
import "encoding/json" |
||||
import "errors" |
||||
import "strings" |
||||
import "github.com/sneak/feta/jsonapis" |
||||
import "github.com/davecgh/go-spew/spew" |
||||
import "github.com/rs/zerolog/log" |
||||
|
||||
//import "encoding/hex"
|
||||
import mh "github.com/multiformats/go-multihash" |
||||
import mhopts "github.com/multiformats/go-multihash/opts" |
||||
|
||||
type TootHash string |
||||
|
||||
type Toot struct { |
||||
Original []byte |
||||
Parsed *jsonapis.APISerializedToot |
||||
Hash TootHash |
||||
FromHost string |
||||
} |
||||
|
||||
func NewTootCollectionFromMastodonAPIResponse(in []byte, hostname string) (*[]Toot, error) { |
||||
var rt []json.RawMessage |
||||
err := json.Unmarshal(in, &rt) |
||||
if err != nil { |
||||
return nil, errors.New("unable to parse api response") |
||||
} |
||||
|
||||
var tc []Toot |
||||
|
||||
// iterate over rawtoots from api
|
||||
for _, item := range rt { |
||||
parsed := new(jsonapis.APISerializedToot) |
||||
err := json.Unmarshal(item, parsed) |
||||
if err != nil { |
||||
log.Error().Msg("unable to parse toot, skipping") |
||||
continue |
||||
} |
||||
t := new(Toot) |
||||
t.Parsed = parsed |
||||
o, err := item.MarshalJSON() |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
t.Original = o |
||||
t.FromHost = hostname |
||||
t.calcHash() |
||||
tc = append(tc, *t) |
||||
} |
||||
spew.Dump(tc) |
||||
panic("") |
||||
return &tc, nil |
||||
} |
||||
|
||||
func (t *Toot) String() string { |
||||
return fmt.Sprintf("%#v", t) |
||||
} |
||||
|
||||
func (t *Toot) multiHash(in []byte) string { |
||||
opts := new(mhopts.Options) |
||||
opts.Algorithm = "sha2-256" |
||||
opts.Encoding = "base58" |
||||
var found bool |
||||
opts.AlgorithmCode, found = mh.Names[opts.Algorithm] |
||||
if !found { |
||||
panic("oops") |
||||
} |
||||
opts.Length = mh.DefaultLengths[opts.AlgorithmCode] |
||||
r := strings.NewReader(string(in)) |
||||
h, err := opts.Multihash(r) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
return h.B58String() |
||||
} |
||||
|
||||
func (t *Toot) DiskStoragePath() string { |
||||
// FIXME make this error if fields are missing
|
||||
// '/YYYYMMDD/example.com/username/YYYY-MM-DD.HHMMSS.username@fromHost.multihash.json'
|
||||
return fmt.Sprintf("%s/%s/%s/%s.%s@%s.%s.json", |
||||
t.Parsed.CreatedAt.Format("20060102"), |
||||
strings.ToLower(t.FromHost), |
||||
t.Parsed.Account.Acct, |
||||
t.Parsed.CreatedAt.Format("2006-01-02.150405"), |
||||
t.Parsed.Account.Acct, |
||||
strings.ToLower(t.FromHost), |
||||
t.Hash, |
||||
) |
||||
} |
||||
|
||||
func (t *Toot) identityHashInput() string { |
||||
return fmt.Sprintf( |
||||
"%s.%s.%s.%s.%s", |
||||
t.Parsed.Account.URL, |
||||
t.Parsed.CreatedAt, |
||||
t.Parsed.ID, |
||||
t.Parsed.Content, |
||||
strings.ToLower(t.FromHost), |
||||
) |
||||
} |
||||
|
||||
func (t *Toot) calcHash() { |
||||
hi := t.identityHashInput() |
||||
t.Hash = TootHash(t.multiHash([]byte(hi))) |
||||
} |
Loading…
Reference in new issue