sneak/feta
1
0
Fork 0
Code Issues Pull Requests 1 Releases Wiki Activity

builds again, not sure how i broke it, also:
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source 
* fixes truncated content col
* adds text_content for plain text (has space/tag strip bug)
* update readme
This commit is contained in:
Jeffrey Paul 2020-03-27 20:18:55 -07:00
parent 2ecd833726
commit 9655265d85
5 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -7,11 +7,14 @@ archives the fediverse
* scan toots for mentions and feed to locator * scan toots for mentions and feed to locator
* put toots in a separate db file * put toots in a separate db file
* test with a real database * test with a real database
* save instances to store more often * save instancelist to store more often (maybe on each new one added not
during initial load)
* verify instances load properly on startup * verify instances load properly on startup
* do some simple in-memory dedupe for toot storage * do some simple in-memory dedupe for toot storage
* make some templates using pongo2 and a simple website * make some templates using pongo2 and a simple website
* update APIs * update json APIs
* index hashtags
* index seen urls
# status # status

2
database/model.go
View File

@ -19,8 +19,10 @@ type StoredToot struct {
ServerCreated time.Time ServerCreated time.Time
Acct string Acct string
Content []byte Content []byte
TextContent []byte
URL string URL string
Hostname string Hostname string
Fetched time.Time
} }
type APInstance struct { type APInstance struct {

9
database/storageconnector.go
View File

@ -2,11 +2,14 @@ package database
import ( import (
"fmt" "fmt"
"html"
"strings" "strings"
"time"
"git.eeqj.de/sneak/feta/toot" "git.eeqj.de/sneak/feta/toot"
"github.com/google/uuid" "github.com/google/uuid"
hstg "github.com/grokify/html-strip-tags-go"
_ "github.com/jinzhu/gorm/dialects/sqlite" _ "github.com/jinzhu/gorm/dialects/sqlite"
) )
@ -28,9 +31,13 @@ func (m *Manager) StoreToot(t *toot.Toot) error {
// FIXME normalize this, check for @ and append hostname if none // FIXME normalize this, check for @ and append hostname if none
nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost)) nt.Acct = fmt.Sprintf("%s@%s", t.Parsed.Account.Acct, strings.ToLower(t.FromHost))
nt.URL = t.Parsed.URL nt.URL = t.Parsed.URL
nt.Content = t.Parsed.Content nt.Content = []byte(t.Parsed.Content)
// FIXME replace tags with spaces, don't just strip them, otherwise text
// gets messed up.
nt.TextContent = []byte(html.UnescapeString(hstg.StripTags(t.Parsed.Content)))
nt.Hostname = strings.ToLower(t.FromHost) nt.Hostname = strings.ToLower(t.FromHost)
nt.Hash = t.GetHash() nt.Hash = t.GetHash()
nt.Fetched = time.Now()
r := m.db.Create(&nt) r := m.db.Create(&nt)
//panic(fmt.Sprintf("%+v", t)) //panic(fmt.Sprintf("%+v", t))
return r.Error return r.Error

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1 github.com/dn365/gin-zerolog v0.0.0-20171227063204-b43714b00db1
github.com/gin-gonic/gin v1.6.2 github.com/gin-gonic/gin v1.6.2
github.com/google/uuid v1.1.1 github.com/google/uuid v1.1.1
github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119
github.com/jinzhu/gorm v1.9.12 github.com/jinzhu/gorm v1.9.12
github.com/k0kubun/pp v3.0.1+incompatible github.com/k0kubun/pp v3.0.1+incompatible
github.com/looplab/fsm v0.1.0 github.com/looplab/fsm v0.1.0

2
go.sum
View File

@ -57,6 +57,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119 h1:h3iGUlU8HyW4baKd6D+h1mwOHnM2kwskSuG6Bv4tSbc=
github.com/grokify/html-strip-tags-go v0.0.0-20200322061010-ea0c1cf2f119/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=