From e6647e47f74ae41b36550fc13d0b0b511ff2c382 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 5 Apr 2020 22:37:22 -0700 Subject: [PATCH] incorporate postgres patch with minor changes --- Dockerfile | 2 +- README.md | 22 +++++++++++- database/manager.go | 65 ++++++++++++++++++++++++++++-------- database/storageconnector.go | 2 +- docker-compose.yml | 25 ++++++++++++++ go.sum | 1 + process/feta.go | 4 ++- 7 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 5c0568b..ce5e6d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ FROM alpine # here are the levers ENV FETA_HOSTDISCOVERYPARALLELISM 20 ENV FETA_FSSTORAGELOCATION /state/tootstore -ENV FETA_DBSTORAGELOCATION /state/feta.state.sqlite3 +ENV FETA_DBURL sqlite:///state/feta.state.sqlite3 ENV FETA_TOOTSTODISK false ENV FETA_TOOTSTODB true ENV FETA_DEBUG false diff --git a/README.md b/README.md index 9d2f1b0..e7fc91b 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,25 @@ archives the fediverse [![Build Status](https://drone.datavi.be/api/badges/sneak/feta/status.svg)](https://drone.datavi.be/sneak/feta) +# getting started +## sqlite + +*using default file location:* +`./feta` + +*using file:* +`FETA_DBURL=sqlite:// ./feta` + +*using memory:* +`FETA_DBURL=sqlite://:memory: ./feta` + +## postgres + +1. `docker-compose up .` +2. `FETA_DBURL=postgres://feta_user:password@localhost:5432/feta?sslmode=disable ./feta` + +Access the pgweb dashboard at `http://localhost:8081` + # ethics statement It seems that some splinter groups are not well acquainted with the norms of @@ -41,6 +60,7 @@ legitimately-obtained files from the hard drives of other people. # Author -Jeffrey Paul <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)> +Jeffrey Paul <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)> and +others [@sneak@sneak.berlin](https://s.sneak.berlin/@sneak) diff --git a/database/manager.go b/database/manager.go index 385dfeb..988ea18 100644 --- a/database/manager.go +++ b/database/manager.go @@ -1,14 +1,18 @@ package database import ( + "net/url" "path/filepath" + "strings" "sync" - "github.com/jinzhu/gorm" + u "git.eeqj.de/sneak/goutil" + "github.com/golang/groupcache/lru" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/rs/zerolog/log" "github.com/spf13/viper" - "github.com/golang/groupcache/lru" ) const cacheEntries = 1000000 @@ -28,7 +32,7 @@ func New() *Manager { } func (m *Manager) init() { - m.open() + m.open(viper.GetString("DBURL")) // breaks stuff, do not use: //m.db.SingularTable(true) m.db.LogMode(false) @@ -38,22 +42,55 @@ func (m *Manager) init() { m.recentlyInsertedTootHashCache = lru.New(cacheEntries) } -func (m *Manager) open() { +func (m *Manager) open(dbURL string) { log.Info().Msg("opening database") - dirname := filepath.Dir(viper.GetString("DbStorageLocation")) - err := u.Mkdirp(dirname) + dsn, err := url.Parse(dbURL) if err != nil { log.Panic(). Err(err). - Msg("db path erro") + Msg("error parsing dbURL") + } + log.Info(). + Str("scheme", dsn.Scheme). + Str("user", dsn.User.Username()). + Str("host", dsn.Host). + Str("db", dsn.Path). + Str("args", dsn.RawQuery). + Msg("db connection values") + switch { + case strings.HasPrefix(dbURL, "postgres://"): + log.Info().Msg("using postgres db") + db, err := gorm.Open("postgres", dbURL) + if err != nil { + log.Panic(). + Err(err). + Msg("failed to open database") + } + m.db = db + case strings.HasPrefix(dbURL, "sqlite://"): + log.Info().Msg("using sqlite db") + if !strings.HasSuffix(dbURL, ":memory:") { + dirname := filepath.Dir(strings.TrimPrefix(dbURL, "sqlite://")) + err := u.Mkdirp(dirname) + if err != nil { + log.Panic(). + Err(err). + Msg("db path error") + } + } + db, err := gorm.Open("sqlite3", strings.TrimPrefix(dbURL, "sqlite://")) + if err != nil { + log.Panic(). + Err(err). + Str("dbURL", dbURL). + Msg("failed to open database") + } + m.db = db + default: + log.Panic(). + Str("driver", dsn.Scheme). + Msg("unsupported driver in database url, must be 'postgres' or 'sqlite'") } - db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation")) - if err != nil { - log.Panic(). - Err(err). - Msg("failed to open database") - } - m.db = db m.doMigrations() } diff --git a/database/storageconnector.go b/database/storageconnector.go index 600c079..19f5f15 100644 --- a/database/storageconnector.go +++ b/database/storageconnector.go @@ -7,7 +7,7 @@ import ( "git.eeqj.de/sneak/feta/toot" "github.com/google/uuid" hstg "github.com/grokify/html-strip-tags-go" - _ "github.com/jinzhu/gorm/dialects/sqlite" + _ "github.com/jinzhu/gorm/dialects/postgres" ) func (m *Manager) TootInsertHashCacheSize() uint { diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..32e57f1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +version: '3.1' + +services: + postgres: + image: postgres:12 + restart: always + container_name: postgres + ports: + - "5432:5432" + environment: + POSTGRES_PASSWORD: password + POSTGRES_USER: feta_user + POSTGRES_DB: feta + pgweb: + image: sosedoff/pgweb + restart: always + container_name: pgweb + ports: + - "8081:8081" + links: + - postgres:postgres + environment: + - DATABASE_URL=postgres://feta_user:password@postgres:5432/feta?sslmode=disable + depends_on: + - postgres diff --git a/go.sum b/go.sum index 55dbde5..62aa566 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,7 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20= github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI= diff --git a/process/feta.go b/process/feta.go index 5ffc2f8..f9bea47 100644 --- a/process/feta.go +++ b/process/feta.go @@ -1,6 +1,7 @@ package process import ( + "fmt" "os" "time" @@ -9,6 +10,7 @@ import ( "git.eeqj.de/sneak/feta/locator" "git.eeqj.de/sneak/feta/manager" "git.eeqj.de/sneak/feta/storage" + _ "github.com/jinzhu/gorm/dialects/postgres" _ "github.com/jinzhu/gorm/dialects/sqlite" "github.com/k0kubun/pp" "github.com/mattn/go-isatty" @@ -56,7 +58,7 @@ func (f *Feta) configure() { viper.SetDefault("TootsToDB", true) viper.SetDefault("HostDiscoveryParallelism", 5) viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d")) - viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")) + viper.SetDefault("DBURL", fmt.Sprintf("sqlite://%s", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))) viper.SetDefault("LogReportInterval", time.Second*10) if err := viper.ReadInConfig(); err != nil {