incorporate postgres patch with minor changes

Dockerfile

@@ -17,7 +17,7 @@ FROM alpine
 # here are the levers
 ENV FETA_HOSTDISCOVERYPARALLELISM 20
 ENV FETA_FSSTORAGELOCATION /state/tootstore
-ENV FETA_DBSTORAGELOCATION /state/feta.state.sqlite3
+ENV FETA_DBURL sqlite:///state/feta.state.sqlite3
 ENV FETA_TOOTSTODISK false
 ENV FETA_TOOTSTODB true
 ENV FETA_DEBUG false

README.md

@@ -20,6 +20,25 @@ archives the fediverse
 
 [![Build Status](https://drone.datavi.be/api/badges/sneak/feta/status.svg)](https://drone.datavi.be/sneak/feta)
 
+# getting started
+## sqlite
+
+*using default file location:*
+`./feta`
+
+*using file:*
+`FETA_DBURL=sqlite://<abs path to file>  ./feta`
+
+*using memory:*
+`FETA_DBURL=sqlite://:memory:  ./feta`
+
+## postgres
+
+1. `docker-compose up .`
+2. `FETA_DBURL=postgres://feta_user:password@localhost:5432/feta?sslmode=disable  ./feta`
+
+Access the pgweb dashboard at `http://localhost:8081`
+
 # ethics statement
 
 It seems that some splinter groups are not well acquainted with the norms of
@@ -41,6 +60,7 @@ legitimately-obtained files from the hard drives of other people.
 
 # Author
 
-Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt;
+Jeffrey Paul &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt; and
+others
 
 [@sneak@sneak.berlin](https://s.sneak.berlin/@sneak)

database/manager.go

@@ -1,14 +1,18 @@
 package database
 
 import (
+	"net/url"
 	"path/filepath"
+	"strings"
 	"sync"
-	"github.com/jinzhu/gorm"
+
 	u "git.eeqj.de/sneak/goutil"
+	"github.com/golang/groupcache/lru"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
-	"github.com/golang/groupcache/lru"
 )
 
 const cacheEntries = 1000000
@@ -28,7 +32,7 @@ func New() *Manager {
 }
 
 func (m *Manager) init() {
-	m.open()
+	m.open(viper.GetString("DBURL"))
 	// breaks stuff, do not use:
 	//m.db.SingularTable(true)
 	m.db.LogMode(false)
@@ -38,22 +42,55 @@ func (m *Manager) init() {
 	m.recentlyInsertedTootHashCache = lru.New(cacheEntries)
 }
 
-func (m *Manager) open() {
+func (m *Manager) open(dbURL string) {
 	log.Info().Msg("opening database")
-	dirname := filepath.Dir(viper.GetString("DbStorageLocation"))
-	err := u.Mkdirp(dirname)
+	dsn, err := url.Parse(dbURL)
 	if err != nil {
 		log.Panic().
 			Err(err).
-			Msg("db path erro")
+			Msg("error parsing dbURL")
 	}
-
-	db, err := gorm.Open("sqlite3", viper.GetString("DbStorageLocation"))
+	log.Info().
+		Str("scheme", dsn.Scheme).
+		Str("user", dsn.User.Username()).
+		Str("host", dsn.Host).
+		Str("db", dsn.Path).
+		Str("args", dsn.RawQuery).
+		Msg("db connection values")
+	switch {
+	case strings.HasPrefix(dbURL, "postgres://"):
+		log.Info().Msg("using postgres db")
+		db, err := gorm.Open("postgres", dbURL)
 		if err != nil {
 			log.Panic().
 				Err(err).
 				Msg("failed to open database")
 		}
 		m.db = db
+	case strings.HasPrefix(dbURL, "sqlite://"):
+		log.Info().Msg("using sqlite db")
+		if !strings.HasSuffix(dbURL, ":memory:") {
+			dirname := filepath.Dir(strings.TrimPrefix(dbURL, "sqlite://"))
+			err := u.Mkdirp(dirname)
+			if err != nil {
+				log.Panic().
+					Err(err).
+					Msg("db path error")
+			}
+		}
+		db, err := gorm.Open("sqlite3", strings.TrimPrefix(dbURL, "sqlite://"))
+		if err != nil {
+			log.Panic().
+				Err(err).
+				Str("dbURL", dbURL).
+				Msg("failed to open database")
+		}
+		m.db = db
+	default:
+		log.Panic().
+			Str("driver", dsn.Scheme).
+			Msg("unsupported driver in database url, must be 'postgres' or 'sqlite'")
+	}
+
 	m.doMigrations()
 }

database/storageconnector.go

@@ -7,7 +7,7 @@ import (
 	"git.eeqj.de/sneak/feta/toot"
 	"github.com/google/uuid"
 	hstg "github.com/grokify/html-strip-tags-go"
-	_ "github.com/jinzhu/gorm/dialects/sqlite"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 )
 
 func (m *Manager) TootInsertHashCacheSize() uint {

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3.1'
+
+services:
+  postgres:
+    image: postgres:12
+    restart: always
+    container_name: postgres
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: feta_user
+      POSTGRES_DB: feta
+  pgweb:
+    image: sosedoff/pgweb
+    restart: always
+    container_name: pgweb
+    ports: 
+        - "8081:8081" 
+    links: 
+        - postgres:postgres
+    environment:
+        - DATABASE_URL=postgres://feta_user:password@postgres:5432/feta?sslmode=disable
+    depends_on:
+        - postgres

go.sum

@@ -115,6 +115,7 @@ github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/looplab/fsm v0.1.0 h1:Qte7Zdn/5hBNbXzP7yxVU4OIFHWXBovyTT2LaBTyC20=
 github.com/looplab/fsm v0.1.0/go.mod h1:m2VaOfDHxqXBBMgc26m6yUOwkFn8H2AlJDE+jd/uafI=

process/feta.go

@@ -1,6 +1,7 @@
 package process
 
 import (
+	"fmt"
 	"os"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 	"git.eeqj.de/sneak/feta/locator"
 	"git.eeqj.de/sneak/feta/manager"
 	"git.eeqj.de/sneak/feta/storage"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/k0kubun/pp"
 	"github.com/mattn/go-isatty"
@@ -56,7 +58,7 @@ func (f *Feta) configure() {
 	viper.SetDefault("TootsToDB", true)
 	viper.SetDefault("HostDiscoveryParallelism", 5)
 	viper.SetDefault("FSStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/tootarchive.d"))
-	viper.SetDefault("DBStorageLocation", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db"))
+	viper.SetDefault("DBURL", fmt.Sprintf("sqlite://%s", os.ExpandEnv("$HOME/Library/ApplicationSupport/feta/feta.state.db")))
 	viper.SetDefault("LogReportInterval", time.Second*10)
 
 	if err := viper.ReadInConfig(); err != nil {