From f1dcc7acf406c4dc037b8116956e298364cc49aa Mon Sep 17 00:00:00 2001 From: sneak Date: Sat, 15 Jun 2024 07:41:37 -0700 Subject: [PATCH] checkpoint --- Makefile | 3 + cmd/importer/main.go | 19 ++++++ go.mod | 44 +++++++------ go.sum | 68 ++++++++++++++++++++ internal/database/database.go | 1 + internal/importer/cmd.go | 40 ++++++++++++ internal/importer/importer.go | 106 +------------------------------ internal/importer/json.go | 42 ++++++++++++ internal/importer/opml.go | 35 ++++++++++ internal/store/site_tag.go | 13 ++-- internal/store/store.go | 2 +- internal/store/user.go | 12 ++-- internal/store/user_attribute.go | 9 +-- internal/store/website.go | 10 +-- 14 files changed, 258 insertions(+), 146 deletions(-) create mode 100644 internal/importer/cmd.go create mode 100644 internal/importer/json.go create mode 100644 internal/importer/opml.go diff --git a/Makefile b/Makefile index db2bd0f..8980ae6 100644 --- a/Makefile +++ b/Makefile @@ -37,3 +37,6 @@ docker: ./$(FN): cmd/$(FN)/main.go internal/*/*.go assets/*/* cd ./cmd/$(FN) && \ go build -o ../../$(FN) $(GOFLAGS) . + +./importer: cmd/importer/main.go internal/*/*.go + go build -o ./importer $(GOFLAGS) ./cmd/importer diff --git a/cmd/importer/main.go b/cmd/importer/main.go index abf432d..8394bdf 100644 --- a/cmd/importer/main.go +++ b/cmd/importer/main.go @@ -1,12 +1,16 @@ package main import ( + "fmt" + "runtime/debug" + "go.uber.org/fx" "sneak.berlin/go/directory/internal/config" "sneak.berlin/go/directory/internal/database" "sneak.berlin/go/directory/internal/globals" "sneak.berlin/go/directory/internal/importer" "sneak.berlin/go/directory/internal/logger" + "sneak.berlin/go/directory/internal/store" ) var ( @@ -15,7 +19,21 @@ var ( Buildarch string ) +var Commit = func() string { + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" { + return setting.Value + } + } + } + return "" +}() + func main() { + + fmt.Println("Commit: ", Commit) + globals.Appname = Appname globals.Version = Version globals.Buildarch = Buildarch @@ -24,6 +42,7 @@ func main() { fx.Provide( config.New, database.New, + store.New, globals.New, logger.New, importer.New, diff --git a/go.mod b/go.mod index 4c42630..101993f 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,22 @@ go 1.22.2 require ( github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 + github.com/CloudyKit/jet/v6 v6.2.0 github.com/getsentry/sentry-go v0.28.0 github.com/go-chi/chi v1.5.5 github.com/go-chi/cors v1.2.1 github.com/joho/godotenv v1.5.1 + github.com/mmcdole/gofeed v1.3.0 + github.com/oklog/ulid/v2 v2.1.0 github.com/prometheus/client_golang v1.19.1 github.com/rs/zerolog v1.33.0 + github.com/schollz/progressbar/v3 v3.14.3 github.com/slok/go-http-metrics v0.12.0 + github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 go.uber.org/fx v1.22.0 + golang.org/x/crypto v0.23.0 + gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.5 gorm.io/gorm v1.25.10 sneak.berlin/go/util v0.0.0-20240601223751-9302c14a6cdc @@ -20,58 +27,55 @@ require ( require ( github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect - github.com/CloudyKit/jet/v6 v6.2.0 // indirect - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/PuerkitoBio/goquery v1.9.2 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-chi/chi/v5 v5.0.12 // indirect github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.4.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/mmcdole/gofeed v1.3.0 // indirect - github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/mmcdole/goxpp v1.1.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/oklog/ulid/v2 v2.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.13.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/schollz/progressbar/v3 v3.14.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/dig v1.17.1 // indirect - go.uber.org/multierr v1.10.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/net v0.24.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/postgres v1.5.7 // indirect ) diff --git a/go.sum b/go.sum index 755b12d..44b4dd1 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,12 @@ github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oM github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -28,6 +32,8 @@ github.com/getsentry/sentry-go v0.28.0 h1:7Rqx9M3ythTKy2J6uZLHmc8Sz9OGgIlseuO1iB github.com/getsentry/sentry-go v0.28.0/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -48,8 +54,14 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -63,6 +75,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -81,6 +95,8 @@ github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= +github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -91,6 +107,8 @@ github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNs github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -106,6 +124,8 @@ github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+a github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -148,41 +168,89 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.22.0 h1:pApUK7yL0OUHMd8vkunWSlLxZVFFk70jR2nKde8X2NM= go.uber.org/fx v1.22.0/go.mod h1:HT2M7d7RHo+ebKGh9NRcrsrHHfpZ60nW3QRubMRfv48= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/database/database.go b/internal/database/database.go index 4c01726..bf1c39f 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/directory/internal/logger" _ "github.com/joho/godotenv/autoload" + _ "github.com/lib/pq" // Import the PostgreSQL driver ) type DatabaseParams struct { diff --git a/internal/importer/cmd.go b/internal/importer/cmd.go new file mode 100644 index 0000000..21837af --- /dev/null +++ b/internal/importer/cmd.go @@ -0,0 +1,40 @@ +package importer + +import ( + "github.com/spf13/cobra" +) + +func setupCommands(i *Importer) *cobra.Command { + rootCmd := &cobra.Command{ + Use: "importer", + Short: "Importer is a CLI for importing data into the directory", + } + + importCmd := &cobra.Command{ + Use: "import", + Short: "Import data into the directory", + } + + importJSONCmd := &cobra.Command{ + Use: "json [file]", + Short: "Import data from a JSON file", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + i.importFromJSON(args[0]) + }, + } + + importOPMLCmd := &cobra.Command{ + Use: "opml [file]", + Short: "Import data from an OPML file", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + i.importFromOPML(args[0]) + }, + } + + importCmd.AddCommand(importJSONCmd, importOPMLCmd) + rootCmd.AddCommand(importCmd) + + return rootCmd +} diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 04cd626..94824ba 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -2,24 +2,17 @@ package importer import ( "context" - "encoding/json" - "io/ioutil" "os" "os/signal" "syscall" "time" - "github.com/mmcdole/gofeed" "github.com/rs/zerolog" - "github.com/schollz/progressbar/v3" - "github.com/spf13/cobra" "go.uber.org/fx" "sneak.berlin/go/directory/internal/config" "sneak.berlin/go/directory/internal/globals" "sneak.berlin/go/directory/internal/logger" "sneak.berlin/go/directory/internal/store" - - _ "github.com/joho/godotenv/autoload" ) type ImporterParams struct { @@ -61,36 +54,7 @@ func New(lc fx.Lifecycle, params ImporterParams) (*Importer, error) { func (i *Importer) Run(ctx context.Context) { i.ctx, i.cancelFunc = context.WithCancel(ctx) - rootCmd := &cobra.Command{ - Use: "importer", - Short: "Importer is a CLI for importing data into the directory", - } - - importCmd := &cobra.Command{ - Use: "import", - Short: "Import data into the directory", - } - - importJSONCmd := &cobra.Command{ - Use: "json [file]", - Short: "Import data from a JSON file", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - i.importFromJSON(args[0]) - }, - } - - importOPMLCmd := &cobra.Command{ - Use: "opml [file]", - Short: "Import data from an OPML file", - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - i.importFromOPML(args[0]) - }, - } - - importCmd.AddCommand(importJSONCmd, importOPMLCmd) - rootCmd.AddCommand(importCmd) + rootCmd := setupCommands(i) go func() { c := make(chan os.Signal, 1) @@ -113,74 +77,8 @@ func (i *Importer) Run(ctx context.Context) { i.cleanShutdown(ctx) } -func (i *Importer) importFromJSON(file string) { - i.log.Info().Msgf("importing from JSON file: %s", file) - data, err := ioutil.ReadFile(file) - if err != nil { - i.log.Error().Err(err).Msg("failed to read JSON file") - return - } - - var records []map[string]interface{} - if err := json.Unmarshal(data, &records); err != nil { - i.log.Error().Err(err).Msg("failed to unmarshal JSON") - return - } - - totalRecords := len(records) - bar := progressbar.NewOptions(totalRecords, - progressbar.OptionSetDescription("Importing records"), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionShowElapsedTime(), - progressbar.OptionShowRemainingTime(), - progressbar.OptionSetPredictTime(true), - ) - - for _, record := range records { - // Insert record into the database - // db.InsertRecord(record) // Replace with actual database insertion logic - bar.Add(1) - } - - i.log.Info().Msg("JSON import completed") -} - -func (i *Importer) importFromOPML(file string) { - i.log.Info().Msgf("importing from OPML file: %s", file) - data, err := ioutil.ReadFile(file) - if err != nil { - i.log.Error().Err(err).Msg("failed to read OPML file") - return - } - - fp := gofeed.NewParser() - feed, err := fp.ParseString(string(data)) - if err != nil { - i.log.Error().Err(err).Msg("failed to parse OPML") - return - } - - totalOutlines := len(feed.Items) - bar := progressbar.NewOptions(totalOutlines, - progressbar.OptionSetDescription("Importing outlines"), - progressbar.OptionShowCount(), - progressbar.OptionShowIts(), - progressbar.OptionShowElapsedTime(), - progressbar.OptionShowRemainingTime(), - progressbar.OptionSetPredictTime(true), - ) - - for _, outline := range feed.Items { - // Insert outline into the database - // db.InsertOutline(outline) // Replace with actual database insertion logic - bar.Add(1) - } - - i.log.Info().Msg("OPML import completed") -} - func (i *Importer) cleanShutdown(ctx context.Context) { i.log.Info().Msgf("shutting down") os.Exit(i.exitCode) } + diff --git a/internal/importer/json.go b/internal/importer/json.go new file mode 100644 index 0000000..a2a17d9 --- /dev/null +++ b/internal/importer/json.go @@ -0,0 +1,42 @@ +package importer + +import ( + "encoding/json" + "io/ioutil" +) + +func (i *Importer) importFromJSON(file string) { + i.log.Info().Msgf("importing from JSON file: %s", file) + data, err := ioutil.ReadFile(file) + if err != nil { + i.log.Error().Err(err).Msg("failed to read JSON file") + return + } + + var records []map[string]interface{} + if err := json.Unmarshal(data, &records); err != nil { + i.log.Error().Err(err).Msg("failed to unmarshal JSON") + return + } + + //totalRecords := len(records) + + /* + bar := progressbar.NewOptions(totalRecords, + progressbar.OptionSetDescription("Importing records"), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetPredictTime(true), + ) + */ + + /* + for _, record := range records { + // Insert record into the database + // db.InsertRecord(record) // Replace with actual database insertion logic + bar.Add(1) + } + */ + + i.log.Info().Msg("JSON import completed") +} diff --git a/internal/importer/opml.go b/internal/importer/opml.go new file mode 100644 index 0000000..395cd1f --- /dev/null +++ b/internal/importer/opml.go @@ -0,0 +1,35 @@ +package importer + +func (i *Importer) importFromOPML(file string) { + /* + i.log.Info().Msgf("importing from OPML file: %s", file) + data, err := ioutil.ReadFile(file) + if err != nil { + i.log.Error().Err(err).Msg("failed to read OPML file") + return + } + + fp := gofeed.NewParser() + feed, err := fp.ParseString(string(data)) + if err != nil { + i.log.Error().Err(err).Msg("failed to parse OPML") + return + } + + totalOutlines := len(feed.Items) + bar := progressbar.NewOptions(totalOutlines, + progressbar.OptionSetDescription("Importing outlines"), + progressbar.OptionShowCount(), + progressbar.OptionShowIts(), + progressbar.OptionSetPredictTime(true), + ) + + for _, outline := range feed.Items { + // Insert outline into the database + // db.InsertOutline(outline) // Replace with actual database insertion logic + bar.Add(1) + } + + */ + i.log.Info().Msg("OPML import completed") +} diff --git a/internal/store/site_tag.go b/internal/store/site_tag.go index 55a55d0..6e8e8bd 100644 --- a/internal/store/site_tag.go +++ b/internal/store/site_tag.go @@ -3,15 +3,16 @@ package store import ( "strings" + "github.com/oklog/ulid/v2" "github.com/rs/zerolog/log" "gorm.io/gorm" ) type SiteTag struct { - ID uint `gorm:"primaryKey"` - SiteID string `gorm:"not null"` - Value string `gorm:"not null"` - db *gorm.DB `gorm:"-"` + ID ulid.ULID `gorm:"primaryKey"` + SiteID ulid.ULID `gorm:"not null"` + Value string `gorm:"not null"` + db *gorm.DB `gorm:"-"` } func (SiteTag) TableName() string { @@ -38,7 +39,7 @@ func (s *Site) AddTag(value string) error { } log.Debug(). - Str("siteid", s.ID). + Str("siteid", s.ID.String()). Str("tag", value). Msg("site tag added") return nil @@ -53,7 +54,7 @@ func (s *Site) DeleteAttribute(value string) error { } log.Debug(). - Str("siteid", s.ID). + Str("siteid", s.ID.String()). Str("tag", value). Msg("site tag deleted") return nil diff --git a/internal/store/store.go b/internal/store/store.go index 3d7ab89..706eb9d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -29,7 +29,7 @@ type Store struct { params *StoreParams } -func New(lc fx.Lifecycle, params *StoreParams) (*Store, error) { +func New(lc fx.Lifecycle, params StoreParams) (*Store, error) { s := new(Store) s.logger = params.Logger diff --git a/internal/store/user.go b/internal/store/user.go index 9214152..95ae248 100644 --- a/internal/store/user.go +++ b/internal/store/user.go @@ -13,8 +13,8 @@ import ( ) type User struct { - ID string `gorm:"primaryKey"` - Username string `gorm:"unique"` + ID ulid.ULID `gorm:"primaryKey"` + Username string `gorm:"unique"` Email *string PasswordHash string CreatedAt *time.Time @@ -47,7 +47,7 @@ func (s *Store) AddUser(username, password string) (*User, error) { hashedPassword = string(hashedPasswordBytes) } - userID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader).String() + userID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader) createdAt := time.Now() user := &User{ ID: userID, @@ -87,7 +87,7 @@ func (s *Store) FindUserByUsername(username string) (*User, error) { return &user, nil } -func (s *Store) FindUserByID(userID string) (*User, error) { +func (s *Store) FindUserByID(userID ulid.ULID) (*User, error) { var user User result := s.db.Where("id = ? and disabled = ?", userID, false).First(&user) if result.Error != nil { @@ -98,11 +98,11 @@ func (s *Store) FindUserByID(userID string) (*User, error) { return &user, nil } -func (s *Store) LogFailedAuth(UserID string, r *http.Request) { +func (s *Store) LogFailedAuth(UserID ulid.ULID, r *http.Request) { // FIXME implement } -func (s *Store) LogSuccessfulAuth(UserID string, r *http.Request) { +func (s *Store) LogSuccessfulAuth(UserID ulid.ULID, r *http.Request) { now := time.Now() u, err := s.FindUserByID(UserID) if err != nil { diff --git a/internal/store/user_attribute.go b/internal/store/user_attribute.go index 773d0b2..d2a2fa5 100644 --- a/internal/store/user_attribute.go +++ b/internal/store/user_attribute.go @@ -3,15 +3,16 @@ package store import ( "strings" + "github.com/oklog/ulid/v2" "github.com/rs/zerolog/log" "gorm.io/gorm" ) type UserAttribute struct { - ID uint `gorm:"primaryKey"` - UserID string `gorm:"not null"` - Value string `gorm:"not null"` - db *gorm.DB `gorm:"-"` + ID ulid.ULID `gorm:"primaryKey"` + UserID ulid.ULID `gorm:"not null"` + Value string `gorm:"not null"` + db *gorm.DB `gorm:"-"` } func (UserAttribute) TableName() string { diff --git a/internal/store/website.go b/internal/store/website.go index 36f267b..d1b805c 100644 --- a/internal/store/website.go +++ b/internal/store/website.go @@ -10,8 +10,8 @@ import ( ) type Site struct { - ID string `gorm:"primaryKey"` - Domain string `gorm:"unique"` + ID ulid.ULID `gorm:"primaryKey"` + Domain string `gorm:"unique"` URL string CreatedAt *time.Time FirstSeen *time.Time @@ -25,7 +25,7 @@ func (Site) TableName() string { } func (s *Store) AddSite(url string) (*Site, error) { - ID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader).String() + ID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader) createdAt := time.Now() site := &Site{ ID: ID, @@ -37,14 +37,14 @@ func (s *Store) AddSite(url string) (*Site, error) { result := s.db.Create(site) if result.Error != nil { log.Error(). - Str("siteid", ID). + Str("siteid", ID.String()). Err(result.Error). Msg("unable to insert site into db") return nil, result.Error } log.Debug(). - Str("siteid", ID). + Str("siteid", ID.String()). Str("url", url). Msg("site added to db") return site, nil