sneak/directory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

checkpoint
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 16s
Details

Browse Source
This commit is contained in:
Jeffrey Paul
2024-06-15 07:41:37 -07:00
parent 000b16e487
commit f1dcc7acf4
14 changed files with 258 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
internal/database/database.go
View File

@@ -14,6 +14,7 @@ import (
"sneak.berlin/go/directory/internal/logger"
_ "github.com/joho/godotenv/autoload"
_ "github.com/lib/pq" // Import the PostgreSQL driver
)
type DatabaseParams struct {

40
internal/importer/cmd.go Normal file
View File

@@ -0,0 +1,40 @@
package importer
import (
"github.com/spf13/cobra"
)
func setupCommands(i *Importer) *cobra.Command {
rootCmd := &cobra.Command{
Use: "importer",
Short: "Importer is a CLI for importing data into the directory",
}
importCmd := &cobra.Command{
Use: "import",
Short: "Import data into the directory",
}
importJSONCmd := &cobra.Command{
Use: "json [file]",
Short: "Import data from a JSON file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromJSON(args[0])
},
}
importOPMLCmd := &cobra.Command{
Use: "opml [file]",
Short: "Import data from an OPML file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromOPML(args[0])
},
}
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
rootCmd.AddCommand(importCmd)
return rootCmd
}

106
internal/importer/importer.go
View File

@@ -2,24 +2,17 @@ package importer
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"os/signal"
"syscall"
"time"
"github.com/mmcdole/gofeed"
"github.com/rs/zerolog"
"github.com/schollz/progressbar/v3"
"github.com/spf13/cobra"
"go.uber.org/fx"
"sneak.berlin/go/directory/internal/config"
"sneak.berlin/go/directory/internal/globals"
"sneak.berlin/go/directory/internal/logger"
"sneak.berlin/go/directory/internal/store"
_ "github.com/joho/godotenv/autoload"
)
type ImporterParams struct {
@@ -61,36 +54,7 @@ func New(lc fx.Lifecycle, params ImporterParams) (*Importer, error) {
func (i *Importer) Run(ctx context.Context) {
i.ctx, i.cancelFunc = context.WithCancel(ctx)
rootCmd := &cobra.Command{
Use: "importer",
Short: "Importer is a CLI for importing data into the directory",
}
importCmd := &cobra.Command{
Use: "import",
Short: "Import data into the directory",
}
importJSONCmd := &cobra.Command{
Use: "json [file]",
Short: "Import data from a JSON file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromJSON(args[0])
},
}
importOPMLCmd := &cobra.Command{
Use: "opml [file]",
Short: "Import data from an OPML file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromOPML(args[0])
},
}
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
rootCmd.AddCommand(importCmd)
rootCmd := setupCommands(i)
go func() {
c := make(chan os.Signal, 1)
@@ -113,74 +77,8 @@ func (i *Importer) Run(ctx context.Context) {
i.cleanShutdown(ctx)
}
func (i *Importer) importFromJSON(file string) {
i.log.Info().Msgf("importing from JSON file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read JSON file")
return
}
var records []map[string]interface{}
if err := json.Unmarshal(data, &records); err != nil {
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
return
}
totalRecords := len(records)
bar := progressbar.NewOptions(totalRecords,
progressbar.OptionSetDescription("Importing records"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, record := range records {
// Insert record into the database
// db.InsertRecord(record) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("JSON import completed")
}
func (i *Importer) importFromOPML(file string) {
i.log.Info().Msgf("importing from OPML file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read OPML file")
return
}
fp := gofeed.NewParser()
feed, err := fp.ParseString(string(data))
if err != nil {
i.log.Error().Err(err).Msg("failed to parse OPML")
return
}
totalOutlines := len(feed.Items)
bar := progressbar.NewOptions(totalOutlines,
progressbar.OptionSetDescription("Importing outlines"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, outline := range feed.Items {
// Insert outline into the database
// db.InsertOutline(outline) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("OPML import completed")
}
func (i *Importer) cleanShutdown(ctx context.Context) {
i.log.Info().Msgf("shutting down")
os.Exit(i.exitCode)
}

42
internal/importer/json.go Normal file
View File

@@ -0,0 +1,42 @@
package importer
import (
"encoding/json"
"io/ioutil"
)
func (i *Importer) importFromJSON(file string) {
i.log.Info().Msgf("importing from JSON file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read JSON file")
return
}
var records []map[string]interface{}
if err := json.Unmarshal(data, &records); err != nil {
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
return
}
//totalRecords := len(records)
/*
bar := progressbar.NewOptions(totalRecords,
progressbar.OptionSetDescription("Importing records"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetPredictTime(true),
)
*/
/*
for _, record := range records {
// Insert record into the database
// db.InsertRecord(record) // Replace with actual database insertion logic
bar.Add(1)
}
*/
i.log.Info().Msg("JSON import completed")
}

35
internal/importer/opml.go Normal file
View File

@@ -0,0 +1,35 @@
package importer
func (i *Importer) importFromOPML(file string) {
/*
i.log.Info().Msgf("importing from OPML file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read OPML file")
return
}
fp := gofeed.NewParser()
feed, err := fp.ParseString(string(data))
if err != nil {
i.log.Error().Err(err).Msg("failed to parse OPML")
return
}
totalOutlines := len(feed.Items)
bar := progressbar.NewOptions(totalOutlines,
progressbar.OptionSetDescription("Importing outlines"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetPredictTime(true),
)
for _, outline := range feed.Items {
// Insert outline into the database
// db.InsertOutline(outline) // Replace with actual database insertion logic
bar.Add(1)
}
*/
i.log.Info().Msg("OPML import completed")
}

13
internal/store/site_tag.go
View File

@@ -3,15 +3,16 @@ package store
import (
"strings"
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog/log"
"gorm.io/gorm"
)
type SiteTag struct {
ID uint `gorm:"primaryKey"`
SiteID string `gorm:"not null"`
Value string `gorm:"not null"`
db *gorm.DB `gorm:"-"`
ID ulid.ULID `gorm:"primaryKey"`
SiteID ulid.ULID `gorm:"not null"`
Value string `gorm:"not null"`
db *gorm.DB `gorm:"-"`
}
func (SiteTag) TableName() string {
@@ -38,7 +39,7 @@ func (s *Site) AddTag(value string) error {
}
log.Debug().
Str("siteid", s.ID).
Str("siteid", s.ID.String()).
Str("tag", value).
Msg("site tag added")
return nil
@@ -53,7 +54,7 @@ func (s *Site) DeleteAttribute(value string) error {
}
log.Debug().
Str("siteid", s.ID).
Str("siteid", s.ID.String()).
Str("tag", value).
Msg("site tag deleted")
return nil

2
internal/store/store.go
View File

@@ -29,7 +29,7 @@ type Store struct {
params *StoreParams
}
func New(lc fx.Lifecycle, params *StoreParams) (*Store, error) {
func New(lc fx.Lifecycle, params StoreParams) (*Store, error) {
s := new(Store)
s.logger = params.Logger

12
internal/store/user.go
View File

@@ -13,8 +13,8 @@ import (
)
type User struct {
ID string `gorm:"primaryKey"`
Username string `gorm:"unique"`
ID ulid.ULID `gorm:"primaryKey"`
Username string `gorm:"unique"`
Email *string
PasswordHash string
CreatedAt *time.Time
@@ -47,7 +47,7 @@ func (s *Store) AddUser(username, password string) (*User, error) {
hashedPassword = string(hashedPasswordBytes)
}
userID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader).String()
userID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader)
createdAt := time.Now()
user := &User{
ID: userID,
@@ -87,7 +87,7 @@ func (s *Store) FindUserByUsername(username string) (*User, error) {
return &user, nil
}
func (s *Store) FindUserByID(userID string) (*User, error) {
func (s *Store) FindUserByID(userID ulid.ULID) (*User, error) {
var user User
result := s.db.Where("id = ? and disabled = ?", userID, false).First(&user)
if result.Error != nil {
@@ -98,11 +98,11 @@ func (s *Store) FindUserByID(userID string) (*User, error) {
return &user, nil
}
func (s *Store) LogFailedAuth(UserID string, r *http.Request) {
func (s *Store) LogFailedAuth(UserID ulid.ULID, r *http.Request) {
// FIXME implement
}
func (s *Store) LogSuccessfulAuth(UserID string, r *http.Request) {
func (s *Store) LogSuccessfulAuth(UserID ulid.ULID, r *http.Request) {
now := time.Now()
u, err := s.FindUserByID(UserID)
if err != nil {

9
internal/store/user_attribute.go
View File

@@ -3,15 +3,16 @@ package store
import (
"strings"
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog/log"
"gorm.io/gorm"
)
type UserAttribute struct {
ID uint `gorm:"primaryKey"`
UserID string `gorm:"not null"`
Value string `gorm:"not null"`
db *gorm.DB `gorm:"-"`
ID ulid.ULID `gorm:"primaryKey"`
UserID ulid.ULID `gorm:"not null"`
Value string `gorm:"not null"`
db *gorm.DB `gorm:"-"`
}
func (UserAttribute) TableName() string {

10
internal/store/website.go
View File

@@ -10,8 +10,8 @@ import (
)
type Site struct {
ID string `gorm:"primaryKey"`
Domain string `gorm:"unique"`
ID ulid.ULID `gorm:"primaryKey"`
Domain string `gorm:"unique"`
URL string
CreatedAt *time.Time
FirstSeen *time.Time
@@ -25,7 +25,7 @@ func (Site) TableName() string {
}
func (s *Store) AddSite(url string) (*Site, error) {
ID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader).String()
ID := ulid.MustNew(ulid.Timestamp(time.Now()), rand.Reader)
createdAt := time.Now()
site := &Site{
ID: ID,
@@ -37,14 +37,14 @@ func (s *Store) AddSite(url string) (*Site, error) {
result := s.db.Create(site)
if result.Error != nil {
log.Error().
Str("siteid", ID).
Str("siteid", ID.String()).
Err(result.Error).
Msg("unable to insert site into db")
return nil, result.Error
}
log.Debug().
Str("siteid", ID).
Str("siteid", ID.String()).
Str("url", url).
Msg("site added to db")
return site, nil