checkpoint
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 16s

This commit is contained in:
2024-06-15 07:41:37 -07:00
parent 000b16e487
commit f1dcc7acf4
14 changed files with 258 additions and 146 deletions

40
internal/importer/cmd.go Normal file
View File

@@ -0,0 +1,40 @@
package importer
import (
"github.com/spf13/cobra"
)
func setupCommands(i *Importer) *cobra.Command {
rootCmd := &cobra.Command{
Use: "importer",
Short: "Importer is a CLI for importing data into the directory",
}
importCmd := &cobra.Command{
Use: "import",
Short: "Import data into the directory",
}
importJSONCmd := &cobra.Command{
Use: "json [file]",
Short: "Import data from a JSON file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromJSON(args[0])
},
}
importOPMLCmd := &cobra.Command{
Use: "opml [file]",
Short: "Import data from an OPML file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromOPML(args[0])
},
}
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
rootCmd.AddCommand(importCmd)
return rootCmd
}

View File

@@ -2,24 +2,17 @@ package importer
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"os/signal"
"syscall"
"time"
"github.com/mmcdole/gofeed"
"github.com/rs/zerolog"
"github.com/schollz/progressbar/v3"
"github.com/spf13/cobra"
"go.uber.org/fx"
"sneak.berlin/go/directory/internal/config"
"sneak.berlin/go/directory/internal/globals"
"sneak.berlin/go/directory/internal/logger"
"sneak.berlin/go/directory/internal/store"
_ "github.com/joho/godotenv/autoload"
)
type ImporterParams struct {
@@ -61,36 +54,7 @@ func New(lc fx.Lifecycle, params ImporterParams) (*Importer, error) {
func (i *Importer) Run(ctx context.Context) {
i.ctx, i.cancelFunc = context.WithCancel(ctx)
rootCmd := &cobra.Command{
Use: "importer",
Short: "Importer is a CLI for importing data into the directory",
}
importCmd := &cobra.Command{
Use: "import",
Short: "Import data into the directory",
}
importJSONCmd := &cobra.Command{
Use: "json [file]",
Short: "Import data from a JSON file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromJSON(args[0])
},
}
importOPMLCmd := &cobra.Command{
Use: "opml [file]",
Short: "Import data from an OPML file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromOPML(args[0])
},
}
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
rootCmd.AddCommand(importCmd)
rootCmd := setupCommands(i)
go func() {
c := make(chan os.Signal, 1)
@@ -113,74 +77,8 @@ func (i *Importer) Run(ctx context.Context) {
i.cleanShutdown(ctx)
}
func (i *Importer) importFromJSON(file string) {
i.log.Info().Msgf("importing from JSON file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read JSON file")
return
}
var records []map[string]interface{}
if err := json.Unmarshal(data, &records); err != nil {
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
return
}
totalRecords := len(records)
bar := progressbar.NewOptions(totalRecords,
progressbar.OptionSetDescription("Importing records"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, record := range records {
// Insert record into the database
// db.InsertRecord(record) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("JSON import completed")
}
func (i *Importer) importFromOPML(file string) {
i.log.Info().Msgf("importing from OPML file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read OPML file")
return
}
fp := gofeed.NewParser()
feed, err := fp.ParseString(string(data))
if err != nil {
i.log.Error().Err(err).Msg("failed to parse OPML")
return
}
totalOutlines := len(feed.Items)
bar := progressbar.NewOptions(totalOutlines,
progressbar.OptionSetDescription("Importing outlines"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, outline := range feed.Items {
// Insert outline into the database
// db.InsertOutline(outline) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("OPML import completed")
}
func (i *Importer) cleanShutdown(ctx context.Context) {
i.log.Info().Msgf("shutting down")
os.Exit(i.exitCode)
}

42
internal/importer/json.go Normal file
View File

@@ -0,0 +1,42 @@
package importer
import (
"encoding/json"
"io/ioutil"
)
func (i *Importer) importFromJSON(file string) {
i.log.Info().Msgf("importing from JSON file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read JSON file")
return
}
var records []map[string]interface{}
if err := json.Unmarshal(data, &records); err != nil {
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
return
}
//totalRecords := len(records)
/*
bar := progressbar.NewOptions(totalRecords,
progressbar.OptionSetDescription("Importing records"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetPredictTime(true),
)
*/
/*
for _, record := range records {
// Insert record into the database
// db.InsertRecord(record) // Replace with actual database insertion logic
bar.Add(1)
}
*/
i.log.Info().Msg("JSON import completed")
}

35
internal/importer/opml.go Normal file
View File

@@ -0,0 +1,35 @@
package importer
func (i *Importer) importFromOPML(file string) {
/*
i.log.Info().Msgf("importing from OPML file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read OPML file")
return
}
fp := gofeed.NewParser()
feed, err := fp.ParseString(string(data))
if err != nil {
i.log.Error().Err(err).Msg("failed to parse OPML")
return
}
totalOutlines := len(feed.Items)
bar := progressbar.NewOptions(totalOutlines,
progressbar.OptionSetDescription("Importing outlines"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionSetPredictTime(true),
)
for _, outline := range feed.Items {
// Insert outline into the database
// db.InsertOutline(outline) // Replace with actual database insertion logic
bar.Add(1)
}
*/
i.log.Info().Msg("OPML import completed")
}