checkpoint
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 16s
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 16s
This commit is contained in:
40
internal/importer/cmd.go
Normal file
40
internal/importer/cmd.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func setupCommands(i *Importer) *cobra.Command {
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "importer",
|
||||
Short: "Importer is a CLI for importing data into the directory",
|
||||
}
|
||||
|
||||
importCmd := &cobra.Command{
|
||||
Use: "import",
|
||||
Short: "Import data into the directory",
|
||||
}
|
||||
|
||||
importJSONCmd := &cobra.Command{
|
||||
Use: "json [file]",
|
||||
Short: "Import data from a JSON file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
i.importFromJSON(args[0])
|
||||
},
|
||||
}
|
||||
|
||||
importOPMLCmd := &cobra.Command{
|
||||
Use: "opml [file]",
|
||||
Short: "Import data from an OPML file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
i.importFromOPML(args[0])
|
||||
},
|
||||
}
|
||||
|
||||
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
|
||||
rootCmd.AddCommand(importCmd)
|
||||
|
||||
return rootCmd
|
||||
}
|
||||
@@ -2,24 +2,17 @@ package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/directory/internal/config"
|
||||
"sneak.berlin/go/directory/internal/globals"
|
||||
"sneak.berlin/go/directory/internal/logger"
|
||||
"sneak.berlin/go/directory/internal/store"
|
||||
|
||||
_ "github.com/joho/godotenv/autoload"
|
||||
)
|
||||
|
||||
type ImporterParams struct {
|
||||
@@ -61,36 +54,7 @@ func New(lc fx.Lifecycle, params ImporterParams) (*Importer, error) {
|
||||
func (i *Importer) Run(ctx context.Context) {
|
||||
i.ctx, i.cancelFunc = context.WithCancel(ctx)
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "importer",
|
||||
Short: "Importer is a CLI for importing data into the directory",
|
||||
}
|
||||
|
||||
importCmd := &cobra.Command{
|
||||
Use: "import",
|
||||
Short: "Import data into the directory",
|
||||
}
|
||||
|
||||
importJSONCmd := &cobra.Command{
|
||||
Use: "json [file]",
|
||||
Short: "Import data from a JSON file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
i.importFromJSON(args[0])
|
||||
},
|
||||
}
|
||||
|
||||
importOPMLCmd := &cobra.Command{
|
||||
Use: "opml [file]",
|
||||
Short: "Import data from an OPML file",
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
i.importFromOPML(args[0])
|
||||
},
|
||||
}
|
||||
|
||||
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
|
||||
rootCmd.AddCommand(importCmd)
|
||||
rootCmd := setupCommands(i)
|
||||
|
||||
go func() {
|
||||
c := make(chan os.Signal, 1)
|
||||
@@ -113,74 +77,8 @@ func (i *Importer) Run(ctx context.Context) {
|
||||
i.cleanShutdown(ctx)
|
||||
}
|
||||
|
||||
func (i *Importer) importFromJSON(file string) {
|
||||
i.log.Info().Msgf("importing from JSON file: %s", file)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to read JSON file")
|
||||
return
|
||||
}
|
||||
|
||||
var records []map[string]interface{}
|
||||
if err := json.Unmarshal(data, &records); err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
|
||||
return
|
||||
}
|
||||
|
||||
totalRecords := len(records)
|
||||
bar := progressbar.NewOptions(totalRecords,
|
||||
progressbar.OptionSetDescription("Importing records"),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionShowElapsedTime(),
|
||||
progressbar.OptionShowRemainingTime(),
|
||||
progressbar.OptionSetPredictTime(true),
|
||||
)
|
||||
|
||||
for _, record := range records {
|
||||
// Insert record into the database
|
||||
// db.InsertRecord(record) // Replace with actual database insertion logic
|
||||
bar.Add(1)
|
||||
}
|
||||
|
||||
i.log.Info().Msg("JSON import completed")
|
||||
}
|
||||
|
||||
func (i *Importer) importFromOPML(file string) {
|
||||
i.log.Info().Msgf("importing from OPML file: %s", file)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to read OPML file")
|
||||
return
|
||||
}
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseString(string(data))
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to parse OPML")
|
||||
return
|
||||
}
|
||||
|
||||
totalOutlines := len(feed.Items)
|
||||
bar := progressbar.NewOptions(totalOutlines,
|
||||
progressbar.OptionSetDescription("Importing outlines"),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionShowElapsedTime(),
|
||||
progressbar.OptionShowRemainingTime(),
|
||||
progressbar.OptionSetPredictTime(true),
|
||||
)
|
||||
|
||||
for _, outline := range feed.Items {
|
||||
// Insert outline into the database
|
||||
// db.InsertOutline(outline) // Replace with actual database insertion logic
|
||||
bar.Add(1)
|
||||
}
|
||||
|
||||
i.log.Info().Msg("OPML import completed")
|
||||
}
|
||||
|
||||
func (i *Importer) cleanShutdown(ctx context.Context) {
|
||||
i.log.Info().Msgf("shutting down")
|
||||
os.Exit(i.exitCode)
|
||||
}
|
||||
|
||||
|
||||
42
internal/importer/json.go
Normal file
42
internal/importer/json.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
func (i *Importer) importFromJSON(file string) {
|
||||
i.log.Info().Msgf("importing from JSON file: %s", file)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to read JSON file")
|
||||
return
|
||||
}
|
||||
|
||||
var records []map[string]interface{}
|
||||
if err := json.Unmarshal(data, &records); err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
|
||||
return
|
||||
}
|
||||
|
||||
//totalRecords := len(records)
|
||||
|
||||
/*
|
||||
bar := progressbar.NewOptions(totalRecords,
|
||||
progressbar.OptionSetDescription("Importing records"),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionSetPredictTime(true),
|
||||
)
|
||||
*/
|
||||
|
||||
/*
|
||||
for _, record := range records {
|
||||
// Insert record into the database
|
||||
// db.InsertRecord(record) // Replace with actual database insertion logic
|
||||
bar.Add(1)
|
||||
}
|
||||
*/
|
||||
|
||||
i.log.Info().Msg("JSON import completed")
|
||||
}
|
||||
35
internal/importer/opml.go
Normal file
35
internal/importer/opml.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package importer
|
||||
|
||||
func (i *Importer) importFromOPML(file string) {
|
||||
/*
|
||||
i.log.Info().Msgf("importing from OPML file: %s", file)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to read OPML file")
|
||||
return
|
||||
}
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
feed, err := fp.ParseString(string(data))
|
||||
if err != nil {
|
||||
i.log.Error().Err(err).Msg("failed to parse OPML")
|
||||
return
|
||||
}
|
||||
|
||||
totalOutlines := len(feed.Items)
|
||||
bar := progressbar.NewOptions(totalOutlines,
|
||||
progressbar.OptionSetDescription("Importing outlines"),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionShowIts(),
|
||||
progressbar.OptionSetPredictTime(true),
|
||||
)
|
||||
|
||||
for _, outline := range feed.Items {
|
||||
// Insert outline into the database
|
||||
// db.InsertOutline(outline) // Replace with actual database insertion logic
|
||||
bar.Add(1)
|
||||
}
|
||||
|
||||
*/
|
||||
i.log.Info().Msg("OPML import completed")
|
||||
}
|
||||
Reference in New Issue
Block a user