directory/internal/importer/importer.go

187 lines
4.3 KiB
Go

package importer
import (
"context"
"encoding/json"
"io/ioutil"
"os"
"os/signal"
"syscall"
"time"
"github.com/mmcdole/gofeed"
"github.com/rs/zerolog"
"github.com/schollz/progressbar/v3"
"github.com/spf13/cobra"
"go.uber.org/fx"
"sneak.berlin/go/directory/internal/config"
"sneak.berlin/go/directory/internal/globals"
"sneak.berlin/go/directory/internal/logger"
"sneak.berlin/go/directory/internal/store"
_ "github.com/joho/godotenv/autoload"
)
type ImporterParams struct {
fx.In
Logger *logger.Logger
Globals *globals.Globals
Config *config.Config
Store *store.Store
}
type Importer struct {
startupTime time.Time
exitCode int
log *zerolog.Logger
params ImporterParams
ctx context.Context
cancelFunc context.CancelFunc
}
func New(lc fx.Lifecycle, params ImporterParams) (*Importer, error) {
i := new(Importer)
i.params = params
i.log = params.Logger.Get()
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
i.startupTime = time.Now()
go i.Run(ctx)
return nil
},
OnStop: func(ctx context.Context) error {
i.cleanShutdown(ctx)
return nil
},
})
return i, nil
}
func (i *Importer) Run(ctx context.Context) {
i.ctx, i.cancelFunc = context.WithCancel(ctx)
rootCmd := &cobra.Command{
Use: "importer",
Short: "Importer is a CLI for importing data into the directory",
}
importCmd := &cobra.Command{
Use: "import",
Short: "Import data into the directory",
}
importJSONCmd := &cobra.Command{
Use: "json [file]",
Short: "Import data from a JSON file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromJSON(args[0])
},
}
importOPMLCmd := &cobra.Command{
Use: "opml [file]",
Short: "Import data from an OPML file",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
i.importFromOPML(args[0])
},
}
importCmd.AddCommand(importJSONCmd, importOPMLCmd)
rootCmd.AddCommand(importCmd)
go func() {
c := make(chan os.Signal, 1)
signal.Ignore(syscall.SIGPIPE)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
sig := <-c
i.log.Info().Msgf("signal received: %+v", sig)
if i.cancelFunc != nil {
i.cancelFunc()
}
}()
if err := rootCmd.ExecuteContext(i.ctx); err != nil {
i.log.Error().Err(err).Msg("command execution failed")
i.exitCode = 1
}
<-i.ctx.Done()
i.exitCode = 0
i.cleanShutdown(ctx)
}
func (i *Importer) importFromJSON(file string) {
i.log.Info().Msgf("importing from JSON file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read JSON file")
return
}
var records []map[string]interface{}
if err := json.Unmarshal(data, &records); err != nil {
i.log.Error().Err(err).Msg("failed to unmarshal JSON")
return
}
totalRecords := len(records)
bar := progressbar.NewOptions(totalRecords,
progressbar.OptionSetDescription("Importing records"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, record := range records {
// Insert record into the database
// db.InsertRecord(record) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("JSON import completed")
}
func (i *Importer) importFromOPML(file string) {
i.log.Info().Msgf("importing from OPML file: %s", file)
data, err := ioutil.ReadFile(file)
if err != nil {
i.log.Error().Err(err).Msg("failed to read OPML file")
return
}
fp := gofeed.NewParser()
feed, err := fp.ParseString(string(data))
if err != nil {
i.log.Error().Err(err).Msg("failed to parse OPML")
return
}
totalOutlines := len(feed.Items)
bar := progressbar.NewOptions(totalOutlines,
progressbar.OptionSetDescription("Importing outlines"),
progressbar.OptionShowCount(),
progressbar.OptionShowIts(),
progressbar.OptionShowElapsedTime(),
progressbar.OptionShowRemainingTime(),
progressbar.OptionSetPredictTime(true),
)
for _, outline := range feed.Items {
// Insert outline into the database
// db.InsertOutline(outline) // Replace with actual database insertion logic
bar.Add(1)
}
i.log.Info().Msg("OPML import completed")
}
func (i *Importer) cleanShutdown(ctx context.Context) {
i.log.Info().Msgf("shutting down")
os.Exit(i.exitCode)
}