From 5f5f51c01483e2b7f284efaef4ead763d6f40b61 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 4 Dec 2022 04:33:24 +0400 Subject: [PATCH] latest --- .golangci.yaml | 2 + Makefile | 19 ++++--- README.md | 3 +- cmd/mfer/main.go | 3 +- internal/cli/check.go | 14 +++++ internal/cli/{cli.go => entry.go} | 3 +- internal/cli/gen.go | 28 ++++++++++ internal/cli/manifest.go | 51 ------------------ internal/cli/mfer.go | 90 +++++++++++++++---------------- mfer/manifest.go | 45 +++++++++++----- mfer/mf.proto | 12 ++--- mfer/mfer_test.go | 1 - 12 files changed, 143 insertions(+), 128 deletions(-) create mode 100644 .golangci.yaml create mode 100644 internal/cli/check.go rename internal/cli/{cli.go => entry.go} (79%) create mode 100644 internal/cli/gen.go delete mode 100644 internal/cli/manifest.go diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..29cd34d --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,2 @@ +run: + tests: false diff --git a/Makefile b/Makefile index 121bb96..87ae6be 100644 --- a/Makefile +++ b/Makefile @@ -4,20 +4,24 @@ export PATH := $(PATH):$(GOPATH)/bin PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go ARCH := $(shell uname -m) -VERSION := $(shell git describe --always --dirty=-dirty) +GITREV := $(shell git describe --always --dirty=-dirty) -GOLDFLAGS += -X main.Version=$(VERSION) +GOLDFLAGS += -X main.Version=0.1.0 +GOLDFLAGS += -X main.Gitrev=$(GITREV) GOFLAGS := -ldflags "$(GOLDFLAGS)" default: run run: ./mfer.cmd ./$< - ./$< gen + ./$< gen --ignore-dotfiles $(PROTOC_GEN_GO): test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@latest +fixme: + @grep -nir fixme . | grep -v Makefile + devprereqs: which gofumpt || go install -v mvdan.cc/gofumpt@latest which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest @@ -25,15 +29,16 @@ devprereqs: mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go protoc --version cd mfer && go generate . - cd cmd/mfer && go build -o ../../mfer.cmd $(GOFLAGS) . + cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) . clean: - rm -rfv mfer/*.pb.go ./mfer + rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer -fmt: prereqs +fmt: devprereqs gofumpt -l -w mfer internal cmd golangci-lint run --fix - prettier -w *.json *.md + -prettier -w *.json + -prettier -w *.md lint: golangci-lint run diff --git a/README.md b/README.md index c3fbd22..2a0df0a 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,7 @@ The manifest file would do several important things: - a command line option to zero/omit mtime/ctime, as well as manifest timestamp, and sort all directory listings so that manifest file generation is deterministic/reproducible -- URL format `mfer fetch - https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2` +- URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2` to assert in the URL which PGP signing key should be used in the manifest, so that shared URLs have a cryptographic trust root - a "well-known" key in the manifest that maps well known keys (could reuse diff --git a/cmd/mfer/main.go b/cmd/mfer/main.go index efc3416..b713fd2 100644 --- a/cmd/mfer/main.go +++ b/cmd/mfer/main.go @@ -9,8 +9,9 @@ import ( var ( Appname string = "mfer" Version string + Gitrev string ) func main() { - os.Exit(cli.Run(Appname, Version)) + os.Exit(cli.Run(Appname, Version, Gitrev)) } diff --git a/internal/cli/check.go b/internal/cli/check.go new file mode 100644 index 0000000..146cfc5 --- /dev/null +++ b/internal/cli/check.go @@ -0,0 +1,14 @@ +package cli + +import ( + "errors" + + log "github.com/visionmedia/go-cli-log" + + "github.com/urfave/cli/v2" +) + +func (mfa *CLIApp) checkManifestOperation(c *cli.Context) error { + log.Error(errors.New("unimplemented")) + return nil +} diff --git a/internal/cli/cli.go b/internal/cli/entry.go similarity index 79% rename from internal/cli/cli.go rename to internal/cli/entry.go index af43d4e..21b3cef 100644 --- a/internal/cli/cli.go +++ b/internal/cli/entry.go @@ -13,10 +13,11 @@ func init() { } } -func Run(Appname, Version string) int { +func Run(Appname, Version, Gitrev string) int { m := &CLIApp{} m.appname = Appname m.version = Version + m.gitrev = Gitrev m.exitCode = 0 m.run() diff --git a/internal/cli/gen.go b/internal/cli/gen.go new file mode 100644 index 0000000..4d3dc69 --- /dev/null +++ b/internal/cli/gen.go @@ -0,0 +1,28 @@ +package cli + +import ( + "fmt" + + "git.eeqj.de/sneak/mfer/mfer" + "github.com/davecgh/go-spew/spew" + "github.com/urfave/cli/v2" +) + +func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error { + fmt.Println("generateManifestOperation()") + fmt.Printf("called with arg: %s\n", c.String("input")) + + opts := &mfer.ManifestScanOptions{ + IgnoreDotfiles: c.Bool("IgnoreDotfiles"), + FollowSymLinks: c.Bool("FollowSymLinks"), + } + // FIXME add command flags for ignoring dotfiles and following symlinks + mf, err := mfer.NewFromPath(c.String("input"), opts) + if err != nil { + panic(err) + } + + spew.Dump(mf) + + return nil +} diff --git a/internal/cli/manifest.go b/internal/cli/manifest.go deleted file mode 100644 index 4b8661b..0000000 --- a/internal/cli/manifest.go +++ /dev/null @@ -1,51 +0,0 @@ -package cli - -import ( - "git.eeqj.de/sneak/mfer/mfer" - "github.com/spf13/afero" -) - -type Job struct { - innerpb *mfer.MFFileInner - outerpb *mfer.MFFile - fileCount int64 - totalSize int64 - afs afero.Fs -} - -func (m *Job) scanForFiles() error { - - m.innerpb = &mfer.MFFileInner{} - m.innerpb.Version = mfer.MFFileInner_ONE - return nil -} - -/* - walkErr := filepath.Walk(m.sourcePath, func(itemPath string, info os.FileInfo, err error) error { - - // we do not include the manifest file in the manifest - if itemPath == "index.mf" { - return nil - } - - fpi := mfer.MFFilePath{} - fpi.Path = itemPath - fpi.Size = info.Size() - m.innerpb.Files = append(m.innerpb.Files, &fpi) - m.fileCount++ - m.totalSize += fpi.Size - return nil - }) - - if walkErr != nil { - log.Fatal(walkErr) - return walkErr - } - - fmt.Printf("%#v\n", m.innerpb) - fmt.Printf("filecount = %#v\n", m.fileCount) - fmt.Printf("totalsize = %#v\n", m.totalSize) - return nil -} - -*/ diff --git a/internal/cli/mfer.go b/internal/cli/mfer.go index 9694b1a..00865bd 100644 --- a/internal/cli/mfer.go +++ b/internal/cli/mfer.go @@ -1,15 +1,12 @@ package cli import ( - "errors" "fmt" "os" "time" log "github.com/visionmedia/go-cli-log" - "git.eeqj.de/sneak/mfer/mfer" - "github.com/davecgh/go-spew/spew" "github.com/pterm/pterm" "github.com/urfave/cli/v2" ) @@ -17,10 +14,9 @@ import ( type CLIApp struct { appname string version string - buildarch string + gitrev string startupTime time.Time exitCode int - errorString string app *cli.App } @@ -40,29 +36,57 @@ func (mfa *CLIApp) disableStyling() { pterm.Fatal.Prefix.Text = "" } +func (mfa *CLIApp) VersionString() string { + return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev) +} + func (mfa *CLIApp) run() { + mfa.startupTime = time.Now() if NO_COLOR { // shoutout to rob pike who thinks it's juvenile mfa.disableStyling() } - mfa.printBanner() - mfa.app = &cli.App{ Name: mfa.appname, Usage: "Manifest generator", - Version: mfa.version, + Version: mfa.VersionString(), EnableBashCompletion: true, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Usage: "Verbosity level", + Aliases: []string{"v"}, + }, + &cli.BoolFlag{ + Name: "quiet", + Usage: "don't produce output except on error", + Aliases: []string{"q"}, + }, + }, Commands: []*cli.Command{ { Name: "generate", Aliases: []string{"gen"}, Usage: "Generate manifest file", Action: func(c *cli.Context) error { + if !c.Bool("quiet") { + mfa.printBanner() + } return mfa.generateManifestOperation(c) }, Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "FollowSymLinks", + Aliases: []string{"follow-symlinks"}, + Usage: "Resolve encountered symlinks", + }, + &cli.BoolFlag{ + Name: "IgnoreDotfiles", + Aliases: []string{"ignore-dotfiles"}, + Usage: "Ignore any dot (hidden) files encountered", + }, &cli.StringFlag{ Name: "input", Value: ".", @@ -81,51 +105,27 @@ func (mfa *CLIApp) run() { Name: "check", Usage: "Validate files using manifest file", Action: func(c *cli.Context) error { - return mfa.validateManifestOperation(c) + if !c.Bool("quiet") { + mfa.printBanner() + } + return mfa.checkManifestOperation(c) + }, + }, + { + Name: "version", + Usage: "Show version", + Action: func(c *cli.Context) error { + fmt.Printf("%s\n", mfa.VersionString()) + return nil }, }, }, } + mfa.app.HideVersion = true err := mfa.app.Run(os.Args) - if err != nil { mfa.exitCode = 1 log.Error(err) } } - -func (mfa *CLIApp) validateManifestOperation(c *cli.Context) error { - log.Error(errors.New("unimplemented")) - return nil -} - -func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error { - fmt.Println("generateManifestOperation()") - fmt.Printf("called with arg: %s\n", c.String("input")) - - opts := &mfer.ManifestScanOptions{ - IgnoreDotfiles: true, - FollowSymLinks: false, - } - mf, err := mfer.NewFromPath(c.String("input"), opts) - - if err != nil { - panic(err) - } - - spew.Dump(mf) - - /* - - mgj, err := NewMFGenerationJobFromFilesystem(c.String("input")) - if err != nil { - log.Fatal(err) - return err - } - mgj.scanForFiles() - //mgj.outputFile = c.String("output") - - */ - return nil -} diff --git a/mfer/manifest.go b/mfer/manifest.go index 552f0b4..6499d7a 100644 --- a/mfer/manifest.go +++ b/mfer/manifest.go @@ -4,10 +4,10 @@ import ( "fmt" "io" "io/fs" + "os" "path/filepath" "strings" - "github.com/davecgh/go-spew/spew" "github.com/spf13/afero" ) @@ -28,6 +28,10 @@ type Manifest struct { TotalFileSize int64 } +func (m *Manifest) String() string { + return fmt.Sprintf("", len(m.Files), m.TotalFileSize) +} + type ManifestScanOptions struct { IgnoreDotfiles bool FollowSymLinks bool @@ -39,13 +43,11 @@ func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, err return nil, err } afs := afero.NewBasePathFs(afero.NewOsFs(), abs) - spew.Dump(afs) m, err := NewFromFS(afs, options) if err != nil { return nil, err } m.SourceFSRoot = abs - spew.Dump(m) return m, nil } @@ -54,13 +56,16 @@ func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) { SourceFS: fs, ScanOptions: options, } - m.Scan() + err := m.Scan() + if err != nil { + return nil, err + } return m, nil } -func (m *Manifest) Scan() { - afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error { - +func (m *Manifest) Scan() error { + // FIXME scan and whatever function does the hashing should take ctx + oe := afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error { if m.ScanOptions.IgnoreDotfiles && strings.HasPrefix(path, ".") { // FIXME make this check all path components BUG return nil @@ -71,10 +76,6 @@ func (m *Manifest) Scan() { return nil } - fmt.Printf("path = %s\n", path) - spew.Dump(path) - spew.Dump(info) - fileinfo, staterr := m.SourceFS.Stat(path) if staterr != nil { panic(staterr) @@ -86,12 +87,28 @@ func (m *Manifest) Scan() { } m.Files = append(m.Files, nf) m.TotalFileSize = m.TotalFileSize + info.Size() - fmt.Printf("total file count now %i\n", len(m.Files)) - fmt.Printf("total file size now %i\n", m.TotalFileSize) return nil }) + if oe != nil { + return oe + } + return nil +} + +func (m *Manifest) WriteToFile(path string) error { + // FIXME refuse to overwrite without -f if file exists + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return m.Write(f) } func (m *Manifest) Write(output io.Writer) error { - return nil + // FIXME implement + panic("nope") + return nil // nolint:all } diff --git a/mfer/mf.proto b/mfer/mf.proto index 7cedde3..a6ba326 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -14,12 +14,12 @@ message MFFile { } // required mffile root attributes 1xx - Version version = 1; - bytes innerMessage = 2; + Version version = 101; + bytes innerMessage = 102; // these are used solely to detect corruption/truncation // and not for cryptographic integrity. - int64 size = 3; - bytes sha256 = 4; + int64 size = 103; + bytes sha256 = 104; // 2xx for optional manifest root attributes // think we might use gosignify instead of gpg: @@ -59,10 +59,10 @@ message MFFileInner { NONE = 0; ONE = 1; // only one for now } - Version version = 1; + Version version = 100; // required manifest attributes: - repeated MFFilePath files = 2; + repeated MFFilePath files = 101; // optional manifest attributes 2xx: optional Timestamp createdAt = 201; diff --git a/mfer/mfer_test.go b/mfer/mfer_test.go index 90caefc..9b5c284 100644 --- a/mfer/mfer_test.go +++ b/mfer/mfer_test.go @@ -9,5 +9,4 @@ var ( ) func TestManifestGeneration(t *testing.T) { - }