rollup from next branch #4

Merged
sneak merged 10 commits from next into master 2022-12-04 07:59:37 +00:00
12 changed files with 143 additions and 128 deletions
Showing only changes of commit 5f5f51c014 - Show all commits

2
.golangci.yaml Normal file
View File

@ -0,0 +1,2 @@
run:
tests: false

View File

@ -4,20 +4,24 @@ export PATH := $(PATH):$(GOPATH)/bin
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
ARCH := $(shell uname -m)
VERSION := $(shell git describe --always --dirty=-dirty)
GITREV := $(shell git describe --always --dirty=-dirty)
GOLDFLAGS += -X main.Version=$(VERSION)
GOLDFLAGS += -X main.Version=0.1.0
GOLDFLAGS += -X main.Gitrev=$(GITREV)
GOFLAGS := -ldflags "$(GOLDFLAGS)"
default: run
run: ./mfer.cmd
./$<
./$< gen
./$< gen --ignore-dotfiles
$(PROTOC_GEN_GO):
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@latest
fixme:
@grep -nir fixme . | grep -v Makefile
devprereqs:
which gofumpt || go install -v mvdan.cc/gofumpt@latest
which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
@ -25,15 +29,16 @@ devprereqs:
mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go
protoc --version
cd mfer && go generate .
cd cmd/mfer && go build -o ../../mfer.cmd $(GOFLAGS) .
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .
clean:
rm -rfv mfer/*.pb.go ./mfer
rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer
fmt: prereqs
fmt: devprereqs
gofumpt -l -w mfer internal cmd
golangci-lint run --fix
prettier -w *.json *.md
-prettier -w *.json
-prettier -w *.md
lint:
golangci-lint run

View File

@ -128,8 +128,7 @@ The manifest file would do several important things:
- a command line option to zero/omit mtime/ctime, as well as manifest
timestamp, and sort all directory listings so that manifest file
generation is deterministic/reproducible
- URL format `mfer fetch
https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
- URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
to assert in the URL which PGP signing key should be used in the manifest,
so that shared URLs have a cryptographic trust root
- a "well-known" key in the manifest that maps well known keys (could reuse

View File

@ -9,8 +9,9 @@ import (
var (
Appname string = "mfer"
Version string
Gitrev string
)
func main() {
os.Exit(cli.Run(Appname, Version))
os.Exit(cli.Run(Appname, Version, Gitrev))
}

14
internal/cli/check.go Normal file
View File

@ -0,0 +1,14 @@
package cli
import (
"errors"
log "github.com/visionmedia/go-cli-log"
"github.com/urfave/cli/v2"
)
func (mfa *CLIApp) checkManifestOperation(c *cli.Context) error {
log.Error(errors.New("unimplemented"))
return nil
}

View File

@ -13,10 +13,11 @@ func init() {
}
}
func Run(Appname, Version string) int {
func Run(Appname, Version, Gitrev string) int {
m := &CLIApp{}
m.appname = Appname
m.version = Version
m.gitrev = Gitrev
m.exitCode = 0
m.run()

28
internal/cli/gen.go Normal file
View File

@ -0,0 +1,28 @@
package cli
import (
"fmt"
"git.eeqj.de/sneak/mfer/mfer"
"github.com/davecgh/go-spew/spew"
"github.com/urfave/cli/v2"
)
func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error {
fmt.Println("generateManifestOperation()")
fmt.Printf("called with arg: %s\n", c.String("input"))
opts := &mfer.ManifestScanOptions{
IgnoreDotfiles: c.Bool("IgnoreDotfiles"),
FollowSymLinks: c.Bool("FollowSymLinks"),
}
// FIXME add command flags for ignoring dotfiles and following symlinks
mf, err := mfer.NewFromPath(c.String("input"), opts)
if err != nil {
panic(err)
}
spew.Dump(mf)
return nil
}

View File

@ -1,51 +0,0 @@
package cli
import (
"git.eeqj.de/sneak/mfer/mfer"
"github.com/spf13/afero"
)
type Job struct {
innerpb *mfer.MFFileInner
outerpb *mfer.MFFile
fileCount int64
totalSize int64
afs afero.Fs
}
func (m *Job) scanForFiles() error {
m.innerpb = &mfer.MFFileInner{}
m.innerpb.Version = mfer.MFFileInner_ONE
return nil
}
/*
walkErr := filepath.Walk(m.sourcePath, func(itemPath string, info os.FileInfo, err error) error {
// we do not include the manifest file in the manifest
if itemPath == "index.mf" {
return nil
}
fpi := mfer.MFFilePath{}
fpi.Path = itemPath
fpi.Size = info.Size()
m.innerpb.Files = append(m.innerpb.Files, &fpi)
m.fileCount++
m.totalSize += fpi.Size
return nil
})
if walkErr != nil {
log.Fatal(walkErr)
return walkErr
}
fmt.Printf("%#v\n", m.innerpb)
fmt.Printf("filecount = %#v\n", m.fileCount)
fmt.Printf("totalsize = %#v\n", m.totalSize)
return nil
}
*/

View File

@ -1,15 +1,12 @@
package cli
import (
"errors"
"fmt"
"os"
"time"
log "github.com/visionmedia/go-cli-log"
"git.eeqj.de/sneak/mfer/mfer"
"github.com/davecgh/go-spew/spew"
"github.com/pterm/pterm"
"github.com/urfave/cli/v2"
)
@ -17,10 +14,9 @@ import (
type CLIApp struct {
appname string
version string
buildarch string
gitrev string
startupTime time.Time
exitCode int
errorString string
app *cli.App
}
@ -40,29 +36,57 @@ func (mfa *CLIApp) disableStyling() {
pterm.Fatal.Prefix.Text = ""
}
func (mfa *CLIApp) VersionString() string {
return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev)
}
func (mfa *CLIApp) run() {
mfa.startupTime = time.Now()
if NO_COLOR {
// shoutout to rob pike who thinks it's juvenile
mfa.disableStyling()
}
mfa.printBanner()
mfa.app = &cli.App{
Name: mfa.appname,
Usage: "Manifest generator",
Version: mfa.version,
Version: mfa.VersionString(),
EnableBashCompletion: true,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
Usage: "Verbosity level",
Aliases: []string{"v"},
},
&cli.BoolFlag{
Name: "quiet",
Usage: "don't produce output except on error",
Aliases: []string{"q"},
},
},
Commands: []*cli.Command{
{
Name: "generate",
Aliases: []string{"gen"},
Usage: "Generate manifest file",
Action: func(c *cli.Context) error {
if !c.Bool("quiet") {
mfa.printBanner()
}
return mfa.generateManifestOperation(c)
},
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "FollowSymLinks",
Aliases: []string{"follow-symlinks"},
Usage: "Resolve encountered symlinks",
},
&cli.BoolFlag{
Name: "IgnoreDotfiles",
Aliases: []string{"ignore-dotfiles"},
Usage: "Ignore any dot (hidden) files encountered",
},
&cli.StringFlag{
Name: "input",
Value: ".",
@ -81,51 +105,27 @@ func (mfa *CLIApp) run() {
Name: "check",
Usage: "Validate files using manifest file",
Action: func(c *cli.Context) error {
return mfa.validateManifestOperation(c)
if !c.Bool("quiet") {
mfa.printBanner()
}
return mfa.checkManifestOperation(c)
},
},
{
Name: "version",
Usage: "Show version",
Action: func(c *cli.Context) error {
fmt.Printf("%s\n", mfa.VersionString())
return nil
},
},
},
}
mfa.app.HideVersion = true
err := mfa.app.Run(os.Args)
if err != nil {
mfa.exitCode = 1
log.Error(err)
}
}
func (mfa *CLIApp) validateManifestOperation(c *cli.Context) error {
log.Error(errors.New("unimplemented"))
return nil
}
func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error {
fmt.Println("generateManifestOperation()")
fmt.Printf("called with arg: %s\n", c.String("input"))
opts := &mfer.ManifestScanOptions{
IgnoreDotfiles: true,
FollowSymLinks: false,
}
mf, err := mfer.NewFromPath(c.String("input"), opts)
if err != nil {
panic(err)
}
spew.Dump(mf)
/*
mgj, err := NewMFGenerationJobFromFilesystem(c.String("input"))
if err != nil {
log.Fatal(err)
return err
}
mgj.scanForFiles()
//mgj.outputFile = c.String("output")
*/
return nil
}

View File

@ -4,10 +4,10 @@ import (
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/davecgh/go-spew/spew"
"github.com/spf13/afero"
)
@ -28,6 +28,10 @@ type Manifest struct {
TotalFileSize int64
}
func (m *Manifest) String() string {
return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.Files), m.TotalFileSize)
}
type ManifestScanOptions struct {
IgnoreDotfiles bool
FollowSymLinks bool
@ -39,13 +43,11 @@ func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, err
return nil, err
}
afs := afero.NewBasePathFs(afero.NewOsFs(), abs)
spew.Dump(afs)
m, err := NewFromFS(afs, options)
if err != nil {
return nil, err
}
m.SourceFSRoot = abs
spew.Dump(m)
return m, nil
}
@ -54,13 +56,16 @@ func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) {
SourceFS: fs,
ScanOptions: options,
}
m.Scan()
err := m.Scan()
if err != nil {
return nil, err
}
return m, nil
}
func (m *Manifest) Scan() {
afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error {
func (m *Manifest) Scan() error {
// FIXME scan and whatever function does the hashing should take ctx
oe := afero.Walk(m.SourceFS, "./", func(path string, info fs.FileInfo, err error) error {
if m.ScanOptions.IgnoreDotfiles && strings.HasPrefix(path, ".") {
// FIXME make this check all path components BUG
return nil
@ -71,10 +76,6 @@ func (m *Manifest) Scan() {
return nil
}
fmt.Printf("path = %s\n", path)
spew.Dump(path)
spew.Dump(info)
fileinfo, staterr := m.SourceFS.Stat(path)
if staterr != nil {
panic(staterr)
@ -86,12 +87,28 @@ func (m *Manifest) Scan() {
}
m.Files = append(m.Files, nf)
m.TotalFileSize = m.TotalFileSize + info.Size()
fmt.Printf("total file count now %i\n", len(m.Files))
fmt.Printf("total file size now %i\n", m.TotalFileSize)
return nil
})
if oe != nil {
return oe
}
return nil
}
func (m *Manifest) WriteToFile(path string) error {
// FIXME refuse to overwrite without -f if file exists
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
return m.Write(f)
}
func (m *Manifest) Write(output io.Writer) error {
return nil
// FIXME implement
panic("nope")
return nil // nolint:all
}

View File

@ -14,12 +14,12 @@ message MFFile {
}
// required mffile root attributes 1xx
Version version = 1;
bytes innerMessage = 2;
Version version = 101;
bytes innerMessage = 102;
// these are used solely to detect corruption/truncation
// and not for cryptographic integrity.
int64 size = 3;
bytes sha256 = 4;
int64 size = 103;
bytes sha256 = 104;
// 2xx for optional manifest root attributes
// think we might use gosignify instead of gpg:
@ -59,10 +59,10 @@ message MFFileInner {
NONE = 0;
ONE = 1; // only one for now
}
Version version = 1;
Version version = 100;
// required manifest attributes:
repeated MFFilePath files = 2;
repeated MFFilePath files = 101;
// optional manifest attributes 2xx:
optional Timestamp createdAt = 201;

View File

@ -9,5 +9,4 @@ var (
)
func TestManifestGeneration(t *testing.T) {
}