diff --git a/.dockerignore b/.dockerignore index f2bcbb8..af42053 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ /dockerdeps /tmp *.docker.tzst +*.tmp diff --git a/.gitignore b/.gitignore index d454731..32c0ff9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ mfer/*.pb.go /mfer.cmd vendor /tmp -/buildimage/go.mod -/buildimage/go.sum +*.tmp *.docker.tzst *.tzst +/builddeps/modcache.tar diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f23bddd --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,2 @@ +run: + tests: false diff --git a/Dockerfile b/Dockerfile index 4b1f320..436a751 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,47 +1,53 @@ -FROM golang:1.19.3-bullseye AS cacher - +################################################################################ +#2345678911234567892123456789312345678941234567895123456789612345678971234567898 +################################################################################ +FROM golang:1.19.3-bullseye AS builder ENV GOPATH /go ENV DEBIAN_FRONTEND noninteractive RUN --mount=type=cache,target=/var/cache/apt \ - apt update && apt install -y make zstd curl unzip -RUN mkdir -p /build -WORKDIR /build - + apt update && \ + apt install -y make zstd unzip && \ + mkdir /build +WORKDIR /tmp # install newer protoc -RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \ - unzip *.zip -d /usr/local && rm -v *.zip && protoc --version - -RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 - -COPY ./go.mod . -COPY ./go.sum . - -RUN --mount=type=cache,target=/go/pkg go mod download -x -RUN go env -## build image -FROM golang:1.19.3-bullseye AS builder - -COPY --from=cacher /usr/local /usr/local -COPY --from=cacher /usr/bin /usr/bin -COPY --from=cacher /go/bin /go/bin -COPY --from=cacher /build /build - +COPY ./builddeps/* ./ +RUN unzip protoc-*-linux-$(uname -m).zip -d /usr/local && \ + protoc --version && \ + mkdir -p /go/pkg/mod && \ + cd /go/pkg/mod && \ + tar xvf /tmp/modcache.tar && \ + cd / && \ + go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \ + go install -v mvdan.cc/gofumpt@latest && \ + go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 && \ + rm -rf /tmp/* WORKDIR /build - -COPY ./ ./ -RUN --mount=type=cache,target=/go/pkg go mod download -x && go mod tidy && cd mfer && go generate . -RUN --mount=type=cache,target=/go/pkg make mfer.cmd && go mod vendor -RUN tar -c . | zstdmt -19 > /src.tzst - -## lint image -FROM golangci/golangci-lint:v1.50.1 AS linter - -COPY --from=cacher /build /build +COPY ./go.mod ./go.sum . +RUN \ + go mod download -x +################################################################################ +#### caching phase done +################################################################################ WORKDIR /build -RUN golangci-lint run - +COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum . +COPY ./internal ./internal +COPY ./bin/gitrev.sh ./bin/gitrev.sh +COPY ./mfer ./mfer +COPY ./cmd ./cmd +RUN find /build +ARG GITREV unknown +RUN \ + cd mfer && go generate . && cd .. && \ + GOPACKAGESDEBUG=true golangci-lint run ./... && \ + make mfer.cmd +RUN go mod vendor && tar -c . | zstdmt -19 > /src.tzst +################################################################################ +#2345678911234567892123456789312345678941234567895123456789612345678971234567898 +################################################################################ +## final image +################################################################################ FROM scratch +# we put all the source into the final image for posterity, it's small COPY --from=builder /src.tzst /src.tzst COPY --from=builder /build/mfer.cmd /mfer ENTRYPOINT ["/mfer"] - diff --git a/Makefile b/Makefile index a813846..f70fdd9 100644 --- a/Makefile +++ b/Makefile @@ -3,19 +3,17 @@ export PROGRESS_NO_TRUNC := 1 GOPATH := $(shell go env GOPATH) export PATH := $(PATH):$(GOPATH)/bin PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go - SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum - ARCH := $(shell uname -m) -GITREV := $(shell git describe --always --dirty=-dirty) +GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh) APPNAME := mfer +VERSION := 0.1.0 export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH) - -GOLDFLAGS += -X main.Version=0.1.0 -GOLDFLAGS += -X main.Gitrev=$(GITREV) +GOLDFLAGS += -X main.Version=$(VERSION) +GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD) GOFLAGS := -ldflags "$(GOLDFLAGS)" -.PHONY: docker default run ci test fixme +.PHONY: docker default run ci test fixme default: fmt test @@ -25,7 +23,7 @@ run: ./mfer.cmd ci: test -test: +test: $(SOURCEFILES) mfer/mf.pb.go go test -v --timeout 3s ./... $(PROTOC_GEN_GO): @@ -38,9 +36,11 @@ devprereqs: which gofumpt || go install -v mvdan.cc/gofumpt@latest which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 -mfer.cmd: $(SOURCEFILES) - protoc --version +mfer/mf.pb.go: mfer/mf.proto cd mfer && go generate . + +mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go + protoc --version make test cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) . @@ -62,7 +62,7 @@ docker: sneak-mfer.$(ARCH).docker.tzst sneak-mfer.$(ARCH).docker.tzst: $(SOURCEFILES) bash -x bin/docker-prereqs.sh - docker build --progress plain -t sneak/mfer . + docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer . docker save sneak/mfer | pv | zstdmt -19 > $@ du -sh $@ diff --git a/bin/docker-prereqs.sh b/bin/docker-prereqs.sh index 3ef7f63..40a781f 100644 --- a/bin/docker-prereqs.sh +++ b/bin/docker-prereqs.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euxo pipefail GOI="golang:1.19.3-bullseye" -CII="golangci/golangci-lint:v1.50.1" +#CII="golangci/golangci-lint:v1.50.1" if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then mkdir -p "$DOCKER_IMAGE_CACHE_DIR" @@ -16,21 +16,32 @@ function buildImageCache() { mv $DICD/go.tzst.tmp $DICD/go.tzst fi - if [[ ! -e "$DICD/ci.tzst" ]]; then - docker pull $CII - docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \ - mv $DICD/ci.tzst.tmp $DICD/ci.tzst - fi + #if [[ ! -e "$DICD/ci.tzst" ]]; then + # docker pull $CII + # docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \ + # mv $DICD/ci.tzst.tmp $DICD/ci.tzst + #fi } function loadImageCache() { - docker image ls $CII || \ - zstdmt -d --stdout $DICD/ci.tzst | pv | docker load + # zstdmt -d --stdout $DICD/ci.tzst | pv | docker load + zstdmt -d --stdout $DICD/go.tzst | pv | docker load - docker image ls $GOI || \ - zstdmt -d --stdout $DICD/go.tzst | pv | docker load + #docker image ls $CII + docker image ls $GOI + +} + +function writeModuleCache() { + if [[ ! -e $DICD/modcache.tar ]]; then + cd $(go env GOMODCACHE) + tar -c . | pv > $DICD/modcache.tar.tmp && mv $DICD/modcache.tar.tmp $DICD/modcache.tar + cd - + fi + cp -av $DICD/modcache.tar ./builddeps/modcache.tar.tmp && mv ./builddeps/modcache.tar.tmp ./builddeps/modcache.tar } buildImageCache - loadImageCache +writeModuleCache + diff --git a/bin/gitrev.sh b/bin/gitrev.sh new file mode 100644 index 0000000..02f6821 --- /dev/null +++ b/bin/gitrev.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if [[ ! -z "$GITREV" ]]; then + echo $GITREV +else + git describe --always --dirty=-dirty +fi diff --git a/builddeps/protoc-21.10-linux-aarch64.zip b/builddeps/protoc-21.10-linux-aarch64.zip new file mode 100644 index 0000000..14a83d7 Binary files /dev/null and b/builddeps/protoc-21.10-linux-aarch64.zip differ diff --git a/builddeps/protoc-21.10-linux-x86_64.zip b/builddeps/protoc-21.10-linux-x86_64.zip new file mode 100644 index 0000000..462d2a3 Binary files /dev/null and b/builddeps/protoc-21.10-linux-x86_64.zip differ diff --git a/go.mod b/go.mod index 5eec910..c8a7ebb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.eeqj.de/sneak/mfer -go 1.19 +go 1.17 require ( github.com/apex/log v1.9.0 @@ -8,14 +8,14 @@ require ( github.com/pterm/pterm v0.12.35 github.com/spf13/afero v1.8.0 github.com/stretchr/testify v1.8.1 - github.com/urfave/cli/v2 v2.3.0 + github.com/urfave/cli/v2 v2.23.6 google.golang.org/protobuf v1.28.1 ) require ( github.com/atomicgo/cursor v0.0.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/fatih/color v1.7.0 // indirect github.com/gookit/color v1.4.2 // indirect github.com/mattn/go-colorable v0.1.2 // indirect @@ -24,9 +24,10 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.4 // indirect diff --git a/go.sum b/go.sum index 96493f7..f82af82 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -190,6 +192,8 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -219,8 +223,12 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.23.6 h1:iWmtKD+prGo1nKUtLO0Wg4z9esfBM4rAV4QRLQiEmJ4= +github.com/urfave/cli/v2 v2.23.6/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/cli/fetch.go b/internal/cli/fetch.go index c775a79..e20143b 100644 --- a/internal/cli/fetch.go +++ b/internal/cli/fetch.go @@ -7,5 +7,6 @@ import ( func (mfa *CLIApp) fetchManifestOperation(c *cli.Context) error { log.Debugf("fetchManifestOperation()") - return nil + panic("not implemented") + return nil //nolint } diff --git a/internal/cli/gen.go b/internal/cli/gen.go index fe0d087..c2e1b0d 100644 --- a/internal/cli/gen.go +++ b/internal/cli/gen.go @@ -8,27 +8,31 @@ import ( "github.com/urfave/cli/v2" ) -func (mfa *CLIApp) generateManifestOperation(c *cli.Context) error { +func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error { fmt.Println("generateManifestOperation()") - myArgs := c.Args() + myArgs := ctx.Args() spew.Dump(myArgs) - fmt.Printf("%#v\n", c.Args().First()) - if c.Args().Len() > 0 { - fmt.Printf("%#v\n", c.Args().Get(1)) + fmt.Printf("%#v\n", ctx.Args().First()) + if ctx.Args().Len() > 0 { + fmt.Printf("%#v\n", ctx.Args().Get(1)) } // fmt.Printf("called with arg: %s\n", c.String("input")) opts := &mfer.ManifestScanOptions{ - IgnoreDotfiles: c.Bool("IgnoreDotfiles"), - FollowSymLinks: c.Bool("FollowSymLinks"), + IgnoreDotfiles: ctx.Bool("IgnoreDotfiles"), + FollowSymLinks: ctx.Bool("FollowSymLinks"), } - // FIXME add command flags for ignoring dotfiles and following symlinks - mf, err := mfer.NewFromPath(c.String("input"), opts) + paths := make([]string, ctx.Args().Len()) + for i := 0; i < ctx.Args().Len(); i++ { + paths = append(paths, ctx.Args().Get(i)) + } + mf, err := mfer.NewFromPaths(opts, paths...) if err != nil { panic(err) } + mf.WithContext(ctx) spew.Dump(mf) diff --git a/internal/cli/mfer.go b/internal/cli/mfer.go index b13129f..78ca912 100644 --- a/internal/cli/mfer.go +++ b/internal/cli/mfer.go @@ -5,11 +5,9 @@ import ( "os" "time" + "git.eeqj.de/sneak/mfer/internal/log" "github.com/pterm/pterm" "github.com/urfave/cli/v2" - - "github.com/apex/log" - acli "github.com/apex/log/handlers/cli" ) type CLIApp struct { @@ -26,17 +24,6 @@ func (mfa *CLIApp) printBanner() { pterm.DefaultCenter.Println(s) // Print BigLetters with the default CenterPrinter } -func (mfa *CLIApp) disableStyling() { - pterm.DisableColor() - pterm.DisableStyling() - pterm.Debug.Prefix.Text = "" - pterm.Info.Prefix.Text = "" - pterm.Success.Prefix.Text = "" - pterm.Warning.Prefix.Text = "" - pterm.Error.Prefix.Text = "" - pterm.Fatal.Prefix.Text = "" -} - func (mfa *CLIApp) VersionString() string { return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev) } @@ -46,11 +33,10 @@ func (mfa *CLIApp) run() { if NO_COLOR { // shoutout to rob pike who thinks it's juvenile - mfa.disableStyling() + log.DisableStyling() } - log.SetHandler(acli.Default) - log.SetLevel(log.InfoLevel) + log.Init() mfa.app = &cli.App{ Name: mfa.appname, @@ -71,7 +57,7 @@ func (mfa *CLIApp) run() { }, Action: func(c *cli.Context) error { if c.Bool("verbose") { - log.SetLevel(log.DebugLevel) + log.IncreaseLevel() } return nil }, diff --git a/internal/log/log.go b/internal/log/log.go new file mode 100644 index 0000000..9df89bf --- /dev/null +++ b/internal/log/log.go @@ -0,0 +1,45 @@ +package log + +import ( + "github.com/apex/log" + acli "github.com/apex/log/handlers/cli" + "github.com/pterm/pterm" +) + +func DisableStyling() { + pterm.DisableColor() + pterm.DisableStyling() + pterm.Debug.Prefix.Text = "" + pterm.Info.Prefix.Text = "" + pterm.Success.Prefix.Text = "" + pterm.Warning.Prefix.Text = "" + pterm.Error.Prefix.Text = "" + pterm.Fatal.Prefix.Text = "" +} + +func Init() { + log.SetHandler(acli.Default) + log.SetLevel(log.InfoLevel) +} + +func SetLevel(arg log.Level) { + log.SetLevel(arg) +} + +func GetLevel() log.Level { + if logger, ok := log.Log.(*log.Logger); ok { + return logger.Level + } + return 0 +} + +func IncreaseLevel() { + SetLevel(GetLevel() + 1) +} + +func WithError(e error) *log.Entry { + if logger, ok := log.Log.(*log.Logger); ok { + return logger.WithError(e) + } + return nil +} diff --git a/mfer/manifest.go b/mfer/manifest.go index e250b7d..87c3ab9 100644 --- a/mfer/manifest.go +++ b/mfer/manifest.go @@ -8,29 +8,32 @@ import ( "strings" "github.com/spf13/afero" + "github.com/urfave/cli/v2" ) -type ManifestFile struct { - Path string - FileInfo fs.FileInfo +type manifestFile struct { + path string + info fs.FileInfo } -func (m *ManifestFile) String() string { - return fmt.Sprintf("", m.Path) +func (m *manifestFile) String() string { + return fmt.Sprintf("", m.path) } -type Manifest struct { - SourceFS afero.Fs - SourceFSRoot string - Files []*ManifestFile - ScanOptions *ManifestScanOptions - TotalFileSize int64 - PBInner *MFFile - PBOuter *MFFileOuter +type manifest struct { + sourceFS []afero.Fs + //sourceFSRoot string + files []*manifestFile + scanOptions *ManifestScanOptions + totalFileSize int64 + pbInner *MFFile + pbOuter *MFFileOuter + ctx *cli.Context + errors []*error } -func (m *Manifest) String() string { - return fmt.Sprintf("", len(m.Files), m.TotalFileSize) +func (m *manifest) String() string { + return fmt.Sprintf("", len(m.files), m.totalFileSize) } type ManifestScanOptions struct { @@ -38,38 +41,74 @@ type ManifestScanOptions struct { FollowSymLinks bool } -func NewFromPath(inputPath string, options *ManifestScanOptions) (*Manifest, error) { +func (m *manifest) HasError() bool { + return len(m.errors) > 0 +} + +func (m *manifest) AddError(e error) *manifest { + m.errors = append(m.errors, &e) + return m +} + +func (m *manifest) WithContext(c *cli.Context) *manifest { + m.ctx = c + return m +} + +func (m *manifest) addInputPath(inputPath string) error { abs, err := filepath.Abs(inputPath) if err != nil { - return nil, err + return err } - afs := afero.NewBasePathFs(afero.NewOsFs(), abs) - m, err := NewFromFS(afs, options) - if err != nil { - return nil, err - } - m.SourceFSRoot = abs - return m, nil + // FIXME check to make sure inputPath/abs exists maybe + afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs)) + m.addInputFS(afs) + return nil } -func NewFromFS(fs afero.Fs, options *ManifestScanOptions) (*Manifest, error) { - m := &Manifest{ - SourceFS: fs, - ScanOptions: options, +func (m *manifest) addInputFS(f afero.Fs) error { + if m.sourceFS == nil { + m.sourceFS = make([]afero.Fs, 1) } - err := m.Scan() + m.sourceFS = append(m.sourceFS, f) + // FIXME do some sort of check on f here? + return nil +} + +func New() *manifest { + m := &manifest{} + return m +} + +func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) { + m := New() + m.scanOptions = options + for _, p := range inputPaths { + m.addInputPath(p) + } + err := m.scan() if err != nil { return nil, err } return m, nil } -func (m *Manifest) GetFileCount() int64 { - return int64(len(m.Files)) +func NewFromFS(options *ManifestScanOptions, fs afero.Fs) (*manifest, error) { + m := New() + m.scanOptions = options + err := m.addInputFS(fs) + if err != nil { + return nil, err + } + return m, nil } -func (m *Manifest) GetTotalFileSize() int64 { - return m.TotalFileSize +func (m *manifest) GetFileCount() int64 { + return int64(len(m.files)) +} + +func (m *manifest) GetTotalFileSize() int64 { + return m.totalFileSize } func pathIsHidden(p string) bool { @@ -89,39 +128,41 @@ func pathIsHidden(p string) bool { } } -/* -func timeToTimestamp(t time.Time) *Timestamp { - -} -*/ - -func (m *Manifest) Scan() error { - // FIXME scan and whatever function does the hashing should take ctx - oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error { - if m.ScanOptions.IgnoreDotfiles && pathIsHidden(p) { - return nil - } - - if info != nil && info.IsDir() { - // manifests contain only files, directories are implied. - return nil - } - - fileinfo, staterr := m.SourceFS.Stat(p) - if staterr != nil { - panic(staterr) - } - - nf := &ManifestFile{ - Path: p, - FileInfo: fileinfo, - } - m.Files = append(m.Files, nf) - m.TotalFileSize = m.TotalFileSize + info.Size() +func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error { + if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) { return nil - }) - if oe != nil { - return oe + } + if fi != nil && fi.IsDir() { + // manifests contain only files, directories are implied. + return nil + } + // FIXME test if 'fi' is already result of stat + fileinfo, staterr := m.sourceFS[sfsIndex].Stat(p) + if staterr != nil { + return staterr + } + cleanPath := p + if cleanPath[0:1] == "/" { + cleanPath = cleanPath[1:] + } + nf := &manifestFile{ + path: cleanPath, + info: fileinfo, + } + m.files = append(m.files, nf) + m.totalFileSize = m.totalFileSize + fi.Size() + return nil +} + +func (m *manifest) scan() error { + // FIXME scan and whatever function does the hashing should take ctx + for idx, sfs := range m.sourceFS { + e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error { + return m.addFile(p, info, idx) + }) + if e != nil { + return e + } } return nil } diff --git a/mfer/mf.proto b/mfer/mf.proto index 9bc86b6..5e350a1 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -option go_package = "git.eeqj.de/sneak/mfer"; +option go_package = "git.eeqj.de/sneak/mfer/mfer"; message Timestamp { int64 seconds = 1; diff --git a/mfer/mfer_test.go b/mfer/mfer_test.go index 71b96dd..ca954eb 100644 --- a/mfer/mfer_test.go +++ b/mfer/mfer_test.go @@ -37,9 +37,9 @@ func TestPathHiddenFunc(t *testing.T) { } func TestManifestGenerationOne(t *testing.T) { - m, err := NewFromFS(mf, &ManifestScanOptions{ + m, err := NewFromFS(&ManifestScanOptions{ IgnoreDotfiles: true, - }) + }, mf) assert.Nil(t, err) assert.NotNil(t, m) assert.Equal(t, int64(2), m.GetFileCount()) @@ -47,12 +47,14 @@ func TestManifestGenerationOne(t *testing.T) { } func TestManifestGenerationTwo(t *testing.T) { - m, err := NewFromFS(mf, &ManifestScanOptions{ + m, err := NewFromFS(&ManifestScanOptions{ IgnoreDotfiles: false, - }) + }, mf) assert.Nil(t, err) assert.NotNil(t, m) spew.Dump(m) assert.Equal(t, int64(4), m.GetFileCount()) assert.Equal(t, int64(54), m.GetTotalFileSize()) + err = m.generate() + assert.Nil(t, err) } diff --git a/mfer/output.go b/mfer/output.go index 23702c3..659e61b 100644 --- a/mfer/output.go +++ b/mfer/output.go @@ -5,7 +5,7 @@ import ( "os" ) -func (m *Manifest) WriteToFile(path string) error { +func (m *manifest) WriteToFile(path string) error { // FIXME refuse to overwrite without -f if file exists f, err := os.Create(path) @@ -17,7 +17,7 @@ func (m *Manifest) WriteToFile(path string) error { return m.Write(f) } -func (m *Manifest) Write(output io.Writer) error { +func (m *manifest) Write(output io.Writer) error { // FIXME implement panic("nope") return nil // nolint:all diff --git a/mfer/serialize.go b/mfer/serialize.go index 14d092b..baa4785 100644 --- a/mfer/serialize.go +++ b/mfer/serialize.go @@ -1,12 +1,55 @@ package mfer +import ( + "time" + + "github.com/davecgh/go-spew/spew" + "google.golang.org/protobuf/proto" +) + //go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto -func (m *Manifest) Generate() error { - m.PBInner = &MFFile{ - Version: MFFile_ONE, - // CreatedAt: time.Now(), - Files: []*MFFilePath{}, +func newTimestampFromTime(t time.Time) *Timestamp { + out := &Timestamp{ + Seconds: t.Unix(), + Nanos: int32(t.UnixNano() - (t.Unix() * 1000000000)), } + return out +} + +func (m *manifest) generate() error { + if m.pbInner == nil { + e := m.generateInner() + if e != nil { + return e + } + } + output, err := proto.Marshal(m.pbInner) + if err != nil { + return err + } + spew.Dump(output) + return nil +} + +func (m *manifest) generateOuter(inner *MFFile) error { + return nil +} + +func (m *manifest) generateInner() error { + m.pbInner = &MFFile{ + Version: MFFile_ONE, + CreatedAt: newTimestampFromTime(time.Now()), + Files: []*MFFilePath{}, + } + + for _, f := range m.files { + nf := &MFFilePath{ + Path: f.path, + // FIXME add more stuff + } + m.pbInner.Files = append(m.pbInner.Files, nf) + } + spew.Dump(m.pbInner) return nil }