next #5

Merged
sneak merged 14 commits from next into master 2022-12-09 00:02:35 +00:00
11 changed files with 133 additions and 33 deletions
Showing only changes of commit bd4b135e17 - Show all commits

5
.dockerignore Normal file
View File

@ -0,0 +1,5 @@
*.tzst
*.tar
/buildimage
/dockerdeps
/tmp

4
.gitignore vendored
View File

@ -1,2 +1,6 @@
mfer/*.pb.go
/mfer.cmd
vendor
/tmp
/buildimage/go.mod
/buildimage/go.sum

View File

@ -1,6 +1,5 @@
## lint image
## current as of 2022-01-25
FROM golangci/golangci-lint:latest AS linter
FROM golangci/golangci-lint:v1.50.1
RUN mkdir -p /build
WORKDIR /build
@ -8,19 +7,7 @@ COPY ./ ./
RUN golangci-lint run
## build image:
# this is golang:1.17.6-buster as of 2022-01-27
FROM golang@sha256:52a48e0239f4d645b20ac268a60361703afe7feb2df5697fa89f72052cb87a3e AS builder
#FROM golang:1.16-buster AS builder
ENV DEBIAN_FRONTEND noninteractive
RUN apt update && apt install -y make bzip2 curl unzip
# install newer protoc than what comes with buster
ENV PB_REL https://github.com/protocolbuffers/protobuf/releases
RUN curl -LO $PB_REL/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip && \
unzip protoc-3.19.0-linux-x86_64.zip -d /usr/local
RUN mkdir -p /build
FROM sneak/mfer-build AS builder
WORKDIR /build
COPY go.mod .

View File

@ -1,22 +1,24 @@
export DOCKER_BUILDKIT := 1
GOPATH := $(shell go env GOPATH)
export PATH := $(PATH):$(GOPATH)/bin
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
ARCH := $(shell uname -m)
GITREV := $(shell git describe --always --dirty=-dirty)
APPNAME := mfer
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
GOLDFLAGS += -X main.Version=0.1.0
GOLDFLAGS += -X main.Gitrev=$(GITREV)
GOFLAGS := -ldflags "$(GOLDFLAGS)"
default: run
default: test
run: ./mfer.cmd
./$<
./$< gen --ignore-dotfiles
test:
test: fmt
go test -v --timeout 3s ./...
$(PROTOC_GEN_GO):
@ -29,7 +31,7 @@ devprereqs:
which gofumpt || go install -v mvdan.cc/gofumpt@latest
which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go
mfer.cmd: mfer/*.go internal/*/*.go cmd/*/*.go
protoc --version
cd mfer && go generate .
make test
@ -48,5 +50,10 @@ lint:
golangci-lint run
sh -c 'test -z "$$(gofmt -l .)"'
dockerbuild:
docker:
bash -x bin/docker-prereqs.sh
docker build .
godoc:
open http://127.0.0.1:6060
godoc -http=:6060

50
bin/docker-prereqs.sh Normal file
View File

@ -0,0 +1,50 @@
#!/bin/bash
set -euxo pipefail
GOI="golang:1.19.3-bullseye"
CII="golangci/golangci-lint:v1.50.1"
BII="sneak/mfer-build"
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
fi
DICD="$DOCKER_IMAGE_CACHE_DIR"
function buildImageCache() {
if [[ ! -e "$DICD/go.tzst" ]]; then
docker pull $GOI
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
mv $DICD/go.tzst.tmp $DICD/go.tzst
fi
if [[ ! -e "$DICD/ci.tzst" ]]; then
docker pull $CII
docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
mv $DICD/ci.tzst.tmp $DICD/ci.tzst
fi
if [[ ! -e "$DICD/build.tzst" ]]; then
go mod download -x
cd buildimage
cp ../go.mod ../go.sum .
docker build -t $BII . && rm -rf go.mod go.sum && \
cd .. && \
docker save $BII | pv | zstdmt -19 > $DICD/build.tzst.tmp && \
mv $DICD/build.tzst.tmp $DICD/build.tzst
fi
}
function loadImageCache() {
docker image ls $CII || \
zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
docker image ls $GOI || \
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
docker image ls $BII || \
zstdmt -d --stdout $DICD/build.tzst | pv | docker load
}
buildImageCache
loadImageCache

21
buildimage/Dockerfile Normal file
View File

@ -0,0 +1,21 @@
## build image:
FROM golang:1.19.3-bullseye AS builder
ENV DEBIAN_FRONTEND noninteractive
RUN apt update && apt install -y make bzip2 curl unzip
RUN mkdir -p /build
WORKDIR /build
# install newer protoc
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
RUN go env
COPY ./go.mod .
COPY ./go.sum .
RUN --mount=type=cache,target=/go/pkg go mod download -x
RUN rm -rfv /var/cache/* /var/tmp/*

6
go.mod
View File

@ -3,11 +3,11 @@ module git.eeqj.de/sneak/mfer
go 1.16
require (
github.com/apex/log v1.9.0 // indirect
github.com/apex/log v1.9.0
github.com/davecgh/go-spew v1.1.1
github.com/pterm/pterm v0.12.35
github.com/spf13/afero v1.8.0
github.com/stretchr/testify v1.8.1
github.com/urfave/cli/v2 v2.3.0
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 // indirect
google.golang.org/protobuf v1.27.1
google.golang.org/protobuf v1.28.1
)

20
go.sum
View File

@ -154,8 +154,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
@ -197,13 +199,19 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S
github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
@ -211,8 +219,6 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 h1:RDJ3ggSqBL4Mu/ANRKmxuLrwnupJsvHRDXe4/KI+HWE=
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20/go.mod h1:iljxuLc3m07jsXOWojqehoTp/Fh0XP7irbhV+6sYNGc=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@ -512,10 +518,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
@ -525,8 +532,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -10,8 +10,6 @@ import (
"github.com/spf13/afero"
)
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
type ManifestFile struct {
Path string
FileInfo fs.FileInfo
@ -27,6 +25,8 @@ type Manifest struct {
Files []*ManifestFile
ScanOptions *ManifestScanOptions
TotalFileSize int64
PBInner *MFFile
PBOuter *MFFileOuter
}
func (m *Manifest) String() string {
@ -89,6 +89,12 @@ func pathIsHidden(p string) bool {
}
}
/*
func timeToTimestamp(t time.Time) *Timestamp {
}
*/
func (m *Manifest) Scan() error {
// FIXME scan and whatever function does the hashing should take ctx
oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error {

View File

@ -7,7 +7,7 @@ message Timestamp {
int32 nanos = 2;
}
message MFFile {
message MFFileOuter {
enum Version {
NONE = 0;
ONE = 1; // only one for now
@ -54,7 +54,7 @@ message MFFileChecksum {
bytes multiHash = 1;
}
message MFFileInner {
message MFFile {
enum Version {
NONE = 0;
ONE = 1; // only one for now

12
mfer/serialize.go Normal file
View File

@ -0,0 +1,12 @@
package mfer
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
func (m *Manifest) Generate() error {
m.PBInner = &MFFile{
Version: MFFile_ONE,
// CreatedAt: time.Now(),
Files: []*MFFilePath{},
}
return nil
}