next #5
5
.dockerignore
Normal file
5
.dockerignore
Normal file
@ -0,0 +1,5 @@
|
||||
*.tzst
|
||||
*.tar
|
||||
/buildimage
|
||||
/dockerdeps
|
||||
/tmp
|
||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -1,2 +1,6 @@
|
||||
mfer/*.pb.go
|
||||
/mfer.cmd
|
||||
vendor
|
||||
/tmp
|
||||
/buildimage/go.mod
|
||||
/buildimage/go.sum
|
||||
|
||||
17
Dockerfile
17
Dockerfile
@ -1,6 +1,5 @@
|
||||
## lint image
|
||||
## current as of 2022-01-25
|
||||
FROM golangci/golangci-lint:latest AS linter
|
||||
FROM golangci/golangci-lint:v1.50.1
|
||||
|
||||
RUN mkdir -p /build
|
||||
WORKDIR /build
|
||||
@ -8,19 +7,7 @@ COPY ./ ./
|
||||
RUN golangci-lint run
|
||||
|
||||
## build image:
|
||||
# this is golang:1.17.6-buster as of 2022-01-27
|
||||
FROM golang@sha256:52a48e0239f4d645b20ac268a60361703afe7feb2df5697fa89f72052cb87a3e AS builder
|
||||
#FROM golang:1.16-buster AS builder
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
RUN apt update && apt install -y make bzip2 curl unzip
|
||||
|
||||
# install newer protoc than what comes with buster
|
||||
ENV PB_REL https://github.com/protocolbuffers/protobuf/releases
|
||||
RUN curl -LO $PB_REL/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip && \
|
||||
unzip protoc-3.19.0-linux-x86_64.zip -d /usr/local
|
||||
|
||||
RUN mkdir -p /build
|
||||
FROM sneak/mfer-build AS builder
|
||||
WORKDIR /build
|
||||
|
||||
COPY go.mod .
|
||||
|
||||
17
Makefile
17
Makefile
@ -1,22 +1,24 @@
|
||||
|
||||
export DOCKER_BUILDKIT := 1
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
export PATH := $(PATH):$(GOPATH)/bin
|
||||
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
|
||||
|
||||
ARCH := $(shell uname -m)
|
||||
GITREV := $(shell git describe --always --dirty=-dirty)
|
||||
APPNAME := mfer
|
||||
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
|
||||
|
||||
GOLDFLAGS += -X main.Version=0.1.0
|
||||
GOLDFLAGS += -X main.Gitrev=$(GITREV)
|
||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||
|
||||
default: run
|
||||
default: test
|
||||
|
||||
run: ./mfer.cmd
|
||||
./$<
|
||||
./$< gen --ignore-dotfiles
|
||||
|
||||
test:
|
||||
test: fmt
|
||||
go test -v --timeout 3s ./...
|
||||
|
||||
$(PROTOC_GEN_GO):
|
||||
@ -29,7 +31,7 @@ devprereqs:
|
||||
which gofumpt || go install -v mvdan.cc/gofumpt@latest
|
||||
which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
|
||||
mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go
|
||||
mfer.cmd: mfer/*.go internal/*/*.go cmd/*/*.go
|
||||
protoc --version
|
||||
cd mfer && go generate .
|
||||
make test
|
||||
@ -48,5 +50,10 @@ lint:
|
||||
golangci-lint run
|
||||
sh -c 'test -z "$$(gofmt -l .)"'
|
||||
|
||||
dockerbuild:
|
||||
docker:
|
||||
bash -x bin/docker-prereqs.sh
|
||||
docker build .
|
||||
|
||||
godoc:
|
||||
open http://127.0.0.1:6060
|
||||
godoc -http=:6060
|
||||
|
||||
50
bin/docker-prereqs.sh
Normal file
50
bin/docker-prereqs.sh
Normal file
@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
set -euxo pipefail
|
||||
GOI="golang:1.19.3-bullseye"
|
||||
CII="golangci/golangci-lint:v1.50.1"
|
||||
BII="sneak/mfer-build"
|
||||
|
||||
if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then
|
||||
mkdir -p "$DOCKER_IMAGE_CACHE_DIR"
|
||||
fi
|
||||
|
||||
DICD="$DOCKER_IMAGE_CACHE_DIR"
|
||||
|
||||
function buildImageCache() {
|
||||
if [[ ! -e "$DICD/go.tzst" ]]; then
|
||||
docker pull $GOI
|
||||
docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \
|
||||
mv $DICD/go.tzst.tmp $DICD/go.tzst
|
||||
fi
|
||||
|
||||
if [[ ! -e "$DICD/ci.tzst" ]]; then
|
||||
docker pull $CII
|
||||
docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \
|
||||
mv $DICD/ci.tzst.tmp $DICD/ci.tzst
|
||||
fi
|
||||
|
||||
if [[ ! -e "$DICD/build.tzst" ]]; then
|
||||
go mod download -x
|
||||
cd buildimage
|
||||
cp ../go.mod ../go.sum .
|
||||
docker build -t $BII . && rm -rf go.mod go.sum && \
|
||||
cd .. && \
|
||||
docker save $BII | pv | zstdmt -19 > $DICD/build.tzst.tmp && \
|
||||
mv $DICD/build.tzst.tmp $DICD/build.tzst
|
||||
fi
|
||||
}
|
||||
|
||||
function loadImageCache() {
|
||||
docker image ls $CII || \
|
||||
zstdmt -d --stdout $DICD/ci.tzst | pv | docker load
|
||||
|
||||
docker image ls $GOI || \
|
||||
zstdmt -d --stdout $DICD/go.tzst | pv | docker load
|
||||
|
||||
docker image ls $BII || \
|
||||
zstdmt -d --stdout $DICD/build.tzst | pv | docker load
|
||||
}
|
||||
|
||||
buildImageCache
|
||||
|
||||
loadImageCache
|
||||
21
buildimage/Dockerfile
Normal file
21
buildimage/Dockerfile
Normal file
@ -0,0 +1,21 @@
|
||||
## build image:
|
||||
FROM golang:1.19.3-bullseye AS builder
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
RUN apt update && apt install -y make bzip2 curl unzip
|
||||
RUN mkdir -p /build
|
||||
WORKDIR /build
|
||||
|
||||
# install newer protoc
|
||||
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \
|
||||
unzip *.zip -d /usr/local && rm -v *.zip && protoc --version
|
||||
|
||||
RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
||||
|
||||
RUN go env
|
||||
|
||||
COPY ./go.mod .
|
||||
COPY ./go.sum .
|
||||
|
||||
RUN --mount=type=cache,target=/go/pkg go mod download -x
|
||||
RUN rm -rfv /var/cache/* /var/tmp/*
|
||||
6
go.mod
6
go.mod
@ -3,11 +3,11 @@ module git.eeqj.de/sneak/mfer
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/apex/log v1.9.0 // indirect
|
||||
github.com/apex/log v1.9.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/pterm/pterm v0.12.35
|
||||
github.com/spf13/afero v1.8.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/urfave/cli/v2 v2.3.0
|
||||
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 // indirect
|
||||
google.golang.org/protobuf v1.27.1
|
||||
google.golang.org/protobuf v1.28.1
|
||||
)
|
||||
|
||||
20
go.sum
20
go.sum
@ -154,8 +154,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
|
||||
@ -197,13 +199,19 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S
|
||||
github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
|
||||
github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
|
||||
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
||||
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
|
||||
github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
@ -211,8 +219,6 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
|
||||
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
|
||||
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 h1:RDJ3ggSqBL4Mu/ANRKmxuLrwnupJsvHRDXe4/KI+HWE=
|
||||
github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20/go.mod h1:iljxuLc3m07jsXOWojqehoTp/Fh0XP7irbhV+6sYNGc=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@ -512,10 +518,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
@ -525,8 +532,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@ -10,8 +10,6 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
|
||||
|
||||
type ManifestFile struct {
|
||||
Path string
|
||||
FileInfo fs.FileInfo
|
||||
@ -27,6 +25,8 @@ type Manifest struct {
|
||||
Files []*ManifestFile
|
||||
ScanOptions *ManifestScanOptions
|
||||
TotalFileSize int64
|
||||
PBInner *MFFile
|
||||
PBOuter *MFFileOuter
|
||||
}
|
||||
|
||||
func (m *Manifest) String() string {
|
||||
@ -89,6 +89,12 @@ func pathIsHidden(p string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
func timeToTimestamp(t time.Time) *Timestamp {
|
||||
|
||||
}
|
||||
*/
|
||||
|
||||
func (m *Manifest) Scan() error {
|
||||
// FIXME scan and whatever function does the hashing should take ctx
|
||||
oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error {
|
||||
|
||||
@ -7,7 +7,7 @@ message Timestamp {
|
||||
int32 nanos = 2;
|
||||
}
|
||||
|
||||
message MFFile {
|
||||
message MFFileOuter {
|
||||
enum Version {
|
||||
NONE = 0;
|
||||
ONE = 1; // only one for now
|
||||
@ -54,7 +54,7 @@ message MFFileChecksum {
|
||||
bytes multiHash = 1;
|
||||
}
|
||||
|
||||
message MFFileInner {
|
||||
message MFFile {
|
||||
enum Version {
|
||||
NONE = 0;
|
||||
ONE = 1; // only one for now
|
||||
|
||||
12
mfer/serialize.go
Normal file
12
mfer/serialize.go
Normal file
@ -0,0 +1,12 @@
|
||||
package mfer
|
||||
|
||||
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
|
||||
|
||||
func (m *Manifest) Generate() error {
|
||||
m.PBInner = &MFFile{
|
||||
Version: MFFile_ONE,
|
||||
// CreatedAt: time.Now(),
|
||||
Files: []*MFFilePath{},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user