From bd4b135e177833ca62dd996c4f6c1a72bc9507b2 Mon Sep 17 00:00:00 2001 From: sneak Date: Sun, 4 Dec 2022 22:55:22 +0400 Subject: [PATCH] test me pls --- .dockerignore | 5 +++++ .gitignore | 4 ++++ Dockerfile | 17 ++------------- Makefile | 17 ++++++++++----- bin/docker-prereqs.sh | 50 +++++++++++++++++++++++++++++++++++++++++++ buildimage/Dockerfile | 21 ++++++++++++++++++ go.mod | 6 +++--- go.sum | 20 +++++++++++------ mfer/manifest.go | 10 +++++++-- mfer/mf.proto | 4 ++-- mfer/serialize.go | 12 +++++++++++ 11 files changed, 133 insertions(+), 33 deletions(-) create mode 100644 .dockerignore create mode 100644 bin/docker-prereqs.sh create mode 100644 buildimage/Dockerfile create mode 100644 mfer/serialize.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ecdb971 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +*.tzst +*.tar +/buildimage +/dockerdeps +/tmp diff --git a/.gitignore b/.gitignore index 8c2487d..074e7cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ mfer/*.pb.go /mfer.cmd +vendor +/tmp +/buildimage/go.mod +/buildimage/go.sum diff --git a/Dockerfile b/Dockerfile index 53230f0..41c2019 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ ## lint image -## current as of 2022-01-25 -FROM golangci/golangci-lint:latest AS linter +FROM golangci/golangci-lint:v1.50.1 RUN mkdir -p /build WORKDIR /build @@ -8,19 +7,7 @@ COPY ./ ./ RUN golangci-lint run ## build image: -# this is golang:1.17.6-buster as of 2022-01-27 -FROM golang@sha256:52a48e0239f4d645b20ac268a60361703afe7feb2df5697fa89f72052cb87a3e AS builder -#FROM golang:1.16-buster AS builder - -ENV DEBIAN_FRONTEND noninteractive -RUN apt update && apt install -y make bzip2 curl unzip - -# install newer protoc than what comes with buster -ENV PB_REL https://github.com/protocolbuffers/protobuf/releases -RUN curl -LO $PB_REL/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip && \ - unzip protoc-3.19.0-linux-x86_64.zip -d /usr/local - -RUN mkdir -p /build +FROM sneak/mfer-build AS builder WORKDIR /build COPY go.mod . diff --git a/Makefile b/Makefile index 21fb8a3..40b861e 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,24 @@ - +export DOCKER_BUILDKIT := 1 GOPATH := $(shell go env GOPATH) export PATH := $(PATH):$(GOPATH)/bin PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go ARCH := $(shell uname -m) GITREV := $(shell git describe --always --dirty=-dirty) +APPNAME := mfer +export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH) GOLDFLAGS += -X main.Version=0.1.0 GOLDFLAGS += -X main.Gitrev=$(GITREV) GOFLAGS := -ldflags "$(GOLDFLAGS)" -default: run +default: test run: ./mfer.cmd ./$< ./$< gen --ignore-dotfiles -test: +test: fmt go test -v --timeout 3s ./... $(PROTOC_GEN_GO): @@ -29,7 +31,7 @@ devprereqs: which gofumpt || go install -v mvdan.cc/gofumpt@latest which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@latest -mfer.cmd: $(PROTOC_GEN_GO) mfer/*.go internal/*/*.go cmd/*/*.go +mfer.cmd: mfer/*.go internal/*/*.go cmd/*/*.go protoc --version cd mfer && go generate . make test @@ -48,5 +50,10 @@ lint: golangci-lint run sh -c 'test -z "$$(gofmt -l .)"' -dockerbuild: +docker: + bash -x bin/docker-prereqs.sh docker build . + +godoc: + open http://127.0.0.1:6060 + godoc -http=:6060 diff --git a/bin/docker-prereqs.sh b/bin/docker-prereqs.sh new file mode 100644 index 0000000..2cd672b --- /dev/null +++ b/bin/docker-prereqs.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -euxo pipefail +GOI="golang:1.19.3-bullseye" +CII="golangci/golangci-lint:v1.50.1" +BII="sneak/mfer-build" + +if [[ ! -d "$DOCKER_IMAGE_CACHE_DIR" ]]; then + mkdir -p "$DOCKER_IMAGE_CACHE_DIR" +fi + +DICD="$DOCKER_IMAGE_CACHE_DIR" + +function buildImageCache() { + if [[ ! -e "$DICD/go.tzst" ]]; then + docker pull $GOI + docker save $GOI | pv | zstdmt -19 > $DICD/go.tzst.tmp && \ + mv $DICD/go.tzst.tmp $DICD/go.tzst + fi + + if [[ ! -e "$DICD/ci.tzst" ]]; then + docker pull $CII + docker save $CII | pv | zstdmt -19 > $DICD/ci.tzst.tmp && \ + mv $DICD/ci.tzst.tmp $DICD/ci.tzst + fi + + if [[ ! -e "$DICD/build.tzst" ]]; then + go mod download -x + cd buildimage + cp ../go.mod ../go.sum . + docker build -t $BII . && rm -rf go.mod go.sum && \ + cd .. && \ + docker save $BII | pv | zstdmt -19 > $DICD/build.tzst.tmp && \ + mv $DICD/build.tzst.tmp $DICD/build.tzst + fi +} + +function loadImageCache() { + docker image ls $CII || \ + zstdmt -d --stdout $DICD/ci.tzst | pv | docker load + + docker image ls $GOI || \ + zstdmt -d --stdout $DICD/go.tzst | pv | docker load + + docker image ls $BII || \ + zstdmt -d --stdout $DICD/build.tzst | pv | docker load +} + +buildImageCache + +loadImageCache diff --git a/buildimage/Dockerfile b/buildimage/Dockerfile new file mode 100644 index 0000000..dea8624 --- /dev/null +++ b/buildimage/Dockerfile @@ -0,0 +1,21 @@ +## build image: +FROM golang:1.19.3-bullseye AS builder + +ENV DEBIAN_FRONTEND noninteractive +RUN apt update && apt install -y make bzip2 curl unzip +RUN mkdir -p /build +WORKDIR /build + +# install newer protoc +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v21.10/protoc-21.10-linux-aarch_64.zip && \ + unzip *.zip -d /usr/local && rm -v *.zip && protoc --version + +RUN go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 + +RUN go env + +COPY ./go.mod . +COPY ./go.sum . + +RUN --mount=type=cache,target=/go/pkg go mod download -x +RUN rm -rfv /var/cache/* /var/tmp/* diff --git a/go.mod b/go.mod index 17b1104..92c0feb 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module git.eeqj.de/sneak/mfer go 1.16 require ( - github.com/apex/log v1.9.0 // indirect + github.com/apex/log v1.9.0 github.com/davecgh/go-spew v1.1.1 github.com/pterm/pterm v0.12.35 github.com/spf13/afero v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/urfave/cli/v2 v2.3.0 - github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 // indirect - google.golang.org/protobuf v1.27.1 + google.golang.org/protobuf v1.28.1 ) diff --git a/go.sum b/go.sum index 37c41c3..96493f7 100644 --- a/go.sum +++ b/go.sum @@ -154,8 +154,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= @@ -197,13 +199,19 @@ github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4S github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= @@ -211,8 +219,6 @@ github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPf github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20 h1:RDJ3ggSqBL4Mu/ANRKmxuLrwnupJsvHRDXe4/KI+HWE= -github.com/visionmedia/go-cli-log v0.0.0-20151214173634-914d1b040b20/go.mod h1:iljxuLc3m07jsXOWojqehoTp/Fh0XP7irbhV+6sYNGc= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -512,10 +518,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -525,8 +532,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/mfer/manifest.go b/mfer/manifest.go index 5eb80d1..e250b7d 100644 --- a/mfer/manifest.go +++ b/mfer/manifest.go @@ -10,8 +10,6 @@ import ( "github.com/spf13/afero" ) -//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto - type ManifestFile struct { Path string FileInfo fs.FileInfo @@ -27,6 +25,8 @@ type Manifest struct { Files []*ManifestFile ScanOptions *ManifestScanOptions TotalFileSize int64 + PBInner *MFFile + PBOuter *MFFileOuter } func (m *Manifest) String() string { @@ -89,6 +89,12 @@ func pathIsHidden(p string) bool { } } +/* +func timeToTimestamp(t time.Time) *Timestamp { + +} +*/ + func (m *Manifest) Scan() error { // FIXME scan and whatever function does the hashing should take ctx oe := afero.Walk(m.SourceFS, "/", func(p string, info fs.FileInfo, err error) error { diff --git a/mfer/mf.proto b/mfer/mf.proto index a6ba326..9ea8d2e 100644 --- a/mfer/mf.proto +++ b/mfer/mf.proto @@ -7,7 +7,7 @@ message Timestamp { int32 nanos = 2; } -message MFFile { +message MFFileOuter { enum Version { NONE = 0; ONE = 1; // only one for now @@ -54,7 +54,7 @@ message MFFileChecksum { bytes multiHash = 1; } -message MFFileInner { +message MFFile { enum Version { NONE = 0; ONE = 1; // only one for now diff --git a/mfer/serialize.go b/mfer/serialize.go new file mode 100644 index 0000000..14d092b --- /dev/null +++ b/mfer/serialize.go @@ -0,0 +1,12 @@ +package mfer + +//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto + +func (m *Manifest) Generate() error { + m.PBInner = &MFFile{ + Version: MFFile_ONE, + // CreatedAt: time.Now(), + Files: []*MFFilePath{}, + } + return nil +}