Compare commits

..
1 Commits
Author SHA1 Message Date
sneak 74185ae5d6 Never regenerate mf.pb.go during checks; fail when it is stale (closes #71)
check / check (push) Failing after 2s
The test and format scripts regenerated mfer/mf.pb.go whenever
mfer/mf.proto looked newer by mtime, which a fresh checkout often causes,
so make check could rewrite a committed file and needed protoc. Nothing
regenerates it any more except make generate (script/generate), which
also records the hash of mf.proto in mfer/mf.proto.sha256. A Go test
compares that hash with mf.proto and fails, naming make generate, when
they differ; it needs no protoc. The Makefile's mtime rule for mf.pb.go,
make clean's deletion of it and the Dockerfile's touch workarounds are
removed.

Model: opus-5-5
2026-10-04 10:07:40 +00:00
12 changed files with 77 additions and 350 deletions
+4
View File
@@ -2,6 +2,7 @@ export DOCKER_BUILDKIT := 1
export PROGRESS_NO_TRUNC := 1 export PROGRESS_NO_TRUNC := 1
GOPATH := $(shell go env GOPATH) GOPATH := $(shell go env GOPATH)
export PATH := $(PATH):$(GOPATH)/bin export PATH := $(PATH):$(GOPATH)/bin
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
ARCH := $(shell uname -m) ARCH := $(shell uname -m)
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown) GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
@@ -34,6 +35,9 @@ test:
fuzz: fuzz:
@script/fuzz @script/fuzz
$(PROTOC_GEN_GO):
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
fixme: fixme:
@grep -nir fixme . | grep -v Makefile @grep -nir fixme . | grep -v Makefile
+4 -11
View File
@@ -74,17 +74,10 @@ provide:
such as `script/docker` such as `script/docker`
- `script/test` — run the test suite (`go test`); one test fails when - `script/test` — run the test suite (`go test`); one test fails when
`mfer/mf.proto` no longer matches the hash `script/generate` recorded `mfer/mf.proto` no longer matches the hash `script/generate` recorded
- `script/generate` (`make generate`) — regenerate `mfer/mf.pb.go` from - `script/generate` — regenerate `mfer/mf.pb.go` from `mfer/mf.proto` and record
`mfer/mf.proto` and record the hash of that `mfer/mf.proto` in the hash of that `mfer/mf.proto` in `mfer/mf.proto.sha256`; needs `protoc` and
`mfer/mf.proto.sha256`; the only thing that regenerates the committed `protoc-gen-go`, and is the only thing that regenerates the committed
`mfer/mf.pb.go`. It needs the exact versions that wrote the committed file, `mfer/mf.pb.go`
and refuses to run with any other: `protoc` 33.4 (unpack
`protoc-33.4-<platform>.zip` from
[its release](https://github.com/protocolbuffers/protobuf/releases/tag/v33.4)
and put its `bin/protoc` on `PATH`) and `protoc-gen-go` v1.36.11
(`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, which
installs it in `$(go env GOPATH)/bin`; `make generate` adds that directory to
`PATH`)
- `script/fuzz` — fuzz the manifest parser for one minute; run by hand - `script/fuzz` — fuzz the manifest parser for one minute; run by hand
(`make fuzz`), never by CI, while `script/test` runs its committed seed corpus (`make fuzz`), never by CI, while `script/test` runs its committed seed corpus
as ordinary tests as ordinary tests
+1 -5
View File
@@ -298,11 +298,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Infof("checking manifest %s with base %s", manifestPath, basePath) log.Infof("checking manifest %s with base %s", manifestPath, basePath)
// Create checker // Create checker
chk, err := mfer.NewChecker(&mfer.CheckerOptions{ chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
ManifestPath: manifestPath,
BasePath: basePath,
Fs: mfa.Fs,
})
if err != nil { if err != nil {
return fmt.Errorf("failed to load manifest: %w", err) return fmt.Errorf("failed to load manifest: %w", err)
} }
+2 -8
View File
@@ -504,10 +504,7 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
assert.True(t, exists) assert.True(t, exists)
// Verify manifest only has 1 file (the non-dotfile) // Verify manifest only has 1 file (the non-dotfile)
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{ manifest, err := mfer.NewManifestFromFile(fs, testMF)
Path: testMF,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 1) assert.Len(t, manifest.Files(), 1)
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath()) assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
@@ -531,10 +528,7 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
require.Equal(t, 0, exitCode) require.Equal(t, 0, exitCode)
// Verify manifest has 2 files (including dotfile) // Verify manifest has 2 files (including dotfile)
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{ manifest, err := mfer.NewManifestFromFile(fs, testMF)
Path: testMF,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
} }
+2 -10
View File
@@ -61,11 +61,7 @@ func unsignedChecker(t *testing.T) *mfer.Checker {
require.NoError(t, s.ToManifest(context.Background(), &buf, nil)) require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644)) require.NoError(t, afero.WriteFile(fs, "/d/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker(&mfer.CheckerOptions{ chk, err := mfer.NewChecker("/d/index.mf", "/d", fs)
ManifestPath: "/d/index.mf",
BasePath: "/d",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
require.False(t, chk.IsSigned()) require.False(t, chk.IsSigned())
@@ -171,11 +167,7 @@ func signedChecker(t *testing.T) *mfer.Checker {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644)) require.NoError(t, afero.WriteFile(fs, "/index.mf", buf.Bytes(), 0o644))
chk, err := mfer.NewChecker(&mfer.CheckerOptions{ chk, err := mfer.NewChecker("/index.mf", "/", fs)
ManifestPath: "/index.mf",
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
require.True(t, chk.IsSigned()) require.True(t, chk.IsSigned())
+1 -4
View File
@@ -445,10 +445,7 @@ func (mfa *CLIApp) loadExistingEntries(
log.Infof("loading manifest from %s", manifestPath) log.Infof("loading manifest from %s", manifestPath)
// Load existing manifest // Load existing manifest
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{ manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
Path: manifestPath,
Fs: mfa.Fs,
})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to load manifest: %w", err) return nil, fmt.Errorf("failed to load manifest: %w", err)
} }
+2 -8
View File
@@ -58,10 +58,7 @@ func TestFreshenUnchanged(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Parse manifest to verify // Parse manifest to verify
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{ manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
Path: "/testdir/.index.mf",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
} }
@@ -73,10 +70,7 @@ func TestFreshenWithChanges(t *testing.T) {
setupFreshenDir(t, fs) setupFreshenDir(t, fs)
// Verify initial manifest has 2 files // Verify initial manifest has 2 files
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{ manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
Path: "/testdir/.index.mf",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Len(t, manifest.Files(), 2) assert.Len(t, manifest.Files(), 2)
-25
View File
@@ -5,12 +5,10 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"path/filepath"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -399,29 +397,6 @@ func TestNewManifestFromReaderTruncated(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestNewManifestFromFileRequiresPath(t *testing.T) {
t.Parallel()
_, err := NewManifestFromFile(nil)
require.ErrorIs(t, err, errManifestPathEmpty)
_, err = NewManifestFromFile(&ManifestFromFileOptions{Fs: afero.NewMemMapFs()})
require.ErrorIs(t, err, errManifestPathEmpty)
}
func TestNewManifestFromFileNilFsUsesOsFs(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "index.mf")
createTestManifest(t, afero.NewOsFs(), path, map[string][]byte{
testFileName: []byte("hello"),
})
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
require.NoError(t, err)
assert.Len(t, m.Files(), 1)
}
func TestManifestString(t *testing.T) { func TestManifestString(t *testing.T) {
t.Parallel() t.Parallel()
+8 -34
View File
@@ -14,11 +14,7 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
var ( var errNoSigningPubKey = errors.New("manifest has no signing public key")
errNoSigningPubKey = errors.New("manifest has no signing public key")
errManifestPathEmpty = errors.New("manifest path cannot be empty")
errBasePathEmpty = errors.New("base path cannot be empty")
)
// Result represents the outcome of checking a single file. // Result represents the outcome of checking a single file.
type Result struct { type Result struct {
@@ -86,42 +82,20 @@ type Checker struct {
signingPubKey []byte signingPubKey []byte
} }
// CheckerOptions configures a Checker. // NewChecker creates a new Checker for the given manifest, base path, and filesystem.
type CheckerOptions struct { // The basePath is the directory relative to which manifest paths are resolved.
// ManifestPath is the manifest file to check against (required). // If fs is nil, the real filesystem (OsFs) is used.
ManifestPath string func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) {
// BasePath is the directory relative to which manifest paths are
// resolved (required).
BasePath string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewChecker creates a new Checker with the given options. It returns an
// error if opts is nil or either path is empty.
func NewChecker(opts *CheckerOptions) (*Checker, error) {
if opts == nil || opts.ManifestPath == "" {
return nil, errManifestPathEmpty
}
if opts.BasePath == "" {
return nil, errBasePathEmpty
}
fs := opts.Fs
if fs == nil { if fs == nil {
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
m, err := NewManifestFromFile(&ManifestFromFileOptions{ m, err := NewManifestFromFile(fs, manifestPath)
Path: opts.ManifestPath,
Fs: fs,
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
abs, err := filepath.Abs(opts.BasePath) abs, err := filepath.Abs(basePath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -134,7 +108,7 @@ func NewChecker(opts *CheckerOptions) (*Checker, error) {
} }
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles // Compute manifest's relative path from basePath for exclusion in FindExtraFiles
absManifest, err := filepath.Abs(opts.ManifestPath) absManifest, err := filepath.Abs(manifestPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+46 -190
View File
@@ -5,8 +5,6 @@ import (
"bytes" "bytes"
"context" "context"
"fmt" "fmt"
"os"
"path/filepath"
"testing" "testing"
"time" "time"
@@ -16,11 +14,9 @@ import (
) )
const ( const (
testFile1 = "file1.txt" testFile1 = "file1.txt"
testFile2 = "file2.txt" testFile2 = "file2.txt"
testExistsFile = "exists.txt" testExistsFile = "exists.txt"
testManifestPath = "/manifest.mf"
testDataDir = "/data"
) )
func TestStatusString(t *testing.T) { func TestStatusString(t *testing.T) {
@@ -70,13 +66,15 @@ func createTestManifest(
} }
// createFilesOnDisk creates the given files on the filesystem under // createFilesOnDisk creates the given files on the filesystem under
// testDataDir. // /data.
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) { func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
t.Helper() t.Helper()
basePath := "/data"
for path, content := range files { for path, content := range files {
fullPath := testDataDir + "/" + path fullPath := basePath + "/" + path
require.NoError(t, fs.MkdirAll(testDataDir, 0o755)) require.NoError(t, fs.MkdirAll(basePath, 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
} }
@@ -92,13 +90,9 @@ func TestNewChecker(t *testing.T) {
testFile1: []byte("hello"), testFile1: []byte("hello"),
testFile2: []byte("world"), testFile2: []byte("world"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/", fs)
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.NotNil(t, chk) assert.NotNil(t, chk)
assert.Equal(t, FileCount(2), chk.FileCount()) assert.Equal(t, FileCount(2), chk.FileCount())
@@ -108,11 +102,7 @@ func TestNewChecker(t *testing.T) {
t.Parallel() t.Parallel()
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
_, err := NewChecker(&CheckerOptions{ _, err := NewChecker("/nonexistent.mf", "/", fs)
ManifestPath: "/nonexistent.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err) assert.Error(t, err)
}) })
@@ -121,73 +111,11 @@ func TestNewChecker(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
_, err := NewChecker(&CheckerOptions{ _, err := NewChecker("/bad.mf", "/", fs)
ManifestPath: "/bad.mf",
BasePath: "/",
Fs: fs,
})
assert.Error(t, err) assert.Error(t, err)
}) })
} }
func TestNewCheckerRequiredPaths(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
opts *CheckerOptions
want string
is error
}{
{
name: "nil options",
opts: nil,
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty manifest path",
opts: &CheckerOptions{BasePath: testDataDir},
want: "manifest path cannot be empty",
is: errManifestPathEmpty,
},
{
name: "empty base path",
opts: &CheckerOptions{ManifestPath: testManifestPath},
want: "base path cannot be empty",
is: errBasePathEmpty,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
chk, err := NewChecker(tc.opts)
require.ErrorIs(t, err, tc.is)
require.EqualError(t, err, tc.want)
assert.Nil(t, chk)
})
}
}
func TestNewCheckerNilFsUsesOsFs(t *testing.T) {
t.Parallel()
dir := t.TempDir()
manifestPath := filepath.Join(dir, "index.mf")
content := []byte("hello")
createTestManifest(t, afero.NewOsFs(), manifestPath, map[string][]byte{
testFile1: content,
})
require.NoError(t, os.WriteFile(filepath.Join(dir, testFile1), content, 0o600))
chk, err := NewChecker(&CheckerOptions{ManifestPath: manifestPath, BasePath: dir})
require.NoError(t, err)
results := make(chan Result, 1)
require.NoError(t, chk.Check(context.Background(), results, nil))
assert.Equal(t, StatusOK, (<-results).Status)
}
func TestCheckerFileCountAndTotalBytes(t *testing.T) { func TestCheckerFileCountAndTotalBytes(t *testing.T) {
t.Parallel() t.Parallel()
@@ -197,13 +125,9 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
"medium.txt": []byte("hello world"), "medium.txt": []byte("hello world"),
"large.txt": bytes.Repeat([]byte("x"), 1000), "large.txt": bytes.Repeat([]byte("x"), 1000),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/", fs)
ManifestPath: testManifestPath,
BasePath: "/",
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, FileCount(3), chk.FileCount()) assert.Equal(t, FileCount(3), chk.FileCount())
@@ -218,14 +142,10 @@ func TestCheckAllFilesOK(t *testing.T) {
testFile1: []byte("content one"), testFile1: []byte("content one"),
testFile2: []byte("content two"), testFile2: []byte("content two"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -252,17 +172,13 @@ func TestCheckMissingFile(t *testing.T) {
testExistsFile: []byte("I exist"), testExistsFile: []byte("I exist"),
"missing.txt": []byte("I don't exist on disk"), "missing.txt": []byte("I don't exist on disk"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
// Only create one file // Only create one file
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("I exist"), testExistsFile: []byte("I exist"),
}) })
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -295,17 +211,13 @@ func TestCheckSizeMismatch(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFileName: []byte("original content"), testFileName: []byte("original content"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
// Create file with different size // Create file with different size
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testFileName: []byte("short"), testFileName: []byte("short"),
}) })
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -325,7 +237,7 @@ func TestCheckHashMismatch(t *testing.T) {
files := map[string][]byte{ files := map[string][]byte{
testFileName: originalContent, testFileName: originalContent,
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
// Create file with same size but different content // Create file with same size but different content
differentContent := []byte("different contnt") // same length (16 bytes) but different differentContent := []byte("different contnt") // same length (16 bytes) but different
require.Len(t, differentContent, len(originalContent), "test requires same length") require.Len(t, differentContent, len(originalContent), "test requires same length")
@@ -333,11 +245,7 @@ func TestCheckHashMismatch(t *testing.T) {
testFileName: differentContent, testFileName: differentContent,
}) })
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -357,14 +265,10 @@ func TestCheckWithProgress(t *testing.T) {
testFile1: bytes.Repeat([]byte("a"), 100), testFile1: bytes.Repeat([]byte("a"), 100),
testFile2: bytes.Repeat([]byte("b"), 200), testFile2: bytes.Repeat([]byte("b"), 200),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -401,14 +305,10 @@ func TestCheckContextCancellation(t *testing.T) {
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000) files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -427,7 +327,7 @@ func TestFindExtraFiles(t *testing.T) {
manifestFiles := map[string][]byte{ manifestFiles := map[string][]byte{
testFile1: []byte("in manifest"), testFile1: []byte("in manifest"),
} }
createTestManifest(t, fs, testManifestPath, manifestFiles) createTestManifest(t, fs, "/manifest.mf", manifestFiles)
// Disk has file1 and file2 // Disk has file1 and file2
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
@@ -435,11 +335,7 @@ func TestFindExtraFiles(t *testing.T) {
testFile2: []byte("extra file"), testFile2: []byte("extra file"),
}) })
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -472,14 +368,10 @@ func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
// Create a real extra file // Create a real extra file
require.NoError(t, fs.MkdirAll(testDataDir, 0o755)) require.NoError(t, fs.MkdirAll("/data", 0o755))
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644)) require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/data/.index.mf", "/data", fs)
ManifestPath: "/data/.index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -508,14 +400,10 @@ func TestFindExtraFilesContextCancellation(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -531,14 +419,10 @@ func TestCheckNilChannels(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
// Should not panic with nil channels // Should not panic with nil channels
@@ -551,14 +435,10 @@ func TestFindExtraFilesNilChannel(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
files := map[string][]byte{testFileName: []byte("data")} files := map[string][]byte{testFileName: []byte("data")}
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
// Should not panic with nil channel // Should not panic with nil channel
@@ -575,7 +455,7 @@ func TestCheckSubdirectories(t *testing.T) {
"dir1/dir2/file2.txt": []byte("content2"), "dir1/dir2/file2.txt": []byte("content2"),
"dir1/dir2/dir3/deep.txt": []byte("deep content"), "dir1/dir2/dir3/deep.txt": []byte("deep content"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
// Create files with full directory structure // Create files with full directory structure
for path, content := range files { for path, content := range files {
@@ -585,11 +465,7 @@ func TestCheckSubdirectories(t *testing.T) {
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -617,17 +493,13 @@ func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
testExistsFile: []byte("here"), testExistsFile: []byte("here"),
"missing.txt": []byte("not on disk"), "missing.txt": []byte("not on disk"),
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
// Only create one file on disk // Only create one file on disk
createFilesOnDisk(t, fs, map[string][]byte{ createFilesOnDisk(t, fs, map[string][]byte{
testExistsFile: []byte("here"), testExistsFile: []byte("here"),
}) })
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -665,11 +537,7 @@ func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
require.NoError(t, require.NoError(t,
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644)) afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/data/.index.mf", "/data", fs)
ManifestPath: "/data/.index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -698,11 +566,7 @@ func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
createTestManifest(t, fs, "/data/index.mf", files) createTestManifest(t, fs, "/data/index.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/data/index.mf", "/data", fs)
ManifestPath: "/data/index.mf",
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 10) results := make(chan Result, 10)
@@ -723,13 +587,9 @@ func TestCheckEmptyManifest(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
// Create manifest with no files // Create manifest with no files
createTestManifest(t, fs, testManifestPath, map[string][]byte{}) createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, FileCount(0), chk.FileCount()) assert.Equal(t, FileCount(0), chk.FileCount())
@@ -761,14 +621,10 @@ func TestCheckProgressRateLimited(t *testing.T) {
files[name] = []byte("content") files[name] = []byte("content")
} }
createTestManifest(t, fs, testManifestPath, files) createTestManifest(t, fs, "/manifest.mf", files)
createFilesOnDisk(t, fs, files) createFilesOnDisk(t, fs, files)
chk, err := NewChecker(&CheckerOptions{ chk, err := NewChecker("/manifest.mf", "/data", fs)
ManifestPath: testManifestPath,
BasePath: testDataDir,
Fs: fs,
})
require.NoError(t, err) require.NoError(t, err)
results := make(chan Result, 200) results := make(chan Result, 200)
+4 -17
View File
@@ -352,29 +352,16 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
return m, nil return m, nil
} }
// ManifestFromFileOptions configures NewManifestFromFile. // NewManifestFromFile reads a manifest from a file path using the given filesystem.
type ManifestFromFileOptions struct { // If fs is nil, the real filesystem (OsFs) is used.
// Path is the manifest file to read (required).
Path string
// Fs is the filesystem to use, defaults to OsFs if nil.
Fs afero.Fs
}
// NewManifestFromFile reads a manifest from a file. It returns an error if
// opts is nil or its path is empty.
// //
//nolint:revive // unexported-return: exporting manifest is owner question 13 //nolint:revive // unexported-return: exporting manifest is owner question 13
func NewManifestFromFile(opts *ManifestFromFileOptions) (*manifest, error) { func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
if opts == nil || opts.Path == "" {
return nil, errManifestPathEmpty
}
fs := opts.Fs
if fs == nil { if fs == nil {
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
f, err := fs.Open(opts.Path) f, err := fs.Open(path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+3 -38
View File
@@ -2,51 +2,16 @@
# script/generate: regenerate mfer/mf.pb.go from mfer/mf.proto, and record # script/generate: regenerate mfer/mf.pb.go from mfer/mf.proto, and record
# the hash of that mf.proto in mfer/mf.proto.sha256. Nothing else # the hash of that mf.proto in mfer/mf.proto.sha256. Nothing else
# regenerates mf.pb.go: it is committed, so building and checking need no # regenerates mf.pb.go: it is committed, so building and checking need no
# protoc. A test fails while mf.proto no longer matches the recorded hash. # protoc. Needs protoc and protoc-gen-go on PATH. A test fails while
# # mf.proto no longer matches the recorded hash.
# Needs exactly the protoc and protoc-gen-go versions named in the header of
# the committed mf.pb.go (README.md says how to install them). Another
# version writes a different mf.pb.go, so the script refuses to run.
set -eu set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# protoc 33.4 names itself v6.33.4 in the mf.pb.go header.
PROTOC_VERSION="33.4"
PROTOC_GEN_GO_VERSION="v1.36.11"
# require_version <command> <its exact --version output>
require_version() {
actual="$("$1" --version 2>/dev/null || true)"
if [ "$actual" != "$2" ]; then
echo "generate: needs $2 on PATH, found: ${actual:-none}" >&2
echo " README.md says how to install it." >&2
exit 1
fi
}
# sha256 <file>: print "<hash> <file>", with sha256sum, or with shasum
# where there is no sha256sum.
sha256() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1"
elif command -v shasum >/dev/null 2>&1; then
shasum -a 256 "$1"
else
echo "generate: needs sha256sum or shasum on PATH" >&2
exit 1
fi
}
main() { main() {
cd "$ROOT/mfer" cd "$ROOT/mfer"
require_version protoc "libprotoc $PROTOC_VERSION"
require_version protoc-gen-go "protoc-gen-go $PROTOC_GEN_GO_VERSION"
# Hashed before regenerating, so a missing hash tool stops the script
# before it changes anything. Regenerating leaves mf.proto as it is.
proto_hash="$(sha256 mf.proto)"
go generate . go generate .
echo "$proto_hash" >mf.proto.sha256 shasum -a 256 mf.proto >mf.proto.sha256
} }
main "$@" main "$@"