Author SHA1 Message Date
sneak 76791d6e6c Rewrite script/test to the canonical race-enabled pattern (closes #67)
check / check (push) Failing after 0s
script/test now runs `go test -timeout 30s -race -cover ./...`, quiet on
success and rerunning verbose with `exit 1` on failure, per policy.

-race surfaced a real data race on the process-global apex/log logger:
Init reconfigures it (SetHandler/SetLevel) each CLI run while other
goroutines read it to log. internal/log now mutates the global under the
write lock and reads it under the read lock (new emit helper, DebugReal),
and drops WithError, whose returned Entry logged outside that lock.

That Entry was also the only thing printing a failed command's error to
stderr, so run() now reports it via log.Errorf (shown under -q too).

The corruption fuzz test shrinks (20000->1500 files, 500->100 iters) to
keep the suite under 20s with -race.

Model: opus-4-8
2026-09-21 08:00:35 +00:00
clawbot 6de3f1d714 Add .editorconfig, broaden .gitignore, drop dead Drone refs (closes #72)
check / check (push) Failing after 1s
Add the canonical .editorconfig from sneak/prompts verbatim. Broaden .gitignore to cover secrets (.env, .env.*, *.key, *.pem), OS files, editor files, and Go artifacts while keeping every repo-specific entry; no tracked file becomes ignored, and bin/gitrev.sh stays tracked despite the /bin/ rule. Remove the stale .drone.yml ignore entry and the DRONE_COMMIT_SHA branch from bin/gitrev.sh, left from the Gitea Actions migration; the GITREV and git-describe paths still work.

A reader may trip over .editorconfig saying four spaces for every file while Go files are tab-indented: gofmt governs Go, editorconfig does not, and the file is taken verbatim by policy.

Disclosure: the worker's cold docker build hit the 10s test timeout in the gpg signature test; this change has no Go code, and that timeout is tracked by issues 67 and 62.

Model: opus-4-8 (implementation); fable-5-1 (merge)
2026-09-21 09:56:15 +02:00
11 changed files with 90 additions and 224 deletions
+12
View File
@@ -0,0 +1,12 @@
root = true
[*]
indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[Makefile]
indent_style = tab
+21 -2
View File
@@ -10,5 +10,24 @@ modcache.tzst
# Generated manifest files
.index.mf
# Stale files
.drone.yml
# Secrets
.env
.env.*
*.key
*.pem
# OS files
.DS_Store
Thumbs.db
# Editor files
*.swp
*.swo
*~
.idea/
.vscode/
# Go build artifacts
*.log
*.out
*.test
+6 -3
View File
@@ -24,8 +24,12 @@ only thing left of the `chore/align-repo-policies` branch is the list below.
# Completed Steps
- 2026-09-21: validate manifest entry paths on deserialize so untrusted `.mf`
files cannot make `Checker` stat or read outside `basePath` (#61)
- 2026-09-21: rewrote `script/test` to the canonical pattern (30s timeout,
`-race -cover`, quiet-first with verbose-on-failure rerun) and fixed the
process-global logger data race it surfaced (#67)
- 2026-09-21: added the canonical `.editorconfig`, made `.gitignore` cover
secrets, OS, editor, and Go artifacts, and removed the dead Drone CI
references from `.gitignore` and `bin/gitrev.sh` (#72)
- 2026-08-09: added `.prettierrc`/`.prettierignore`, gave `script/fmt` and
`script/fmt-check` one shared prettier file set via `script/prettier`, dropped
the `|| true` that hid prettier failures, and added a node-based Dockerfile
@@ -64,7 +68,6 @@ only thing left of the `chore/align-repo-policies` branch is the list below.
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md reference
- Pin Makefile-installed Go tools (`protoc-gen-go@v1.28.1`,
`golangci-lint@v2.12.2`) by module hash, not mutable tag
- Set `make test` timeout to 30s (currently 10s)
- Add explicit README "Rationale" heading (content exists under other
names); name the author in the README Description first line
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep or
-5
View File
@@ -1,10 +1,5 @@
#!/bin/bash
#
if [[ ! -z "$DRONE_COMMIT_SHA" ]]; then
echo "${DRONE_COMMIT_SHA:0:7}"
exit 0
fi
if [[ ! -z "$GITREV" ]]; then
echo $GITREV
else
+10 -8
View File
@@ -679,11 +679,13 @@ func TestCheckDetectsManifestCorruption(t *testing.T) {
fs := afero.NewMemMapFs()
rng := rand.New(rand.NewSource(42)) //nolint:gosec // deterministic test data
// Create many small files with random names to generate a ~1MB manifest
// Each manifest entry is roughly 50-60 bytes, so we need ~20000 files
// Create many small files with random names so the manifest has many
// entries and random single-byte flips land at varied offsets. Each
// manifest entry is roughly 50-60 bytes. Kept modest so the suite stays
// within its wall-clock budget under -race.
require.NoError(t, fs.MkdirAll(testDir, 0o755))
numFiles := 20000
numFiles := 1500
for range numFiles {
// Generate random filename
filename := fmt.Sprintf("/testdir/%08x%08x%08x.dat",
@@ -699,11 +701,11 @@ func TestCheckDetectsManifestCorruption(t *testing.T) {
exitCode := runCLI(opts)
require.Equal(t, 0, exitCode, "generate should succeed")
// Read the valid manifest and verify it's approximately 1MB
// Read the valid manifest and verify it has real size.
validManifest, err := afero.ReadFile(fs, testManifest)
require.NoError(t, err)
require.GreaterOrEqual(t, len(validManifest), 1024*1024,
"manifest should be at least 1MB, got %d bytes", len(validManifest))
require.GreaterOrEqual(t, len(validManifest), 64*1024,
"manifest should be at least 64KB, got %d bytes", len(validManifest))
t.Logf("manifest size: %d bytes (%d files)", len(validManifest), numFiles)
// First corruption: truncate the manifest
@@ -726,8 +728,8 @@ func TestCheckDetectsManifestCorruption(t *testing.T) {
exitCode = runCLI(opts)
require.Equal(t, 0, exitCode, "check should pass with valid manifest")
// Now do 500 random corruption iterations
for i := range 500 {
// Now do 100 random corruption iterations
for i := range 100 {
// Corrupt: write a random byte at a random offset
corrupted := make([]byte, len(validManifest))
copy(corrupted, validManifest)
+1 -1
View File
@@ -357,6 +357,6 @@ func (mfa *CLIApp) run(args []string) {
if err != nil {
mfa.exitCode = 1
log.WithError(err).Debugf("exiting")
log.Errorf("%s", err)
}
}
+34 -41
View File
@@ -112,13 +112,16 @@ func DisableStyling() {
}
// Init initializes the logger with the CLI handler and default log level.
//
// It reconfigures the process-global apex/log logger under the write lock so
// the global is never mutated while another goroutine holds the read lock to
// read it in emit. Without this, parallel callers (e.g. the test suite) race
// Init's SetLevel/SetHandler against concurrent log calls.
func Init() {
mu.RLock()
mu.Lock()
defer mu.Unlock()
w := stderr
mu.RUnlock()
log.SetHandler(acli.New(w))
log.SetHandler(acli.New(stderr))
log.SetLevel(log.DebugLevel) // Let apex/log pass everything; we filter ourselves
}
@@ -130,74 +133,66 @@ func isEnabled(l Level) bool {
return l >= currentLevel
}
// emit calls fn while holding the read lock if messages at level l are
// enabled. Holding the read lock across the apex/log call keeps the global
// logger from being read while Init reconfigures it under the write lock.
func emit(l Level, fn func()) {
mu.RLock()
defer mu.RUnlock()
if l >= currentLevel {
fn()
}
}
// Fatalf logs a formatted message at fatal level.
func Fatalf(format string, args ...any) {
if isEnabled(FatalLevel) {
log.Fatalf(format, args...)
}
emit(FatalLevel, func() { log.Fatalf(format, args...) })
}
// Fatal logs a message at fatal level.
func Fatal(arg string) {
if isEnabled(FatalLevel) {
log.Fatal(arg)
}
emit(FatalLevel, func() { log.Fatal(arg) })
}
// Errorf logs a formatted message at error level.
func Errorf(format string, args ...any) {
if isEnabled(ErrorLevel) {
log.Errorf(format, args...)
}
emit(ErrorLevel, func() { log.Errorf(format, args...) })
}
// Error logs a message at error level.
func Error(arg string) {
if isEnabled(ErrorLevel) {
log.Error(arg)
}
emit(ErrorLevel, func() { log.Error(arg) })
}
// Warnf logs a formatted message at warn level.
func Warnf(format string, args ...any) {
if isEnabled(WarnLevel) {
log.Warnf(format, args...)
}
emit(WarnLevel, func() { log.Warnf(format, args...) })
}
// Warn logs a message at warn level.
func Warn(arg string) {
if isEnabled(WarnLevel) {
log.Warn(arg)
}
emit(WarnLevel, func() { log.Warn(arg) })
}
// Infof logs a formatted message at info level.
func Infof(format string, args ...any) {
if isEnabled(InfoLevel) {
log.Infof(format, args...)
}
emit(InfoLevel, func() { log.Infof(format, args...) })
}
// Info logs a message at info level.
func Info(arg string) {
if isEnabled(InfoLevel) {
log.Info(arg)
}
emit(InfoLevel, func() { log.Info(arg) })
}
// Verbosef logs a formatted message at verbose level.
func Verbosef(format string, args ...any) {
if isEnabled(VerboseLevel) {
log.Infof(format, args...)
}
emit(VerboseLevel, func() { log.Infof(format, args...) })
}
// Verbose logs a message at verbose level.
func Verbose(arg string) {
if isEnabled(VerboseLevel) {
log.Info(arg)
}
emit(VerboseLevel, func() { log.Info(arg) })
}
// Debugf logs a formatted message at debug level with caller location.
@@ -216,7 +211,10 @@ func Debug(arg string) {
// DebugReal logs at debug level with caller info from the specified stack depth.
func DebugReal(arg string, cs int) {
if !isEnabled(DebugLevel) {
mu.RLock()
defer mu.RUnlock()
if DebugLevel < currentLevel {
return
}
@@ -275,11 +273,6 @@ func GetLevel() Level {
return currentLevel
}
// WithError returns a log entry with the error attached.
func WithError(e error) *log.Entry {
return log.Log.WithError(e)
}
// Progressf prints a progress message that overwrites the current line.
// Use ProgressDone() when progress is complete to move to the next line.
func Progressf(format string, args ...any) {
-4
View File
@@ -312,10 +312,6 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
}
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
// entry.GetPath() is safe to join here: a manifest's entry paths are
// validated against the path invariants when it is loaded (see
// deserializeInner) or built (see Builder.AddFile), so a traversal or
// absolute path can never reach this point.
absPath := filepath.Join(string(c.basePath), entry.GetPath())
relPath := RelFilePath(entry.GetPath())
-14
View File
@@ -25,7 +25,6 @@ var (
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
errInvalidFileFormat = errors.New("invalid file format")
errInvalidManifestPath = errors.New("manifest contains invalid path")
)
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
@@ -182,19 +181,6 @@ func (m *manifest) deserializeInner() error {
return errUUIDMismatch
}
// Enforce the manifest path invariants on every entry as it is loaded,
// so that no consumer of a manifest — Checker today, any restore or
// extract path tomorrow — acts on a traversal or absolute path from an
// untrusted .mf. Reject loudly on the first offender rather than
// dropping entries, which would let a hostile manifest hide files from a
// check.
for _, f := range m.pbInner.GetFiles() {
err = ValidatePath(f.GetPath())
if err != nil {
return fmt.Errorf("%w: %w", errInvalidManifestPath, err)
}
}
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
return nil
-145
View File
@@ -1,145 +0,0 @@
//nolint:testpackage // white-box tests exercise unexported internals
package mfer
import (
"bytes"
"crypto/sha256"
"fmt"
"testing"
"github.com/google/uuid"
"github.com/klauspost/compress/zstd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/proto"
)
// craftInnerBytes builds the wire bytes of an inner MFFile holding a single
// file entry whose path is exactly pathBytes. It writes the wire form by hand
// so a hostile path — including one that is not valid UTF-8 — can be embedded
// without proto.Marshal's own UTF-8 enforcement rejecting it first.
func craftInnerBytes(id uuid.UUID, pathBytes string) []byte {
entry := protowire.AppendTag(nil, 1, protowire.BytesType) // MFFilePath.path
entry = protowire.AppendString(entry, pathBytes)
inner := protowire.AppendTag(nil, 100, protowire.VarintType) // MFFile.version
inner = protowire.AppendVarint(inner, uint64(MFFile_VERSION_ONE))
inner = protowire.AppendTag(inner, 101, protowire.BytesType) // MFFile.files
inner = protowire.AppendBytes(inner, entry)
inner = protowire.AppendTag(inner, 102, protowire.BytesType) // MFFile.uuid
inner = protowire.AppendBytes(inner, id[:])
return inner
}
// wrapInner wraps inner MFFile wire bytes in a complete, well-formed .mf
// envelope (magic prefix, zstd-compressed payload, matching hash and UUID) so
// that deserialization reaches path validation rather than failing earlier on
// an integrity check.
func wrapInner(t *testing.T, id uuid.UUID, innerData []byte) []byte {
t.Helper()
var cbuf bytes.Buffer
zw, err := zstd.NewWriter(&cbuf, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
require.NoError(t, err)
_, err = zw.Write(innerData)
require.NoError(t, err)
require.NoError(t, zw.Close())
compressed := cbuf.Bytes()
sum := sha256.Sum256(compressed)
outer := &MFFileOuter{
InnerMessage: compressed,
Size: int64(len(innerData)),
Sha256: sum[:],
Uuid: id[:],
Version: MFFileOuter_VERSION_ONE,
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
}
ob, err := proto.Marshal(outer)
require.NoError(t, err)
return append([]byte(MAGIC), ob...)
}
func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
t.Parallel()
tests := []struct {
name string
path string
}{
{"parent traversal", "../escape"},
{"interior traversal", "a/../../escape"},
{"absolute path", "/etc/passwd"},
{"backslash path", `a\b`},
{"double slash", "a//b"},
{"empty path", ""},
{"invalid utf-8", "abc\xff"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
id := uuid.New()
data := wrapInner(t, id, craftInnerBytes(id, tt.path))
_, err := NewManifestFromReader(bytes.NewReader(data))
require.Error(t, err)
if tt.path == "abc\xff" {
// A path that is not valid UTF-8 cannot survive the proto3
// string decoder, which rejects it before path validation
// runs; the manifest is still refused at load time.
return
}
require.ErrorIs(t, err, errInvalidManifestPath)
if tt.path != "" {
// ValidatePath quotes the path with %q; assert against the
// same rendering so escaped characters (e.g. a backslash)
// still match.
assert.Contains(t, err.Error(), fmt.Sprintf("%q", tt.path),
"error must name the offending path")
}
})
}
}
func TestDeserializeValidManifestRoundTrips(t *testing.T) {
t.Parallel()
hash := make([]byte, 34) // multihash: 2-byte prefix + 32-byte SHA-256
b := NewBuilder()
require.NoError(t, b.AddFileWithHash("dir/file.txt", 123, ModTime{}, hash))
var buf bytes.Buffer
require.NoError(t, b.Build(&buf))
m, err := NewManifestFromReader(bytes.NewReader(buf.Bytes()))
require.NoError(t, err)
files := m.Files()
require.Len(t, files, 1)
assert.Equal(t, "dir/file.txt", files[0].GetPath())
assert.Equal(t, int64(123), files[0].GetSize())
}
// TestValidatePathRejectsInvalidUTF8 pins the ValidatePath rule that a manifest
// path must be valid UTF-8, independent of the proto decoder that also enforces
// it on the wire.
func TestValidatePathRejectsInvalidUTF8(t *testing.T) {
t.Parallel()
err := ValidatePath("abc\xff")
require.ErrorIs(t, err, errPathNotUTF8)
assert.Contains(t, err.Error(), "UTF-8")
}
+6 -1
View File
@@ -17,7 +17,12 @@ ensure_pb() {
main() {
cd "$ROOT"
ensure_pb
go test -v --timeout 10s ./...
go test -timeout 30s -race -cover ./... ||
{
echo "--- Rerunning with -v for details ---"
go test -timeout 30s -race -v ./...
exit 1
}
}
main "$@"