Harden the lint-guard shell scanner against silent evasions (closes #121 )

The guard test's shell scanner was weaker than its commit message claimed. Two holes are closed. shellCode now treats `<<` as a here-document only when it is a real redirection: outside single and double quotes, followed by a delimiter word. A `<<` inside a quoted string no longer opens a phantom here-document that swallows the rest of the file, and a here-document still open at end of file is a loud error rather than a silent truncation. assertLinterIsContainerised now cuts the joined line into the simple commands the shell would run -- on `;`, `&&`, `||` and `|` -- and requires the command that names the linter to begin with docker. So `docker info; golangci-lint run` and `docker info || golangci-lint run` are rejected, while script/lint-fix's `docker run ... golangci-lint` still passes. The scanner comment now names the inherent limits of a text scan. Dockerfile.lint's citation is corrected from `lll` to `revive`, the finding the recorded evidence actually named. Model: opus-4-8
2026-09-21 13:07:01 +00:00
9 changed files with 202 additions and 164 deletions
@@ -22,10 +22,8 @@ FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f

 ARG VERSION=dev

-# Build tooling: make, plus a C toolchain because `go test -race` needs cgo.
-# The sqlite driver is pure Go (modernc.org/sqlite), so no sqlite library or
-# CLI is required.
-RUN apk add --no-cache make build-base
+# Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests)
+RUN apk add --no-cache make build-base sqlite

 WORKDIR /src

@@ -72,11 +72,11 @@ RUN [ -n "$CHECK_EPOCH" ] || exit 1
 # running, and exits 0 reporting `0 issues.` on a tree the real config
 # fails. Demonstrated on this repo at this pin, recorded on
 # https://git.eeqj.de/sneak/vaultik/pulls/114: with a planted
-# over-length line, `script/lint` exits 1 naming the `lll` finding with
-# `linters:` and exits 0 with `linterz:`. A set-but-ineffective config
-# quietly falling back to defaults is precisely the false-green class
-# this gate exists to eliminate, so it must not sit in the gate's own
-# configuration.
+# over-length line, `script/lint` exits 1 naming the `revive` finding
+# with `linters:` and exits 0 with `linterz:`. A set-but-ineffective
+# config quietly falling back to defaults is precisely the false-green
+# class this gate exists to eliminate, so it must not sit in the gate's
+# own configuration.
 #
 # `config verify` catches it, and it does so OFFLINE at this pinned
 # version -- verified, not assumed. Under `docker run --network none`
@@ -603,6 +603,7 @@ regardless of color setting (emoji are not color).
  and the pre-commit hook both run it. A `golangci-lint` installed on
  `PATH` is not a substitute and is never used on a host, whatever its
  version.
+* `sqlite3` CLI, which the test suite shells out to
 * S3-compatible object storage (or local filesystem, or rclone remote)

 ## development workflow
@@ -633,8 +634,8 @@ standard: normalized scripts in `script/` are the entrypoints for the
 development workflow, and the Makefile targets are thin shims that call
 them. We provide:

-* `script/bootstrap` — install all development dependencies (go, Go
-  module download). It deliberately does not install `golangci-lint`;
+* `script/bootstrap` — install all development dependencies (go, sqlite3,
+  Go module download). It deliberately does not install `golangci-lint`;
  see `script/lint` below.
 * `script/setup` — make a fresh clone ready for development: runs
  `script/bootstrap`, then `script/install-precommit`
@@ -25,14 +25,6 @@ release" is exactly the contradiction

 # Completed Steps

- 2026-09-21: Made `snapshot create` VACUUM the per-snapshot metadata
-  database through the `modernc.org/sqlite` driver instead of shelling
-  out to the external `sqlite` command-line binary (issue #120). A
-  backup no longer needs that binary on `PATH`, so `make check` passes
-  on a stock `go install` host; `script/bootstrap` and the `Dockerfile`
-  test image no longer install it, and a new test asserts the uploaded
-  database keeps no pages from deleted rows. Dropped the now-false note
-  on the 2026-08-07 entry below that said bootstrap installs it.
 - 2026-09-21: Made `.gitea/workflows/check.yml` run on pushes to `main`
  and `next` and on pull requests against either, so unit PRs (whose
  base is `next`) and `next` itself get a CI run instead of relying on a
@@ -537,7 +529,7 @@ release" is exactly the contradiction
  was green was wrong.
 - 2026-08-07: Added the standard `.golangci.yml` and `.editorconfig`
  (issue #59); lint findings under the new config are tracked in issue
-  #61.
+  #61. `script/bootstrap` now installs sqlite3 (needed by tests).
 - 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
  Makefile shims, README Entrypoints section
 - 2026-07-02: Consolidated CLI verbs, retired overlapping commands; bound
@@ -1,6 +1,8 @@
 package main_test

 import (
+	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -252,29 +254,64 @@ func TestNoHostLintPathRemains(t *testing.T) {
 		}

 		name := filepath.Join("script", entry.Name())
-		for _, line := range shellCode(readRepoFile(t, name)) {
+
+		lines, err := shellCode(readRepoFile(t, name))
+		require.NoError(t, err, "scanning %s", name)
+
+		for _, line := range lines {
 			assertLinterIsContainerised(t, name, line)
 		}
 	}
 }

-// assertLinterIsContainerised fails if the line runs the linter without
-// handing it to docker first. Position matters: docker has to come
-// before the binary, or the line is running the host linter and merely
-// mentioning docker afterwards.
+// assertLinterIsContainerised fails unless every command that names the
+// linter on this joined line is a docker command. Merely mentioning
+// docker somewhere on the line is not enough; see linterRunsInDocker.
 func assertLinterIsContainerised(t *testing.T, name, line string) {
 	t.Helper()

-	at := strings.Index(line, linterBinary)
-	if at < 0 {
-		return
+	assert.True(t, linterRunsInDocker(line),
+		"%s runs %s outside a container; every command that names the"+
+			" linter must begin with docker (line: %s)", name, linterBinary,
+		line)
+}
+
+// linterRunsInDocker reports whether the linter, wherever it appears on
+// this joined shell line, is only ever the argument of a docker command.
+// The line is cut into the simple commands the shell would run -- on
+// `;`, `&&`, `||` and `|` -- and every command that names the linter
+// must begin with `docker`. This is what distinguishes the one
+// legitimate invocation, script/lint-fix's `docker run ... golangci-lint
+// run ...`, from evasions like `docker info; golangci-lint run` or
+// `docker info || golangci-lint run`, where the linter sits in a command
+// of its own that docker does not introduce.
+func linterRunsInDocker(line string) bool {
+	for _, command := range splitShellCommands(line) {
+		if !strings.Contains(command, linterBinary) {
+			continue
+		}
+
+		if !strings.HasPrefix(strings.TrimSpace(command), "docker") {
+			return false
+		}
 	}

-	docker := strings.Index(line, "docker")
+	return true
+}

-	assert.True(t, docker >= 0 && docker < at,
-		"%s runs %s on the host; every lint run happens in a container"+
-			" (line: %s)", name, linterBinary, line)
+// splitShellCommands breaks a joined shell line into the separate simple
+// commands the shell would run, cutting at the `;`, `&&`, `||` and `|`
+// operators (`||` before `|`, so the two-character operator is not split
+// twice). It is deliberately blind to quoting and to `$(...)`: no line
+// under guard puts one of these operators inside a string, and a scan
+// that tried to account for that would be the kind of half-parser this
+// file avoids.
+func splitShellCommands(line string) []string {
+	for _, op := range []string{"&&", "||", "|", ";"} {
+		line = strings.ReplaceAll(line, op, "\n")
+	}
+
+	return strings.Split(line, "\n")
 }

 // TestShellCodeSeesCodeAndNotProse keeps the scanner above honest. It
@@ -287,20 +324,70 @@ func assertLinterIsContainerised(t *testing.T, name, line string) {
 func TestShellCodeSeesCodeAndNotProse(t *testing.T) {
 	t.Parallel()

+	// A `<<` inside quotes is not a here-document, so the code after it
+	// is still scanned; a real `<<EOF` opens one and its body is dropped.
 	script := strings.Join([]string{
 		"#!/bin/sh",
 		"# a comment naming golangci-lint",
 		"cat >&2 <<EOF",
 		"prose naming golangci-lint, printed not executed",
 		"EOF",
+		`echo "a left shift << is not a here-document"`,
 		"docker run --rm \\",
 		"    \"$image\" \\",
 		"    golangci-lint run ./...",
 	}, "\n")

+	lines, err := shellCode(script)
+	require.NoError(t, err)
 	assert.Equal(t,
-		[]string{"cat >&2 <<EOF", `docker run --rm "$image" golangci-lint run ./...`},
-		shellCode(script))
+		[]string{
+			"cat >&2 <<EOF",
+			`echo "a left shift << is not a here-document"`,
+			`docker run --rm "$image" golangci-lint run ./...`,
+		},
+		lines)
+
+	// A here-document still open at end of file must be a loud error,
+	// not a silent truncation of everything the scanner has yet to see.
+	unterminated := strings.Join([]string{
+		"cat <<EOF",
+		"body line naming golangci-lint, no terminator follows",
+	}, "\n")
+
+	_, err = shellCode(unterminated)
+	require.Error(t, err)
+}
+
+// TestLinterCommandMustBeginWithDocker pins the property that a mention
+// of docker somewhere on the line is not enough: the command that
+// actually runs the linter has to be a docker command. The two evasions
+// from the issue place the linter in a command of its own, joined to a
+// harmless docker command by `;` or `||`; both must be rejected. The
+// containerised invocation script/lint-fix writes -- docker run with the
+// linter as its argument -- must still be accepted.
+func TestLinterCommandMustBeginWithDocker(t *testing.T) {
+	t.Parallel()
+
+	rejected := []string{
+		"docker info >/dev/null; golangci-lint run ./...",
+		"docker info || golangci-lint run ./...",
+		"docker build . && golangci-lint run ./... | tee log",
+	}
+	for _, line := range rejected {
+		assert.False(t, linterRunsInDocker(line),
+			"a linter command docker does not introduce must be rejected: %s",
+			line)
+	}
+
+	accepted := []string{
+		`docker run --rm "$image" golangci-lint run ./...`,
+		`docker run --rm --user x --volume "$ROOT:/src" img golangci-lint run --fix ./...`,
+	}
+	for _, line := range accepted {
+		assert.True(t, linterRunsInDocker(line),
+			"a docker-introduced linter command must be accepted: %s", line)
+	}
 }

 // assertEpochExpandedInto fails unless some instruction runs the named
@@ -410,7 +497,9 @@ func indexContaining(found []string, want string) int {
 // shellCode returns a POSIX shell script's executable lines: comments
 // dropped, here-document bodies dropped, and backslash continuations
 // joined so a multi-line command is a single string. Whitespace is
-// collapsed, as it is for Dockerfile instructions.
+// collapsed, as it is for Dockerfile instructions. A here-document left
+// open at end of file is an error rather than a silent truncation of
+// everything after its opener.
 //
 // Both exclusions are load-bearing rather than tidiness. The scripts
 // name golangci-lint in prose to state that the host binary is never
@@ -418,7 +507,11 @@ func indexContaining(found []string, want string) int {
 // container invocation -- script/lint-fix's `docker run`, whose linter
 // command sits several lines below the word `docker` -- be recognised
 // as containerised.
-func shellCode(contents string) []string {
+//
+// This is a text scan, not a shell: it cannot see a linter name
+// assembled at runtime, one split across a continuation, a script in a
+// subdirectory of script/, or anything in the Makefile.
+func shellCode(contents string) ([]string, error) {
 	var (
 		out       []string
 		joined    string
@@ -452,23 +545,82 @@ func shellCode(contents string) []string {
 		joined = ""
 	}

-	return out
-}
-
-// heredocTerminator returns the terminator of the here-document a
-// command opens, or "" if it opens none. Only the first on a line is
-// recognised; nothing in script/ opens two.
-func heredocTerminator(line string) string {
-	_, after, opens := strings.Cut(line, "<<")
-	if !opens {
-		return ""
+	if terminate != "" {
+		return nil, fmt.Errorf("%w: terminator %q", errUnterminatedHeredoc,
+			terminate)
 	}

-	// `<<-` strips leading tabs from the body; the terminator word is
-	// the same either way, and callers compare against trimmed lines.
-	word, _, _ := strings.Cut(strings.TrimPrefix(after, "-"), " ")
+	return out, nil
+}

-	return strings.Trim(word, `'"`)
+// errUnterminatedHeredoc is what shellCode returns when a here-document
+// is still open at end of file. Its callers require its absence, so an
+// unterminated body -- which would otherwise be swallowed silently --
+// fails the guard loudly.
+var errUnterminatedHeredoc = errors.New(
+	"here-document opened but never closed before end of file")
+
+// heredocTerminator returns the delimiter word of the here-document the
+// command opens, or "" if it opens none. A `<<` only opens one when it
+// is a real redirection: outside single and double quotes, and followed
+// by a delimiter word. A `<<` inside a quoted string, or an arithmetic
+// left shift like `$((x << 2))`, is not a here-document; the former is
+// the case this guards, the latter appears in no script here. Only the
+// first opener on a line is recognised; nothing in script/ opens two.
+func heredocTerminator(line string) string {
+	var quote byte // 0 when outside quotes, else '\'' or '"'
+
+	for i := 0; i+1 < len(line); i++ {
+		c := line[i]
+
+		switch {
+		case quote != 0:
+			if c == quote {
+				quote = 0
+			}
+		case c == '\'' || c == '"':
+			quote = c
+		case c == '<' && line[i+1] == '<':
+			return heredocWord(line[i+2:])
+		}
+	}
+
+	return ""
+}
+
+// heredocWord extracts the delimiter that follows `<<` or `<<-`: it drops
+// an optional `-`, skips blanks, then reads the delimiter -- quoted or
+// bare -- and returns it with quotes removed. `<<-'EOF'` and `<< EOF`
+// both yield "EOF". It returns "" when no word follows, so a bare `<<`
+// opens nothing.
+func heredocWord(after string) string {
+	after = strings.TrimLeft(strings.TrimPrefix(after, "-"), " \t")
+
+	var (
+		word  strings.Builder
+		quote byte
+	)
+
+	for i := range len(after) {
+		c := after[i]
+
+		switch {
+		case quote != 0:
+			if c == quote {
+				quote = 0
+			} else {
+				word.WriteByte(c)
+			}
+		case c == '\'' || c == '"':
+			quote = c
+		case c == ' ' || c == '\t':
+			return word.String()
+		default:
+			word.WriteByte(c)
+		}
+	}
+
+	return word.String()
 }

 // readRepoFile reads a file by its path relative to the repository
@@ -192,7 +192,7 @@ Tracks blob upload metrics.
 After a snapshot is completed:
 1. Copy database to temporary file
 2. Clean temporary database to contain only current snapshot data
-3. VACUUM the trimmed database so deleted rows leave no pages behind
+3. Export to SQL dump using sqlite3
 4. Compress with zstd and encrypt with age
 5. Upload to S3 as `metadata/{snapshot-id}/db.zst.age`
 6. Generate blob manifest and upload as `metadata/{snapshot-id}/manifest.json.zst`
@@ -44,6 +44,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -668,31 +669,14 @@ func (sm *SnapshotManager) collectCleanupStats(

 // vacuumDatabase runs VACUUM on the database to remove deleted data and compact
 // This is critical for security - ensures no stale/deleted data pages are uploaded
-//
-// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
-// connection with no transaction in flight (VACUUM cannot run inside one).
-// The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
-// checkpoint on Close flushes it into the main file, which is the file we
-// then compress and upload.
 func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
 	log.Debug("Running VACUUM on database", "path", dbPath)
+	//nolint:gosec // G204: fixed argv; dbPath is our own temp file path
+	cmd := exec.CommandContext(ctx, "sqlite3", dbPath, "VACUUM;")

-	db, err := database.New(ctx, dbPath)
+	output, err := cmd.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("opening database for VACUUM: %w", err)
-	}
-
-	defer func() {
-		cerr := db.Close()
-		if cerr != nil {
-			log.Debug("Failed to close database after VACUUM",
-				"path", dbPath, "error", cerr)
-		}
-	}()
-
-	_, err = db.ExecWithLog(ctx, "VACUUM")
-	if err != nil {
-		return fmt.Errorf("running VACUUM: %w", err)
+		return fmt.Errorf("running VACUUM: %w (output: %s)", err, string(output))
 	}

 	return nil
@@ -2,7 +2,6 @@
 package snapshot

 import (
-	"bytes"
 	"context"
 	"database/sql"
 	"io"
@@ -97,97 +96,6 @@ func verifyCleanedDB(
 	}
 }

-// TestVacuumDatabaseRemovesDeletedData proves the export path uploads a
-// compacted database: after rows carrying a recognizable marker are deleted
-// and vacuumDatabase runs, no page holding that marker survives in the file
-// on disk (the file compressFile later reads for upload).
-func TestVacuumDatabaseRemovesDeletedData(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	ctx := context.Background()
-	fs := afero.NewOsFs()
-
-	tempDir := t.TempDir()
-	dbPath := filepath.Join(tempDir, "snapshot.db")
-
-	db, err := database.New(ctx, dbPath)
-	if err != nil {
-		t.Fatalf("failed to create database: %v", err)
-	}
-
-	// A marker distinctive enough that its presence in the raw file can only
-	// come from the rows inserted below.
-	marker := []byte("VACUUM_PROBE_DEADBEEF_DELETED_ROW")
-	payload := bytes.Repeat(marker, 128) // ~4 KiB per row
-
-	_, err = db.Conn().ExecContext(ctx,
-		"CREATE TABLE vacuum_probe (id INTEGER PRIMARY KEY, payload BLOB)")
-	if err != nil {
-		t.Fatalf("failed to create probe table: %v", err)
-	}
-
-	for range 512 {
-		_, err = db.Conn().ExecContext(ctx,
-			"INSERT INTO vacuum_probe (payload) VALUES (?)", payload)
-		if err != nil {
-			t.Fatalf("failed to insert probe row: %v", err)
-		}
-	}
-
-	_, err = db.Conn().ExecContext(ctx, "DELETE FROM vacuum_probe")
-	if err != nil {
-		t.Fatalf("failed to delete probe rows: %v", err)
-	}
-
-	// Close so the deletes reach the main file, mirroring the state
-	// prepareExportDB hands to vacuumDatabase.
-	err = db.Close()
-	if err != nil {
-		t.Fatalf("failed to close database: %v", err)
-	}
-
-	beforeInfo, err := fs.Stat(dbPath)
-	if err != nil {
-		t.Fatalf("failed to stat database before vacuum: %v", err)
-	}
-
-	beforeBytes, err := afero.ReadFile(fs, dbPath)
-	if err != nil {
-		t.Fatalf("failed to read database before vacuum: %v", err)
-	}
-
-	if !bytes.Contains(beforeBytes, marker) {
-		t.Fatalf("expected deleted-row data to linger before vacuum")
-	}
-
-	sm := &SnapshotManager{fs: fs}
-
-	err = sm.vacuumDatabase(ctx, dbPath)
-	if err != nil {
-		t.Fatalf("vacuumDatabase failed: %v", err)
-	}
-
-	afterBytes, err := afero.ReadFile(fs, dbPath)
-	if err != nil {
-		t.Fatalf("failed to read database after vacuum: %v", err)
-	}
-
-	if bytes.Contains(afterBytes, marker) {
-		t.Fatalf("deleted-row data survived vacuum in the uploaded file")
-	}
-
-	afterInfo, err := fs.Stat(dbPath)
-	if err != nil {
-		t.Fatalf("failed to stat database after vacuum: %v", err)
-	}
-
-	if afterInfo.Size() >= beforeInfo.Size() {
-		t.Fatalf("expected vacuum to shrink the file: before=%d after=%d",
-			beforeInfo.Size(), afterInfo.Size())
-	}
-}
-
 func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
 	// Initialize logger
 	log.Initialize(log.Config{})
@@ -114,6 +114,9 @@ main() {
    # from CI. Nothing on the host is ever used as a linter, at any
    # version, so installing one here would buy nothing.

+    # sqlite3 CLI: the test suite shells out to it (VACUUM).
+    if missing sqlite3; then pkg_install sqlite sqlite3 sqlite sqlite; fi
+
    # goreleaser, at the version pinned by script/install-goreleaser and
    # verified against a hardcoded sha256. Package managers are not used
    # for it: they ship whatever version they happen to carry, and the