bring the repo up to org standards (closes #1) #7

Open
clawbot wants to merge 1 commits from clawbot/dcf:issue-1-repo-standards into master
32 changed files with 1622 additions and 152 deletions

20
.dockerignore Normal file
View File

@@ -0,0 +1,20 @@
# Never exclude Go sources, go.mod, go.sum or .golangci.yml: the lint
# and test stages only ever examine what reaches the build context, and
# an exclusion here makes them pass over a tree that is missing a
# package. script/assert-context-complete exists to catch exactly that,
# and will fail the build rather than let it happen silently.
.git/
.gitea/
README.md
LICENSE
.editorconfig
.prettierrc
.prettierignore
.env
.env.*
.DS_Store
.idea/
.vscode/
.claude/
tmp/
temp/

15
.editorconfig Normal file
View File

@@ -0,0 +1,15 @@
root = true
[*]
indent_style = space
indent_size = 4
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[*.go]
indent_style = tab
[Makefile]
indent_style = tab

View File

@@ -0,0 +1,18 @@
name: check
on:
push:
branches:
- "**"
jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
# The image build is the gate: it runs gofmt, config verify,
# lint and test, and script/cibuild refuses a build in which
# any of them was cached or skipped.
- name: cibuild
run: script/cibuild

29
.gitignore vendored
View File

@@ -1 +1,28 @@
dcfinfo # Build artifacts
/bin/
*.test
*.out
*.log
# Environment / secrets
.env
.env.*
*.key
*.pem
# OS
.DS_Store
Thumbs.db
# Editor and assistant tooling directories. These are per-developer
# state and never belong in the repository.
*~
*.swp
*.swo
.idea/
.vscode/
.claude/
.cursor/
.aider*
.continue/
.windsurf/

34
.golangci.yml Normal file
View File

@@ -0,0 +1,34 @@
version: "2"
# Config schema uses the golangci-lint v2 layout (settings live under
# linters.settings, not top-level linters-settings) so that the
# thresholds below are actually applied by golangci-lint >= v2.
run:
timeout: 5m
modules-download-mode: readonly
linters:
default: all
disable:
# Genuinely incompatible with project patterns
- exhaustruct # Requires all struct fields
- depguard # Dependency allow/block lists
- godot # Requires comments to end with periods
- wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go
settings:
lll:
line-length: 88
funlen:
lines: 80
statements: 50
cyclop:
max-complexity: 15
dupl:
threshold: 100
issues:
max-issues-per-linter: 0
max-same-issues: 0

6
.prettierignore Normal file
View File

@@ -0,0 +1,6 @@
go.sum
LICENSE
# The vendored golangci-lint config is canonical and must stay
# byte-identical to sneak/prompts. Formatting it here would fork it.
.golangci.yml

4
.prettierrc Normal file
View File

@@ -0,0 +1,4 @@
{
"tabWidth": 4,
"proseWrap": "always"
}

83
Dockerfile Normal file
View File

@@ -0,0 +1,83 @@
# This repo is a library: there is no cmd/ and no binary to ship, so the
# build ends at the stage that compiles and tests the packages. There is
# no runtime stage because there is nothing to run.
# Lint stage — fast feedback on formatting and lint issues. Tools are
# invoked directly (not via make/script): the docker build is its own
# single path.
# This stage must stay the one that runs golangci-lint, and its name must
# match $lint_stage in script/cibuild, which cache-busts it by name.
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
WORKDIR /src
# Copy go mod files first for better layer caching
COPY go.mod go.sum ./
RUN go mod download
# Copy source code
COPY . .
# Inventory of the sources that actually arrived here. script/cibuild
# reads these lines out of the build log and compares them against the
# git index, so a .dockerignore entry or a narrowed COPY that hides a
# package fails the run instead of yielding a clean report over a tree
# the linter never saw. Keep it immediately after `COPY . .`, and keep
# `echo context-manifest-begin` as its first command:
# script/assert-context-complete matches the step by that prefix.
RUN echo context-manifest-begin; \
{ find . -type f -name '*.go'; \
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
if [ -f "$f" ]; then echo "./$f"; fi; \
done; } \
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
echo context-manifest-end
# Formatting check, config check, linter
RUN test -z "$(gofmt -s -l .)" || { echo "gofmt needed on:"; gofmt -s -l .; exit 1; }
RUN golangci-lint config verify --config .golangci.yml
RUN golangci-lint run --config .golangci.yml ./...
# Build stage, and the last one: script/cibuild passes no --target, so
# BuildKit builds whichever stage is last and appending one drops lint,
# builder and their checks out of the run. Must stay the stage that runs
# go test, and its name must match $test_stage in script/cibuild, which
# cache-busts it by name.
# golang:1.25.7-bookworm (Debian-based: the race detector used by the
# test run requires glibc), 2026-08-30
FROM golang:1.25.7-bookworm@sha256:564e366a28ad1d70f460a2b97d1d299a562f08707eb0ecb24b659e5bd6c108e1 AS builder
# Depend on lint stage passing (forces BuildKit ordering)
COPY --from=lint /src/go.sum /dev/null
WORKDIR /build
# Copy go mod files first for better layer caching
COPY go.mod go.sum ./
RUN go mod download
# Copy source code
COPY . .
# Same inventory as the lint stage, and separately checked: this stage
# has its own COPY, so an intact context over there is no evidence about
# the tree `go test ./...` is about to walk here. A package that did not
# arrive is a package the tests never run, and the run still ends `ok`.
RUN echo context-manifest-begin; \
{ find . -type f -name '*.go'; \
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
if [ -f "$f" ]; then echo "./$f"; fi; \
done; } \
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
echo context-manifest-end
# Run tests: quiet first, verbose rerun on failure (and still fail).
# -count=1 disables the test result cache, matching script/test.
RUN go test -count=1 -timeout 90s -race -cover ./... || \
{ echo "--- Rerunning with -v for details ---"; \
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
# No binary to emit, so the compile check is the artifact: it proves every
# package builds for the target the library claims to support.
RUN CGO_ENABLED=0 go build -trimpath ./...

35
Dockerfile.lint Normal file
View File

@@ -0,0 +1,35 @@
# Lint image, built by script/lint: golangci-lint runs as a build step, so
# a successful build is a clean lint. Works with a remote docker daemon,
# where bind mounts are impossible.
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
# This stage must stay the last one and the one that runs golangci-lint,
# and its name must match $stage in script/lint.
FROM deps AS lint
COPY . .
# Inventory of the sources that actually arrived here. script/lint reads
# these lines out of the build log and compares them against the git
# index, so a .dockerignore entry or a narrowed COPY that hides a
# package fails the run instead of yielding a clean report over a tree
# the linter never saw. Keep it immediately after `COPY . .`, and keep
# `echo context-manifest-begin` as its first command:
# script/assert-context-complete matches the step by that prefix.
RUN echo context-manifest-begin; \
{ find . -type f -name '*.go'; \
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
if [ -f "$f" ]; then echo "./$f"; fi; \
done; } \
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
echo context-manifest-end
RUN golangci-lint config verify --config .golangci.yml
RUN golangci-lint run --config .golangci.yml ./...

View File

@@ -1,14 +1,50 @@
default: test .PHONY: bootstrap setup test lint fmt fmt-check check build deps docker cibuild clean hooks
# Every target is a thin shim over script/. The scripts are the real
# entrypoints: they carry the project-specific knowledge (flags,
# timeouts, docker-only linting) that a raw `go test` or `golangci-lint`
# invocation silently bypasses. Never call the underlying tools
# directly.
.DEFAULT_GOAL := check
bootstrap:
@script/bootstrap
setup:
@script/setup
test: test:
go test -v ./... @script/test
run: build lint:
DEBUG=1 ./dcfinfo @script/lint
fmt:
@script/fmt
fmt-check:
@script/fmt-check
check:
@script/check
# This repo is a library: there is no cmd/ and nothing to link, so
# `build` is the compile check over every package.
build: build:
go env -w CGO_ENABLED="0" go build ./...
cd cmd/dcfinfo && go build -o ../../dcfinfo
deps:
go mod download
go mod tidy
docker:
@script/docker
cibuild:
@script/cibuild
clean: clean:
rm -f dcfinfo go clean ./...
hooks:
@script/install-precommit

151
README.md
View File

@@ -1,33 +1,158 @@
# dcf # dcf
Golang reader implementation of the so-called "Design rule for Camera File `dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that
system" or DCF, aka JEITA (Japan Electronics and Information Technology reads the so-called "Design rule for Camera File system", or DCF, aka JEITA
Industries Association) specification number CP-3461. (Japan Electronics and Information Technology Industries Association)
specification number CP-3461.
[wikipedia.org/wiki/Design_rule_for_Camera_File_system](https://en.wikipedia.org/wiki/Design_rule_for_Camera_File_system) [wikipedia.org/wiki/Design_rule_for_Camera_File_system](https://en.wikipedia.org/wiki/Design_rule_for_Camera_File_system)
The DCF specification is why your digital camera puts images and videos in The DCF specification is why your digital camera puts images and videos in
`DCIM` and `PRIVATE/M4ROOT` directories on the memory card. `DCIM` and `PRIVATE/M4ROOT` directories on the memory card.
# status ## Status
incomplete, under development, does not work yet incomplete, under development, does not work yet
# why ## Getting started
```sh
go get git.eeqj.de/sneak/dcf
```
```go
package main
import (
"fmt"
"git.eeqj.de/sneak/dcf/pkg/dcf"
)
func main() {
stores, err := dcf.GetDCFStores(0)
if err != nil {
panic(err)
}
for _, store := range *stores {
fmt.Println(store.String())
}
}
```
To work on the library itself:
```sh
git clone git@git.eeqj.de:sneak/dcf.git
cd dcf
script/setup # dependencies plus the git pre-commit hook
make check # test, lint, fmt-check
```
## Entrypoints
This repo adheres to the
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
standard. Each `script/` entrypoint has a thin `make` shim; the scripts are
where the project's knowledge about flags, timeouts and docker-only linting
lives, so use them rather than invoking `go` or `golangci-lint` directly.
- `script/bootstrap` (`make bootstrap`) — install build dependencies (git, make,
go) idempotently. Linting additionally needs docker; markdown formatting needs
docker or a local `prettier`.
- `script/setup` (`make setup`) — `bootstrap` plus the git pre-commit hook.
- `script/test` (`make test`) — `go test` with the race detector and coverage,
`-count=1`, 90s timeout; quiet on success, verbose rerun on failure.
- `script/lint` (`make lint`) — `golangci-lint` via docker only, against the
digest-pinned image in `Dockerfile.lint`, then `script/assert-step-ran` and
`script/assert-context-complete` over the build log. Nothing is installed
locally and nothing runs on the host.
- `script/fmt` (`make fmt`) — format Go with `gofmt` and everything else with
`prettier` (writes).
- `script/fmt-check` (`make fmt-check`) — the same scope, read-only.
- `script/check` (`make check`) — `test` + `lint` + `fmt-check`. What the
pre-commit hook runs. Never modifies files.
- `script/docker` (`make docker`) — build the image, tagged with
`script/projectname`.
- `script/cibuild` (`make cibuild`) — the CI gate:
`docker build --progress=plain --no-cache-filter=lint --no-cache-filter=builder .`
followed by assertions that the lint step and the test step really ran, and
that both stages really received the whole repository. The Gitea workflow runs
this on every push.
- `script/precommit` — the hook body: `go mod tidy` must not change
`go.mod`/`go.sum`, then `check`.
- `script/install-precommit` (`make hooks`) — installs the hook.
- `script/projectname` — prints the project name; other scripts call it so they
can stay identical across repos.
- `script/prettier`, `script/assert-step-ran`, `script/assert-context-complete`
and `script/repo-source-manifest` are helpers, not entrypoints.
`make build`, `make deps` and `make clean` are the ordinary conveniences.
`make build` compiles every package; this repo is a library and ships no binary,
so there is no `cmd/` and nothing to link.
### Why the lint assertions exist
A green `docker build` is not evidence that the checks ran. On an unchanged tree
every layer comes from cache and the build exits 0 having executed nothing;
BuildKit silently ignores a `--no-cache-filter` naming a stage that no longer
exists; and a `.dockerignore` entry can remove a package from the build context,
after which the linter genuinely runs, genuinely examines what it was handed,
and genuinely reports `0 issues.` over a repository with a violation in it.
`script/assert-step-ran` and `script/assert-context-complete` close both holes;
read the comments in them before changing either.
## Rationale
I wanted to copy images off my memory cards for processing and like I wanted to copy images off my memory cards for processing and like
overengineering and reusable code. overengineering and reusable code.
# bugs ## Design
Everything lives in `pkg/dcf`, a single library package with no command
entrypoints.
- `dcf.go` — the public surface. `Store` is one DCF store (one mounted
filesystem); `Object` is one file in it, embedded in `Image` and `Video`.
`GetDCFStores` finds the stores on the system and walks each one, classifying
files by extension.
- `helpers.go` — mountpoint discovery. `findAllMountPoints` enumerates the
system's physical filesystems via `gopsutil`; `findDCFMountPoints` keeps only
those whose root holds a marker directory.
Detection is by directory name at the filesystem root, so no filesystem is ever
mounted and nothing outside a filesystem root is searched.
## Bugs
does not yet handle "dcf file groups" where multiple objects share the same does not yet handle "dcf file groups" where multiple objects share the same
index number. if you need this, send me a link to a zip or tar of a memory index number. if you need this, send me a link to a zip or tar of a memory card
card that uses it and i'll see what i can do. that uses it and i'll see what i can do.
# author ## TODO
The work in flight is on the tracker; this is the reading order for picking it
up.
- [#2](https://git.eeqj.de/sneak/dcf/issues/2) — parse the CP-3461 structure
rather than grepping every filesystem root for media extensions.
- [#3](https://git.eeqj.de/sneak/dcf/issues/3) — DCF file groups: objects
sharing an index number are one logical object.
- [#4](https://git.eeqj.de/sneak/dcf/issues/4) — API cleanup: `fs.FS`-based
store access, no pointer-to-slice returns, contexts on walks. This is also
what makes the filesystem walk testable; today it is reachable only through
`GetDCFStores`, which needs real mounted media, so it has no test coverage.
- [#5](https://git.eeqj.de/sneak/dcf/issues/5) — checksummed import off the
card, which is what the library is for.
- [#6](https://git.eeqj.de/sneak/dcf/issues/6) — mountpoint detection checks
`M4ROOT` at the filesystem root rather than `PRIVATE/M4ROOT`, and
`findDCFMountPoints` stops one store short of `requestedCount`.
## License
WTFPL. See [LICENSE](LICENSE).
## Author
sneak <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)> sneak <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)>
# license
WTFPL

3
go.mod
View File

@@ -1,6 +1,6 @@
module git.eeqj.de/sneak/dcf module git.eeqj.de/sneak/dcf
go 1.22.1 go 1.25.7
require ( require (
github.com/dustin/go-humanize v1.0.1 github.com/dustin/go-humanize v1.0.1
@@ -9,7 +9,6 @@ require (
require ( require (
github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect
github.com/lmittmann/tint v1.0.4 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/sys v0.20.0 // indirect golang.org/x/sys v0.20.0 // indirect
) )

2
go.sum
View File

@@ -3,8 +3,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=

View File

@@ -1,30 +1,34 @@
// Package dcf reads DCF stores: the directory structures described by
// the "Design rule for Camera File system" (JEITA CP-3461) that digital
// cameras write to the root of removable media.
package dcf package dcf
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/hex"
"fmt" "fmt"
"io" "io"
"io/fs"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
) )
var imageExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw"} // Store represents a DCF store, which is a directory structure that
var videoExtensions = []string{".mp4", ".mov", ".avi", ".mkv", ".wmv"} // contains images and videos in the root of a mounted filesystem.
type Store struct {
// DCFStore represents a DCF store, which is a directory sturcture that contains
// images and videos in the root of a mounted filesystem.
type DCFStore struct {
RootDirectory string RootDirectory string
Images []*Image Images []*Image
Videos []*Video Videos []*Video
} }
// DCFObject represents a file in a DCF store. It is embedded in Image and Video types. // Object represents a file in a DCF store. It is embedded in the Image
type DCFObject struct { // and Video types.
DCFStoreRoot string type Object struct {
StoreRoot string
Path string Path string
Size int64 Size int64
Extension string Extension string
@@ -32,146 +36,224 @@ type DCFObject struct {
// Image represents an image file on a DCF store. // Image represents an image file on a DCF store.
type Image struct { type Image struct {
DCFObject Object
} }
// Video represents a video file on a DCF store. // Video represents a video file on a DCF store.
type Video struct { type Video struct {
DCFObject Object
} }
func (d *DCFObject) FullFilePath() string { // FullFilePath returns the path of the object on the local filesystem,
return filepath.Join(d.DCFStoreRoot, d.Path) // which is its store-relative path resolved against the store root.
func (d *Object) FullFilePath() string {
return filepath.Join(d.StoreRoot, d.Path)
} }
// VideosCount returns the number of videos in the DCF store. // VideosCount returns the number of videos in the DCF store.
func (d *DCFStore) VideosCount() int { func (d *Store) VideosCount() int {
return len(d.Videos) return len(d.Videos)
} }
// ImagesCount returns the number of images in the DCF store. // ImagesCount returns the number of images in the DCF store.
func (d *DCFStore) ImagesCount() int { func (d *Store) ImagesCount() int {
return len(d.Images) return len(d.Images)
} }
func (d *DCFStore) TotalImageSize() int64 { // TotalImageSize returns the summed size in bytes of the store's images.
func (d *Store) TotalImageSize() int64 {
totalSize := int64(0) totalSize := int64(0)
for _, image := range d.Images { for _, image := range d.Images {
totalSize += image.Size totalSize += image.Size
} }
return totalSize return totalSize
} }
func (d *DCFStore) TotalVideoSize() int64 { // TotalVideoSize returns the summed size in bytes of the store's videos.
func (d *Store) TotalVideoSize() int64 {
totalSize := int64(0) totalSize := int64(0)
for _, video := range d.Videos { for _, video := range d.Videos {
totalSize += video.Size totalSize += video.Size
} }
return totalSize return totalSize
} }
func (d *DCFStore) TotalSize() int64 { // TotalSize returns the summed size in bytes of every object in the
// store.
func (d *Store) TotalSize() int64 {
return d.TotalImageSize() + d.TotalVideoSize() return d.TotalImageSize() + d.TotalVideoSize()
} }
func (d *DCFStore) String() string { // String renders the store as a single human-readable summary line.
return fmt.Sprintf("DCFStore{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}", func (d *Store) String() string {
return fmt.Sprintf(
"Store{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}",
d.RootDirectory, d.RootDirectory,
d.ImagesCount(), d.ImagesCount(),
d.VideosCount(), d.VideosCount(),
humanize.Bytes(uint64(d.TotalSize())), humanize.Bytes(d.totalSizeBytes()),
) )
} }
// totalSizeBytes returns TotalSize as an unsigned byte count for
// formatting. Sizes come from the filesystem and are never negative,
// but the conversion is guarded rather than assumed.
func (d *Store) totalSizeBytes() uint64 {
total := d.TotalSize()
if total < 0 {
return 0
}
return uint64(total)
}
// String renders the video as a single human-readable line.
func (v *Video) String() string { func (v *Video) String() string {
return fmt.Sprintf("Video{Path: %s, Size: %d, Extension: %s}", v.Path, v.Size, v.Extension) return fmt.Sprintf(
"Video{Path: %s, Size: %d, Extension: %s}",
v.Path, v.Size, v.Extension,
)
} }
// String renders the image as a single human-readable line.
func (i *Image) String() string { func (i *Image) String() string {
return fmt.Sprintf("Image{Path: %s, Size: %d, Extension: %s}", i.Path, i.Size, i.Extension) return fmt.Sprintf(
"Image{Path: %s, Size: %d, Extension: %s}",
i.Path, i.Size, i.Extension,
)
} }
// Hash() returns the SHA256 hash of the file as a hex string. // Hash returns the SHA-256 digest of the object's file as a hex string.
func (d *DCFObject) Hash() (string, error) { func (d *Object) Hash() (string, error) {
return pathToSHA256(d.FullFilePath()) return pathToSHA256(d.FullFilePath())
} }
// pathToSHA256 returns the SHA-256 digest of the file at path as a hex
// string.
func pathToSHA256(path string) (string, error) { func pathToSHA256(path string) (string, error) {
file, err := os.Open(path) // The path is supplied by the caller by design: this library exists
// to hash files the caller pointed it at.
file, err := os.Open(filepath.Clean(path))
if err != nil { if err != nil {
return "", err return "", err
} }
defer file.Close()
defer func() { _ = file.Close() }()
hash := sha256.New() hash := sha256.New()
if _, err := io.Copy(hash, file); err != nil {
_, err = io.Copy(hash, file)
if err != nil {
return "", err return "", err
} }
return fmt.Sprintf("%x", hash.Sum(nil)), nil
return hex.EncodeToString(hash.Sum(nil)), nil
} }
// GetDCFStores returns a list of DCF stores found on the system in the root of any mounted filesystems. // GetDCFStores returns the DCF stores found in the root directories of
// the integer argument is the number of DCF stores to return. If the argument is 0, all DCF stores are returned. // the system's mounted filesystems. requestedCount bounds how many
// It does not mount filesystems or search outside of filesystem root directories. It does, however, // stores are returned; 0 returns all of them.
// walk the filesystems to find images and videos and populate the returned DCFStores. //
func GetDCFStores(requestedCount int) (*[]DCFStore, error) { // It neither mounts filesystems nor searches outside filesystem root
dcfStorePaths, err := findDCFMountPoints(requestedCount) // directories. It does walk each store it finds, populating the
// returned stores with the images and videos they contain.
func GetDCFStores(requestedCount int) (*[]Store, error) {
storePaths, err := findDCFMountPoints(requestedCount)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores := []DCFStore{}
for _, dcfStorePath := range dcfStorePaths { stores := []Store{}
dcfStore := DCFStore{
RootDirectory: dcfStorePath, for _, storePath := range storePaths {
store, err := scanStore(storePath)
if err != nil {
return nil, err
}
stores = append(stores, *store)
}
return &stores, nil
}
// scanStore walks root and returns a Store holding every image and
// video beneath it.
func scanStore(root string) (*Store, error) {
store := Store{
RootDirectory: root,
Images: make([]*Image, 0), Images: make([]*Image, 0),
Videos: make([]*Video, 0), Videos: make([]*Video, 0),
} }
err := filepath.Walk(dcfStorePath, func(path string, info os.FileInfo, err error) error { walk := func(path string, entry fs.DirEntry, err error) error {
if err != nil { if err != nil {
return err return err
} }
if !info.IsDir() && isImageFile(path) {
image := Image{} if entry.IsDir() {
image.Path = strings.TrimPrefix(path, dcfStorePath+"/")
image.Size = info.Size()
image.Extension = filepath.Ext(path)
image.DCFStoreRoot = dcfStorePath
dcfStore.Images = append(dcfStore.Images, &image)
}
if !info.IsDir() && isVideoFile(path) {
video := Video{}
video.Path = strings.TrimPrefix(path, dcfStorePath+"/")
video.Size = info.Size()
video.Extension = filepath.Ext(path)
video.DCFStoreRoot = dcfStorePath
dcfStore.Videos = append(dcfStore.Videos, &video)
}
return nil return nil
}) }
return store.addFile(root, path, entry)
}
err := filepath.WalkDir(root, walk)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dcfStores = append(dcfStores, dcfStore)
} return &store, nil
return &dcfStores, nil
} }
// addFile classifies path by extension and appends it to the store's
// images or videos. A file that is neither is ignored.
func (d *Store) addFile(root, path string, entry fs.DirEntry) error {
isImage := isImageFile(path)
if !isImage && !isVideoFile(path) {
return nil
}
info, err := entry.Info()
if err != nil {
return err
}
object := Object{
StoreRoot: root,
Path: strings.TrimPrefix(path, root+string(filepath.Separator)),
Size: info.Size(),
Extension: filepath.Ext(path),
}
if isImage {
d.Images = append(d.Images, &Image{Object: object})
return nil
}
d.Videos = append(d.Videos, &Video{Object: object})
return nil
}
// isImageFile reports whether path names a file this package treats as
// an image.
func isImageFile(path string) bool { func isImageFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path)) return hasExtension(path,
for _, imageExt := range imageExtensions { ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw")
if ext == imageExt {
return true
}
}
return false
} }
// isVideoFile reports whether path names a file this package treats as
// a video.
func isVideoFile(path string) bool { func isVideoFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path)) return hasExtension(path, ".mp4", ".mov", ".avi", ".mkv", ".wmv")
for _, videoExt := range videoExtensions { }
if ext == videoExt {
return true // hasExtension reports whether path's extension, lowercased, is one of
} // the given extensions. Extensions are given with their leading dot.
} func hasExtension(path string, extensions ...string) bool {
return false return slices.Contains(extensions, strings.ToLower(filepath.Ext(path)))
} }

View File

@@ -1,10 +1,177 @@
package dcf package dcf_test
import ( import (
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing" "testing"
"git.eeqj.de/sneak/dcf/pkg/dcf"
) )
func TestCompile(t *testing.T) { // newStore builds a store with two images and one video of known sizes,
// This is a placeholder test to ensure that the module compiles successfully. // so the count and size accessors have something deterministic to
// You can add more meaningful tests here once you start implementing your code. // report.
func newStore(root string) *dcf.Store {
return &dcf.Store{
RootDirectory: root,
Images: []*dcf.Image{
{Object: dcf.Object{
StoreRoot: root,
Path: "DCIM/100MSDCF/DSC00001.ARW",
Size: 2000,
Extension: ".ARW",
}},
{Object: dcf.Object{
StoreRoot: root,
Path: "DCIM/100MSDCF/DSC00002.JPG",
Size: 500,
Extension: ".JPG",
}},
},
Videos: []*dcf.Video{
{Object: dcf.Object{
StoreRoot: root,
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
Size: 9000,
Extension: ".MP4",
}},
},
}
}
func TestObjectFullFilePath(t *testing.T) {
t.Parallel()
object := dcf.Object{
StoreRoot: filepath.Join("/mnt", "card"),
Path: filepath.Join("DCIM", "100MSDCF", "DSC00001.ARW"),
}
want := filepath.Join("/mnt", "card", "DCIM", "100MSDCF", "DSC00001.ARW")
if got := object.FullFilePath(); got != want {
t.Fatalf("FullFilePath() = %q, want %q", got, want)
}
}
func TestStoreCounts(t *testing.T) {
t.Parallel()
store := newStore("/mnt/card")
if got := store.ImagesCount(); got != 2 {
t.Errorf("ImagesCount() = %d, want 2", got)
}
if got := store.VideosCount(); got != 1 {
t.Errorf("VideosCount() = %d, want 1", got)
}
}
func TestStoreSizes(t *testing.T) {
t.Parallel()
store := newStore("/mnt/card")
if got := store.TotalImageSize(); got != 2500 {
t.Errorf("TotalImageSize() = %d, want 2500", got)
}
if got := store.TotalVideoSize(); got != 9000 {
t.Errorf("TotalVideoSize() = %d, want 9000", got)
}
if got := store.TotalSize(); got != 11500 {
t.Errorf("TotalSize() = %d, want 11500", got)
}
}
func TestEmptyStoreSizesAreZero(t *testing.T) {
t.Parallel()
store := dcf.Store{RootDirectory: "/mnt/empty"}
if got := store.TotalSize(); got != 0 {
t.Fatalf("TotalSize() = %d, want 0", got)
}
}
func TestStoreStringReportsHumanizedTotal(t *testing.T) {
t.Parallel()
got := newStore("/mnt/card").String()
for _, want := range []string{"/mnt/card", "Images: 2", "Videos: 1", "12 kB"} {
if !strings.Contains(got, want) {
t.Errorf("String() = %q, want it to contain %q", got, want)
}
}
}
func TestImageAndVideoString(t *testing.T) {
t.Parallel()
image := dcf.Image{Object: dcf.Object{
Path: "DCIM/100MSDCF/DSC00001.ARW",
Size: 2000,
Extension: ".ARW",
}}
want := "Image{Path: DCIM/100MSDCF/DSC00001.ARW, Size: 2000, " +
"Extension: .ARW}"
if got := image.String(); got != want {
t.Errorf("Image.String() = %q, want %q", got, want)
}
video := dcf.Video{Object: dcf.Object{
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
Size: 9000,
Extension: ".MP4",
}}
want = "Video{Path: PRIVATE/M4ROOT/CLIP/C0001.MP4, Size: 9000, " +
"Extension: .MP4}"
if got := video.String(); got != want {
t.Errorf("Video.String() = %q, want %q", got, want)
}
}
func TestObjectHashMatchesSHA256OfFile(t *testing.T) {
t.Parallel()
root := t.TempDir()
contents := []byte("DSC00001.ARW contents")
err := os.MkdirAll(filepath.Join(root, "DCIM"), 0o750)
if err != nil {
t.Fatalf("MkdirAll: %v", err)
}
err = os.WriteFile(filepath.Join(root, "DCIM", "a.arw"), contents, 0o600)
if err != nil {
t.Fatalf("WriteFile: %v", err)
}
object := dcf.Object{StoreRoot: root, Path: filepath.Join("DCIM", "a.arw")}
got, err := object.Hash()
if err != nil {
t.Fatalf("Hash() returned error: %v", err)
}
sum := sha256.Sum256(contents)
if want := hex.EncodeToString(sum[:]); got != want {
t.Fatalf("Hash() = %q, want %q", got, want)
}
}
func TestObjectHashOnMissingFileReturnsError(t *testing.T) {
t.Parallel()
object := dcf.Object{StoreRoot: t.TempDir(), Path: "absent.jpg"}
_, err := object.Hash()
if err == nil {
t.Fatal("Hash() on a missing file returned no error")
}
} }

View File

@@ -1,75 +1,84 @@
package dcf package dcf
import ( import (
"fmt"
"log/slog" "log/slog"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "slices"
"github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/disk"
) )
// dcfMarkerDirectories are the directory names whose presence in a
// filesystem root identifies that filesystem as a DCF store.
func dcfMarkerDirectories() []string {
return []string{"DCIM", "M4ROOT"}
}
// findAllMountPoints returns the distinct mountpoints of the system's
// physical filesystems.
func findAllMountPoints() ([]string, error) { func findAllMountPoints() ([]string, error) {
mountpoints := []string{} partitions, err := disk.Partitions(false) // physical devices only
partitions, err := disk.Partitions(false) // physical devices only, so false
if err != nil { if err != nil {
return nil, err return nil, err
} }
mountpoints := []string{}
for _, partition := range partitions { for _, partition := range partitions {
if !contains(mountpoints, partition.Mountpoint) { if !slices.Contains(mountpoints, partition.Mountpoint) {
mountpoints = append(mountpoints, partition.Mountpoint) mountpoints = append(mountpoints, partition.Mountpoint)
} }
} }
return mountpoints, nil return mountpoints, nil
} }
func dump(x interface{}) { // findDCFMountPoints returns the paths of the mountpoints that contain
_, file, line, _ := runtime.Caller(1) // one of the marker directories a DCF store is identified by. Its only
slog.Debug(fmt.Sprintf("%s:%d %#v\n", file, line, x)) // argument is how many mountpoints to return; 0 returns all of them.
}
// findDCFMountPoints returns a list of strings of the paths of mountpoints that
// contain a DCIM or PRIVATE directory, which is how we detect DCF stores. Its
// only argument is an integer of how many mountpoints to return. If the
// argument is 0, all mountpoints are returned.
func findDCFMountPoints(requestedCount int) ([]string, error) { func findDCFMountPoints(requestedCount int) ([]string, error) {
filteredMountpoints := []string{}
var erro error
mountpoints, err := findAllMountPoints() mountpoints, err := findAllMountPoints()
if err != nil {
if erro != nil {
return nil, err return nil, err
} }
filteredMountpoints := []string{}
for _, mountpoint := range mountpoints { for _, mountpoint := range mountpoints {
dcimPath := filepath.Join(mountpoint, "DCIM") if !isDCFMountPoint(mountpoint) {
privatePath := filepath.Join(mountpoint, "PRIVATE") continue
privatePath = filepath.Join(mountpoint, "M4ROOT")
shouldKeep := false
if _, err := os.Stat(dcimPath); err == nil {
shouldKeep = true
slog.Debug(fmt.Sprintf("Found DCIM directory at %s\n", dcimPath))
} }
if _, err := os.Stat(privatePath); err == nil {
shouldKeep = true slog.Debug("keeping mountpoint", "mountpoint", mountpoint)
slog.Debug(fmt.Sprintf("Found M4ROOT directory at %s\n", privatePath))
}
if shouldKeep {
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
filteredMountpoints = append(filteredMountpoints, mountpoint) filteredMountpoints = append(filteredMountpoints, mountpoint)
if (requestedCount > 0) && (len(filteredMountpoints)+1 >= requestedCount) {
if requestedCount > 0 && len(filteredMountpoints)+1 >= requestedCount {
break break
} }
} }
}
return filteredMountpoints, nil return filteredMountpoints, nil
} }
func contains(slice []string, str string) bool { // isDCFMountPoint reports whether mountpoint holds any of the marker
for _, s := range slice { // directories a DCF store is identified by.
if s == str { func isDCFMountPoint(mountpoint string) bool {
return true found := false
for _, marker := range dcfMarkerDirectories() {
markerPath := filepath.Join(mountpoint, marker)
_, err := os.Stat(markerPath)
if err != nil {
continue
} }
slog.Debug("found marker directory", "path", markerPath)
found = true
} }
return false
return found
} }

166
script/assert-context-complete Executable file
View File

@@ -0,0 +1,166 @@
#!/bin/sh
# script/assert-context-complete: fail unless the build context a stage
# was given contained every source file the repository has.
#
# Usage: script/assert-context-complete LOG STAGE EXPECTED
#
# LOG must be `docker build --progress=plain` output. EXPECTED is a list
# of repo-relative paths, one per line, as script/repo-source-manifest
# prints it.
#
# script/assert-step-ran answers "did the tool execute". This answers
# the other half: "was the tool given the tree". They are different
# questions, and a green build can satisfy the first while failing the
# second — a .dockerignore entry, or a COPY that brings in less than the
# whole tree, removes files from the context silently, and golangci-lint
# then genuinely runs, genuinely examines what it was handed, and
# genuinely reports `0 issues.` over a repo that has a violation in it.
# `go test ./...` likewise never runs a package that did not arrive.
#
# How it knows: each source-consuming stage emits, right after its
# `COPY . .`, an inventory of the Go files actually present —
#
# context-manifest-begin
# context-file: cmd/gotemplate/main.go
# ...
# context-manifest-end
#
# — and this compares that against EXPECTED, which comes from the git
# index rather than from the same build. The two sources are
# independent: .dockerignore decides the first and cannot touch the
# second. An expectation read from the evidence would prove nothing.
#
# The manifest lines are only believed under the same discipline
# script/assert-step-ran applies to a tool's success line: they must be
# attributed to the id of a step whose header is "#ID [STAGE n/m] CMD"
# with CMD matching the manifest command, which BuildKit reported DONE,
# and both sentinels must be present. So another step's output does not
# count, a header does not count, a cached step (which writes nothing)
# does not count, and a truncated log fails rather than passing with a
# short list.
#
# Only one direction is checked: everything the repo has must have
# arrived. Files in the context that git does not track are not a
# failure — untracked local work is normal and hides nothing.
#
# What this does not cover: a Dockerfile edit to the manifest step
# itself, and evidence forged inside a matched command. Both are visible
# in the Dockerfile in plain sight, and both are equally outside
# script/assert-step-ran. This is not an exhaustive list of ways a green
# run can be untrue; it is the ones known.
set -eu
die() {
echo "script/assert-context-complete: $*" >&2
exit 1
}
[ $# -eq 3 ] || die "usage: $0 LOG STAGE EXPECTED"
[ -r "$1" ] || die "cannot read $1"
[ -r "$3" ] || die "cannot read $3"
# The stage and the expected-list path travel in the environment, not in
# -v: awk expands escape sequences in a -v assignment.
ACC_STAGE="$2" ACC_EXPECTED="$3" awk '
function fail(msg) {
printf "script/assert-context-complete: %s\n", msg | "cat 1>&2"
close("cat 1>&2")
exit 1
}
# The package a missing file belongs to — its directory, which is what a
# reader needs named. A bare "." for a root-level file reads as a typo
# next to the sentence punctuation, so it is spelled out.
function pkgof(path, i) {
i = length(path)
while (i > 0 && substr(path, i, 1) != "/") i--
return i == 0 ? "the repository root" : substr(path, 1, i - 1)
}
function preview(list, n, limit, i, out) {
for (i = 1; i <= n && i <= limit; i++)
out = out (i > 1 ? ", " : "") list[i]
if (n > limit)
out = out sprintf(", and %d more", n - limit)
return out
}
BEGIN {
stage = ENVIRON["ACC_STAGE"]
# Fixed by the protocol the Dockerfile emits, so callers cannot
# drift from it.
step = "^RUN echo context-manifest-begin"
beginmark = "context-manifest-begin"
endmark = "context-manifest-end"
prefix = "context-file: "
expected_path = ENVIRON["ACC_EXPECTED"]
while ((getline line < expected_path) > 0)
if (line != "") expected[++nexpected] = line
close(expected_path)
if (nexpected == 0)
fail(sprintf("the expected-file list (%s) is empty, so any build context would satisfy this check", expected_path))
}
# Every progress line is "#ID " and then a step header, a status, or one
# line the step itself wrote.
/^#[0-9]+ / {
id = substr($1, 2)
rest = substr($0, length($1) + 2)
if (rest == "CACHED") { cached[id] = 1; next }
if (rest ~ /^DONE /) { done[id] = 1; next }
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
# build names a platform. Bracketed spans with no n/m are BuildKit
# internals, not steps.
if (substr(rest, 1, 1) == "[") {
p = index(rest, "] ")
if (p == 0) next
n = split(substr(rest, 2, p - 2), part, " ")
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
if (part[n - 1] != stage) next
if (substr(rest, p + 2) ~ step) matched[id] = 1
next
}
# Output: "#ID 0.31 <the line the step wrote>".
if (rest ~ /^[0-9]+[.][0-9]+ /) {
sub(/^[0-9]+[.][0-9]+ /, "", rest)
if (rest == beginmark) begun[id] = 1
else if (rest == endmark) ended[id] = 1
else if (index(rest, prefix) == 1)
arrived[id SUBSEP substr(rest, length(prefix) + 1)] = 1
}
}
END {
for (id in matched) {
nmatched++
if (cached[id]) ncached++
if (done[id] && begun[id] && ended[id]) chosen = id
}
if (nmatched == 0)
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so the build context was never inventoried. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or the manifest step was taken out of it", step, stage))
if (chosen == "" && ncached == nmatched)
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so its inventory describes an earlier tree rather than this one; the cache bust for that stage is not taking effect", step, stage))
if (chosen == "")
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but wrote no complete inventory between %s and %s. The log is truncated, the build was not run with --progress=plain, or that step no longer emits the manifest", step, stage, beginmark, endmark))
for (i = 1; i <= nexpected; i++) {
if (arrived[chosen SUBSEP expected[i]]) continue
missing[++nmissing] = expected[i]
pkg = pkgof(expected[i])
if (!(pkg in seenpkg)) { seenpkg[pkg] = 1; pkgs[++npkgs] = pkg }
}
if (nmissing == 0) exit 0
fail(sprintf("%d of the %d source files this repository tracks never reached the build context of stage \"%s\", so nothing examined them. Missing package(s): %s. Missing file(s): %s. A .dockerignore entry, or a COPY that brings in less than the whole tree, drops files silently: the tool still runs, still finds nothing wrong with what it was handed, and still reports success", nmissing, nexpected, stage, preview(pkgs, npkgs, 10), preview(missing, nmissing, 10)))
}
' "$1"

125
script/assert-step-ran Executable file
View File

@@ -0,0 +1,125 @@
#!/bin/sh
# script/assert-step-ran: fail unless a docker build actually executed a
# named step, proved by output the step's own tool wrote.
#
# Usage: script/assert-step-ran LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT
#
# LOG must be `docker build --progress=plain` output. The default tty
# renderer rewrites lines in place and keeps only the tail of a step's
# output, so the flag is load-bearing wherever this is used.
#
# It passes only if some step in LOG satisfies all three of:
#
# 1. its header is "#ID [STAGE n/m] CMD" with CMD matching STEP_REGEX,
# 2. one of the lines that step wrote matches EVIDENCE_REGEX,
# 3. BuildKit reported "#ID DONE".
#
# This observes the build that happened; it parses no Dockerfile. A
# stage renamed, moved, made unreachable, deleted, split by a whitespace
# byte BuildKit treats as a separator, or an instruction rewritten —
# none of it can make this pass a build in which the step did not run,
# because a step that did not run wrote no output, and a step served
# from cache writes none either ("#ID CACHED", no output lines).
#
# The stage name in a header is BuildKit's label for that vertex, not a
# reading of the file being built. A vertex shared with an earlier build
# of a DIFFERENT Dockerfile is replayed under the label it was first
# recorded with, so a `Dockerfile.lint` run can print `[lint 1/8]` for
# the main `Dockerfile`'s lint stage. A borrowed label cannot produce a
# pass — a replayed vertex prints CACHED and writes no evidence, and a
# vertex that really executes really ran the command in the header — but
# it does mean a header alone proves nothing about which stages the file
# declares. Hence one failure message naming both causes rather than a
# split that would claim to know which.
#
# What this guard cannot see — not an exhaustive list of ways a green can
# be untrue, only the ones known:
#
# - Evidence forged inside the matched step, e.g.
# `RUN golangci-lint run ... || echo "0 issues."`. Condition 1 ties
# the evidence to a step whose own command is in the log, so the
# forgery has to be written into that command in the Dockerfile, in
# plain sight.
# This guards against a step falling silently out of the build. It does
# not certify that what ran examined everything it should have — a
# `.dockerignore` entry excluding a package makes the linter genuinely
# run and genuinely print `0 issues.` while a real violation sits
# unexamined in the repo. That is a different question, asked separately
# by script/assert-context-complete, which every caller of this script
# also runs.
#
# What it depends on — BuildKit's plain progress format and the tool's
# own success wording — fails the caller loudly if it drifts, because
# drift removes a match rather than creating one.
set -eu
die() {
echo "script/assert-step-ran: $*" >&2
exit 1
}
[ $# -eq 5 ] || die "usage: $0 LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT"
[ -r "$1" ] || die "cannot read $1"
# The regexes travel in the environment, not in -v: awk expands escape
# sequences in a -v assignment, which would eat a backslash before the
# regex ever sees it.
ASR_STAGE="$2" ASR_STEP="$3" ASR_EVIDENCE="$4" ASR_WHAT="$5" awk '
function fail(msg) {
printf "script/assert-step-ran: %s\n", msg | "cat 1>&2"
close("cat 1>&2")
exit 1
}
BEGIN {
stage = ENVIRON["ASR_STAGE"]
step = ENVIRON["ASR_STEP"]
evidence = ENVIRON["ASR_EVIDENCE"]
what = ENVIRON["ASR_WHAT"]
}
# Every progress line is "#ID " and then a step header, a status, or one
# line the step itself wrote.
/^#[0-9]+ / {
id = substr($1, 2)
rest = substr($0, length($1) + 2)
if (rest == "CACHED") { cached[id] = 1; next }
if (rest ~ /^DONE /) { done[id] = 1; next }
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
# build names a platform. Bracketed spans with no n/m are BuildKit
# internals ("[internal] load build context"), not steps.
if (substr(rest, 1, 1) == "[") {
p = index(rest, "] ")
if (p == 0) next
n = split(substr(rest, 2, p - 2), part, " ")
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
if (part[n - 1] != stage) next
if (substr(rest, p + 2) ~ step) matched[id] = 1
next
}
# Output: "#ID 41.80 <the line the step wrote>".
if (rest ~ /^[0-9]+[.][0-9]+ /) {
sub(/^[0-9]+[.][0-9]+ /, "", rest)
if (rest ~ evidence) emitted[id] = 1
}
}
END {
for (id in matched) {
nmatched++
if (done[id] && emitted[id]) exit 0
if (cached[id]) ncached++
}
if (nmatched == 0)
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so %s did not run. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or it was built without that command among its steps", step, stage, what))
if (ncached == nmatched)
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so %s did not run on this tree; the cache bust for that stage is not taking effect", step, stage, what))
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but never wrote a line matching /%s/, so there is no evidence %s did the work; the command or the tool that produces that line has changed", step, stage, evidence, what))
}
' "$1"

84
script/bootstrap Executable file
View File

@@ -0,0 +1,84 @@
#!/bin/sh
# script/bootstrap: install all dependencies needed to build and develop
# this repo. Idempotent: every install is guarded by a check so already
# installed tools are skipped. Base tooling comes from nix, apt, brew,
# or apk (detected in that order); assumes NOTHING is present (not git,
# make, or go). The linter is NOT installed locally: golangci-lint runs
# via docker only (script/lint), pinned by image digest, so the only
# lint prerequisite is a working docker.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
PKGMGR=""
SUDO=""
detect_pkgmgr() {
[ -n "$PKGMGR" ] && return 0
if command -v nix-env >/dev/null 2>&1; then
PKGMGR="nix"
elif command -v apt-get >/dev/null 2>&1; then
PKGMGR="apt"
elif command -v brew >/dev/null 2>&1; then
PKGMGR="brew"
elif command -v apk >/dev/null 2>&1; then
PKGMGR="apk"
else
echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
exit 1
fi
if [ "$PKGMGR" = "apt" ]; then
export DEBIAN_FRONTEND=noninteractive
if [ "$(id -u)" != "0" ]; then
SUDO="sudo"
fi
fi
}
# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
pkg_install() {
detect_pkgmgr
case "$PKGMGR" in
nix) nix-env -iA "nixpkgs.$1" ;;
apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;;
brew) brew install "$3" ;;
apk) apk add --no-cache "$4" ;;
esac
}
missing() {
! command -v "$1" >/dev/null 2>&1
}
main() {
cd "$ROOT"
# Base tooling
if missing git; then pkg_install git git git git; fi
if missing make; then pkg_install gnumake make make make; fi
# Go toolchain
if missing go; then pkg_install go golang go go; fi
# Linting runs via docker only (script/lint). Warn, don't fail:
# everything except `make lint` works without it.
if missing docker; then
echo "bootstrap: WARNING: docker not found; make lint and" >&2
echo "bootstrap: make docker require it. Install docker to" >&2
echo "bootstrap: run the linter." >&2
fi
# Markdown/CSS/JS formatting uses prettier. script/prettier falls
# back to a pinned docker image when it is not on PATH, so this is
# a convenience, not a requirement.
if missing prettier && missing docker; then
echo "bootstrap: WARNING: neither prettier nor docker found;" >&2
echo "bootstrap: markdown formatting will be skipped." >&2
fi
go mod download
echo "bootstrap complete"
}
main "$@"

15
script/check Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/sh
# script/check: run all checks (test, lint, fmt-check). Our own
# extension to scripts-to-rule-them-all. Must not modify any files.
# Generic: usually needs no adaptation.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
main() {
"$SCRIPT_DIR/test"
"$SCRIPT_DIR/lint"
"$SCRIPT_DIR/fmt-check"
}
main "$@"

89
script/cibuild Executable file
View File

@@ -0,0 +1,89 @@
#!/bin/sh
# script/cibuild: run the CI build. The Dockerfile runs the checks
# (gofmt, config verify, lint, test), so a successful build implies a
# green repo. The Gitea workflow runs this on push.
#
# Only if the checks actually ran, though, and a green `docker build` is
# no evidence that they did:
#
# 1. On an unchanged tree every layer comes from cache and the build
# exits 0 in under a second having executed nothing. Hence the two
# --no-cache-filter flags.
#
# 2. BuildKit silently ignores a --no-cache-filter naming a stage that
# does not exist, so a rename restores that green no-op.
#
# 3. BuildKit builds only the final stage's dependency graph. A stage
# nothing references is never built, and with no --target the final
# stage is whichever is last in the file — so appending a stage, or
# moving a reference into a stage that is itself unreachable, drops
# the checks out of the build while every reference to them is still
# there to read.
#
# Rather than predict any of that from the Dockerfile's text, this asks
# the finished build what it did: script/assert-step-ran requires that
# the log contain a lint step and a test step that ran, and that each
# wrote the line its own tool writes on success. Every trap above ends
# with the step absent from the log or served from cache, and both fail
# that assertion. See that script for what it does not cover.
#
# 4. A step that ran is not a step that saw the repo. .dockerignore, or
# a COPY narrower than the tree, removes files from the build
# context, and the linter then reports `0 issues.` over what is left
# while `go test ./...` never compiles the package that went missing.
# So each source-consuming stage inventories what reached it, and
# script/assert-context-complete compares that inventory against the
# git index — a source .dockerignore cannot reach.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
dockerfile=Dockerfile
lint_stage=lint
test_stage=builder
# `ok <pkg> 1.234s`: go test prints `(cached)` in place of the duration
# when it replays a result, so requiring the duration requires a package
# that was really exercised.
test_ran='^ok[[:space:]]+[^[:space:]]+[[:space:]]+[0-9]+[.][0-9]+s'
main() {
cd "$ROOT"
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-cibuild.XXXXXX")"
trap 'rm -rf "$tmp"' EXIT INT TERM
# --progress=plain is load-bearing: the assertions below read the
# steps' own output out of this log, and the tty renderer discards
# all but the tail of it. The exit status travels through a file
# because a pipeline's status is the last command's; `set +e` is
# what lets the recording line run at all, since errexit would
# otherwise abandon the subshell on a failing build. A missing file
# reads as failure.
(
set +e
docker build \
--progress=plain \
--no-cache-filter="$lint_stage" \
--no-cache-filter="$test_stage" \
-f "$dockerfile" . 2>&1
echo "$?" >"$tmp/status"
) | tee "$tmp/build.log"
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
script/assert-step-ran "$tmp/build.log" "$lint_stage" \
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
script/assert-step-ran "$tmp/build.log" "$test_stage" \
'^RUN go test' "$test_ran" 'the tests'
# Both stages, separately: each has its own COPY, so an intact
# context in one is no evidence about the other.
script/repo-source-manifest >"$tmp/expected"
script/assert-context-complete "$tmp/build.log" "$lint_stage" "$tmp/expected"
script/assert-context-complete "$tmp/build.log" "$test_stage" "$tmp/expected"
}
main "$@"

15
script/docker Executable file
View File

@@ -0,0 +1,15 @@
#!/bin/sh
# script/docker: build the Docker image tagged with the project name.
# Identical in all repos; the tag comes from script/projectname.
# Generic: needs no adaptation.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
docker build -t "$("$SCRIPT_DIR/projectname")" .
}
main "$@"

26
script/fmt Executable file
View File

@@ -0,0 +1,26 @@
#!/bin/sh
# script/fmt: format all files (writes). Go via gofmt, everything else
# (markdown, CSS, JS, YAML, JSON) via prettier with the repo's
# .prettierrc. Never hand-rolled substitutions: this script is the only
# sanctioned way to reformat the tree.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
gofmt -s -w .
if command -v goimports >/dev/null 2>&1; then
goimports -w .
fi
# A missing runner (exit 2) is a warning here: script/prettier has
# already said so on stderr, and failing `make fmt` over it would
# block work that has nothing to do with markdown.
"$SCRIPT_DIR/prettier" --write . || [ $? -eq 2 ]
}
main "$@"

25
script/fmt-check Executable file
View File

@@ -0,0 +1,25 @@
#!/bin/sh
# script/fmt-check: check formatting (read-only). Same scope as
# script/fmt, but fails instead of writing.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
if [ -n "$(gofmt -s -l .)" ]; then
echo "gofmt needed on:"
gofmt -s -l .
exit 1
fi
# Exit 2 means no prettier and no docker: reported by
# script/prettier on stderr and not treated as a failure, so this
# check still runs on a machine that has neither. The container
# build is the gate that always has both.
"$SCRIPT_DIR/prettier" --check . || [ $? -eq 2 ]
}
main "$@"

16
script/install-precommit Executable file
View File

@@ -0,0 +1,16 @@
#!/bin/sh
# script/install-precommit: install the git pre-commit hook that runs
# script/precommit. Our own extension to scripts-to-rule-them-all.
# Generic: needs no adaptation.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
chmod +x .git/hooks/pre-commit
echo "pre-commit hook installed: runs script/precommit"
}
main "$@"

71
script/lint Executable file
View File

@@ -0,0 +1,71 @@
#!/bin/sh
# script/lint: run the linter. golangci-lint is never installed locally: it
# runs via docker only, one way, everywhere.
#
# Traps, each of which yields a green run over an unlinted or partly linted
# tree:
#
# 1. A bare `docker build -f Dockerfile.lint .` serves the lint layer from
# cache on an unchanged tree and exits 0 having linted nothing, and
# BuildKit ignores a --no-cache-filter whose stage name matches
# nothing, restoring that no-op after a rename. So the run is not
# trusted: script/assert-step-ran requires the log to show the lint
# step executing and golangci-lint's own success line coming out of
# it. A cached or absent step fails that.
#
# 2. .dockerignore decides what reaches the container, and only what
# reaches it is linted, so excluding a Go file drops it from the lint
# with the linter still reporting `0 issues.` over what it was
# handed. So the context is not trusted either: the lint stage
# inventories the sources that reached it, and
# script/assert-context-complete compares that against the git index,
# which .dockerignore cannot touch. Never exclude Go sources,
# go.mod/go.sum or .golangci.yml — and now nothing silently does.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
dockerfile=Dockerfile.lint
# Must match the stage name in Dockerfile.lint.
stage=lint
main() {
cd "$ROOT"
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-lint.XXXXXX")"
trap 'rm -rf "$tmp"' EXIT INT TERM
# No --target: whatever stage is last is the one BuildKit builds, and
# the assertion below is what decides whether the linter was in it.
# --progress=plain is load-bearing — the tty renderer discards all but
# the tail of a step's output. The exit status travels through a file
# because a pipeline's status is the last command's; `set +e` is what
# lets the recording line run at all, since errexit would otherwise
# abandon the subshell on a failing build. A missing file reads as
# failure.
(
set +e
docker build \
--progress=plain \
--no-cache-filter="$stage" \
--output=type=cacheonly \
-f "$dockerfile" . 2>&1
echo "$?" >"$tmp/status"
) | tee "$tmp/build.log"
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
script/assert-step-ran "$tmp/build.log" "$stage" \
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
# That the linter ran says nothing about what it was given. The
# expectation comes from git and the evidence from the build, which
# is the only reason comparing them means anything.
script/repo-source-manifest >"$tmp/expected"
script/assert-context-complete "$tmp/build.log" "$stage" "$tmp/expected"
}
main "$@"

21
script/precommit Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/sh
# script/precommit: run by the git pre-commit hook; fails the commit if
# checks fail. Our own extension to scripts-to-rule-them-all. Go repo
# extras: go mod tidy must not change go.mod/go.sum.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
main() {
cd "$ROOT"
go mod tidy
git diff --exit-code -- go.mod go.sum || {
echo "precommit: go mod tidy changed go.mod/go.sum;" \
"stage the changes and retry" >&2
exit 1
}
"$SCRIPT_DIR/check"
}
main "$@"

52
script/prettier Executable file
View File

@@ -0,0 +1,52 @@
#!/bin/sh
# script/prettier: run prettier over this repo with the repo's own
# .prettierrc. Helper, not an entrypoint: script/fmt and
# script/fmt-check call it.
#
# Prettier from PATH if it is there, otherwise a hash-pinned node image
# via docker, so a checkout with neither node nor a global prettier
# still formats identically. If neither is available the caller is told
# and the markdown pass is skipped rather than silently passing — a
# formatter that is not present cannot certify formatting.
#
# Exit status: prettier's own, 2 when it was skipped for lack of a
# runner. script/fmt-check treats 2 as a warning, not a failure, so
# that `make check` still works on a machine without docker.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# node:24-alpine, 2026-08-22. Bumping the pin is a deliberate edit.
NODE_IMAGE="node:24-alpine@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43"
# Pinned exactly: prettier's default formatting changes between minor
# releases, so an unpinned version turns fmt-check into a coin flip.
PRETTIER_VERSION="3.9.6"
main() {
cd "$ROOT"
if command -v prettier >/dev/null 2>&1; then
prettier "$@"
return $?
fi
if command -v docker >/dev/null 2>&1; then
# --user keeps written files owned by the invoking user rather
# than root; the container only ever touches the bind mount.
docker run --rm \
--user "$(id -u):$(id -g)" \
-v "$ROOT:/work" \
-w /work \
"$NODE_IMAGE" \
npx --yes "prettier@$PRETTIER_VERSION" "$@"
return $?
fi
echo "script/prettier: neither prettier nor docker found;" >&2
echo "script/prettier: skipping markdown/CSS/JS formatting." >&2
return 2
}
main "$@"

12
script/projectname Executable file
View File

@@ -0,0 +1,12 @@
#!/bin/sh
# script/projectname: output the name of this project. Our own
# extension to scripts-to-rule-them-all. Other scripts that need the
# name (e.g. script/docker) call this, so they can stay identical
# across all repos.
set -eu
main() {
echo "dcf"
}
main "$@"

60
script/repo-source-manifest Executable file
View File

@@ -0,0 +1,60 @@
#!/bin/sh
# script/repo-source-manifest: print every file this repository contains
# whose absence from a docker build context would go unnoticed — one
# repo-relative path per line, LC_ALL=C-sorted.
#
# That is the Go sources. A missing go.mod, go.sum or .golangci.yml
# fails the build loudly (the COPY errors, or golangci-lint refuses to
# start), so they cannot hide anything; they are listed anyway because
# it costs two lines and makes the manifest the linter's whole input
# rather than most of it.
#
# The list comes from the git index, never from a walk of the working
# tree, and that is the point: script/assert-context-complete compares
# it against the inventory the build itself emitted, and a check that
# reads its expectation from the same place it reads its evidence proves
# nothing. .dockerignore governs what docker sends; it cannot touch what
# git tracks.
#
# Tracked files only, and only those present in the worktree — a file
# staged for deletion is not something the build context is missing.
# Untracked files are not expected either, so local scratch work in the
# tree is not a failure.
#
# A path containing a newline or a quote is quoted by git and will not
# match the plain path the build emits, so it fails loudly rather than
# passing silently. No such path exists here, and none should.
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
die() {
echo "script/repo-source-manifest: $*" >&2
exit 1
}
main() {
cd "$ROOT"
git rev-parse --is-inside-work-tree >/dev/null 2>&1 ||
die "not a git work tree, so there is nothing to compare a build context against"
# Captured before the loop so that a git failure is the script's
# exit status: in a pipeline only the last command's status counts.
tracked="$(git ls-files -- '*.go' go.mod go.sum .golangci.yml .golangci.yaml)"
manifest="$(
printf '%s\n' "$tracked" | while IFS= read -r f; do
if [ -n "$f" ] && [ -f "$f" ]; then
printf '%s\n' "$f"
fi
done | LC_ALL=C sort
)"
[ -n "$manifest" ] ||
die "the git index lists no Go sources, so any build context would satisfy the check"
printf '%s\n' "$manifest"
}
main "$@"

14
script/setup Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/sh
# script/setup: set up the repo for development after a fresh clone:
# installs dependencies (script/bootstrap) and the git pre-commit hook.
# Add any repo-specific initialization (db init, .env template) here.
set -eu
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
main() {
"$SCRIPT_DIR/bootstrap"
"$SCRIPT_DIR/install-precommit"
}
main "$@"

21
script/test Executable file
View File

@@ -0,0 +1,21 @@
#!/bin/sh
# script/test: run the test suite. Quiet on success; on failure, rerun
# with -v for full diagnostic output (and still fail — the first run
# already proved the tests are broken).
set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
main() {
cd "$ROOT"
# -count=1 disables the test result cache. Without it a repeat run on
# an unchanged tree reports every package `ok ... (cached)` and exits
# 0 having executed nothing — a gate that cannot fail.
go test -count=1 -timeout 90s -race -cover ./... || {
echo "--- Rerunning with -v for details ---"
go test -count=1 -timeout 90s -race -v ./...
exit 1
}
}
main "$@"