bring the repo up to org standards
Adopt the standard tooling: a Makefile of thin shims over a full scripts-to-rule-them-all `script/` set, the canonical `.golangci.yml` vendored byte-identical from `sneak/prompts`, docker-only linting via `Dockerfile.lint`, a `Dockerfile` and Gitea workflow that gate every push, and prettier/editorconfig/dockerignore config. `make build` pointed at a `cmd/dcfinfo` that is not in the tree and could never have succeeded; this repo is a library, so `build` is now the compile check over every package. Clear the 57 findings the canonical linter config reports on `pkg/dcf`. Two were real: `findDCFMountPoints` checked a never-assigned `erro` instead of the error from `findAllMountPoints`, discarding it, and `privatePath` was computed twice so the first computation was dead. Mountpoint selection otherwise behaves exactly as before; the defects that survive are filed as issue #6, not fixed here. Rename `DCFStore` to `Store` and `DCFObject` to `Object` (with its `DCFStoreRoot` field to `StoreRoot`), which revive's stutter rule requires and which the `fs.FS` rework in issue #4 will build on. Replace the placeholder test with tests over the exported surface. The filesystem walk stays uncovered: it is reachable only through `GetDCFStores`, which needs real mounted media. README keeps its content, reorganised into the required sections and gaining Entrypoints. (closes #1)
This commit is contained in:
20
.dockerignore
Normal file
20
.dockerignore
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
# Never exclude Go sources, go.mod, go.sum or .golangci.yml: the lint
|
||||||
|
# and test stages only ever examine what reaches the build context, and
|
||||||
|
# an exclusion here makes them pass over a tree that is missing a
|
||||||
|
# package. script/assert-context-complete exists to catch exactly that,
|
||||||
|
# and will fail the build rather than let it happen silently.
|
||||||
|
.git/
|
||||||
|
.gitea/
|
||||||
|
README.md
|
||||||
|
LICENSE
|
||||||
|
.editorconfig
|
||||||
|
.prettierrc
|
||||||
|
.prettierignore
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
.DS_Store
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
.claude/
|
||||||
|
tmp/
|
||||||
|
temp/
|
||||||
15
.editorconfig
Normal file
15
.editorconfig
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 4
|
||||||
|
end_of_line = lf
|
||||||
|
charset = utf-8
|
||||||
|
trim_trailing_whitespace = true
|
||||||
|
insert_final_newline = true
|
||||||
|
|
||||||
|
[*.go]
|
||||||
|
indent_style = tab
|
||||||
|
|
||||||
|
[Makefile]
|
||||||
|
indent_style = tab
|
||||||
18
.gitea/workflows/check.yml
Normal file
18
.gitea/workflows/check.yml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
name: check
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- "**"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||||
|
# The image build is the gate: it runs gofmt, config verify,
|
||||||
|
# lint and test, and script/cibuild refuses a build in which
|
||||||
|
# any of them was cached or skipped.
|
||||||
|
- name: cibuild
|
||||||
|
run: script/cibuild
|
||||||
29
.gitignore
vendored
29
.gitignore
vendored
@@ -1 +1,28 @@
|
|||||||
dcfinfo
|
# Build artifacts
|
||||||
|
/bin/
|
||||||
|
*.test
|
||||||
|
*.out
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Environment / secrets
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
*.key
|
||||||
|
*.pem
|
||||||
|
|
||||||
|
# OS
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Editor and assistant tooling directories. These are per-developer
|
||||||
|
# state and never belong in the repository.
|
||||||
|
*~
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
.idea/
|
||||||
|
.vscode/
|
||||||
|
.claude/
|
||||||
|
.cursor/
|
||||||
|
.aider*
|
||||||
|
.continue/
|
||||||
|
.windsurf/
|
||||||
|
|||||||
34
.golangci.yml
Normal file
34
.golangci.yml
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
version: "2"
|
||||||
|
|
||||||
|
# Config schema uses the golangci-lint v2 layout (settings live under
|
||||||
|
# linters.settings, not top-level linters-settings) so that the
|
||||||
|
# thresholds below are actually applied by golangci-lint >= v2.
|
||||||
|
|
||||||
|
run:
|
||||||
|
timeout: 5m
|
||||||
|
modules-download-mode: readonly
|
||||||
|
|
||||||
|
linters:
|
||||||
|
default: all
|
||||||
|
disable:
|
||||||
|
# Genuinely incompatible with project patterns
|
||||||
|
- exhaustruct # Requires all struct fields
|
||||||
|
- depguard # Dependency allow/block lists
|
||||||
|
- godot # Requires comments to end with periods
|
||||||
|
- wsl # Deprecated, replaced by wsl_v5
|
||||||
|
- wrapcheck # Too verbose for internal packages
|
||||||
|
- varnamelen # Short names like db, id are idiomatic Go
|
||||||
|
settings:
|
||||||
|
lll:
|
||||||
|
line-length: 88
|
||||||
|
funlen:
|
||||||
|
lines: 80
|
||||||
|
statements: 50
|
||||||
|
cyclop:
|
||||||
|
max-complexity: 15
|
||||||
|
dupl:
|
||||||
|
threshold: 100
|
||||||
|
|
||||||
|
issues:
|
||||||
|
max-issues-per-linter: 0
|
||||||
|
max-same-issues: 0
|
||||||
6
.prettierignore
Normal file
6
.prettierignore
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
go.sum
|
||||||
|
LICENSE
|
||||||
|
|
||||||
|
# The vendored golangci-lint config is canonical and must stay
|
||||||
|
# byte-identical to sneak/prompts. Formatting it here would fork it.
|
||||||
|
.golangci.yml
|
||||||
4
.prettierrc
Normal file
4
.prettierrc
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"tabWidth": 4,
|
||||||
|
"proseWrap": "always"
|
||||||
|
}
|
||||||
83
Dockerfile
Normal file
83
Dockerfile
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
# This repo is a library: there is no cmd/ and no binary to ship, so the
|
||||||
|
# build ends at the stage that compiles and tests the packages. There is
|
||||||
|
# no runtime stage because there is nothing to run.
|
||||||
|
|
||||||
|
# Lint stage — fast feedback on formatting and lint issues. Tools are
|
||||||
|
# invoked directly (not via make/script): the docker build is its own
|
||||||
|
# single path.
|
||||||
|
# This stage must stay the one that runs golangci-lint, and its name must
|
||||||
|
# match $lint_stage in script/cibuild, which cache-busts it by name.
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Inventory of the sources that actually arrived here. script/cibuild
|
||||||
|
# reads these lines out of the build log and compares them against the
|
||||||
|
# git index, so a .dockerignore entry or a narrowed COPY that hides a
|
||||||
|
# package fails the run instead of yielding a clean report over a tree
|
||||||
|
# the linter never saw. Keep it immediately after `COPY . .`, and keep
|
||||||
|
# `echo context-manifest-begin` as its first command:
|
||||||
|
# script/assert-context-complete matches the step by that prefix.
|
||||||
|
RUN echo context-manifest-begin; \
|
||||||
|
{ find . -type f -name '*.go'; \
|
||||||
|
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||||
|
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||||
|
done; } \
|
||||||
|
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||||
|
echo context-manifest-end
|
||||||
|
|
||||||
|
# Formatting check, config check, linter
|
||||||
|
RUN test -z "$(gofmt -s -l .)" || { echo "gofmt needed on:"; gofmt -s -l .; exit 1; }
|
||||||
|
RUN golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
|
# Build stage, and the last one: script/cibuild passes no --target, so
|
||||||
|
# BuildKit builds whichever stage is last and appending one drops lint,
|
||||||
|
# builder and their checks out of the run. Must stay the stage that runs
|
||||||
|
# go test, and its name must match $test_stage in script/cibuild, which
|
||||||
|
# cache-busts it by name.
|
||||||
|
# golang:1.25.7-bookworm (Debian-based: the race detector used by the
|
||||||
|
# test run requires glibc), 2026-08-30
|
||||||
|
FROM golang:1.25.7-bookworm@sha256:564e366a28ad1d70f460a2b97d1d299a562f08707eb0ecb24b659e5bd6c108e1 AS builder
|
||||||
|
|
||||||
|
# Depend on lint stage passing (forces BuildKit ordering)
|
||||||
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
|
WORKDIR /build
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
# Copy source code
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Same inventory as the lint stage, and separately checked: this stage
|
||||||
|
# has its own COPY, so an intact context over there is no evidence about
|
||||||
|
# the tree `go test ./...` is about to walk here. A package that did not
|
||||||
|
# arrive is a package the tests never run, and the run still ends `ok`.
|
||||||
|
RUN echo context-manifest-begin; \
|
||||||
|
{ find . -type f -name '*.go'; \
|
||||||
|
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||||
|
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||||
|
done; } \
|
||||||
|
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||||
|
echo context-manifest-end
|
||||||
|
|
||||||
|
# Run tests: quiet first, verbose rerun on failure (and still fail).
|
||||||
|
# -count=1 disables the test result cache, matching script/test.
|
||||||
|
RUN go test -count=1 -timeout 90s -race -cover ./... || \
|
||||||
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
|
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||||
|
|
||||||
|
# No binary to emit, so the compile check is the artifact: it proves every
|
||||||
|
# package builds for the target the library claims to support.
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath ./...
|
||||||
35
Dockerfile.lint
Normal file
35
Dockerfile.lint
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
# Lint image, built by script/lint: golangci-lint runs as a build step, so
|
||||||
|
# a successful build is a clean lint. Works with a remote docker daemon,
|
||||||
|
# where bind mounts are impossible.
|
||||||
|
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
# This stage must stay the last one and the one that runs golangci-lint,
|
||||||
|
# and its name must match $stage in script/lint.
|
||||||
|
FROM deps AS lint
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Inventory of the sources that actually arrived here. script/lint reads
|
||||||
|
# these lines out of the build log and compares them against the git
|
||||||
|
# index, so a .dockerignore entry or a narrowed COPY that hides a
|
||||||
|
# package fails the run instead of yielding a clean report over a tree
|
||||||
|
# the linter never saw. Keep it immediately after `COPY . .`, and keep
|
||||||
|
# `echo context-manifest-begin` as its first command:
|
||||||
|
# script/assert-context-complete matches the step by that prefix.
|
||||||
|
RUN echo context-manifest-begin; \
|
||||||
|
{ find . -type f -name '*.go'; \
|
||||||
|
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||||
|
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||||
|
done; } \
|
||||||
|
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||||
|
echo context-manifest-end
|
||||||
|
|
||||||
|
RUN golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN golangci-lint run --config .golangci.yml ./...
|
||||||
50
Makefile
50
Makefile
@@ -1,14 +1,50 @@
|
|||||||
default: test
|
.PHONY: bootstrap setup test lint fmt fmt-check check build deps docker cibuild clean hooks
|
||||||
|
|
||||||
|
# Every target is a thin shim over script/. The scripts are the real
|
||||||
|
# entrypoints: they carry the project-specific knowledge (flags,
|
||||||
|
# timeouts, docker-only linting) that a raw `go test` or `golangci-lint`
|
||||||
|
# invocation silently bypasses. Never call the underlying tools
|
||||||
|
# directly.
|
||||||
|
.DEFAULT_GOAL := check
|
||||||
|
|
||||||
|
bootstrap:
|
||||||
|
@script/bootstrap
|
||||||
|
|
||||||
|
setup:
|
||||||
|
@script/setup
|
||||||
|
|
||||||
test:
|
test:
|
||||||
go test -v ./...
|
@script/test
|
||||||
|
|
||||||
run: build
|
lint:
|
||||||
DEBUG=1 ./dcfinfo
|
@script/lint
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
@script/fmt
|
||||||
|
|
||||||
|
fmt-check:
|
||||||
|
@script/fmt-check
|
||||||
|
|
||||||
|
check:
|
||||||
|
@script/check
|
||||||
|
|
||||||
|
# This repo is a library: there is no cmd/ and nothing to link, so
|
||||||
|
# `build` is the compile check over every package.
|
||||||
build:
|
build:
|
||||||
go env -w CGO_ENABLED="0"
|
go build ./...
|
||||||
cd cmd/dcfinfo && go build -o ../../dcfinfo
|
|
||||||
|
deps:
|
||||||
|
go mod download
|
||||||
|
go mod tidy
|
||||||
|
|
||||||
|
docker:
|
||||||
|
@script/docker
|
||||||
|
|
||||||
|
cibuild:
|
||||||
|
@script/cibuild
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -f dcfinfo
|
go clean ./...
|
||||||
|
|
||||||
|
hooks:
|
||||||
|
@script/install-precommit
|
||||||
|
|||||||
151
README.md
151
README.md
@@ -1,33 +1,158 @@
|
|||||||
# dcf
|
# dcf
|
||||||
|
|
||||||
Golang reader implementation of the so-called "Design rule for Camera File
|
`dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that
|
||||||
system" or DCF, aka JEITA (Japan Electronics and Information Technology
|
reads the so-called "Design rule for Camera File system", or DCF, aka JEITA
|
||||||
Industries Association) specification number CP-3461.
|
(Japan Electronics and Information Technology Industries Association)
|
||||||
|
specification number CP-3461.
|
||||||
|
|
||||||
[wikipedia.org/wiki/Design_rule_for_Camera_File_system](https://en.wikipedia.org/wiki/Design_rule_for_Camera_File_system)
|
[wikipedia.org/wiki/Design_rule_for_Camera_File_system](https://en.wikipedia.org/wiki/Design_rule_for_Camera_File_system)
|
||||||
|
|
||||||
The DCF specification is why your digital camera puts images and videos in
|
The DCF specification is why your digital camera puts images and videos in
|
||||||
`DCIM` and `PRIVATE/M4ROOT` directories on the memory card.
|
`DCIM` and `PRIVATE/M4ROOT` directories on the memory card.
|
||||||
|
|
||||||
# status
|
## Status
|
||||||
|
|
||||||
incomplete, under development, does not work yet
|
incomplete, under development, does not work yet
|
||||||
|
|
||||||
# why
|
## Getting started
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get git.eeqj.de/sneak/dcf
|
||||||
|
```
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/dcf/pkg/dcf"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
stores, err := dcf.GetDCFStores(0)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, store := range *stores {
|
||||||
|
fmt.Println(store.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
To work on the library itself:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git clone git@git.eeqj.de:sneak/dcf.git
|
||||||
|
cd dcf
|
||||||
|
script/setup # dependencies plus the git pre-commit hook
|
||||||
|
make check # test, lint, fmt-check
|
||||||
|
```
|
||||||
|
|
||||||
|
## Entrypoints
|
||||||
|
|
||||||
|
This repo adheres to the
|
||||||
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
|
standard. Each `script/` entrypoint has a thin `make` shim; the scripts are
|
||||||
|
where the project's knowledge about flags, timeouts and docker-only linting
|
||||||
|
lives, so use them rather than invoking `go` or `golangci-lint` directly.
|
||||||
|
|
||||||
|
- `script/bootstrap` (`make bootstrap`) — install build dependencies (git, make,
|
||||||
|
go) idempotently. Linting additionally needs docker; markdown formatting needs
|
||||||
|
docker or a local `prettier`.
|
||||||
|
- `script/setup` (`make setup`) — `bootstrap` plus the git pre-commit hook.
|
||||||
|
- `script/test` (`make test`) — `go test` with the race detector and coverage,
|
||||||
|
`-count=1`, 90s timeout; quiet on success, verbose rerun on failure.
|
||||||
|
- `script/lint` (`make lint`) — `golangci-lint` via docker only, against the
|
||||||
|
digest-pinned image in `Dockerfile.lint`, then `script/assert-step-ran` and
|
||||||
|
`script/assert-context-complete` over the build log. Nothing is installed
|
||||||
|
locally and nothing runs on the host.
|
||||||
|
- `script/fmt` (`make fmt`) — format Go with `gofmt` and everything else with
|
||||||
|
`prettier` (writes).
|
||||||
|
- `script/fmt-check` (`make fmt-check`) — the same scope, read-only.
|
||||||
|
- `script/check` (`make check`) — `test` + `lint` + `fmt-check`. What the
|
||||||
|
pre-commit hook runs. Never modifies files.
|
||||||
|
- `script/docker` (`make docker`) — build the image, tagged with
|
||||||
|
`script/projectname`.
|
||||||
|
- `script/cibuild` (`make cibuild`) — the CI gate:
|
||||||
|
`docker build --progress=plain --no-cache-filter=lint --no-cache-filter=builder .`
|
||||||
|
followed by assertions that the lint step and the test step really ran, and
|
||||||
|
that both stages really received the whole repository. The Gitea workflow runs
|
||||||
|
this on every push.
|
||||||
|
- `script/precommit` — the hook body: `go mod tidy` must not change
|
||||||
|
`go.mod`/`go.sum`, then `check`.
|
||||||
|
- `script/install-precommit` (`make hooks`) — installs the hook.
|
||||||
|
- `script/projectname` — prints the project name; other scripts call it so they
|
||||||
|
can stay identical across repos.
|
||||||
|
- `script/prettier`, `script/assert-step-ran`, `script/assert-context-complete`
|
||||||
|
and `script/repo-source-manifest` are helpers, not entrypoints.
|
||||||
|
|
||||||
|
`make build`, `make deps` and `make clean` are the ordinary conveniences.
|
||||||
|
`make build` compiles every package; this repo is a library and ships no binary,
|
||||||
|
so there is no `cmd/` and nothing to link.
|
||||||
|
|
||||||
|
### Why the lint assertions exist
|
||||||
|
|
||||||
|
A green `docker build` is not evidence that the checks ran. On an unchanged tree
|
||||||
|
every layer comes from cache and the build exits 0 having executed nothing;
|
||||||
|
BuildKit silently ignores a `--no-cache-filter` naming a stage that no longer
|
||||||
|
exists; and a `.dockerignore` entry can remove a package from the build context,
|
||||||
|
after which the linter genuinely runs, genuinely examines what it was handed,
|
||||||
|
and genuinely reports `0 issues.` over a repository with a violation in it.
|
||||||
|
`script/assert-step-ran` and `script/assert-context-complete` close both holes;
|
||||||
|
read the comments in them before changing either.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
I wanted to copy images off my memory cards for processing and like
|
I wanted to copy images off my memory cards for processing and like
|
||||||
overengineering and reusable code.
|
overengineering and reusable code.
|
||||||
|
|
||||||
# bugs
|
## Design
|
||||||
|
|
||||||
|
Everything lives in `pkg/dcf`, a single library package with no command
|
||||||
|
entrypoints.
|
||||||
|
|
||||||
|
- `dcf.go` — the public surface. `Store` is one DCF store (one mounted
|
||||||
|
filesystem); `Object` is one file in it, embedded in `Image` and `Video`.
|
||||||
|
`GetDCFStores` finds the stores on the system and walks each one, classifying
|
||||||
|
files by extension.
|
||||||
|
- `helpers.go` — mountpoint discovery. `findAllMountPoints` enumerates the
|
||||||
|
system's physical filesystems via `gopsutil`; `findDCFMountPoints` keeps only
|
||||||
|
those whose root holds a marker directory.
|
||||||
|
|
||||||
|
Detection is by directory name at the filesystem root, so no filesystem is ever
|
||||||
|
mounted and nothing outside a filesystem root is searched.
|
||||||
|
|
||||||
|
## Bugs
|
||||||
|
|
||||||
does not yet handle "dcf file groups" where multiple objects share the same
|
does not yet handle "dcf file groups" where multiple objects share the same
|
||||||
index number. if you need this, send me a link to a zip or tar of a memory
|
index number. if you need this, send me a link to a zip or tar of a memory card
|
||||||
card that uses it and i'll see what i can do.
|
that uses it and i'll see what i can do.
|
||||||
|
|
||||||
# author
|
## TODO
|
||||||
|
|
||||||
|
The work in flight is on the tracker; this is the reading order for picking it
|
||||||
|
up.
|
||||||
|
|
||||||
|
- [#2](https://git.eeqj.de/sneak/dcf/issues/2) — parse the CP-3461 structure
|
||||||
|
rather than grepping every filesystem root for media extensions.
|
||||||
|
- [#3](https://git.eeqj.de/sneak/dcf/issues/3) — DCF file groups: objects
|
||||||
|
sharing an index number are one logical object.
|
||||||
|
- [#4](https://git.eeqj.de/sneak/dcf/issues/4) — API cleanup: `fs.FS`-based
|
||||||
|
store access, no pointer-to-slice returns, contexts on walks. This is also
|
||||||
|
what makes the filesystem walk testable; today it is reachable only through
|
||||||
|
`GetDCFStores`, which needs real mounted media, so it has no test coverage.
|
||||||
|
- [#5](https://git.eeqj.de/sneak/dcf/issues/5) — checksummed import off the
|
||||||
|
card, which is what the library is for.
|
||||||
|
- [#6](https://git.eeqj.de/sneak/dcf/issues/6) — mountpoint detection checks
|
||||||
|
`M4ROOT` at the filesystem root rather than `PRIVATE/M4ROOT`, and
|
||||||
|
`findDCFMountPoints` stops one store short of `requestedCount`.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
WTFPL. See [LICENSE](LICENSE).
|
||||||
|
|
||||||
|
## Author
|
||||||
|
|
||||||
sneak <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)>
|
sneak <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)>
|
||||||
|
|
||||||
# license
|
|
||||||
|
|
||||||
WTFPL
|
|
||||||
|
|||||||
3
go.mod
3
go.mod
@@ -1,6 +1,6 @@
|
|||||||
module git.eeqj.de/sneak/dcf
|
module git.eeqj.de/sneak/dcf
|
||||||
|
|
||||||
go 1.22.1
|
go 1.25.7
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/dustin/go-humanize v1.0.1
|
github.com/dustin/go-humanize v1.0.1
|
||||||
@@ -9,7 +9,6 @@ require (
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||||
github.com/lmittmann/tint v1.0.4 // indirect
|
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
golang.org/x/sys v0.20.0 // indirect
|
golang.org/x/sys v0.20.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -3,8 +3,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
|
|||||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||||
github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
|
|
||||||
github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
|
|
||||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||||
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||||
|
|||||||
228
pkg/dcf/dcf.go
228
pkg/dcf/dcf.go
@@ -1,30 +1,34 @@
|
|||||||
|
// Package dcf reads DCF stores: the directory structures described by
|
||||||
|
// the "Design rule for Camera File system" (JEITA CP-3461) that digital
|
||||||
|
// cameras write to the root of removable media.
|
||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
)
|
)
|
||||||
|
|
||||||
var imageExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw"}
|
// Store represents a DCF store, which is a directory structure that
|
||||||
var videoExtensions = []string{".mp4", ".mov", ".avi", ".mkv", ".wmv"}
|
// contains images and videos in the root of a mounted filesystem.
|
||||||
|
type Store struct {
|
||||||
// DCFStore represents a DCF store, which is a directory sturcture that contains
|
|
||||||
// images and videos in the root of a mounted filesystem.
|
|
||||||
type DCFStore struct {
|
|
||||||
RootDirectory string
|
RootDirectory string
|
||||||
Images []*Image
|
Images []*Image
|
||||||
Videos []*Video
|
Videos []*Video
|
||||||
}
|
}
|
||||||
|
|
||||||
// DCFObject represents a file in a DCF store. It is embedded in Image and Video types.
|
// Object represents a file in a DCF store. It is embedded in the Image
|
||||||
type DCFObject struct {
|
// and Video types.
|
||||||
DCFStoreRoot string
|
type Object struct {
|
||||||
|
StoreRoot string
|
||||||
Path string
|
Path string
|
||||||
Size int64
|
Size int64
|
||||||
Extension string
|
Extension string
|
||||||
@@ -32,146 +36,224 @@ type DCFObject struct {
|
|||||||
|
|
||||||
// Image represents an image file on a DCF store.
|
// Image represents an image file on a DCF store.
|
||||||
type Image struct {
|
type Image struct {
|
||||||
DCFObject
|
Object
|
||||||
}
|
}
|
||||||
|
|
||||||
// Video represents a video file on a DCF store.
|
// Video represents a video file on a DCF store.
|
||||||
type Video struct {
|
type Video struct {
|
||||||
DCFObject
|
Object
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DCFObject) FullFilePath() string {
|
// FullFilePath returns the path of the object on the local filesystem,
|
||||||
return filepath.Join(d.DCFStoreRoot, d.Path)
|
// which is its store-relative path resolved against the store root.
|
||||||
|
func (d *Object) FullFilePath() string {
|
||||||
|
return filepath.Join(d.StoreRoot, d.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// VideosCount returns the number of videos in the DCF store.
|
// VideosCount returns the number of videos in the DCF store.
|
||||||
func (d *DCFStore) VideosCount() int {
|
func (d *Store) VideosCount() int {
|
||||||
return len(d.Videos)
|
return len(d.Videos)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ImagesCount returns the number of images in the DCF store.
|
// ImagesCount returns the number of images in the DCF store.
|
||||||
func (d *DCFStore) ImagesCount() int {
|
func (d *Store) ImagesCount() int {
|
||||||
return len(d.Images)
|
return len(d.Images)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DCFStore) TotalImageSize() int64 {
|
// TotalImageSize returns the summed size in bytes of the store's images.
|
||||||
|
func (d *Store) TotalImageSize() int64 {
|
||||||
totalSize := int64(0)
|
totalSize := int64(0)
|
||||||
for _, image := range d.Images {
|
for _, image := range d.Images {
|
||||||
totalSize += image.Size
|
totalSize += image.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
return totalSize
|
return totalSize
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DCFStore) TotalVideoSize() int64 {
|
// TotalVideoSize returns the summed size in bytes of the store's videos.
|
||||||
|
func (d *Store) TotalVideoSize() int64 {
|
||||||
totalSize := int64(0)
|
totalSize := int64(0)
|
||||||
for _, video := range d.Videos {
|
for _, video := range d.Videos {
|
||||||
totalSize += video.Size
|
totalSize += video.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
return totalSize
|
return totalSize
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DCFStore) TotalSize() int64 {
|
// TotalSize returns the summed size in bytes of every object in the
|
||||||
|
// store.
|
||||||
|
func (d *Store) TotalSize() int64 {
|
||||||
return d.TotalImageSize() + d.TotalVideoSize()
|
return d.TotalImageSize() + d.TotalVideoSize()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *DCFStore) String() string {
|
// String renders the store as a single human-readable summary line.
|
||||||
return fmt.Sprintf("DCFStore{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}",
|
func (d *Store) String() string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"Store{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}",
|
||||||
d.RootDirectory,
|
d.RootDirectory,
|
||||||
d.ImagesCount(),
|
d.ImagesCount(),
|
||||||
d.VideosCount(),
|
d.VideosCount(),
|
||||||
humanize.Bytes(uint64(d.TotalSize())),
|
humanize.Bytes(d.totalSizeBytes()),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// totalSizeBytes returns TotalSize as an unsigned byte count for
|
||||||
|
// formatting. Sizes come from the filesystem and are never negative,
|
||||||
|
// but the conversion is guarded rather than assumed.
|
||||||
|
func (d *Store) totalSizeBytes() uint64 {
|
||||||
|
total := d.TotalSize()
|
||||||
|
if total < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return uint64(total)
|
||||||
|
}
|
||||||
|
|
||||||
|
// String renders the video as a single human-readable line.
|
||||||
func (v *Video) String() string {
|
func (v *Video) String() string {
|
||||||
return fmt.Sprintf("Video{Path: %s, Size: %d, Extension: %s}", v.Path, v.Size, v.Extension)
|
return fmt.Sprintf(
|
||||||
|
"Video{Path: %s, Size: %d, Extension: %s}",
|
||||||
|
v.Path, v.Size, v.Extension,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// String renders the image as a single human-readable line.
|
||||||
func (i *Image) String() string {
|
func (i *Image) String() string {
|
||||||
return fmt.Sprintf("Image{Path: %s, Size: %d, Extension: %s}", i.Path, i.Size, i.Extension)
|
return fmt.Sprintf(
|
||||||
|
"Image{Path: %s, Size: %d, Extension: %s}",
|
||||||
|
i.Path, i.Size, i.Extension,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hash() returns the SHA256 hash of the file as a hex string.
|
// Hash returns the SHA-256 digest of the object's file as a hex string.
|
||||||
func (d *DCFObject) Hash() (string, error) {
|
func (d *Object) Hash() (string, error) {
|
||||||
return pathToSHA256(d.FullFilePath())
|
return pathToSHA256(d.FullFilePath())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pathToSHA256 returns the SHA-256 digest of the file at path as a hex
|
||||||
|
// string.
|
||||||
func pathToSHA256(path string) (string, error) {
|
func pathToSHA256(path string) (string, error) {
|
||||||
file, err := os.Open(path)
|
// The path is supplied by the caller by design: this library exists
|
||||||
|
// to hash files the caller pointed it at.
|
||||||
|
file, err := os.Open(filepath.Clean(path))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
defer file.Close()
|
|
||||||
|
defer func() { _ = file.Close() }()
|
||||||
|
|
||||||
hash := sha256.New()
|
hash := sha256.New()
|
||||||
if _, err := io.Copy(hash, file); err != nil {
|
|
||||||
|
_, err = io.Copy(hash, file)
|
||||||
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetDCFStores returns a list of DCF stores found on the system in the root of any mounted filesystems.
|
// GetDCFStores returns the DCF stores found in the root directories of
|
||||||
// the integer argument is the number of DCF stores to return. If the argument is 0, all DCF stores are returned.
|
// the system's mounted filesystems. requestedCount bounds how many
|
||||||
// It does not mount filesystems or search outside of filesystem root directories. It does, however,
|
// stores are returned; 0 returns all of them.
|
||||||
// walk the filesystems to find images and videos and populate the returned DCFStores.
|
//
|
||||||
func GetDCFStores(requestedCount int) (*[]DCFStore, error) {
|
// It neither mounts filesystems nor searches outside filesystem root
|
||||||
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
// directories. It does walk each store it finds, populating the
|
||||||
|
// returned stores with the images and videos they contain.
|
||||||
|
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
||||||
|
storePaths, err := findDCFMountPoints(requestedCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
dcfStores := []DCFStore{}
|
|
||||||
for _, dcfStorePath := range dcfStorePaths {
|
stores := []Store{}
|
||||||
dcfStore := DCFStore{
|
|
||||||
RootDirectory: dcfStorePath,
|
for _, storePath := range storePaths {
|
||||||
|
store, err := scanStore(storePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
stores = append(stores, *store)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &stores, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// scanStore walks root and returns a Store holding every image and
|
||||||
|
// video beneath it.
|
||||||
|
func scanStore(root string) (*Store, error) {
|
||||||
|
store := Store{
|
||||||
|
RootDirectory: root,
|
||||||
Images: make([]*Image, 0),
|
Images: make([]*Image, 0),
|
||||||
Videos: make([]*Video, 0),
|
Videos: make([]*Video, 0),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := filepath.Walk(dcfStorePath, func(path string, info os.FileInfo, err error) error {
|
walk := func(path string, entry fs.DirEntry, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !info.IsDir() && isImageFile(path) {
|
|
||||||
image := Image{}
|
if entry.IsDir() {
|
||||||
image.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
|
||||||
image.Size = info.Size()
|
|
||||||
image.Extension = filepath.Ext(path)
|
|
||||||
image.DCFStoreRoot = dcfStorePath
|
|
||||||
dcfStore.Images = append(dcfStore.Images, &image)
|
|
||||||
}
|
|
||||||
if !info.IsDir() && isVideoFile(path) {
|
|
||||||
video := Video{}
|
|
||||||
video.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
|
||||||
video.Size = info.Size()
|
|
||||||
video.Extension = filepath.Ext(path)
|
|
||||||
video.DCFStoreRoot = dcfStorePath
|
|
||||||
dcfStore.Videos = append(dcfStore.Videos, &video)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
})
|
}
|
||||||
|
|
||||||
|
return store.addFile(root, path, entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := filepath.WalkDir(root, walk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
dcfStores = append(dcfStores, dcfStore)
|
|
||||||
}
|
return &store, nil
|
||||||
return &dcfStores, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// addFile classifies path by extension and appends it to the store's
|
||||||
|
// images or videos. A file that is neither is ignored.
|
||||||
|
func (d *Store) addFile(root, path string, entry fs.DirEntry) error {
|
||||||
|
isImage := isImageFile(path)
|
||||||
|
if !isImage && !isVideoFile(path) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
info, err := entry.Info()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
object := Object{
|
||||||
|
StoreRoot: root,
|
||||||
|
Path: strings.TrimPrefix(path, root+string(filepath.Separator)),
|
||||||
|
Size: info.Size(),
|
||||||
|
Extension: filepath.Ext(path),
|
||||||
|
}
|
||||||
|
|
||||||
|
if isImage {
|
||||||
|
d.Images = append(d.Images, &Image{Object: object})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
d.Videos = append(d.Videos, &Video{Object: object})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isImageFile reports whether path names a file this package treats as
|
||||||
|
// an image.
|
||||||
func isImageFile(path string) bool {
|
func isImageFile(path string) bool {
|
||||||
ext := strings.ToLower(filepath.Ext(path))
|
return hasExtension(path,
|
||||||
for _, imageExt := range imageExtensions {
|
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw")
|
||||||
if ext == imageExt {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isVideoFile reports whether path names a file this package treats as
|
||||||
|
// a video.
|
||||||
func isVideoFile(path string) bool {
|
func isVideoFile(path string) bool {
|
||||||
ext := strings.ToLower(filepath.Ext(path))
|
return hasExtension(path, ".mp4", ".mov", ".avi", ".mkv", ".wmv")
|
||||||
for _, videoExt := range videoExtensions {
|
|
||||||
if ext == videoExt {
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return false
|
// hasExtension reports whether path's extension, lowercased, is one of
|
||||||
|
// the given extensions. Extensions are given with their leading dot.
|
||||||
|
func hasExtension(path string, extensions ...string) bool {
|
||||||
|
return slices.Contains(extensions, strings.ToLower(filepath.Ext(path)))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,177 @@
|
|||||||
package dcf
|
package dcf_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/dcf/pkg/dcf"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCompile(t *testing.T) {
|
// newStore builds a store with two images and one video of known sizes,
|
||||||
// This is a placeholder test to ensure that the module compiles successfully.
|
// so the count and size accessors have something deterministic to
|
||||||
// You can add more meaningful tests here once you start implementing your code.
|
// report.
|
||||||
|
func newStore(root string) *dcf.Store {
|
||||||
|
return &dcf.Store{
|
||||||
|
RootDirectory: root,
|
||||||
|
Images: []*dcf.Image{
|
||||||
|
{Object: dcf.Object{
|
||||||
|
StoreRoot: root,
|
||||||
|
Path: "DCIM/100MSDCF/DSC00001.ARW",
|
||||||
|
Size: 2000,
|
||||||
|
Extension: ".ARW",
|
||||||
|
}},
|
||||||
|
{Object: dcf.Object{
|
||||||
|
StoreRoot: root,
|
||||||
|
Path: "DCIM/100MSDCF/DSC00002.JPG",
|
||||||
|
Size: 500,
|
||||||
|
Extension: ".JPG",
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
Videos: []*dcf.Video{
|
||||||
|
{Object: dcf.Object{
|
||||||
|
StoreRoot: root,
|
||||||
|
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||||
|
Size: 9000,
|
||||||
|
Extension: ".MP4",
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectFullFilePath(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
object := dcf.Object{
|
||||||
|
StoreRoot: filepath.Join("/mnt", "card"),
|
||||||
|
Path: filepath.Join("DCIM", "100MSDCF", "DSC00001.ARW"),
|
||||||
|
}
|
||||||
|
|
||||||
|
want := filepath.Join("/mnt", "card", "DCIM", "100MSDCF", "DSC00001.ARW")
|
||||||
|
if got := object.FullFilePath(); got != want {
|
||||||
|
t.Fatalf("FullFilePath() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreCounts(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newStore("/mnt/card")
|
||||||
|
|
||||||
|
if got := store.ImagesCount(); got != 2 {
|
||||||
|
t.Errorf("ImagesCount() = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.VideosCount(); got != 1 {
|
||||||
|
t.Errorf("VideosCount() = %d, want 1", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreSizes(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := newStore("/mnt/card")
|
||||||
|
|
||||||
|
if got := store.TotalImageSize(); got != 2500 {
|
||||||
|
t.Errorf("TotalImageSize() = %d, want 2500", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.TotalVideoSize(); got != 9000 {
|
||||||
|
t.Errorf("TotalVideoSize() = %d, want 9000", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := store.TotalSize(); got != 11500 {
|
||||||
|
t.Errorf("TotalSize() = %d, want 11500", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEmptyStoreSizesAreZero(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
store := dcf.Store{RootDirectory: "/mnt/empty"}
|
||||||
|
if got := store.TotalSize(); got != 0 {
|
||||||
|
t.Fatalf("TotalSize() = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreStringReportsHumanizedTotal(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got := newStore("/mnt/card").String()
|
||||||
|
for _, want := range []string{"/mnt/card", "Images: 2", "Videos: 1", "12 kB"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("String() = %q, want it to contain %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImageAndVideoString(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
image := dcf.Image{Object: dcf.Object{
|
||||||
|
Path: "DCIM/100MSDCF/DSC00001.ARW",
|
||||||
|
Size: 2000,
|
||||||
|
Extension: ".ARW",
|
||||||
|
}}
|
||||||
|
|
||||||
|
want := "Image{Path: DCIM/100MSDCF/DSC00001.ARW, Size: 2000, " +
|
||||||
|
"Extension: .ARW}"
|
||||||
|
if got := image.String(); got != want {
|
||||||
|
t.Errorf("Image.String() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
video := dcf.Video{Object: dcf.Object{
|
||||||
|
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||||
|
Size: 9000,
|
||||||
|
Extension: ".MP4",
|
||||||
|
}}
|
||||||
|
|
||||||
|
want = "Video{Path: PRIVATE/M4ROOT/CLIP/C0001.MP4, Size: 9000, " +
|
||||||
|
"Extension: .MP4}"
|
||||||
|
if got := video.String(); got != want {
|
||||||
|
t.Errorf("Video.String() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectHashMatchesSHA256OfFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
contents := []byte("DSC00001.ARW contents")
|
||||||
|
|
||||||
|
err := os.MkdirAll(filepath.Join(root, "DCIM"), 0o750)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MkdirAll: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = os.WriteFile(filepath.Join(root, "DCIM", "a.arw"), contents, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("WriteFile: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
object := dcf.Object{StoreRoot: root, Path: filepath.Join("DCIM", "a.arw")}
|
||||||
|
|
||||||
|
got, err := object.Hash()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Hash() returned error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sum := sha256.Sum256(contents)
|
||||||
|
if want := hex.EncodeToString(sum[:]); got != want {
|
||||||
|
t.Fatalf("Hash() = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestObjectHashOnMissingFileReturnsError(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
object := dcf.Object{StoreRoot: t.TempDir(), Path: "absent.jpg"}
|
||||||
|
|
||||||
|
_, err := object.Hash()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("Hash() on a missing file returned no error")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,75 +1,84 @@
|
|||||||
package dcf
|
package dcf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"slices"
|
||||||
|
|
||||||
"github.com/shirou/gopsutil/disk"
|
"github.com/shirou/gopsutil/disk"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// dcfMarkerDirectories are the directory names whose presence in a
|
||||||
|
// filesystem root identifies that filesystem as a DCF store.
|
||||||
|
func dcfMarkerDirectories() []string {
|
||||||
|
return []string{"DCIM", "M4ROOT"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// findAllMountPoints returns the distinct mountpoints of the system's
|
||||||
|
// physical filesystems.
|
||||||
func findAllMountPoints() ([]string, error) {
|
func findAllMountPoints() ([]string, error) {
|
||||||
mountpoints := []string{}
|
partitions, err := disk.Partitions(false) // physical devices only
|
||||||
partitions, err := disk.Partitions(false) // physical devices only, so false
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mountpoints := []string{}
|
||||||
|
|
||||||
for _, partition := range partitions {
|
for _, partition := range partitions {
|
||||||
if !contains(mountpoints, partition.Mountpoint) {
|
if !slices.Contains(mountpoints, partition.Mountpoint) {
|
||||||
mountpoints = append(mountpoints, partition.Mountpoint)
|
mountpoints = append(mountpoints, partition.Mountpoint)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return mountpoints, nil
|
return mountpoints, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func dump(x interface{}) {
|
// findDCFMountPoints returns the paths of the mountpoints that contain
|
||||||
_, file, line, _ := runtime.Caller(1)
|
// one of the marker directories a DCF store is identified by. Its only
|
||||||
slog.Debug(fmt.Sprintf("%s:%d %#v\n", file, line, x))
|
// argument is how many mountpoints to return; 0 returns all of them.
|
||||||
}
|
|
||||||
|
|
||||||
// findDCFMountPoints returns a list of strings of the paths of mountpoints that
|
|
||||||
// contain a DCIM or PRIVATE directory, which is how we detect DCF stores. Its
|
|
||||||
// only argument is an integer of how many mountpoints to return. If the
|
|
||||||
// argument is 0, all mountpoints are returned.
|
|
||||||
func findDCFMountPoints(requestedCount int) ([]string, error) {
|
func findDCFMountPoints(requestedCount int) ([]string, error) {
|
||||||
filteredMountpoints := []string{}
|
|
||||||
var erro error
|
|
||||||
mountpoints, err := findAllMountPoints()
|
mountpoints, err := findAllMountPoints()
|
||||||
|
if err != nil {
|
||||||
if erro != nil {
|
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
filteredMountpoints := []string{}
|
||||||
|
|
||||||
for _, mountpoint := range mountpoints {
|
for _, mountpoint := range mountpoints {
|
||||||
dcimPath := filepath.Join(mountpoint, "DCIM")
|
if !isDCFMountPoint(mountpoint) {
|
||||||
privatePath := filepath.Join(mountpoint, "PRIVATE")
|
continue
|
||||||
privatePath = filepath.Join(mountpoint, "M4ROOT")
|
|
||||||
shouldKeep := false
|
|
||||||
if _, err := os.Stat(dcimPath); err == nil {
|
|
||||||
shouldKeep = true
|
|
||||||
slog.Debug(fmt.Sprintf("Found DCIM directory at %s\n", dcimPath))
|
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(privatePath); err == nil {
|
|
||||||
shouldKeep = true
|
slog.Debug("keeping mountpoint", "mountpoint", mountpoint)
|
||||||
slog.Debug(fmt.Sprintf("Found M4ROOT directory at %s\n", privatePath))
|
|
||||||
}
|
|
||||||
if shouldKeep {
|
|
||||||
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
|
|
||||||
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
||||||
if (requestedCount > 0) && (len(filteredMountpoints)+1 >= requestedCount) {
|
|
||||||
|
if requestedCount > 0 && len(filteredMountpoints)+1 >= requestedCount {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return filteredMountpoints, nil
|
return filteredMountpoints, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func contains(slice []string, str string) bool {
|
// isDCFMountPoint reports whether mountpoint holds any of the marker
|
||||||
for _, s := range slice {
|
// directories a DCF store is identified by.
|
||||||
if s == str {
|
func isDCFMountPoint(mountpoint string) bool {
|
||||||
return true
|
found := false
|
||||||
|
|
||||||
|
for _, marker := range dcfMarkerDirectories() {
|
||||||
|
markerPath := filepath.Join(mountpoint, marker)
|
||||||
|
|
||||||
|
_, err := os.Stat(markerPath)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
slog.Debug("found marker directory", "path", markerPath)
|
||||||
|
|
||||||
|
found = true
|
||||||
}
|
}
|
||||||
return false
|
|
||||||
|
return found
|
||||||
}
|
}
|
||||||
|
|||||||
166
script/assert-context-complete
Executable file
166
script/assert-context-complete
Executable file
@@ -0,0 +1,166 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/assert-context-complete: fail unless the build context a stage
|
||||||
|
# was given contained every source file the repository has.
|
||||||
|
#
|
||||||
|
# Usage: script/assert-context-complete LOG STAGE EXPECTED
|
||||||
|
#
|
||||||
|
# LOG must be `docker build --progress=plain` output. EXPECTED is a list
|
||||||
|
# of repo-relative paths, one per line, as script/repo-source-manifest
|
||||||
|
# prints it.
|
||||||
|
#
|
||||||
|
# script/assert-step-ran answers "did the tool execute". This answers
|
||||||
|
# the other half: "was the tool given the tree". They are different
|
||||||
|
# questions, and a green build can satisfy the first while failing the
|
||||||
|
# second — a .dockerignore entry, or a COPY that brings in less than the
|
||||||
|
# whole tree, removes files from the context silently, and golangci-lint
|
||||||
|
# then genuinely runs, genuinely examines what it was handed, and
|
||||||
|
# genuinely reports `0 issues.` over a repo that has a violation in it.
|
||||||
|
# `go test ./...` likewise never runs a package that did not arrive.
|
||||||
|
#
|
||||||
|
# How it knows: each source-consuming stage emits, right after its
|
||||||
|
# `COPY . .`, an inventory of the Go files actually present —
|
||||||
|
#
|
||||||
|
# context-manifest-begin
|
||||||
|
# context-file: cmd/gotemplate/main.go
|
||||||
|
# ...
|
||||||
|
# context-manifest-end
|
||||||
|
#
|
||||||
|
# — and this compares that against EXPECTED, which comes from the git
|
||||||
|
# index rather than from the same build. The two sources are
|
||||||
|
# independent: .dockerignore decides the first and cannot touch the
|
||||||
|
# second. An expectation read from the evidence would prove nothing.
|
||||||
|
#
|
||||||
|
# The manifest lines are only believed under the same discipline
|
||||||
|
# script/assert-step-ran applies to a tool's success line: they must be
|
||||||
|
# attributed to the id of a step whose header is "#ID [STAGE n/m] CMD"
|
||||||
|
# with CMD matching the manifest command, which BuildKit reported DONE,
|
||||||
|
# and both sentinels must be present. So another step's output does not
|
||||||
|
# count, a header does not count, a cached step (which writes nothing)
|
||||||
|
# does not count, and a truncated log fails rather than passing with a
|
||||||
|
# short list.
|
||||||
|
#
|
||||||
|
# Only one direction is checked: everything the repo has must have
|
||||||
|
# arrived. Files in the context that git does not track are not a
|
||||||
|
# failure — untracked local work is normal and hides nothing.
|
||||||
|
#
|
||||||
|
# What this does not cover: a Dockerfile edit to the manifest step
|
||||||
|
# itself, and evidence forged inside a matched command. Both are visible
|
||||||
|
# in the Dockerfile in plain sight, and both are equally outside
|
||||||
|
# script/assert-step-ran. This is not an exhaustive list of ways a green
|
||||||
|
# run can be untrue; it is the ones known.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "script/assert-context-complete: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ $# -eq 3 ] || die "usage: $0 LOG STAGE EXPECTED"
|
||||||
|
[ -r "$1" ] || die "cannot read $1"
|
||||||
|
[ -r "$3" ] || die "cannot read $3"
|
||||||
|
|
||||||
|
# The stage and the expected-list path travel in the environment, not in
|
||||||
|
# -v: awk expands escape sequences in a -v assignment.
|
||||||
|
ACC_STAGE="$2" ACC_EXPECTED="$3" awk '
|
||||||
|
function fail(msg) {
|
||||||
|
printf "script/assert-context-complete: %s\n", msg | "cat 1>&2"
|
||||||
|
close("cat 1>&2")
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# The package a missing file belongs to — its directory, which is what a
|
||||||
|
# reader needs named. A bare "." for a root-level file reads as a typo
|
||||||
|
# next to the sentence punctuation, so it is spelled out.
|
||||||
|
function pkgof(path, i) {
|
||||||
|
i = length(path)
|
||||||
|
while (i > 0 && substr(path, i, 1) != "/") i--
|
||||||
|
return i == 0 ? "the repository root" : substr(path, 1, i - 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
function preview(list, n, limit, i, out) {
|
||||||
|
for (i = 1; i <= n && i <= limit; i++)
|
||||||
|
out = out (i > 1 ? ", " : "") list[i]
|
||||||
|
if (n > limit)
|
||||||
|
out = out sprintf(", and %d more", n - limit)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
BEGIN {
|
||||||
|
stage = ENVIRON["ACC_STAGE"]
|
||||||
|
|
||||||
|
# Fixed by the protocol the Dockerfile emits, so callers cannot
|
||||||
|
# drift from it.
|
||||||
|
step = "^RUN echo context-manifest-begin"
|
||||||
|
beginmark = "context-manifest-begin"
|
||||||
|
endmark = "context-manifest-end"
|
||||||
|
prefix = "context-file: "
|
||||||
|
|
||||||
|
expected_path = ENVIRON["ACC_EXPECTED"]
|
||||||
|
while ((getline line < expected_path) > 0)
|
||||||
|
if (line != "") expected[++nexpected] = line
|
||||||
|
close(expected_path)
|
||||||
|
|
||||||
|
if (nexpected == 0)
|
||||||
|
fail(sprintf("the expected-file list (%s) is empty, so any build context would satisfy this check", expected_path))
|
||||||
|
}
|
||||||
|
|
||||||
|
# Every progress line is "#ID " and then a step header, a status, or one
|
||||||
|
# line the step itself wrote.
|
||||||
|
/^#[0-9]+ / {
|
||||||
|
id = substr($1, 2)
|
||||||
|
rest = substr($0, length($1) + 2)
|
||||||
|
|
||||||
|
if (rest == "CACHED") { cached[id] = 1; next }
|
||||||
|
if (rest ~ /^DONE /) { done[id] = 1; next }
|
||||||
|
|
||||||
|
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
|
||||||
|
# build names a platform. Bracketed spans with no n/m are BuildKit
|
||||||
|
# internals, not steps.
|
||||||
|
if (substr(rest, 1, 1) == "[") {
|
||||||
|
p = index(rest, "] ")
|
||||||
|
if (p == 0) next
|
||||||
|
n = split(substr(rest, 2, p - 2), part, " ")
|
||||||
|
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
|
||||||
|
if (part[n - 1] != stage) next
|
||||||
|
if (substr(rest, p + 2) ~ step) matched[id] = 1
|
||||||
|
next
|
||||||
|
}
|
||||||
|
|
||||||
|
# Output: "#ID 0.31 <the line the step wrote>".
|
||||||
|
if (rest ~ /^[0-9]+[.][0-9]+ /) {
|
||||||
|
sub(/^[0-9]+[.][0-9]+ /, "", rest)
|
||||||
|
if (rest == beginmark) begun[id] = 1
|
||||||
|
else if (rest == endmark) ended[id] = 1
|
||||||
|
else if (index(rest, prefix) == 1)
|
||||||
|
arrived[id SUBSEP substr(rest, length(prefix) + 1)] = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
END {
|
||||||
|
for (id in matched) {
|
||||||
|
nmatched++
|
||||||
|
if (cached[id]) ncached++
|
||||||
|
if (done[id] && begun[id] && ended[id]) chosen = id
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nmatched == 0)
|
||||||
|
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so the build context was never inventoried. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or the manifest step was taken out of it", step, stage))
|
||||||
|
|
||||||
|
if (chosen == "" && ncached == nmatched)
|
||||||
|
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so its inventory describes an earlier tree rather than this one; the cache bust for that stage is not taking effect", step, stage))
|
||||||
|
|
||||||
|
if (chosen == "")
|
||||||
|
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but wrote no complete inventory between %s and %s. The log is truncated, the build was not run with --progress=plain, or that step no longer emits the manifest", step, stage, beginmark, endmark))
|
||||||
|
|
||||||
|
for (i = 1; i <= nexpected; i++) {
|
||||||
|
if (arrived[chosen SUBSEP expected[i]]) continue
|
||||||
|
missing[++nmissing] = expected[i]
|
||||||
|
pkg = pkgof(expected[i])
|
||||||
|
if (!(pkg in seenpkg)) { seenpkg[pkg] = 1; pkgs[++npkgs] = pkg }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nmissing == 0) exit 0
|
||||||
|
|
||||||
|
fail(sprintf("%d of the %d source files this repository tracks never reached the build context of stage \"%s\", so nothing examined them. Missing package(s): %s. Missing file(s): %s. A .dockerignore entry, or a COPY that brings in less than the whole tree, drops files silently: the tool still runs, still finds nothing wrong with what it was handed, and still reports success", nmissing, nexpected, stage, preview(pkgs, npkgs, 10), preview(missing, nmissing, 10)))
|
||||||
|
}
|
||||||
|
' "$1"
|
||||||
125
script/assert-step-ran
Executable file
125
script/assert-step-ran
Executable file
@@ -0,0 +1,125 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/assert-step-ran: fail unless a docker build actually executed a
|
||||||
|
# named step, proved by output the step's own tool wrote.
|
||||||
|
#
|
||||||
|
# Usage: script/assert-step-ran LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT
|
||||||
|
#
|
||||||
|
# LOG must be `docker build --progress=plain` output. The default tty
|
||||||
|
# renderer rewrites lines in place and keeps only the tail of a step's
|
||||||
|
# output, so the flag is load-bearing wherever this is used.
|
||||||
|
#
|
||||||
|
# It passes only if some step in LOG satisfies all three of:
|
||||||
|
#
|
||||||
|
# 1. its header is "#ID [STAGE n/m] CMD" with CMD matching STEP_REGEX,
|
||||||
|
# 2. one of the lines that step wrote matches EVIDENCE_REGEX,
|
||||||
|
# 3. BuildKit reported "#ID DONE".
|
||||||
|
#
|
||||||
|
# This observes the build that happened; it parses no Dockerfile. A
|
||||||
|
# stage renamed, moved, made unreachable, deleted, split by a whitespace
|
||||||
|
# byte BuildKit treats as a separator, or an instruction rewritten —
|
||||||
|
# none of it can make this pass a build in which the step did not run,
|
||||||
|
# because a step that did not run wrote no output, and a step served
|
||||||
|
# from cache writes none either ("#ID CACHED", no output lines).
|
||||||
|
#
|
||||||
|
# The stage name in a header is BuildKit's label for that vertex, not a
|
||||||
|
# reading of the file being built. A vertex shared with an earlier build
|
||||||
|
# of a DIFFERENT Dockerfile is replayed under the label it was first
|
||||||
|
# recorded with, so a `Dockerfile.lint` run can print `[lint 1/8]` for
|
||||||
|
# the main `Dockerfile`'s lint stage. A borrowed label cannot produce a
|
||||||
|
# pass — a replayed vertex prints CACHED and writes no evidence, and a
|
||||||
|
# vertex that really executes really ran the command in the header — but
|
||||||
|
# it does mean a header alone proves nothing about which stages the file
|
||||||
|
# declares. Hence one failure message naming both causes rather than a
|
||||||
|
# split that would claim to know which.
|
||||||
|
#
|
||||||
|
# What this guard cannot see — not an exhaustive list of ways a green can
|
||||||
|
# be untrue, only the ones known:
|
||||||
|
#
|
||||||
|
# - Evidence forged inside the matched step, e.g.
|
||||||
|
# `RUN golangci-lint run ... || echo "0 issues."`. Condition 1 ties
|
||||||
|
# the evidence to a step whose own command is in the log, so the
|
||||||
|
# forgery has to be written into that command in the Dockerfile, in
|
||||||
|
# plain sight.
|
||||||
|
# This guards against a step falling silently out of the build. It does
|
||||||
|
# not certify that what ran examined everything it should have — a
|
||||||
|
# `.dockerignore` entry excluding a package makes the linter genuinely
|
||||||
|
# run and genuinely print `0 issues.` while a real violation sits
|
||||||
|
# unexamined in the repo. That is a different question, asked separately
|
||||||
|
# by script/assert-context-complete, which every caller of this script
|
||||||
|
# also runs.
|
||||||
|
#
|
||||||
|
# What it depends on — BuildKit's plain progress format and the tool's
|
||||||
|
# own success wording — fails the caller loudly if it drifts, because
|
||||||
|
# drift removes a match rather than creating one.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "script/assert-step-ran: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
[ $# -eq 5 ] || die "usage: $0 LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT"
|
||||||
|
[ -r "$1" ] || die "cannot read $1"
|
||||||
|
|
||||||
|
# The regexes travel in the environment, not in -v: awk expands escape
|
||||||
|
# sequences in a -v assignment, which would eat a backslash before the
|
||||||
|
# regex ever sees it.
|
||||||
|
ASR_STAGE="$2" ASR_STEP="$3" ASR_EVIDENCE="$4" ASR_WHAT="$5" awk '
|
||||||
|
function fail(msg) {
|
||||||
|
printf "script/assert-step-ran: %s\n", msg | "cat 1>&2"
|
||||||
|
close("cat 1>&2")
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
BEGIN {
|
||||||
|
stage = ENVIRON["ASR_STAGE"]
|
||||||
|
step = ENVIRON["ASR_STEP"]
|
||||||
|
evidence = ENVIRON["ASR_EVIDENCE"]
|
||||||
|
what = ENVIRON["ASR_WHAT"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Every progress line is "#ID " and then a step header, a status, or one
|
||||||
|
# line the step itself wrote.
|
||||||
|
/^#[0-9]+ / {
|
||||||
|
id = substr($1, 2)
|
||||||
|
rest = substr($0, length($1) + 2)
|
||||||
|
|
||||||
|
if (rest == "CACHED") { cached[id] = 1; next }
|
||||||
|
if (rest ~ /^DONE /) { done[id] = 1; next }
|
||||||
|
|
||||||
|
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
|
||||||
|
# build names a platform. Bracketed spans with no n/m are BuildKit
|
||||||
|
# internals ("[internal] load build context"), not steps.
|
||||||
|
if (substr(rest, 1, 1) == "[") {
|
||||||
|
p = index(rest, "] ")
|
||||||
|
if (p == 0) next
|
||||||
|
n = split(substr(rest, 2, p - 2), part, " ")
|
||||||
|
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
|
||||||
|
if (part[n - 1] != stage) next
|
||||||
|
if (substr(rest, p + 2) ~ step) matched[id] = 1
|
||||||
|
next
|
||||||
|
}
|
||||||
|
|
||||||
|
# Output: "#ID 41.80 <the line the step wrote>".
|
||||||
|
if (rest ~ /^[0-9]+[.][0-9]+ /) {
|
||||||
|
sub(/^[0-9]+[.][0-9]+ /, "", rest)
|
||||||
|
if (rest ~ evidence) emitted[id] = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
END {
|
||||||
|
for (id in matched) {
|
||||||
|
nmatched++
|
||||||
|
if (done[id] && emitted[id]) exit 0
|
||||||
|
if (cached[id]) ncached++
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nmatched == 0)
|
||||||
|
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so %s did not run. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or it was built without that command among its steps", step, stage, what))
|
||||||
|
|
||||||
|
if (ncached == nmatched)
|
||||||
|
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so %s did not run on this tree; the cache bust for that stage is not taking effect", step, stage, what))
|
||||||
|
|
||||||
|
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but never wrote a line matching /%s/, so there is no evidence %s did the work; the command or the tool that produces that line has changed", step, stage, evidence, what))
|
||||||
|
}
|
||||||
|
' "$1"
|
||||||
84
script/bootstrap
Executable file
84
script/bootstrap
Executable file
@@ -0,0 +1,84 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/bootstrap: install all dependencies needed to build and develop
|
||||||
|
# this repo. Idempotent: every install is guarded by a check so already
|
||||||
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
|
# make, or go). The linter is NOT installed locally: golangci-lint runs
|
||||||
|
# via docker only (script/lint), pinned by image digest, so the only
|
||||||
|
# lint prerequisite is a working docker.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
PKGMGR=""
|
||||||
|
SUDO=""
|
||||||
|
|
||||||
|
detect_pkgmgr() {
|
||||||
|
[ -n "$PKGMGR" ] && return 0
|
||||||
|
if command -v nix-env >/dev/null 2>&1; then
|
||||||
|
PKGMGR="nix"
|
||||||
|
elif command -v apt-get >/dev/null 2>&1; then
|
||||||
|
PKGMGR="apt"
|
||||||
|
elif command -v brew >/dev/null 2>&1; then
|
||||||
|
PKGMGR="brew"
|
||||||
|
elif command -v apk >/dev/null 2>&1; then
|
||||||
|
PKGMGR="apk"
|
||||||
|
else
|
||||||
|
echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$PKGMGR" = "apt" ]; then
|
||||||
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
|
if [ "$(id -u)" != "0" ]; then
|
||||||
|
SUDO="sudo"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
|
||||||
|
pkg_install() {
|
||||||
|
detect_pkgmgr
|
||||||
|
case "$PKGMGR" in
|
||||||
|
nix) nix-env -iA "nixpkgs.$1" ;;
|
||||||
|
apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;;
|
||||||
|
brew) brew install "$3" ;;
|
||||||
|
apk) apk add --no-cache "$4" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
missing() {
|
||||||
|
! command -v "$1" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
# Base tooling
|
||||||
|
if missing git; then pkg_install git git git git; fi
|
||||||
|
if missing make; then pkg_install gnumake make make make; fi
|
||||||
|
|
||||||
|
# Go toolchain
|
||||||
|
if missing go; then pkg_install go golang go go; fi
|
||||||
|
|
||||||
|
# Linting runs via docker only (script/lint). Warn, don't fail:
|
||||||
|
# everything except `make lint` works without it.
|
||||||
|
if missing docker; then
|
||||||
|
echo "bootstrap: WARNING: docker not found; make lint and" >&2
|
||||||
|
echo "bootstrap: make docker require it. Install docker to" >&2
|
||||||
|
echo "bootstrap: run the linter." >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Markdown/CSS/JS formatting uses prettier. script/prettier falls
|
||||||
|
# back to a pinned docker image when it is not on PATH, so this is
|
||||||
|
# a convenience, not a requirement.
|
||||||
|
if missing prettier && missing docker; then
|
||||||
|
echo "bootstrap: WARNING: neither prettier nor docker found;" >&2
|
||||||
|
echo "bootstrap: markdown formatting will be skipped." >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
go mod download
|
||||||
|
|
||||||
|
echo "bootstrap complete"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
15
script/check
Executable file
15
script/check
Executable file
@@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||||
|
# extension to scripts-to-rule-them-all. Must not modify any files.
|
||||||
|
# Generic: usually needs no adaptation.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
"$SCRIPT_DIR/test"
|
||||||
|
"$SCRIPT_DIR/lint"
|
||||||
|
"$SCRIPT_DIR/fmt-check"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
89
script/cibuild
Executable file
89
script/cibuild
Executable file
@@ -0,0 +1,89 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/cibuild: run the CI build. The Dockerfile runs the checks
|
||||||
|
# (gofmt, config verify, lint, test), so a successful build implies a
|
||||||
|
# green repo. The Gitea workflow runs this on push.
|
||||||
|
#
|
||||||
|
# Only if the checks actually ran, though, and a green `docker build` is
|
||||||
|
# no evidence that they did:
|
||||||
|
#
|
||||||
|
# 1. On an unchanged tree every layer comes from cache and the build
|
||||||
|
# exits 0 in under a second having executed nothing. Hence the two
|
||||||
|
# --no-cache-filter flags.
|
||||||
|
#
|
||||||
|
# 2. BuildKit silently ignores a --no-cache-filter naming a stage that
|
||||||
|
# does not exist, so a rename restores that green no-op.
|
||||||
|
#
|
||||||
|
# 3. BuildKit builds only the final stage's dependency graph. A stage
|
||||||
|
# nothing references is never built, and with no --target the final
|
||||||
|
# stage is whichever is last in the file — so appending a stage, or
|
||||||
|
# moving a reference into a stage that is itself unreachable, drops
|
||||||
|
# the checks out of the build while every reference to them is still
|
||||||
|
# there to read.
|
||||||
|
#
|
||||||
|
# Rather than predict any of that from the Dockerfile's text, this asks
|
||||||
|
# the finished build what it did: script/assert-step-ran requires that
|
||||||
|
# the log contain a lint step and a test step that ran, and that each
|
||||||
|
# wrote the line its own tool writes on success. Every trap above ends
|
||||||
|
# with the step absent from the log or served from cache, and both fail
|
||||||
|
# that assertion. See that script for what it does not cover.
|
||||||
|
#
|
||||||
|
# 4. A step that ran is not a step that saw the repo. .dockerignore, or
|
||||||
|
# a COPY narrower than the tree, removes files from the build
|
||||||
|
# context, and the linter then reports `0 issues.` over what is left
|
||||||
|
# while `go test ./...` never compiles the package that went missing.
|
||||||
|
# So each source-consuming stage inventories what reached it, and
|
||||||
|
# script/assert-context-complete compares that inventory against the
|
||||||
|
# git index — a source .dockerignore cannot reach.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
dockerfile=Dockerfile
|
||||||
|
lint_stage=lint
|
||||||
|
test_stage=builder
|
||||||
|
|
||||||
|
# `ok <pkg> 1.234s`: go test prints `(cached)` in place of the duration
|
||||||
|
# when it replays a result, so requiring the duration requires a package
|
||||||
|
# that was really exercised.
|
||||||
|
test_ran='^ok[[:space:]]+[^[:space:]]+[[:space:]]+[0-9]+[.][0-9]+s'
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-cibuild.XXXXXX")"
|
||||||
|
trap 'rm -rf "$tmp"' EXIT INT TERM
|
||||||
|
|
||||||
|
# --progress=plain is load-bearing: the assertions below read the
|
||||||
|
# steps' own output out of this log, and the tty renderer discards
|
||||||
|
# all but the tail of it. The exit status travels through a file
|
||||||
|
# because a pipeline's status is the last command's; `set +e` is
|
||||||
|
# what lets the recording line run at all, since errexit would
|
||||||
|
# otherwise abandon the subshell on a failing build. A missing file
|
||||||
|
# reads as failure.
|
||||||
|
(
|
||||||
|
set +e
|
||||||
|
docker build \
|
||||||
|
--progress=plain \
|
||||||
|
--no-cache-filter="$lint_stage" \
|
||||||
|
--no-cache-filter="$test_stage" \
|
||||||
|
-f "$dockerfile" . 2>&1
|
||||||
|
echo "$?" >"$tmp/status"
|
||||||
|
) | tee "$tmp/build.log"
|
||||||
|
|
||||||
|
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
|
||||||
|
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
|
||||||
|
|
||||||
|
script/assert-step-ran "$tmp/build.log" "$lint_stage" \
|
||||||
|
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
|
||||||
|
script/assert-step-ran "$tmp/build.log" "$test_stage" \
|
||||||
|
'^RUN go test' "$test_ran" 'the tests'
|
||||||
|
|
||||||
|
# Both stages, separately: each has its own COPY, so an intact
|
||||||
|
# context in one is no evidence about the other.
|
||||||
|
script/repo-source-manifest >"$tmp/expected"
|
||||||
|
script/assert-context-complete "$tmp/build.log" "$lint_stage" "$tmp/expected"
|
||||||
|
script/assert-context-complete "$tmp/build.log" "$test_stage" "$tmp/expected"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
15
script/docker
Executable file
15
script/docker
Executable file
@@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/docker: build the Docker image tagged with the project name.
|
||||||
|
# Identical in all repos; the tag comes from script/projectname.
|
||||||
|
# Generic: needs no adaptation.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
docker build -t "$("$SCRIPT_DIR/projectname")" .
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
26
script/fmt
Executable file
26
script/fmt
Executable file
@@ -0,0 +1,26 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fmt: format all files (writes). Go via gofmt, everything else
|
||||||
|
# (markdown, CSS, JS, YAML, JSON) via prettier with the repo's
|
||||||
|
# .prettierrc. Never hand-rolled substitutions: this script is the only
|
||||||
|
# sanctioned way to reformat the tree.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
gofmt -s -w .
|
||||||
|
|
||||||
|
if command -v goimports >/dev/null 2>&1; then
|
||||||
|
goimports -w .
|
||||||
|
fi
|
||||||
|
|
||||||
|
# A missing runner (exit 2) is a warning here: script/prettier has
|
||||||
|
# already said so on stderr, and failing `make fmt` over it would
|
||||||
|
# block work that has nothing to do with markdown.
|
||||||
|
"$SCRIPT_DIR/prettier" --write . || [ $? -eq 2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
25
script/fmt-check
Executable file
25
script/fmt-check
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/fmt-check: check formatting (read-only). Same scope as
|
||||||
|
# script/fmt, but fails instead of writing.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
if [ -n "$(gofmt -s -l .)" ]; then
|
||||||
|
echo "gofmt needed on:"
|
||||||
|
gofmt -s -l .
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Exit 2 means no prettier and no docker: reported by
|
||||||
|
# script/prettier on stderr and not treated as a failure, so this
|
||||||
|
# check still runs on a machine that has neither. The container
|
||||||
|
# build is the gate that always has both.
|
||||||
|
"$SCRIPT_DIR/prettier" --check . || [ $? -eq 2 ]
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
16
script/install-precommit
Executable file
16
script/install-precommit
Executable file
@@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/install-precommit: install the git pre-commit hook that runs
|
||||||
|
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
||||||
|
# Generic: needs no adaptation.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
||||||
|
chmod +x .git/hooks/pre-commit
|
||||||
|
echo "pre-commit hook installed: runs script/precommit"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
71
script/lint
Executable file
71
script/lint
Executable file
@@ -0,0 +1,71 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/lint: run the linter. golangci-lint is never installed locally: it
|
||||||
|
# runs via docker only, one way, everywhere.
|
||||||
|
#
|
||||||
|
# Traps, each of which yields a green run over an unlinted or partly linted
|
||||||
|
# tree:
|
||||||
|
#
|
||||||
|
# 1. A bare `docker build -f Dockerfile.lint .` serves the lint layer from
|
||||||
|
# cache on an unchanged tree and exits 0 having linted nothing, and
|
||||||
|
# BuildKit ignores a --no-cache-filter whose stage name matches
|
||||||
|
# nothing, restoring that no-op after a rename. So the run is not
|
||||||
|
# trusted: script/assert-step-ran requires the log to show the lint
|
||||||
|
# step executing and golangci-lint's own success line coming out of
|
||||||
|
# it. A cached or absent step fails that.
|
||||||
|
#
|
||||||
|
# 2. .dockerignore decides what reaches the container, and only what
|
||||||
|
# reaches it is linted, so excluding a Go file drops it from the lint
|
||||||
|
# with the linter still reporting `0 issues.` over what it was
|
||||||
|
# handed. So the context is not trusted either: the lint stage
|
||||||
|
# inventories the sources that reached it, and
|
||||||
|
# script/assert-context-complete compares that against the git index,
|
||||||
|
# which .dockerignore cannot touch. Never exclude Go sources,
|
||||||
|
# go.mod/go.sum or .golangci.yml — and now nothing silently does.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
dockerfile=Dockerfile.lint
|
||||||
|
|
||||||
|
# Must match the stage name in Dockerfile.lint.
|
||||||
|
stage=lint
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-lint.XXXXXX")"
|
||||||
|
trap 'rm -rf "$tmp"' EXIT INT TERM
|
||||||
|
|
||||||
|
# No --target: whatever stage is last is the one BuildKit builds, and
|
||||||
|
# the assertion below is what decides whether the linter was in it.
|
||||||
|
# --progress=plain is load-bearing — the tty renderer discards all but
|
||||||
|
# the tail of a step's output. The exit status travels through a file
|
||||||
|
# because a pipeline's status is the last command's; `set +e` is what
|
||||||
|
# lets the recording line run at all, since errexit would otherwise
|
||||||
|
# abandon the subshell on a failing build. A missing file reads as
|
||||||
|
# failure.
|
||||||
|
(
|
||||||
|
set +e
|
||||||
|
docker build \
|
||||||
|
--progress=plain \
|
||||||
|
--no-cache-filter="$stage" \
|
||||||
|
--output=type=cacheonly \
|
||||||
|
-f "$dockerfile" . 2>&1
|
||||||
|
echo "$?" >"$tmp/status"
|
||||||
|
) | tee "$tmp/build.log"
|
||||||
|
|
||||||
|
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
|
||||||
|
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
|
||||||
|
|
||||||
|
script/assert-step-ran "$tmp/build.log" "$stage" \
|
||||||
|
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
|
||||||
|
|
||||||
|
# That the linter ran says nothing about what it was given. The
|
||||||
|
# expectation comes from git and the evidence from the build, which
|
||||||
|
# is the only reason comparing them means anything.
|
||||||
|
script/repo-source-manifest >"$tmp/expected"
|
||||||
|
script/assert-context-complete "$tmp/build.log" "$stage" "$tmp/expected"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
21
script/precommit
Executable file
21
script/precommit
Executable file
@@ -0,0 +1,21 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/precommit: run by the git pre-commit hook; fails the commit if
|
||||||
|
# checks fail. Our own extension to scripts-to-rule-them-all. Go repo
|
||||||
|
# extras: go mod tidy must not change go.mod/go.sum.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
go mod tidy
|
||||||
|
git diff --exit-code -- go.mod go.sum || {
|
||||||
|
echo "precommit: go mod tidy changed go.mod/go.sum;" \
|
||||||
|
"stage the changes and retry" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
"$SCRIPT_DIR/check"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
52
script/prettier
Executable file
52
script/prettier
Executable file
@@ -0,0 +1,52 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/prettier: run prettier over this repo with the repo's own
|
||||||
|
# .prettierrc. Helper, not an entrypoint: script/fmt and
|
||||||
|
# script/fmt-check call it.
|
||||||
|
#
|
||||||
|
# Prettier from PATH if it is there, otherwise a hash-pinned node image
|
||||||
|
# via docker, so a checkout with neither node nor a global prettier
|
||||||
|
# still formats identically. If neither is available the caller is told
|
||||||
|
# and the markdown pass is skipped rather than silently passing — a
|
||||||
|
# formatter that is not present cannot certify formatting.
|
||||||
|
#
|
||||||
|
# Exit status: prettier's own, 2 when it was skipped for lack of a
|
||||||
|
# runner. script/fmt-check treats 2 as a warning, not a failure, so
|
||||||
|
# that `make check` still works on a machine without docker.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
# node:24-alpine, 2026-08-22. Bumping the pin is a deliberate edit.
|
||||||
|
NODE_IMAGE="node:24-alpine@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43"
|
||||||
|
|
||||||
|
# Pinned exactly: prettier's default formatting changes between minor
|
||||||
|
# releases, so an unpinned version turns fmt-check into a coin flip.
|
||||||
|
PRETTIER_VERSION="3.9.6"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
if command -v prettier >/dev/null 2>&1; then
|
||||||
|
prettier "$@"
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
# --user keeps written files owned by the invoking user rather
|
||||||
|
# than root; the container only ever touches the bind mount.
|
||||||
|
docker run --rm \
|
||||||
|
--user "$(id -u):$(id -g)" \
|
||||||
|
-v "$ROOT:/work" \
|
||||||
|
-w /work \
|
||||||
|
"$NODE_IMAGE" \
|
||||||
|
npx --yes "prettier@$PRETTIER_VERSION" "$@"
|
||||||
|
return $?
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "script/prettier: neither prettier nor docker found;" >&2
|
||||||
|
echo "script/prettier: skipping markdown/CSS/JS formatting." >&2
|
||||||
|
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
12
script/projectname
Executable file
12
script/projectname
Executable file
@@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/projectname: output the name of this project. Our own
|
||||||
|
# extension to scripts-to-rule-them-all. Other scripts that need the
|
||||||
|
# name (e.g. script/docker) call this, so they can stay identical
|
||||||
|
# across all repos.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
main() {
|
||||||
|
echo "dcf"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
60
script/repo-source-manifest
Executable file
60
script/repo-source-manifest
Executable file
@@ -0,0 +1,60 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/repo-source-manifest: print every file this repository contains
|
||||||
|
# whose absence from a docker build context would go unnoticed — one
|
||||||
|
# repo-relative path per line, LC_ALL=C-sorted.
|
||||||
|
#
|
||||||
|
# That is the Go sources. A missing go.mod, go.sum or .golangci.yml
|
||||||
|
# fails the build loudly (the COPY errors, or golangci-lint refuses to
|
||||||
|
# start), so they cannot hide anything; they are listed anyway because
|
||||||
|
# it costs two lines and makes the manifest the linter's whole input
|
||||||
|
# rather than most of it.
|
||||||
|
#
|
||||||
|
# The list comes from the git index, never from a walk of the working
|
||||||
|
# tree, and that is the point: script/assert-context-complete compares
|
||||||
|
# it against the inventory the build itself emitted, and a check that
|
||||||
|
# reads its expectation from the same place it reads its evidence proves
|
||||||
|
# nothing. .dockerignore governs what docker sends; it cannot touch what
|
||||||
|
# git tracks.
|
||||||
|
#
|
||||||
|
# Tracked files only, and only those present in the worktree — a file
|
||||||
|
# staged for deletion is not something the build context is missing.
|
||||||
|
# Untracked files are not expected either, so local scratch work in the
|
||||||
|
# tree is not a failure.
|
||||||
|
#
|
||||||
|
# A path containing a newline or a quote is quoted by git and will not
|
||||||
|
# match the plain path the build emits, so it fails loudly rather than
|
||||||
|
# passing silently. No such path exists here, and none should.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
die() {
|
||||||
|
echo "script/repo-source-manifest: $*" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
|
||||||
|
git rev-parse --is-inside-work-tree >/dev/null 2>&1 ||
|
||||||
|
die "not a git work tree, so there is nothing to compare a build context against"
|
||||||
|
|
||||||
|
# Captured before the loop so that a git failure is the script's
|
||||||
|
# exit status: in a pipeline only the last command's status counts.
|
||||||
|
tracked="$(git ls-files -- '*.go' go.mod go.sum .golangci.yml .golangci.yaml)"
|
||||||
|
|
||||||
|
manifest="$(
|
||||||
|
printf '%s\n' "$tracked" | while IFS= read -r f; do
|
||||||
|
if [ -n "$f" ] && [ -f "$f" ]; then
|
||||||
|
printf '%s\n' "$f"
|
||||||
|
fi
|
||||||
|
done | LC_ALL=C sort
|
||||||
|
)"
|
||||||
|
|
||||||
|
[ -n "$manifest" ] ||
|
||||||
|
die "the git index lists no Go sources, so any build context would satisfy the check"
|
||||||
|
|
||||||
|
printf '%s\n' "$manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
14
script/setup
Executable file
14
script/setup
Executable file
@@ -0,0 +1,14 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/setup: set up the repo for development after a fresh clone:
|
||||||
|
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
||||||
|
# Add any repo-specific initialization (db init, .env template) here.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
"$SCRIPT_DIR/bootstrap"
|
||||||
|
"$SCRIPT_DIR/install-precommit"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
21
script/test
Executable file
21
script/test
Executable file
@@ -0,0 +1,21 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# script/test: run the test suite. Quiet on success; on failure, rerun
|
||||||
|
# with -v for full diagnostic output (and still fail — the first run
|
||||||
|
# already proved the tests are broken).
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
|
main() {
|
||||||
|
cd "$ROOT"
|
||||||
|
# -count=1 disables the test result cache. Without it a repeat run on
|
||||||
|
# an unchanged tree reports every package `ok ... (cached)` and exits
|
||||||
|
# 0 having executed nothing — a gate that cannot fail.
|
||||||
|
go test -count=1 -timeout 90s -race -cover ./... || {
|
||||||
|
echo "--- Rerunning with -v for details ---"
|
||||||
|
go test -count=1 -timeout 90s -race -v ./...
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user