Compare commits
1 Commits
master
...
issue-1-re
| Author | SHA1 | Date | |
|---|---|---|---|
| ed95e66f7d |
20
.dockerignore
Normal file
20
.dockerignore
Normal file
@@ -0,0 +1,20 @@
|
||||
# Never exclude Go sources, go.mod, go.sum or .golangci.yml: the lint
|
||||
# and test stages only ever examine what reaches the build context, and
|
||||
# an exclusion here makes them pass over a tree that is missing a
|
||||
# package. script/assert-context-complete exists to catch exactly that,
|
||||
# and will fail the build rather than let it happen silently.
|
||||
.git/
|
||||
.gitea/
|
||||
README.md
|
||||
LICENSE
|
||||
.editorconfig
|
||||
.prettierrc
|
||||
.prettierignore
|
||||
.env
|
||||
.env.*
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
.claude/
|
||||
tmp/
|
||||
temp/
|
||||
15
.editorconfig
Normal file
15
.editorconfig
Normal file
@@ -0,0 +1,15 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.go]
|
||||
indent_style = tab
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
18
.gitea/workflows/check.yml
Normal file
18
.gitea/workflows/check.yml
Normal file
@@ -0,0 +1,18 @@
|
||||
name: check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||
# The image build is the gate: it runs gofmt, config verify,
|
||||
# lint and test, and script/cibuild refuses a build in which
|
||||
# any of them was cached or skipped.
|
||||
- name: cibuild
|
||||
run: script/cibuild
|
||||
29
.gitignore
vendored
29
.gitignore
vendored
@@ -1 +1,28 @@
|
||||
dcfinfo
|
||||
# Build artifacts
|
||||
/bin/
|
||||
*.test
|
||||
*.out
|
||||
*.log
|
||||
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.*
|
||||
*.key
|
||||
*.pem
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Editor and assistant tooling directories. These are per-developer
|
||||
# state and never belong in the repository.
|
||||
*~
|
||||
*.swp
|
||||
*.swo
|
||||
.idea/
|
||||
.vscode/
|
||||
.claude/
|
||||
.cursor/
|
||||
.aider*
|
||||
.continue/
|
||||
.windsurf/
|
||||
|
||||
34
.golangci.yml
Normal file
34
.golangci.yml
Normal file
@@ -0,0 +1,34 @@
|
||||
version: "2"
|
||||
|
||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
||||
# linters.settings, not top-level linters-settings) so that the
|
||||
# thresholds below are actually applied by golangci-lint >= v2.
|
||||
|
||||
run:
|
||||
timeout: 5m
|
||||
modules-download-mode: readonly
|
||||
|
||||
linters:
|
||||
default: all
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
funlen:
|
||||
lines: 80
|
||||
statements: 50
|
||||
cyclop:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
6
.prettierignore
Normal file
6
.prettierignore
Normal file
@@ -0,0 +1,6 @@
|
||||
go.sum
|
||||
LICENSE
|
||||
|
||||
# The vendored golangci-lint config is canonical and must stay
|
||||
# byte-identical to sneak/prompts. Formatting it here would fork it.
|
||||
.golangci.yml
|
||||
4
.prettierrc
Normal file
4
.prettierrc
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"tabWidth": 4,
|
||||
"proseWrap": "always"
|
||||
}
|
||||
83
Dockerfile
Normal file
83
Dockerfile
Normal file
@@ -0,0 +1,83 @@
|
||||
# This repo is a library: there is no cmd/ and no binary to ship, so the
|
||||
# build ends at the stage that compiles and tests the packages. There is
|
||||
# no runtime stage because there is nothing to run.
|
||||
|
||||
# Lint stage — fast feedback on formatting and lint issues. Tools are
|
||||
# invoked directly (not via make/script): the docker build is its own
|
||||
# single path.
|
||||
# This stage must stay the one that runs golangci-lint, and its name must
|
||||
# match $lint_stage in script/cibuild, which cache-busts it by name.
|
||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
|
||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Inventory of the sources that actually arrived here. script/cibuild
|
||||
# reads these lines out of the build log and compares them against the
|
||||
# git index, so a .dockerignore entry or a narrowed COPY that hides a
|
||||
# package fails the run instead of yielding a clean report over a tree
|
||||
# the linter never saw. Keep it immediately after `COPY . .`, and keep
|
||||
# `echo context-manifest-begin` as its first command:
|
||||
# script/assert-context-complete matches the step by that prefix.
|
||||
RUN echo context-manifest-begin; \
|
||||
{ find . -type f -name '*.go'; \
|
||||
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||
done; } \
|
||||
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||
echo context-manifest-end
|
||||
|
||||
# Formatting check, config check, linter
|
||||
RUN test -z "$(gofmt -s -l .)" || { echo "gofmt needed on:"; gofmt -s -l .; exit 1; }
|
||||
RUN golangci-lint config verify --config .golangci.yml
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
|
||||
# Build stage, and the last one: script/cibuild passes no --target, so
|
||||
# BuildKit builds whichever stage is last and appending one drops lint,
|
||||
# builder and their checks out of the run. Must stay the stage that runs
|
||||
# go test, and its name must match $test_stage in script/cibuild, which
|
||||
# cache-busts it by name.
|
||||
# golang:1.25.7-bookworm (Debian-based: the race detector used by the
|
||||
# test run requires glibc), 2026-08-30
|
||||
FROM golang:1.25.7-bookworm@sha256:564e366a28ad1d70f460a2b97d1d299a562f08707eb0ecb24b659e5bd6c108e1 AS builder
|
||||
|
||||
# Depend on lint stage passing (forces BuildKit ordering)
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Same inventory as the lint stage, and separately checked: this stage
|
||||
# has its own COPY, so an intact context over there is no evidence about
|
||||
# the tree `go test ./...` is about to walk here. A package that did not
|
||||
# arrive is a package the tests never run, and the run still ends `ok`.
|
||||
RUN echo context-manifest-begin; \
|
||||
{ find . -type f -name '*.go'; \
|
||||
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||
done; } \
|
||||
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||
echo context-manifest-end
|
||||
|
||||
# Run tests: quiet first, verbose rerun on failure (and still fail).
|
||||
# -count=1 disables the test result cache, matching script/test.
|
||||
RUN go test -count=1 -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
go test -count=1 -timeout 90s -race -v ./...; exit 1; }
|
||||
|
||||
# No binary to emit, so the compile check is the artifact: it proves every
|
||||
# package builds for the target the library claims to support.
|
||||
RUN CGO_ENABLED=0 go build -trimpath ./...
|
||||
35
Dockerfile.lint
Normal file
35
Dockerfile.lint
Normal file
@@ -0,0 +1,35 @@
|
||||
# Lint image, built by script/lint: golangci-lint runs as a build step, so
|
||||
# a successful build is a clean lint. Works with a remote docker daemon,
|
||||
# where bind mounts are impossible.
|
||||
|
||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-30
|
||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# This stage must stay the last one and the one that runs golangci-lint,
|
||||
# and its name must match $stage in script/lint.
|
||||
FROM deps AS lint
|
||||
|
||||
COPY . .
|
||||
|
||||
# Inventory of the sources that actually arrived here. script/lint reads
|
||||
# these lines out of the build log and compares them against the git
|
||||
# index, so a .dockerignore entry or a narrowed COPY that hides a
|
||||
# package fails the run instead of yielding a clean report over a tree
|
||||
# the linter never saw. Keep it immediately after `COPY . .`, and keep
|
||||
# `echo context-manifest-begin` as its first command:
|
||||
# script/assert-context-complete matches the step by that prefix.
|
||||
RUN echo context-manifest-begin; \
|
||||
{ find . -type f -name '*.go'; \
|
||||
for f in go.mod go.sum .golangci.yml .golangci.yaml; do \
|
||||
if [ -f "$f" ]; then echo "./$f"; fi; \
|
||||
done; } \
|
||||
| sed 's|^\./||' | LC_ALL=C sort | sed 's|^|context-file: |'; \
|
||||
echo context-manifest-end
|
||||
|
||||
RUN golangci-lint config verify --config .golangci.yml
|
||||
RUN golangci-lint run --config .golangci.yml ./...
|
||||
50
Makefile
50
Makefile
@@ -1,14 +1,50 @@
|
||||
default: test
|
||||
.PHONY: bootstrap setup test lint fmt fmt-check check build deps docker cibuild clean hooks
|
||||
|
||||
# Every target is a thin shim over script/. The scripts are the real
|
||||
# entrypoints: they carry the project-specific knowledge (flags,
|
||||
# timeouts, docker-only linting) that a raw `go test` or `golangci-lint`
|
||||
# invocation silently bypasses. Never call the underlying tools
|
||||
# directly.
|
||||
.DEFAULT_GOAL := check
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
|
||||
setup:
|
||||
@script/setup
|
||||
|
||||
test:
|
||||
go test -v ./...
|
||||
@script/test
|
||||
|
||||
run: build
|
||||
DEBUG=1 ./dcfinfo
|
||||
lint:
|
||||
@script/lint
|
||||
|
||||
fmt:
|
||||
@script/fmt
|
||||
|
||||
fmt-check:
|
||||
@script/fmt-check
|
||||
|
||||
check:
|
||||
@script/check
|
||||
|
||||
# This repo is a library: there is no cmd/ and nothing to link, so
|
||||
# `build` is the compile check over every package.
|
||||
build:
|
||||
go env -w CGO_ENABLED="0"
|
||||
cd cmd/dcfinfo && go build -o ../../dcfinfo
|
||||
go build ./...
|
||||
|
||||
deps:
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
docker:
|
||||
@script/docker
|
||||
|
||||
cibuild:
|
||||
@script/cibuild
|
||||
|
||||
clean:
|
||||
rm -f dcfinfo
|
||||
go clean ./...
|
||||
|
||||
hooks:
|
||||
@script/install-precommit
|
||||
|
||||
151
README.md
151
README.md
@@ -1,33 +1,158 @@
|
||||
# dcf
|
||||
|
||||
Golang reader implementation of the so-called "Design rule for Camera File
|
||||
system" or DCF, aka JEITA (Japan Electronics and Information Technology
|
||||
Industries Association) specification number CP-3461.
|
||||
`dcf` is a WTFPL-licensed Go library by [@sneak](https://sneak.berlin) that
|
||||
reads the so-called "Design rule for Camera File system", or DCF, aka JEITA
|
||||
(Japan Electronics and Information Technology Industries Association)
|
||||
specification number CP-3461.
|
||||
|
||||
[wikipedia.org/wiki/Design_rule_for_Camera_File_system](https://en.wikipedia.org/wiki/Design_rule_for_Camera_File_system)
|
||||
|
||||
The DCF specification is why your digital camera puts images and videos in
|
||||
`DCIM` and `PRIVATE/M4ROOT` directories on the memory card.
|
||||
|
||||
# status
|
||||
## Status
|
||||
|
||||
incomplete, under development, does not work yet
|
||||
|
||||
# why
|
||||
## Getting started
|
||||
|
||||
```sh
|
||||
go get git.eeqj.de/sneak/dcf
|
||||
```
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.eeqj.de/sneak/dcf/pkg/dcf"
|
||||
)
|
||||
|
||||
func main() {
|
||||
stores, err := dcf.GetDCFStores(0)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, store := range *stores {
|
||||
fmt.Println(store.String())
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To work on the library itself:
|
||||
|
||||
```sh
|
||||
git clone git@git.eeqj.de:sneak/dcf.git
|
||||
cd dcf
|
||||
script/setup # dependencies plus the git pre-commit hook
|
||||
make check # test, lint, fmt-check
|
||||
```
|
||||
|
||||
## Entrypoints
|
||||
|
||||
This repo adheres to the
|
||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||
standard. Each `script/` entrypoint has a thin `make` shim; the scripts are
|
||||
where the project's knowledge about flags, timeouts and docker-only linting
|
||||
lives, so use them rather than invoking `go` or `golangci-lint` directly.
|
||||
|
||||
- `script/bootstrap` (`make bootstrap`) — install build dependencies (git, make,
|
||||
go) idempotently. Linting additionally needs docker; markdown formatting needs
|
||||
docker or a local `prettier`.
|
||||
- `script/setup` (`make setup`) — `bootstrap` plus the git pre-commit hook.
|
||||
- `script/test` (`make test`) — `go test` with the race detector and coverage,
|
||||
`-count=1`, 90s timeout; quiet on success, verbose rerun on failure.
|
||||
- `script/lint` (`make lint`) — `golangci-lint` via docker only, against the
|
||||
digest-pinned image in `Dockerfile.lint`, then `script/assert-step-ran` and
|
||||
`script/assert-context-complete` over the build log. Nothing is installed
|
||||
locally and nothing runs on the host.
|
||||
- `script/fmt` (`make fmt`) — format Go with `gofmt` and everything else with
|
||||
`prettier` (writes).
|
||||
- `script/fmt-check` (`make fmt-check`) — the same scope, read-only.
|
||||
- `script/check` (`make check`) — `test` + `lint` + `fmt-check`. What the
|
||||
pre-commit hook runs. Never modifies files.
|
||||
- `script/docker` (`make docker`) — build the image, tagged with
|
||||
`script/projectname`.
|
||||
- `script/cibuild` (`make cibuild`) — the CI gate:
|
||||
`docker build --progress=plain --no-cache-filter=lint --no-cache-filter=builder .`
|
||||
followed by assertions that the lint step and the test step really ran, and
|
||||
that both stages really received the whole repository. The Gitea workflow runs
|
||||
this on every push.
|
||||
- `script/precommit` — the hook body: `go mod tidy` must not change
|
||||
`go.mod`/`go.sum`, then `check`.
|
||||
- `script/install-precommit` (`make hooks`) — installs the hook.
|
||||
- `script/projectname` — prints the project name; other scripts call it so they
|
||||
can stay identical across repos.
|
||||
- `script/prettier`, `script/assert-step-ran`, `script/assert-context-complete`
|
||||
and `script/repo-source-manifest` are helpers, not entrypoints.
|
||||
|
||||
`make build`, `make deps` and `make clean` are the ordinary conveniences.
|
||||
`make build` compiles every package; this repo is a library and ships no binary,
|
||||
so there is no `cmd/` and nothing to link.
|
||||
|
||||
### Why the lint assertions exist
|
||||
|
||||
A green `docker build` is not evidence that the checks ran. On an unchanged tree
|
||||
every layer comes from cache and the build exits 0 having executed nothing;
|
||||
BuildKit silently ignores a `--no-cache-filter` naming a stage that no longer
|
||||
exists; and a `.dockerignore` entry can remove a package from the build context,
|
||||
after which the linter genuinely runs, genuinely examines what it was handed,
|
||||
and genuinely reports `0 issues.` over a repository with a violation in it.
|
||||
`script/assert-step-ran` and `script/assert-context-complete` close both holes;
|
||||
read the comments in them before changing either.
|
||||
|
||||
## Rationale
|
||||
|
||||
I wanted to copy images off my memory cards for processing and like
|
||||
overengineering and reusable code.
|
||||
|
||||
# bugs
|
||||
## Design
|
||||
|
||||
Everything lives in `pkg/dcf`, a single library package with no command
|
||||
entrypoints.
|
||||
|
||||
- `dcf.go` — the public surface. `Store` is one DCF store (one mounted
|
||||
filesystem); `Object` is one file in it, embedded in `Image` and `Video`.
|
||||
`GetDCFStores` finds the stores on the system and walks each one, classifying
|
||||
files by extension.
|
||||
- `helpers.go` — mountpoint discovery. `findAllMountPoints` enumerates the
|
||||
system's physical filesystems via `gopsutil`; `findDCFMountPoints` keeps only
|
||||
those whose root holds a marker directory.
|
||||
|
||||
Detection is by directory name at the filesystem root, so no filesystem is ever
|
||||
mounted and nothing outside a filesystem root is searched.
|
||||
|
||||
## Bugs
|
||||
|
||||
does not yet handle "dcf file groups" where multiple objects share the same
|
||||
index number. if you need this, send me a link to a zip or tar of a memory
|
||||
card that uses it and i'll see what i can do.
|
||||
index number. if you need this, send me a link to a zip or tar of a memory card
|
||||
that uses it and i'll see what i can do.
|
||||
|
||||
# author
|
||||
## TODO
|
||||
|
||||
The work in flight is on the tracker; this is the reading order for picking it
|
||||
up.
|
||||
|
||||
- [#2](https://git.eeqj.de/sneak/dcf/issues/2) — parse the CP-3461 structure
|
||||
rather than grepping every filesystem root for media extensions.
|
||||
- [#3](https://git.eeqj.de/sneak/dcf/issues/3) — DCF file groups: objects
|
||||
sharing an index number are one logical object.
|
||||
- [#4](https://git.eeqj.de/sneak/dcf/issues/4) — API cleanup: `fs.FS`-based
|
||||
store access, no pointer-to-slice returns, contexts on walks. This is also
|
||||
what makes the filesystem walk testable; today it is reachable only through
|
||||
`GetDCFStores`, which needs real mounted media, so it has no test coverage.
|
||||
- [#5](https://git.eeqj.de/sneak/dcf/issues/5) — checksummed import off the
|
||||
card, which is what the library is for.
|
||||
- [#6](https://git.eeqj.de/sneak/dcf/issues/6) — mountpoint detection checks
|
||||
`M4ROOT` at the filesystem root rather than `PRIVATE/M4ROOT`, and
|
||||
`findDCFMountPoints` stops one store short of `requestedCount`.
|
||||
|
||||
## License
|
||||
|
||||
WTFPL. See [LICENSE](LICENSE).
|
||||
|
||||
## Author
|
||||
|
||||
sneak <[sneak@sneak.berlin](mailto:sneak@sneak.berlin)>
|
||||
|
||||
# license
|
||||
|
||||
WTFPL
|
||||
|
||||
3
go.mod
3
go.mod
@@ -1,6 +1,6 @@
|
||||
module git.eeqj.de/sneak/dcf
|
||||
|
||||
go 1.22.1
|
||||
go 1.25.7
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
@@ -9,7 +9,6 @@ require (
|
||||
|
||||
require (
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/lmittmann/tint v1.0.4 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
)
|
||||
|
||||
2
go.sum
2
go.sum
@@ -3,8 +3,6 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
|
||||
github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
|
||||
230
pkg/dcf/dcf.go
230
pkg/dcf/dcf.go
@@ -1,30 +1,34 @@
|
||||
// Package dcf reads DCF stores: the directory structures described by
|
||||
// the "Design rule for Camera File system" (JEITA CP-3461) that digital
|
||||
// cameras write to the root of removable media.
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
var imageExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw"}
|
||||
var videoExtensions = []string{".mp4", ".mov", ".avi", ".mkv", ".wmv"}
|
||||
|
||||
// DCFStore represents a DCF store, which is a directory sturcture that contains
|
||||
// images and videos in the root of a mounted filesystem.
|
||||
type DCFStore struct {
|
||||
// Store represents a DCF store, which is a directory structure that
|
||||
// contains images and videos in the root of a mounted filesystem.
|
||||
type Store struct {
|
||||
RootDirectory string
|
||||
Images []*Image
|
||||
Videos []*Video
|
||||
}
|
||||
|
||||
// DCFObject represents a file in a DCF store. It is embedded in Image and Video types.
|
||||
type DCFObject struct {
|
||||
DCFStoreRoot string
|
||||
// Object represents a file in a DCF store. It is embedded in the Image
|
||||
// and Video types.
|
||||
type Object struct {
|
||||
StoreRoot string
|
||||
Path string
|
||||
Size int64
|
||||
Extension string
|
||||
@@ -32,146 +36,224 @@ type DCFObject struct {
|
||||
|
||||
// Image represents an image file on a DCF store.
|
||||
type Image struct {
|
||||
DCFObject
|
||||
Object
|
||||
}
|
||||
|
||||
// Video represents a video file on a DCF store.
|
||||
type Video struct {
|
||||
DCFObject
|
||||
Object
|
||||
}
|
||||
|
||||
func (d *DCFObject) FullFilePath() string {
|
||||
return filepath.Join(d.DCFStoreRoot, d.Path)
|
||||
// FullFilePath returns the path of the object on the local filesystem,
|
||||
// which is its store-relative path resolved against the store root.
|
||||
func (d *Object) FullFilePath() string {
|
||||
return filepath.Join(d.StoreRoot, d.Path)
|
||||
}
|
||||
|
||||
// VideosCount returns the number of videos in the DCF store.
|
||||
func (d *DCFStore) VideosCount() int {
|
||||
func (d *Store) VideosCount() int {
|
||||
return len(d.Videos)
|
||||
}
|
||||
|
||||
// ImagesCount returns the number of images in the DCF store.
|
||||
func (d *DCFStore) ImagesCount() int {
|
||||
func (d *Store) ImagesCount() int {
|
||||
return len(d.Images)
|
||||
}
|
||||
|
||||
func (d *DCFStore) TotalImageSize() int64 {
|
||||
// TotalImageSize returns the summed size in bytes of the store's images.
|
||||
func (d *Store) TotalImageSize() int64 {
|
||||
totalSize := int64(0)
|
||||
for _, image := range d.Images {
|
||||
totalSize += image.Size
|
||||
}
|
||||
|
||||
return totalSize
|
||||
}
|
||||
|
||||
func (d *DCFStore) TotalVideoSize() int64 {
|
||||
// TotalVideoSize returns the summed size in bytes of the store's videos.
|
||||
func (d *Store) TotalVideoSize() int64 {
|
||||
totalSize := int64(0)
|
||||
for _, video := range d.Videos {
|
||||
totalSize += video.Size
|
||||
}
|
||||
|
||||
return totalSize
|
||||
}
|
||||
|
||||
func (d *DCFStore) TotalSize() int64 {
|
||||
// TotalSize returns the summed size in bytes of every object in the
|
||||
// store.
|
||||
func (d *Store) TotalSize() int64 {
|
||||
return d.TotalImageSize() + d.TotalVideoSize()
|
||||
}
|
||||
|
||||
func (d *DCFStore) String() string {
|
||||
return fmt.Sprintf("DCFStore{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}",
|
||||
// String renders the store as a single human-readable summary line.
|
||||
func (d *Store) String() string {
|
||||
return fmt.Sprintf(
|
||||
"Store{RootDirectory: %s, Images: %d, Videos: %d, TotalSize: %s}",
|
||||
d.RootDirectory,
|
||||
d.ImagesCount(),
|
||||
d.VideosCount(),
|
||||
humanize.Bytes(uint64(d.TotalSize())),
|
||||
humanize.Bytes(d.totalSizeBytes()),
|
||||
)
|
||||
}
|
||||
|
||||
// totalSizeBytes returns TotalSize as an unsigned byte count for
|
||||
// formatting. Sizes come from the filesystem and are never negative,
|
||||
// but the conversion is guarded rather than assumed.
|
||||
func (d *Store) totalSizeBytes() uint64 {
|
||||
total := d.TotalSize()
|
||||
if total < 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return uint64(total)
|
||||
}
|
||||
|
||||
// String renders the video as a single human-readable line.
|
||||
func (v *Video) String() string {
|
||||
return fmt.Sprintf("Video{Path: %s, Size: %d, Extension: %s}", v.Path, v.Size, v.Extension)
|
||||
return fmt.Sprintf(
|
||||
"Video{Path: %s, Size: %d, Extension: %s}",
|
||||
v.Path, v.Size, v.Extension,
|
||||
)
|
||||
}
|
||||
|
||||
// String renders the image as a single human-readable line.
|
||||
func (i *Image) String() string {
|
||||
return fmt.Sprintf("Image{Path: %s, Size: %d, Extension: %s}", i.Path, i.Size, i.Extension)
|
||||
return fmt.Sprintf(
|
||||
"Image{Path: %s, Size: %d, Extension: %s}",
|
||||
i.Path, i.Size, i.Extension,
|
||||
)
|
||||
}
|
||||
|
||||
// Hash() returns the SHA256 hash of the file as a hex string.
|
||||
func (d *DCFObject) Hash() (string, error) {
|
||||
// Hash returns the SHA-256 digest of the object's file as a hex string.
|
||||
func (d *Object) Hash() (string, error) {
|
||||
return pathToSHA256(d.FullFilePath())
|
||||
}
|
||||
|
||||
// pathToSHA256 returns the SHA-256 digest of the file at path as a hex
|
||||
// string.
|
||||
func pathToSHA256(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
// The path is supplied by the caller by design: this library exists
|
||||
// to hash files the caller pointed it at.
|
||||
file, err := os.Open(filepath.Clean(path))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
|
||||
_, err = io.Copy(hash, file)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", hash.Sum(nil)), nil
|
||||
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// GetDCFStores returns a list of DCF stores found on the system in the root of any mounted filesystems.
|
||||
// the integer argument is the number of DCF stores to return. If the argument is 0, all DCF stores are returned.
|
||||
// It does not mount filesystems or search outside of filesystem root directories. It does, however,
|
||||
// walk the filesystems to find images and videos and populate the returned DCFStores.
|
||||
func GetDCFStores(requestedCount int) (*[]DCFStore, error) {
|
||||
dcfStorePaths, err := findDCFMountPoints(requestedCount)
|
||||
// GetDCFStores returns the DCF stores found in the root directories of
|
||||
// the system's mounted filesystems. requestedCount bounds how many
|
||||
// stores are returned; 0 returns all of them.
|
||||
//
|
||||
// It neither mounts filesystems nor searches outside filesystem root
|
||||
// directories. It does walk each store it finds, populating the
|
||||
// returned stores with the images and videos they contain.
|
||||
func GetDCFStores(requestedCount int) (*[]Store, error) {
|
||||
storePaths, err := findDCFMountPoints(requestedCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dcfStores := []DCFStore{}
|
||||
for _, dcfStorePath := range dcfStorePaths {
|
||||
dcfStore := DCFStore{
|
||||
RootDirectory: dcfStorePath,
|
||||
|
||||
stores := []Store{}
|
||||
|
||||
for _, storePath := range storePaths {
|
||||
store, err := scanStore(storePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stores = append(stores, *store)
|
||||
}
|
||||
|
||||
return &stores, nil
|
||||
}
|
||||
|
||||
// scanStore walks root and returns a Store holding every image and
|
||||
// video beneath it.
|
||||
func scanStore(root string) (*Store, error) {
|
||||
store := Store{
|
||||
RootDirectory: root,
|
||||
Images: make([]*Image, 0),
|
||||
Videos: make([]*Video, 0),
|
||||
}
|
||||
|
||||
err := filepath.Walk(dcfStorePath, func(path string, info os.FileInfo, err error) error {
|
||||
walk := func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && isImageFile(path) {
|
||||
image := Image{}
|
||||
image.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
||||
image.Size = info.Size()
|
||||
image.Extension = filepath.Ext(path)
|
||||
image.DCFStoreRoot = dcfStorePath
|
||||
dcfStore.Images = append(dcfStore.Images, &image)
|
||||
}
|
||||
if !info.IsDir() && isVideoFile(path) {
|
||||
video := Video{}
|
||||
video.Path = strings.TrimPrefix(path, dcfStorePath+"/")
|
||||
video.Size = info.Size()
|
||||
video.Extension = filepath.Ext(path)
|
||||
video.DCFStoreRoot = dcfStorePath
|
||||
dcfStore.Videos = append(dcfStore.Videos, &video)
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
return store.addFile(root, path, entry)
|
||||
}
|
||||
|
||||
err := filepath.WalkDir(root, walk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dcfStores = append(dcfStores, dcfStore)
|
||||
}
|
||||
return &dcfStores, nil
|
||||
|
||||
return &store, nil
|
||||
}
|
||||
|
||||
// addFile classifies path by extension and appends it to the store's
|
||||
// images or videos. A file that is neither is ignored.
|
||||
func (d *Store) addFile(root, path string, entry fs.DirEntry) error {
|
||||
isImage := isImageFile(path)
|
||||
if !isImage && !isVideoFile(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
object := Object{
|
||||
StoreRoot: root,
|
||||
Path: strings.TrimPrefix(path, root+string(filepath.Separator)),
|
||||
Size: info.Size(),
|
||||
Extension: filepath.Ext(path),
|
||||
}
|
||||
|
||||
if isImage {
|
||||
d.Images = append(d.Images, &Image{Object: object})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
d.Videos = append(d.Videos, &Video{Object: object})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isImageFile reports whether path names a file this package treats as
|
||||
// an image.
|
||||
func isImageFile(path string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
for _, imageExt := range imageExtensions {
|
||||
if ext == imageExt {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return hasExtension(path,
|
||||
".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".raw", ".arw")
|
||||
}
|
||||
|
||||
// isVideoFile reports whether path names a file this package treats as
|
||||
// a video.
|
||||
func isVideoFile(path string) bool {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
for _, videoExt := range videoExtensions {
|
||||
if ext == videoExt {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
return hasExtension(path, ".mp4", ".mov", ".avi", ".mkv", ".wmv")
|
||||
}
|
||||
|
||||
// hasExtension reports whether path's extension, lowercased, is one of
|
||||
// the given extensions. Extensions are given with their leading dot.
|
||||
func hasExtension(path string, extensions ...string) bool {
|
||||
return slices.Contains(extensions, strings.ToLower(filepath.Ext(path)))
|
||||
}
|
||||
|
||||
@@ -1,10 +1,177 @@
|
||||
package dcf
|
||||
package dcf_test
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/dcf/pkg/dcf"
|
||||
)
|
||||
|
||||
func TestCompile(t *testing.T) {
|
||||
// This is a placeholder test to ensure that the module compiles successfully.
|
||||
// You can add more meaningful tests here once you start implementing your code.
|
||||
// newStore builds a store with two images and one video of known sizes,
|
||||
// so the count and size accessors have something deterministic to
|
||||
// report.
|
||||
func newStore(root string) *dcf.Store {
|
||||
return &dcf.Store{
|
||||
RootDirectory: root,
|
||||
Images: []*dcf.Image{
|
||||
{Object: dcf.Object{
|
||||
StoreRoot: root,
|
||||
Path: "DCIM/100MSDCF/DSC00001.ARW",
|
||||
Size: 2000,
|
||||
Extension: ".ARW",
|
||||
}},
|
||||
{Object: dcf.Object{
|
||||
StoreRoot: root,
|
||||
Path: "DCIM/100MSDCF/DSC00002.JPG",
|
||||
Size: 500,
|
||||
Extension: ".JPG",
|
||||
}},
|
||||
},
|
||||
Videos: []*dcf.Video{
|
||||
{Object: dcf.Object{
|
||||
StoreRoot: root,
|
||||
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||
Size: 9000,
|
||||
Extension: ".MP4",
|
||||
}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectFullFilePath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
object := dcf.Object{
|
||||
StoreRoot: filepath.Join("/mnt", "card"),
|
||||
Path: filepath.Join("DCIM", "100MSDCF", "DSC00001.ARW"),
|
||||
}
|
||||
|
||||
want := filepath.Join("/mnt", "card", "DCIM", "100MSDCF", "DSC00001.ARW")
|
||||
if got := object.FullFilePath(); got != want {
|
||||
t.Fatalf("FullFilePath() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := newStore("/mnt/card")
|
||||
|
||||
if got := store.ImagesCount(); got != 2 {
|
||||
t.Errorf("ImagesCount() = %d, want 2", got)
|
||||
}
|
||||
|
||||
if got := store.VideosCount(); got != 1 {
|
||||
t.Errorf("VideosCount() = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := newStore("/mnt/card")
|
||||
|
||||
if got := store.TotalImageSize(); got != 2500 {
|
||||
t.Errorf("TotalImageSize() = %d, want 2500", got)
|
||||
}
|
||||
|
||||
if got := store.TotalVideoSize(); got != 9000 {
|
||||
t.Errorf("TotalVideoSize() = %d, want 9000", got)
|
||||
}
|
||||
|
||||
if got := store.TotalSize(); got != 11500 {
|
||||
t.Errorf("TotalSize() = %d, want 11500", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyStoreSizesAreZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store := dcf.Store{RootDirectory: "/mnt/empty"}
|
||||
if got := store.TotalSize(); got != 0 {
|
||||
t.Fatalf("TotalSize() = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreStringReportsHumanizedTotal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := newStore("/mnt/card").String()
|
||||
for _, want := range []string{"/mnt/card", "Images: 2", "Videos: 1", "12 kB"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("String() = %q, want it to contain %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestImageAndVideoString(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
image := dcf.Image{Object: dcf.Object{
|
||||
Path: "DCIM/100MSDCF/DSC00001.ARW",
|
||||
Size: 2000,
|
||||
Extension: ".ARW",
|
||||
}}
|
||||
|
||||
want := "Image{Path: DCIM/100MSDCF/DSC00001.ARW, Size: 2000, " +
|
||||
"Extension: .ARW}"
|
||||
if got := image.String(); got != want {
|
||||
t.Errorf("Image.String() = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
video := dcf.Video{Object: dcf.Object{
|
||||
Path: "PRIVATE/M4ROOT/CLIP/C0001.MP4",
|
||||
Size: 9000,
|
||||
Extension: ".MP4",
|
||||
}}
|
||||
|
||||
want = "Video{Path: PRIVATE/M4ROOT/CLIP/C0001.MP4, Size: 9000, " +
|
||||
"Extension: .MP4}"
|
||||
if got := video.String(); got != want {
|
||||
t.Errorf("Video.String() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashMatchesSHA256OfFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := t.TempDir()
|
||||
contents := []byte("DSC00001.ARW contents")
|
||||
|
||||
err := os.MkdirAll(filepath.Join(root, "DCIM"), 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(filepath.Join(root, "DCIM", "a.arw"), contents, 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
object := dcf.Object{StoreRoot: root, Path: filepath.Join("DCIM", "a.arw")}
|
||||
|
||||
got, err := object.Hash()
|
||||
if err != nil {
|
||||
t.Fatalf("Hash() returned error: %v", err)
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(contents)
|
||||
if want := hex.EncodeToString(sum[:]); got != want {
|
||||
t.Fatalf("Hash() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectHashOnMissingFileReturnsError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
object := dcf.Object{StoreRoot: t.TempDir(), Path: "absent.jpg"}
|
||||
|
||||
_, err := object.Hash()
|
||||
if err == nil {
|
||||
t.Fatal("Hash() on a missing file returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,75 +1,84 @@
|
||||
package dcf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
|
||||
"github.com/shirou/gopsutil/disk"
|
||||
)
|
||||
|
||||
// dcfMarkerDirectories are the directory names whose presence in a
|
||||
// filesystem root identifies that filesystem as a DCF store.
|
||||
func dcfMarkerDirectories() []string {
|
||||
return []string{"DCIM", "M4ROOT"}
|
||||
}
|
||||
|
||||
// findAllMountPoints returns the distinct mountpoints of the system's
|
||||
// physical filesystems.
|
||||
func findAllMountPoints() ([]string, error) {
|
||||
mountpoints := []string{}
|
||||
partitions, err := disk.Partitions(false) // physical devices only, so false
|
||||
partitions, err := disk.Partitions(false) // physical devices only
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mountpoints := []string{}
|
||||
|
||||
for _, partition := range partitions {
|
||||
if !contains(mountpoints, partition.Mountpoint) {
|
||||
if !slices.Contains(mountpoints, partition.Mountpoint) {
|
||||
mountpoints = append(mountpoints, partition.Mountpoint)
|
||||
}
|
||||
}
|
||||
|
||||
return mountpoints, nil
|
||||
}
|
||||
|
||||
func dump(x interface{}) {
|
||||
_, file, line, _ := runtime.Caller(1)
|
||||
slog.Debug(fmt.Sprintf("%s:%d %#v\n", file, line, x))
|
||||
}
|
||||
|
||||
// findDCFMountPoints returns a list of strings of the paths of mountpoints that
|
||||
// contain a DCIM or PRIVATE directory, which is how we detect DCF stores. Its
|
||||
// only argument is an integer of how many mountpoints to return. If the
|
||||
// argument is 0, all mountpoints are returned.
|
||||
// findDCFMountPoints returns the paths of the mountpoints that contain
|
||||
// one of the marker directories a DCF store is identified by. Its only
|
||||
// argument is how many mountpoints to return; 0 returns all of them.
|
||||
func findDCFMountPoints(requestedCount int) ([]string, error) {
|
||||
filteredMountpoints := []string{}
|
||||
var erro error
|
||||
mountpoints, err := findAllMountPoints()
|
||||
|
||||
if erro != nil {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
filteredMountpoints := []string{}
|
||||
|
||||
for _, mountpoint := range mountpoints {
|
||||
dcimPath := filepath.Join(mountpoint, "DCIM")
|
||||
privatePath := filepath.Join(mountpoint, "PRIVATE")
|
||||
privatePath = filepath.Join(mountpoint, "M4ROOT")
|
||||
shouldKeep := false
|
||||
if _, err := os.Stat(dcimPath); err == nil {
|
||||
shouldKeep = true
|
||||
slog.Debug(fmt.Sprintf("Found DCIM directory at %s\n", dcimPath))
|
||||
if !isDCFMountPoint(mountpoint) {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(privatePath); err == nil {
|
||||
shouldKeep = true
|
||||
slog.Debug(fmt.Sprintf("Found M4ROOT directory at %s\n", privatePath))
|
||||
}
|
||||
if shouldKeep {
|
||||
slog.Debug(fmt.Sprintf("Keeping mountpoint %s\n", mountpoint))
|
||||
|
||||
slog.Debug("keeping mountpoint", "mountpoint", mountpoint)
|
||||
|
||||
filteredMountpoints = append(filteredMountpoints, mountpoint)
|
||||
if (requestedCount > 0) && (len(filteredMountpoints)+1 >= requestedCount) {
|
||||
|
||||
if requestedCount > 0 && len(filteredMountpoints)+1 >= requestedCount {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return filteredMountpoints, nil
|
||||
}
|
||||
|
||||
func contains(slice []string, str string) bool {
|
||||
for _, s := range slice {
|
||||
if s == str {
|
||||
return true
|
||||
// isDCFMountPoint reports whether mountpoint holds any of the marker
|
||||
// directories a DCF store is identified by.
|
||||
func isDCFMountPoint(mountpoint string) bool {
|
||||
found := false
|
||||
|
||||
for _, marker := range dcfMarkerDirectories() {
|
||||
markerPath := filepath.Join(mountpoint, marker)
|
||||
|
||||
_, err := os.Stat(markerPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Debug("found marker directory", "path", markerPath)
|
||||
|
||||
found = true
|
||||
}
|
||||
return false
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
166
script/assert-context-complete
Executable file
166
script/assert-context-complete
Executable file
@@ -0,0 +1,166 @@
|
||||
#!/bin/sh
|
||||
# script/assert-context-complete: fail unless the build context a stage
|
||||
# was given contained every source file the repository has.
|
||||
#
|
||||
# Usage: script/assert-context-complete LOG STAGE EXPECTED
|
||||
#
|
||||
# LOG must be `docker build --progress=plain` output. EXPECTED is a list
|
||||
# of repo-relative paths, one per line, as script/repo-source-manifest
|
||||
# prints it.
|
||||
#
|
||||
# script/assert-step-ran answers "did the tool execute". This answers
|
||||
# the other half: "was the tool given the tree". They are different
|
||||
# questions, and a green build can satisfy the first while failing the
|
||||
# second — a .dockerignore entry, or a COPY that brings in less than the
|
||||
# whole tree, removes files from the context silently, and golangci-lint
|
||||
# then genuinely runs, genuinely examines what it was handed, and
|
||||
# genuinely reports `0 issues.` over a repo that has a violation in it.
|
||||
# `go test ./...` likewise never runs a package that did not arrive.
|
||||
#
|
||||
# How it knows: each source-consuming stage emits, right after its
|
||||
# `COPY . .`, an inventory of the Go files actually present —
|
||||
#
|
||||
# context-manifest-begin
|
||||
# context-file: cmd/gotemplate/main.go
|
||||
# ...
|
||||
# context-manifest-end
|
||||
#
|
||||
# — and this compares that against EXPECTED, which comes from the git
|
||||
# index rather than from the same build. The two sources are
|
||||
# independent: .dockerignore decides the first and cannot touch the
|
||||
# second. An expectation read from the evidence would prove nothing.
|
||||
#
|
||||
# The manifest lines are only believed under the same discipline
|
||||
# script/assert-step-ran applies to a tool's success line: they must be
|
||||
# attributed to the id of a step whose header is "#ID [STAGE n/m] CMD"
|
||||
# with CMD matching the manifest command, which BuildKit reported DONE,
|
||||
# and both sentinels must be present. So another step's output does not
|
||||
# count, a header does not count, a cached step (which writes nothing)
|
||||
# does not count, and a truncated log fails rather than passing with a
|
||||
# short list.
|
||||
#
|
||||
# Only one direction is checked: everything the repo has must have
|
||||
# arrived. Files in the context that git does not track are not a
|
||||
# failure — untracked local work is normal and hides nothing.
|
||||
#
|
||||
# What this does not cover: a Dockerfile edit to the manifest step
|
||||
# itself, and evidence forged inside a matched command. Both are visible
|
||||
# in the Dockerfile in plain sight, and both are equally outside
|
||||
# script/assert-step-ran. This is not an exhaustive list of ways a green
|
||||
# run can be untrue; it is the ones known.
|
||||
set -eu
|
||||
|
||||
die() {
|
||||
echo "script/assert-context-complete: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ $# -eq 3 ] || die "usage: $0 LOG STAGE EXPECTED"
|
||||
[ -r "$1" ] || die "cannot read $1"
|
||||
[ -r "$3" ] || die "cannot read $3"
|
||||
|
||||
# The stage and the expected-list path travel in the environment, not in
|
||||
# -v: awk expands escape sequences in a -v assignment.
|
||||
ACC_STAGE="$2" ACC_EXPECTED="$3" awk '
|
||||
function fail(msg) {
|
||||
printf "script/assert-context-complete: %s\n", msg | "cat 1>&2"
|
||||
close("cat 1>&2")
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The package a missing file belongs to — its directory, which is what a
|
||||
# reader needs named. A bare "." for a root-level file reads as a typo
|
||||
# next to the sentence punctuation, so it is spelled out.
|
||||
function pkgof(path, i) {
|
||||
i = length(path)
|
||||
while (i > 0 && substr(path, i, 1) != "/") i--
|
||||
return i == 0 ? "the repository root" : substr(path, 1, i - 1)
|
||||
}
|
||||
|
||||
function preview(list, n, limit, i, out) {
|
||||
for (i = 1; i <= n && i <= limit; i++)
|
||||
out = out (i > 1 ? ", " : "") list[i]
|
||||
if (n > limit)
|
||||
out = out sprintf(", and %d more", n - limit)
|
||||
return out
|
||||
}
|
||||
|
||||
BEGIN {
|
||||
stage = ENVIRON["ACC_STAGE"]
|
||||
|
||||
# Fixed by the protocol the Dockerfile emits, so callers cannot
|
||||
# drift from it.
|
||||
step = "^RUN echo context-manifest-begin"
|
||||
beginmark = "context-manifest-begin"
|
||||
endmark = "context-manifest-end"
|
||||
prefix = "context-file: "
|
||||
|
||||
expected_path = ENVIRON["ACC_EXPECTED"]
|
||||
while ((getline line < expected_path) > 0)
|
||||
if (line != "") expected[++nexpected] = line
|
||||
close(expected_path)
|
||||
|
||||
if (nexpected == 0)
|
||||
fail(sprintf("the expected-file list (%s) is empty, so any build context would satisfy this check", expected_path))
|
||||
}
|
||||
|
||||
# Every progress line is "#ID " and then a step header, a status, or one
|
||||
# line the step itself wrote.
|
||||
/^#[0-9]+ / {
|
||||
id = substr($1, 2)
|
||||
rest = substr($0, length($1) + 2)
|
||||
|
||||
if (rest == "CACHED") { cached[id] = 1; next }
|
||||
if (rest ~ /^DONE /) { done[id] = 1; next }
|
||||
|
||||
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
|
||||
# build names a platform. Bracketed spans with no n/m are BuildKit
|
||||
# internals, not steps.
|
||||
if (substr(rest, 1, 1) == "[") {
|
||||
p = index(rest, "] ")
|
||||
if (p == 0) next
|
||||
n = split(substr(rest, 2, p - 2), part, " ")
|
||||
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
|
||||
if (part[n - 1] != stage) next
|
||||
if (substr(rest, p + 2) ~ step) matched[id] = 1
|
||||
next
|
||||
}
|
||||
|
||||
# Output: "#ID 0.31 <the line the step wrote>".
|
||||
if (rest ~ /^[0-9]+[.][0-9]+ /) {
|
||||
sub(/^[0-9]+[.][0-9]+ /, "", rest)
|
||||
if (rest == beginmark) begun[id] = 1
|
||||
else if (rest == endmark) ended[id] = 1
|
||||
else if (index(rest, prefix) == 1)
|
||||
arrived[id SUBSEP substr(rest, length(prefix) + 1)] = 1
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
for (id in matched) {
|
||||
nmatched++
|
||||
if (cached[id]) ncached++
|
||||
if (done[id] && begun[id] && ended[id]) chosen = id
|
||||
}
|
||||
|
||||
if (nmatched == 0)
|
||||
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so the build context was never inventoried. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or the manifest step was taken out of it", step, stage))
|
||||
|
||||
if (chosen == "" && ncached == nmatched)
|
||||
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so its inventory describes an earlier tree rather than this one; the cache bust for that stage is not taking effect", step, stage))
|
||||
|
||||
if (chosen == "")
|
||||
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but wrote no complete inventory between %s and %s. The log is truncated, the build was not run with --progress=plain, or that step no longer emits the manifest", step, stage, beginmark, endmark))
|
||||
|
||||
for (i = 1; i <= nexpected; i++) {
|
||||
if (arrived[chosen SUBSEP expected[i]]) continue
|
||||
missing[++nmissing] = expected[i]
|
||||
pkg = pkgof(expected[i])
|
||||
if (!(pkg in seenpkg)) { seenpkg[pkg] = 1; pkgs[++npkgs] = pkg }
|
||||
}
|
||||
|
||||
if (nmissing == 0) exit 0
|
||||
|
||||
fail(sprintf("%d of the %d source files this repository tracks never reached the build context of stage \"%s\", so nothing examined them. Missing package(s): %s. Missing file(s): %s. A .dockerignore entry, or a COPY that brings in less than the whole tree, drops files silently: the tool still runs, still finds nothing wrong with what it was handed, and still reports success", nmissing, nexpected, stage, preview(pkgs, npkgs, 10), preview(missing, nmissing, 10)))
|
||||
}
|
||||
' "$1"
|
||||
125
script/assert-step-ran
Executable file
125
script/assert-step-ran
Executable file
@@ -0,0 +1,125 @@
|
||||
#!/bin/sh
|
||||
# script/assert-step-ran: fail unless a docker build actually executed a
|
||||
# named step, proved by output the step's own tool wrote.
|
||||
#
|
||||
# Usage: script/assert-step-ran LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT
|
||||
#
|
||||
# LOG must be `docker build --progress=plain` output. The default tty
|
||||
# renderer rewrites lines in place and keeps only the tail of a step's
|
||||
# output, so the flag is load-bearing wherever this is used.
|
||||
#
|
||||
# It passes only if some step in LOG satisfies all three of:
|
||||
#
|
||||
# 1. its header is "#ID [STAGE n/m] CMD" with CMD matching STEP_REGEX,
|
||||
# 2. one of the lines that step wrote matches EVIDENCE_REGEX,
|
||||
# 3. BuildKit reported "#ID DONE".
|
||||
#
|
||||
# This observes the build that happened; it parses no Dockerfile. A
|
||||
# stage renamed, moved, made unreachable, deleted, split by a whitespace
|
||||
# byte BuildKit treats as a separator, or an instruction rewritten —
|
||||
# none of it can make this pass a build in which the step did not run,
|
||||
# because a step that did not run wrote no output, and a step served
|
||||
# from cache writes none either ("#ID CACHED", no output lines).
|
||||
#
|
||||
# The stage name in a header is BuildKit's label for that vertex, not a
|
||||
# reading of the file being built. A vertex shared with an earlier build
|
||||
# of a DIFFERENT Dockerfile is replayed under the label it was first
|
||||
# recorded with, so a `Dockerfile.lint` run can print `[lint 1/8]` for
|
||||
# the main `Dockerfile`'s lint stage. A borrowed label cannot produce a
|
||||
# pass — a replayed vertex prints CACHED and writes no evidence, and a
|
||||
# vertex that really executes really ran the command in the header — but
|
||||
# it does mean a header alone proves nothing about which stages the file
|
||||
# declares. Hence one failure message naming both causes rather than a
|
||||
# split that would claim to know which.
|
||||
#
|
||||
# What this guard cannot see — not an exhaustive list of ways a green can
|
||||
# be untrue, only the ones known:
|
||||
#
|
||||
# - Evidence forged inside the matched step, e.g.
|
||||
# `RUN golangci-lint run ... || echo "0 issues."`. Condition 1 ties
|
||||
# the evidence to a step whose own command is in the log, so the
|
||||
# forgery has to be written into that command in the Dockerfile, in
|
||||
# plain sight.
|
||||
# This guards against a step falling silently out of the build. It does
|
||||
# not certify that what ran examined everything it should have — a
|
||||
# `.dockerignore` entry excluding a package makes the linter genuinely
|
||||
# run and genuinely print `0 issues.` while a real violation sits
|
||||
# unexamined in the repo. That is a different question, asked separately
|
||||
# by script/assert-context-complete, which every caller of this script
|
||||
# also runs.
|
||||
#
|
||||
# What it depends on — BuildKit's plain progress format and the tool's
|
||||
# own success wording — fails the caller loudly if it drifts, because
|
||||
# drift removes a match rather than creating one.
|
||||
set -eu
|
||||
|
||||
die() {
|
||||
echo "script/assert-step-ran: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ $# -eq 5 ] || die "usage: $0 LOG STAGE STEP_REGEX EVIDENCE_REGEX WHAT"
|
||||
[ -r "$1" ] || die "cannot read $1"
|
||||
|
||||
# The regexes travel in the environment, not in -v: awk expands escape
|
||||
# sequences in a -v assignment, which would eat a backslash before the
|
||||
# regex ever sees it.
|
||||
ASR_STAGE="$2" ASR_STEP="$3" ASR_EVIDENCE="$4" ASR_WHAT="$5" awk '
|
||||
function fail(msg) {
|
||||
printf "script/assert-step-ran: %s\n", msg | "cat 1>&2"
|
||||
close("cat 1>&2")
|
||||
exit 1
|
||||
}
|
||||
|
||||
BEGIN {
|
||||
stage = ENVIRON["ASR_STAGE"]
|
||||
step = ENVIRON["ASR_STEP"]
|
||||
evidence = ENVIRON["ASR_EVIDENCE"]
|
||||
what = ENVIRON["ASR_WHAT"]
|
||||
}
|
||||
|
||||
# Every progress line is "#ID " and then a step header, a status, or one
|
||||
# line the step itself wrote.
|
||||
/^#[0-9]+ / {
|
||||
id = substr($1, 2)
|
||||
rest = substr($0, length($1) + 2)
|
||||
|
||||
if (rest == "CACHED") { cached[id] = 1; next }
|
||||
if (rest ~ /^DONE /) { done[id] = 1; next }
|
||||
|
||||
# Header: "[STAGE n/m] CMD", or "[PLATFORM STAGE n/m] CMD" when the
|
||||
# build names a platform. Bracketed spans with no n/m are BuildKit
|
||||
# internals ("[internal] load build context"), not steps.
|
||||
if (substr(rest, 1, 1) == "[") {
|
||||
p = index(rest, "] ")
|
||||
if (p == 0) next
|
||||
n = split(substr(rest, 2, p - 2), part, " ")
|
||||
if (n < 2 || part[n] !~ /^[0-9]+\/[0-9]+$/) next
|
||||
if (part[n - 1] != stage) next
|
||||
if (substr(rest, p + 2) ~ step) matched[id] = 1
|
||||
next
|
||||
}
|
||||
|
||||
# Output: "#ID 41.80 <the line the step wrote>".
|
||||
if (rest ~ /^[0-9]+[.][0-9]+ /) {
|
||||
sub(/^[0-9]+[.][0-9]+ /, "", rest)
|
||||
if (rest ~ evidence) emitted[id] = 1
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
for (id in matched) {
|
||||
nmatched++
|
||||
if (done[id] && emitted[id]) exit 0
|
||||
if (cached[id]) ncached++
|
||||
}
|
||||
|
||||
if (nmatched == 0)
|
||||
fail(sprintf("the build ran no step matching /%s/ in stage \"%s\", so %s did not run. Either the stage is not in the build graph — renamed, deleted, or nothing the final stage builds depends on it any more — or it was built without that command among its steps", step, stage, what))
|
||||
|
||||
if (ncached == nmatched)
|
||||
fail(sprintf("every step matching /%s/ in stage \"%s\" was served from cache, so %s did not run on this tree; the cache bust for that stage is not taking effect", step, stage, what))
|
||||
|
||||
fail(sprintf("a step matching /%s/ ran in stage \"%s\" but never wrote a line matching /%s/, so there is no evidence %s did the work; the command or the tool that produces that line has changed", step, stage, evidence, what))
|
||||
}
|
||||
' "$1"
|
||||
84
script/bootstrap
Executable file
84
script/bootstrap
Executable file
@@ -0,0 +1,84 @@
|
||||
#!/bin/sh
|
||||
# script/bootstrap: install all dependencies needed to build and develop
|
||||
# this repo. Idempotent: every install is guarded by a check so already
|
||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||
# make, or go). The linter is NOT installed locally: golangci-lint runs
|
||||
# via docker only (script/lint), pinned by image digest, so the only
|
||||
# lint prerequisite is a working docker.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
PKGMGR=""
|
||||
SUDO=""
|
||||
|
||||
detect_pkgmgr() {
|
||||
[ -n "$PKGMGR" ] && return 0
|
||||
if command -v nix-env >/dev/null 2>&1; then
|
||||
PKGMGR="nix"
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
PKGMGR="apt"
|
||||
elif command -v brew >/dev/null 2>&1; then
|
||||
PKGMGR="brew"
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
PKGMGR="apk"
|
||||
else
|
||||
echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PKGMGR" = "apt" ]; then
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if [ "$(id -u)" != "0" ]; then
|
||||
SUDO="sudo"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
|
||||
pkg_install() {
|
||||
detect_pkgmgr
|
||||
case "$PKGMGR" in
|
||||
nix) nix-env -iA "nixpkgs.$1" ;;
|
||||
apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;;
|
||||
brew) brew install "$3" ;;
|
||||
apk) apk add --no-cache "$4" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
missing() {
|
||||
! command -v "$1" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
# Base tooling
|
||||
if missing git; then pkg_install git git git git; fi
|
||||
if missing make; then pkg_install gnumake make make make; fi
|
||||
|
||||
# Go toolchain
|
||||
if missing go; then pkg_install go golang go go; fi
|
||||
|
||||
# Linting runs via docker only (script/lint). Warn, don't fail:
|
||||
# everything except `make lint` works without it.
|
||||
if missing docker; then
|
||||
echo "bootstrap: WARNING: docker not found; make lint and" >&2
|
||||
echo "bootstrap: make docker require it. Install docker to" >&2
|
||||
echo "bootstrap: run the linter." >&2
|
||||
fi
|
||||
|
||||
# Markdown/CSS/JS formatting uses prettier. script/prettier falls
|
||||
# back to a pinned docker image when it is not on PATH, so this is
|
||||
# a convenience, not a requirement.
|
||||
if missing prettier && missing docker; then
|
||||
echo "bootstrap: WARNING: neither prettier nor docker found;" >&2
|
||||
echo "bootstrap: markdown formatting will be skipped." >&2
|
||||
fi
|
||||
|
||||
go mod download
|
||||
|
||||
echo "bootstrap complete"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
15
script/check
Executable file
15
script/check
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
||||
# Generic: usually needs no adaptation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
main() {
|
||||
"$SCRIPT_DIR/test"
|
||||
"$SCRIPT_DIR/lint"
|
||||
"$SCRIPT_DIR/fmt-check"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
89
script/cibuild
Executable file
89
script/cibuild
Executable file
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# script/cibuild: run the CI build. The Dockerfile runs the checks
|
||||
# (gofmt, config verify, lint, test), so a successful build implies a
|
||||
# green repo. The Gitea workflow runs this on push.
|
||||
#
|
||||
# Only if the checks actually ran, though, and a green `docker build` is
|
||||
# no evidence that they did:
|
||||
#
|
||||
# 1. On an unchanged tree every layer comes from cache and the build
|
||||
# exits 0 in under a second having executed nothing. Hence the two
|
||||
# --no-cache-filter flags.
|
||||
#
|
||||
# 2. BuildKit silently ignores a --no-cache-filter naming a stage that
|
||||
# does not exist, so a rename restores that green no-op.
|
||||
#
|
||||
# 3. BuildKit builds only the final stage's dependency graph. A stage
|
||||
# nothing references is never built, and with no --target the final
|
||||
# stage is whichever is last in the file — so appending a stage, or
|
||||
# moving a reference into a stage that is itself unreachable, drops
|
||||
# the checks out of the build while every reference to them is still
|
||||
# there to read.
|
||||
#
|
||||
# Rather than predict any of that from the Dockerfile's text, this asks
|
||||
# the finished build what it did: script/assert-step-ran requires that
|
||||
# the log contain a lint step and a test step that ran, and that each
|
||||
# wrote the line its own tool writes on success. Every trap above ends
|
||||
# with the step absent from the log or served from cache, and both fail
|
||||
# that assertion. See that script for what it does not cover.
|
||||
#
|
||||
# 4. A step that ran is not a step that saw the repo. .dockerignore, or
|
||||
# a COPY narrower than the tree, removes files from the build
|
||||
# context, and the linter then reports `0 issues.` over what is left
|
||||
# while `go test ./...` never compiles the package that went missing.
|
||||
# So each source-consuming stage inventories what reached it, and
|
||||
# script/assert-context-complete compares that inventory against the
|
||||
# git index — a source .dockerignore cannot reach.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
dockerfile=Dockerfile
|
||||
lint_stage=lint
|
||||
test_stage=builder
|
||||
|
||||
# `ok <pkg> 1.234s`: go test prints `(cached)` in place of the duration
|
||||
# when it replays a result, so requiring the duration requires a package
|
||||
# that was really exercised.
|
||||
test_ran='^ok[[:space:]]+[^[:space:]]+[[:space:]]+[0-9]+[.][0-9]+s'
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-cibuild.XXXXXX")"
|
||||
trap 'rm -rf "$tmp"' EXIT INT TERM
|
||||
|
||||
# --progress=plain is load-bearing: the assertions below read the
|
||||
# steps' own output out of this log, and the tty renderer discards
|
||||
# all but the tail of it. The exit status travels through a file
|
||||
# because a pipeline's status is the last command's; `set +e` is
|
||||
# what lets the recording line run at all, since errexit would
|
||||
# otherwise abandon the subshell on a failing build. A missing file
|
||||
# reads as failure.
|
||||
(
|
||||
set +e
|
||||
docker build \
|
||||
--progress=plain \
|
||||
--no-cache-filter="$lint_stage" \
|
||||
--no-cache-filter="$test_stage" \
|
||||
-f "$dockerfile" . 2>&1
|
||||
echo "$?" >"$tmp/status"
|
||||
) | tee "$tmp/build.log"
|
||||
|
||||
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
|
||||
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
|
||||
|
||||
script/assert-step-ran "$tmp/build.log" "$lint_stage" \
|
||||
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
|
||||
script/assert-step-ran "$tmp/build.log" "$test_stage" \
|
||||
'^RUN go test' "$test_ran" 'the tests'
|
||||
|
||||
# Both stages, separately: each has its own COPY, so an intact
|
||||
# context in one is no evidence about the other.
|
||||
script/repo-source-manifest >"$tmp/expected"
|
||||
script/assert-context-complete "$tmp/build.log" "$lint_stage" "$tmp/expected"
|
||||
script/assert-context-complete "$tmp/build.log" "$test_stage" "$tmp/expected"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
15
script/docker
Executable file
15
script/docker
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# script/docker: build the Docker image tagged with the project name.
|
||||
# Identical in all repos; the tag comes from script/projectname.
|
||||
# Generic: needs no adaptation.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
docker build -t "$("$SCRIPT_DIR/projectname")" .
|
||||
}
|
||||
|
||||
main "$@"
|
||||
26
script/fmt
Executable file
26
script/fmt
Executable file
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
# script/fmt: format all files (writes). Go via gofmt, everything else
|
||||
# (markdown, CSS, JS, YAML, JSON) via prettier with the repo's
|
||||
# .prettierrc. Never hand-rolled substitutions: this script is the only
|
||||
# sanctioned way to reformat the tree.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
gofmt -s -w .
|
||||
|
||||
if command -v goimports >/dev/null 2>&1; then
|
||||
goimports -w .
|
||||
fi
|
||||
|
||||
# A missing runner (exit 2) is a warning here: script/prettier has
|
||||
# already said so on stderr, and failing `make fmt` over it would
|
||||
# block work that has nothing to do with markdown.
|
||||
"$SCRIPT_DIR/prettier" --write . || [ $? -eq 2 ]
|
||||
}
|
||||
|
||||
main "$@"
|
||||
25
script/fmt-check
Executable file
25
script/fmt-check
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/bin/sh
|
||||
# script/fmt-check: check formatting (read-only). Same scope as
|
||||
# script/fmt, but fails instead of writing.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
if [ -n "$(gofmt -s -l .)" ]; then
|
||||
echo "gofmt needed on:"
|
||||
gofmt -s -l .
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Exit 2 means no prettier and no docker: reported by
|
||||
# script/prettier on stderr and not treated as a failure, so this
|
||||
# check still runs on a machine that has neither. The container
|
||||
# build is the gate that always has both.
|
||||
"$SCRIPT_DIR/prettier" --check . || [ $? -eq 2 ]
|
||||
}
|
||||
|
||||
main "$@"
|
||||
16
script/install-precommit
Executable file
16
script/install-precommit
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# script/install-precommit: install the git pre-commit hook that runs
|
||||
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
||||
# Generic: needs no adaptation.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
||||
chmod +x .git/hooks/pre-commit
|
||||
echo "pre-commit hook installed: runs script/precommit"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
71
script/lint
Executable file
71
script/lint
Executable file
@@ -0,0 +1,71 @@
|
||||
#!/bin/sh
|
||||
# script/lint: run the linter. golangci-lint is never installed locally: it
|
||||
# runs via docker only, one way, everywhere.
|
||||
#
|
||||
# Traps, each of which yields a green run over an unlinted or partly linted
|
||||
# tree:
|
||||
#
|
||||
# 1. A bare `docker build -f Dockerfile.lint .` serves the lint layer from
|
||||
# cache on an unchanged tree and exits 0 having linted nothing, and
|
||||
# BuildKit ignores a --no-cache-filter whose stage name matches
|
||||
# nothing, restoring that no-op after a rename. So the run is not
|
||||
# trusted: script/assert-step-ran requires the log to show the lint
|
||||
# step executing and golangci-lint's own success line coming out of
|
||||
# it. A cached or absent step fails that.
|
||||
#
|
||||
# 2. .dockerignore decides what reaches the container, and only what
|
||||
# reaches it is linted, so excluding a Go file drops it from the lint
|
||||
# with the linter still reporting `0 issues.` over what it was
|
||||
# handed. So the context is not trusted either: the lint stage
|
||||
# inventories the sources that reached it, and
|
||||
# script/assert-context-complete compares that against the git index,
|
||||
# which .dockerignore cannot touch. Never exclude Go sources,
|
||||
# go.mod/go.sum or .golangci.yml — and now nothing silently does.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
dockerfile=Dockerfile.lint
|
||||
|
||||
# Must match the stage name in Dockerfile.lint.
|
||||
stage=lint
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
tmp="$(mktemp -d "${TMPDIR:-/tmp}/$("$SCRIPT_DIR/projectname")-lint.XXXXXX")"
|
||||
trap 'rm -rf "$tmp"' EXIT INT TERM
|
||||
|
||||
# No --target: whatever stage is last is the one BuildKit builds, and
|
||||
# the assertion below is what decides whether the linter was in it.
|
||||
# --progress=plain is load-bearing — the tty renderer discards all but
|
||||
# the tail of a step's output. The exit status travels through a file
|
||||
# because a pipeline's status is the last command's; `set +e` is what
|
||||
# lets the recording line run at all, since errexit would otherwise
|
||||
# abandon the subshell on a failing build. A missing file reads as
|
||||
# failure.
|
||||
(
|
||||
set +e
|
||||
docker build \
|
||||
--progress=plain \
|
||||
--no-cache-filter="$stage" \
|
||||
--output=type=cacheonly \
|
||||
-f "$dockerfile" . 2>&1
|
||||
echo "$?" >"$tmp/status"
|
||||
) | tee "$tmp/build.log"
|
||||
|
||||
status="$(cat "$tmp/status" 2>/dev/null || echo 1)"
|
||||
[ "${status:-1}" -eq 0 ] || exit "${status:-1}"
|
||||
|
||||
script/assert-step-ran "$tmp/build.log" "$stage" \
|
||||
'^RUN golangci-lint run' '^0 issues[.]$' 'the linter'
|
||||
|
||||
# That the linter ran says nothing about what it was given. The
|
||||
# expectation comes from git and the evidence from the build, which
|
||||
# is the only reason comparing them means anything.
|
||||
script/repo-source-manifest >"$tmp/expected"
|
||||
script/assert-context-complete "$tmp/build.log" "$stage" "$tmp/expected"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
21
script/precommit
Executable file
21
script/precommit
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# script/precommit: run by the git pre-commit hook; fails the commit if
|
||||
# checks fail. Our own extension to scripts-to-rule-them-all. Go repo
|
||||
# extras: go mod tidy must not change go.mod/go.sum.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
go mod tidy
|
||||
git diff --exit-code -- go.mod go.sum || {
|
||||
echo "precommit: go mod tidy changed go.mod/go.sum;" \
|
||||
"stage the changes and retry" >&2
|
||||
exit 1
|
||||
}
|
||||
"$SCRIPT_DIR/check"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
52
script/prettier
Executable file
52
script/prettier
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
# script/prettier: run prettier over this repo with the repo's own
|
||||
# .prettierrc. Helper, not an entrypoint: script/fmt and
|
||||
# script/fmt-check call it.
|
||||
#
|
||||
# Prettier from PATH if it is there, otherwise a hash-pinned node image
|
||||
# via docker, so a checkout with neither node nor a global prettier
|
||||
# still formats identically. If neither is available the caller is told
|
||||
# and the markdown pass is skipped rather than silently passing — a
|
||||
# formatter that is not present cannot certify formatting.
|
||||
#
|
||||
# Exit status: prettier's own, 2 when it was skipped for lack of a
|
||||
# runner. script/fmt-check treats 2 as a warning, not a failure, so
|
||||
# that `make check` still works on a machine without docker.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
# node:24-alpine, 2026-08-22. Bumping the pin is a deliberate edit.
|
||||
NODE_IMAGE="node:24-alpine@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43"
|
||||
|
||||
# Pinned exactly: prettier's default formatting changes between minor
|
||||
# releases, so an unpinned version turns fmt-check into a coin flip.
|
||||
PRETTIER_VERSION="3.9.6"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
if command -v prettier >/dev/null 2>&1; then
|
||||
prettier "$@"
|
||||
return $?
|
||||
fi
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
# --user keeps written files owned by the invoking user rather
|
||||
# than root; the container only ever touches the bind mount.
|
||||
docker run --rm \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
-v "$ROOT:/work" \
|
||||
-w /work \
|
||||
"$NODE_IMAGE" \
|
||||
npx --yes "prettier@$PRETTIER_VERSION" "$@"
|
||||
return $?
|
||||
fi
|
||||
|
||||
echo "script/prettier: neither prettier nor docker found;" >&2
|
||||
echo "script/prettier: skipping markdown/CSS/JS formatting." >&2
|
||||
|
||||
return 2
|
||||
}
|
||||
|
||||
main "$@"
|
||||
12
script/projectname
Executable file
12
script/projectname
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# script/projectname: output the name of this project. Our own
|
||||
# extension to scripts-to-rule-them-all. Other scripts that need the
|
||||
# name (e.g. script/docker) call this, so they can stay identical
|
||||
# across all repos.
|
||||
set -eu
|
||||
|
||||
main() {
|
||||
echo "dcf"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
60
script/repo-source-manifest
Executable file
60
script/repo-source-manifest
Executable file
@@ -0,0 +1,60 @@
|
||||
#!/bin/sh
|
||||
# script/repo-source-manifest: print every file this repository contains
|
||||
# whose absence from a docker build context would go unnoticed — one
|
||||
# repo-relative path per line, LC_ALL=C-sorted.
|
||||
#
|
||||
# That is the Go sources. A missing go.mod, go.sum or .golangci.yml
|
||||
# fails the build loudly (the COPY errors, or golangci-lint refuses to
|
||||
# start), so they cannot hide anything; they are listed anyway because
|
||||
# it costs two lines and makes the manifest the linter's whole input
|
||||
# rather than most of it.
|
||||
#
|
||||
# The list comes from the git index, never from a walk of the working
|
||||
# tree, and that is the point: script/assert-context-complete compares
|
||||
# it against the inventory the build itself emitted, and a check that
|
||||
# reads its expectation from the same place it reads its evidence proves
|
||||
# nothing. .dockerignore governs what docker sends; it cannot touch what
|
||||
# git tracks.
|
||||
#
|
||||
# Tracked files only, and only those present in the worktree — a file
|
||||
# staged for deletion is not something the build context is missing.
|
||||
# Untracked files are not expected either, so local scratch work in the
|
||||
# tree is not a failure.
|
||||
#
|
||||
# A path containing a newline or a quote is quoted by git and will not
|
||||
# match the plain path the build emits, so it fails loudly rather than
|
||||
# passing silently. No such path exists here, and none should.
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
die() {
|
||||
echo "script/repo-source-manifest: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
|
||||
git rev-parse --is-inside-work-tree >/dev/null 2>&1 ||
|
||||
die "not a git work tree, so there is nothing to compare a build context against"
|
||||
|
||||
# Captured before the loop so that a git failure is the script's
|
||||
# exit status: in a pipeline only the last command's status counts.
|
||||
tracked="$(git ls-files -- '*.go' go.mod go.sum .golangci.yml .golangci.yaml)"
|
||||
|
||||
manifest="$(
|
||||
printf '%s\n' "$tracked" | while IFS= read -r f; do
|
||||
if [ -n "$f" ] && [ -f "$f" ]; then
|
||||
printf '%s\n' "$f"
|
||||
fi
|
||||
done | LC_ALL=C sort
|
||||
)"
|
||||
|
||||
[ -n "$manifest" ] ||
|
||||
die "the git index lists no Go sources, so any build context would satisfy the check"
|
||||
|
||||
printf '%s\n' "$manifest"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
14
script/setup
Executable file
14
script/setup
Executable file
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# script/setup: set up the repo for development after a fresh clone:
|
||||
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
||||
# Add any repo-specific initialization (db init, .env template) here.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
|
||||
main() {
|
||||
"$SCRIPT_DIR/bootstrap"
|
||||
"$SCRIPT_DIR/install-precommit"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
21
script/test
Executable file
21
script/test
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# script/test: run the test suite. Quiet on success; on failure, rerun
|
||||
# with -v for full diagnostic output (and still fail — the first run
|
||||
# already proved the tests are broken).
|
||||
set -eu
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
# -count=1 disables the test result cache. Without it a repeat run on
|
||||
# an unchanged tree reports every package `ok ... (cached)` and exits
|
||||
# 0 having executed nothing — a gate that cannot fail.
|
||||
go test -count=1 -timeout 90s -race -cover ./... || {
|
||||
echo "--- Rerunning with -v for details ---"
|
||||
go test -count=1 -timeout 90s -race -v ./...
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user