Compare commits
1 Commits
next
...
fix/go-ver
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7cdb1fe0a8 |
@@ -1,4 +1,3 @@
|
|||||||
*.tmp
|
*.tmp
|
||||||
*.dockerimage
|
*.dockerimage
|
||||||
.git
|
.git
|
||||||
node_modules
|
|
||||||
|
|||||||
23
.drone.yml
Normal file
23
.drone.yml
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
kind: pipeline
|
||||||
|
name: test-docker-build
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: test-docker-build
|
||||||
|
image: plugins/docker
|
||||||
|
network_mode: bridge
|
||||||
|
settings:
|
||||||
|
repo: sneak/mfer
|
||||||
|
build_args_from_env: [ DRONE_COMMIT_SHA ]
|
||||||
|
dry_run: true
|
||||||
|
custom_dns: [ 116.202.204.30 ]
|
||||||
|
tags:
|
||||||
|
- ${DRONE_COMMIT_SHA:0:7}
|
||||||
|
- ${DRONE_BRANCH}
|
||||||
|
- latest
|
||||||
|
- name: notify
|
||||||
|
image: plugins/slack
|
||||||
|
settings:
|
||||||
|
webhook:
|
||||||
|
from_secret: SLACK_WEBHOOK_URL
|
||||||
|
when:
|
||||||
|
event: pull_request
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
name: check
|
|
||||||
on: [push]
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
# actions/checkout v4.2.2, 2026-03-16
|
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
|
||||||
- run: script/cibuild
|
|
||||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -1,14 +1,6 @@
|
|||||||
/bin/
|
mfer/*.pb.go
|
||||||
|
/mfer.cmd
|
||||||
/tmp
|
/tmp
|
||||||
/node_modules/
|
|
||||||
*.tmp
|
*.tmp
|
||||||
*.dockerimage
|
*.dockerimage
|
||||||
/vendor
|
/vendor
|
||||||
vendor.tzst
|
|
||||||
modcache.tzst
|
|
||||||
|
|
||||||
# Generated manifest files
|
|
||||||
.index.mf
|
|
||||||
|
|
||||||
# Stale files
|
|
||||||
.drone.yml
|
|
||||||
|
|||||||
2
.golangci.yaml
Normal file
2
.golangci.yaml
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
run:
|
||||||
|
tests: false
|
||||||
@@ -1,34 +1,2 @@
|
|||||||
version: "2"
|
|
||||||
|
|
||||||
# Config schema uses the golangci-lint v2 layout (settings live under
|
|
||||||
# linters.settings, not top-level linters-settings) so that the
|
|
||||||
# thresholds below are actually applied by golangci-lint >= v2.
|
|
||||||
|
|
||||||
run:
|
run:
|
||||||
timeout: 5m
|
tests: false
|
||||||
modules-download-mode: readonly
|
|
||||||
|
|
||||||
linters:
|
|
||||||
default: all
|
|
||||||
disable:
|
|
||||||
# Genuinely incompatible with project patterns
|
|
||||||
- exhaustruct # Requires all struct fields
|
|
||||||
- depguard # Dependency allow/block lists
|
|
||||||
- godot # Requires comments to end with periods
|
|
||||||
- wsl # Deprecated, replaced by wsl_v5
|
|
||||||
- wrapcheck # Too verbose for internal packages
|
|
||||||
- varnamelen # Short names like db, id are idiomatic Go
|
|
||||||
settings:
|
|
||||||
lll:
|
|
||||||
line-length: 88
|
|
||||||
funlen:
|
|
||||||
lines: 80
|
|
||||||
statements: 50
|
|
||||||
cyclop:
|
|
||||||
max-complexity: 15
|
|
||||||
dupl:
|
|
||||||
threshold: 100
|
|
||||||
|
|
||||||
issues:
|
|
||||||
max-issues-per-linter: 0
|
|
||||||
max-same-issues: 0
|
|
||||||
|
|||||||
@@ -1,14 +0,0 @@
|
|||||||
# REPO_POLICIES.md is a verbatim copy of an authoritative upstream
|
|
||||||
# document (sneak/prompts). Local tooling must never rewrite it: any
|
|
||||||
# reformatting is silent drift from the source of truth.
|
|
||||||
REPO_POLICIES.md
|
|
||||||
|
|
||||||
# User-owned configuration, copied verbatim from upstream. Not matched by
|
|
||||||
# the current prettier file set, but listed so widening that set can never
|
|
||||||
# start rewriting it.
|
|
||||||
.golangci.yml
|
|
||||||
|
|
||||||
# Dependencies and build output
|
|
||||||
node_modules/
|
|
||||||
vendor/
|
|
||||||
bin/
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"tabWidth": 4,
|
|
||||||
"proseWrap": "always"
|
|
||||||
}
|
|
||||||
31
AGENTS.md
31
AGENTS.md
@@ -1,31 +0,0 @@
|
|||||||
# Agent Instructions
|
|
||||||
|
|
||||||
Read `REPO_POLICIES.md` before making any changes. It is the authoritative
|
|
||||||
source for coding standards, formatting, linting, and workflow rules.
|
|
||||||
|
|
||||||
## Workflow
|
|
||||||
|
|
||||||
- When fixing a bug, write a failing test FIRST. Only after the test fails,
|
|
||||||
write the code to fix the bug. Then ensure the test passes. Leave the test in
|
|
||||||
place and commit it with the bugfix. Don't run shell commands to test bugfixes
|
|
||||||
or reproduce bugs. Write tests!
|
|
||||||
|
|
||||||
- After each change, run `make fmt`, then `make test`, then `make lint`. Fix any
|
|
||||||
failures before committing.
|
|
||||||
|
|
||||||
- After each change, commit only the files you've changed. Push after
|
|
||||||
committing.
|
|
||||||
|
|
||||||
## Attribution
|
|
||||||
|
|
||||||
- Never mention Claude, Anthropic, or any AI/LLM tooling in commit messages. Do
|
|
||||||
not use attribution.
|
|
||||||
|
|
||||||
## Repository-Specific Notes
|
|
||||||
|
|
||||||
- This is a Go library + CLI tool for generating `.mf` manifest files.
|
|
||||||
- The proto definition is in `mfer/mf.proto`; generated `.pb.go` files are
|
|
||||||
committed (required for `go get` compatibility).
|
|
||||||
- The format specification is in `FORMAT.md`.
|
|
||||||
- See the TODO section in `README.md` for the 1.0 implementation plan and open
|
|
||||||
design questions.
|
|
||||||
86
Dockerfile
86
Dockerfile
@@ -1,55 +1,37 @@
|
|||||||
# Lint stage — fast feedback on formatting and lint issues
|
################################################################################
|
||||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
|
||||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
################################################################################
|
||||||
|
FROM sneak/builder:2022-12-08 AS builder
|
||||||
WORKDIR /src
|
ENV DEBIAN_FRONTEND noninteractive
|
||||||
COPY go.mod go.sum ./
|
WORKDIR /build
|
||||||
RUN go mod download
|
COPY ./Makefile ./.golangci.yml ./go.mod ./go.sum /build/
|
||||||
|
COPY ./vendor.tzst /build/vendor.tzst
|
||||||
COPY . .
|
COPY ./modcache.tzst /build/modcache.tzst
|
||||||
|
COPY ./internal ./internal
|
||||||
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
COPY ./bin/gitrev.sh ./bin/gitrev.sh
|
||||||
RUN touch mfer/mf.pb.go
|
COPY ./mfer ./mfer
|
||||||
|
COPY ./cmd ./cmd
|
||||||
# Go half of fmt-check only: this image has no node, so no prettier. The
|
ARG GITREV unknown
|
||||||
# markdown half runs in the mdfmt stage below.
|
ARG DRONE_COMMIT_SHA unknown
|
||||||
RUN make fmt-check-go
|
|
||||||
RUN make lint
|
|
||||||
|
|
||||||
# Markdown/JSON format stage — prettier needs node, which the Go images
|
|
||||||
# do not have. node:22.17.0-bookworm-slim (2026-08-09); ships node
|
|
||||||
# 22.17.0 and yarn 1.22.22, the versions script/bootstrap pins.
|
|
||||||
FROM node@sha256:b04ce4ae4e95b522112c2e5c52f781471a5cbc3b594527bcddedee9bc48c03a0 AS mdfmt
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY package.json yarn.lock ./
|
|
||||||
RUN yarn install --frozen-lockfile
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# No make in this image; call the script entrypoint directly.
|
|
||||||
RUN script/prettier --check
|
|
||||||
|
|
||||||
# Build stage — tests and compilation
|
|
||||||
# golang:1.23 (2026-03-14)
|
|
||||||
FROM golang@sha256:60deed95d3888cc5e4d9ff8a10c54e5edc008c6ae3fba6187be6fb592e19e8c0 AS builder
|
|
||||||
|
|
||||||
# Force BuildKit to run the lint and mdfmt stages by creating stage dependencies
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
COPY --from=mdfmt /src/go.sum /dev/null
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# Touch .pb.go so make does not try to regenerate via protoc (file is committed)
|
|
||||||
RUN touch mfer/mf.pb.go
|
|
||||||
|
|
||||||
RUN make test
|
|
||||||
RUN cd cmd/mfer && go build -tags urfave_cli_no_docs -o /mfer .
|
|
||||||
|
|
||||||
|
RUN mkdir -p "$(go env GOMODCACHE)" && cd "$(go env GOMODCACHE)" && \
|
||||||
|
zstdmt -d --stdout /build/modcache.tzst | tar xf - && \
|
||||||
|
rm /build/modcache.tzst && cd /build
|
||||||
|
RUN \
|
||||||
|
cd mfer && go generate . && cd .. && \
|
||||||
|
GOPACKAGESDEBUG=true golangci-lint run ./... && \
|
||||||
|
mkdir vendor && cd vendor && \
|
||||||
|
zstdmt -d --stdout /build/vendor.tzst | tar xf - && rm /build/vendor.tzst && \
|
||||||
|
cd .. && \
|
||||||
|
make mfer.cmd
|
||||||
|
RUN rm -rf /build/vendor && go mod vendor && tar -c . | zstdmt -19 > /src.tzst
|
||||||
|
################################################################################
|
||||||
|
#2345678911234567892123456789312345678941234567895123456789612345678971234567898
|
||||||
|
################################################################################
|
||||||
|
## final image
|
||||||
|
################################################################################
|
||||||
FROM scratch
|
FROM scratch
|
||||||
COPY --from=builder /mfer /mfer
|
# we put all the source into the final image for posterity, it's small
|
||||||
|
COPY --from=builder /src.tzst /src.tzst
|
||||||
|
COPY --from=builder /build/mfer.cmd /mfer
|
||||||
ENTRYPOINT ["/mfer"]
|
ENTRYPOINT ["/mfer"]
|
||||||
|
|||||||
144
FORMAT.md
144
FORMAT.md
@@ -1,144 +0,0 @@
|
|||||||
# .mf File Format Specification
|
|
||||||
|
|
||||||
Version 1.0
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
|
|
||||||
An `.mf` file is a binary manifest that describes a directory tree of files,
|
|
||||||
including their paths, sizes, and cryptographic checksums. It supports optional
|
|
||||||
GPG signatures for integrity verification and optional timestamps for metadata
|
|
||||||
preservation.
|
|
||||||
|
|
||||||
## File Structure
|
|
||||||
|
|
||||||
An `.mf` file consists of two parts, concatenated:
|
|
||||||
|
|
||||||
1. **Magic bytes** (8 bytes): the ASCII string `ZNAVSRFG`
|
|
||||||
2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message
|
|
||||||
|
|
||||||
There is no length prefix or version byte between the magic and the protobuf
|
|
||||||
message. The protobuf message extends to the end of the file.
|
|
||||||
|
|
||||||
See [`mfer/mf.proto`](mfer/mf.proto) for exact field numbers and types.
|
|
||||||
|
|
||||||
## Outer Message (`MFFileOuter`)
|
|
||||||
|
|
||||||
The outer message contains:
|
|
||||||
|
|
||||||
| Field | Number | Type | Description |
|
|
||||||
| ----------------- | ------ | ---------------- | ------------------------------------------------------------------------ |
|
|
||||||
| `version` | 101 | enum | Must be `VERSION_ONE` (1) |
|
|
||||||
| `compressionType` | 102 | enum | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1) |
|
|
||||||
| `size` | 103 | int64 | Uncompressed size of `innerMessage` (corruption detection) |
|
|
||||||
| `sha256` | 104 | bytes | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) |
|
|
||||||
| `uuid` | 105 | bytes | Random v4 UUID; must match the inner message UUID |
|
|
||||||
| `innerMessage` | 199 | bytes | Zstd-compressed serialized `MFFile` message |
|
|
||||||
| `signature` | 201 | bytes (optional) | GPG signature (ASCII-armored or binary) |
|
|
||||||
| `signer` | 202 | bytes (optional) | Full GPG key ID of the signer |
|
|
||||||
| `signingPubKey` | 203 | bytes (optional) | Full GPG signing public key |
|
|
||||||
|
|
||||||
### SHA-256 Hash
|
|
||||||
|
|
||||||
The `sha256` field (104) covers the **compressed** `innerMessage` bytes. This
|
|
||||||
allows verifying data integrity before decompression.
|
|
||||||
|
|
||||||
## Compression
|
|
||||||
|
|
||||||
The `innerMessage` field is compressed with
|
|
||||||
[Zstandard (zstd)](https://facebook.github.io/zstd/). Implementations must
|
|
||||||
enforce a decompression size limit to prevent decompression bombs. The reference
|
|
||||||
implementation limits decompressed size to 256 MB.
|
|
||||||
|
|
||||||
## Inner Message (`MFFile`)
|
|
||||||
|
|
||||||
After decompressing `innerMessage`, the result is a serialized `MFFile`
|
|
||||||
(referred to as the manifest):
|
|
||||||
|
|
||||||
| Field | Number | Type | Description |
|
|
||||||
| ----------- | ------ | --------------------- | ------------------------------------- |
|
|
||||||
| `version` | 100 | enum | Must be `VERSION_ONE` (1) |
|
|
||||||
| `files` | 101 | repeated `MFFilePath` | List of files in the manifest |
|
|
||||||
| `uuid` | 102 | bytes | Random v4 UUID; must match outer UUID |
|
|
||||||
| `createdAt` | 201 | Timestamp (optional) | When the manifest was created |
|
|
||||||
|
|
||||||
## File Entries (`MFFilePath`)
|
|
||||||
|
|
||||||
Each file entry contains:
|
|
||||||
|
|
||||||
| Field | Number | Type | Description |
|
|
||||||
| ---------- | ------ | ------------------------- | ----------------------------------- |
|
|
||||||
| `path` | 1 | string | Relative file path (see Path Rules) |
|
|
||||||
| `size` | 2 | int64 | File size in bytes |
|
|
||||||
| `hashes` | 3 | repeated `MFFileChecksum` | At least one hash required |
|
|
||||||
| `mimeType` | 301 | string (optional) | MIME type |
|
|
||||||
| `mtime` | 302 | Timestamp (optional) | Modification time |
|
|
||||||
| `ctime` | 303 | Timestamp (optional) | Change time (inode metadata change) |
|
|
||||||
|
|
||||||
Field 304 (`atime`) has been removed from the specification. Access time is
|
|
||||||
volatile and non-deterministic; it is not useful for integrity verification.
|
|
||||||
|
|
||||||
## Path Rules
|
|
||||||
|
|
||||||
All `path` values must satisfy these invariants:
|
|
||||||
|
|
||||||
- **UTF-8**: paths must be valid UTF-8
|
|
||||||
- **Forward slashes**: use `/` as the path separator (never `\`)
|
|
||||||
- **Relative only**: no leading `/`
|
|
||||||
- **No parent traversal**: no `..` path segments
|
|
||||||
- **No empty segments**: no `//` sequences
|
|
||||||
- **No trailing slash**: paths refer to files, not directories
|
|
||||||
|
|
||||||
Implementations must validate these invariants when reading and writing
|
|
||||||
manifests. Paths that violate these rules must be rejected.
|
|
||||||
|
|
||||||
## Hash Format (`MFFileChecksum`)
|
|
||||||
|
|
||||||
Each checksum is a single `bytes multiHash` field containing a
|
|
||||||
[multihash](https://multiformats.io/multihash/)-encoded value. Multihash is
|
|
||||||
self-describing: the encoded bytes include a varint algorithm identifier
|
|
||||||
followed by a varint digest length followed by the digest itself.
|
|
||||||
|
|
||||||
The 1.0 implementation writes SHA-256 multihashes (`0x12` algorithm code).
|
|
||||||
Implementations must be able to verify SHA-256 multihashes at minimum.
|
|
||||||
|
|
||||||
## Signature Scheme
|
|
||||||
|
|
||||||
Signing is optional. When present, the signature covers a canonical string
|
|
||||||
constructed as:
|
|
||||||
|
|
||||||
```
|
|
||||||
ZNAVSRFG-<UUID>-<SHA256>
|
|
||||||
```
|
|
||||||
|
|
||||||
Where:
|
|
||||||
|
|
||||||
- `ZNAVSRFG` is the magic bytes string (literal ASCII)
|
|
||||||
- `<UUID>` is the hex-encoded UUID from the outer message
|
|
||||||
- `<SHA256>` is the hex-encoded SHA-256 hash from the outer message (covering
|
|
||||||
compressed data)
|
|
||||||
|
|
||||||
Components are separated by hyphens. The signature is produced by GPG over this
|
|
||||||
canonical string and stored in the `signature` field of the outer message.
|
|
||||||
|
|
||||||
## Deterministic Serialization
|
|
||||||
|
|
||||||
By default, manifests are generated deterministically:
|
|
||||||
|
|
||||||
- File entries are sorted by `path` in **lexicographic byte order**
|
|
||||||
- `createdAt` is omitted unless explicitly requested
|
|
||||||
- `atime` is never included (field removed from schema)
|
|
||||||
|
|
||||||
This ensures that two independent runs over the same directory tree produce
|
|
||||||
byte-identical `.mf` files (assuming file contents and metadata have not
|
|
||||||
changed).
|
|
||||||
|
|
||||||
## MIME Type
|
|
||||||
|
|
||||||
The recommended MIME type for `.mf` files is `application/octet-stream`. The
|
|
||||||
`.mf` file extension is the canonical identifier.
|
|
||||||
|
|
||||||
## Reference
|
|
||||||
|
|
||||||
- Proto definition: [`mfer/mf.proto`](mfer/mf.proto)
|
|
||||||
- Reference implementation:
|
|
||||||
[git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
|
||||||
57
Makefile
57
Makefile
@@ -5,7 +5,7 @@ export PATH := $(PATH):$(GOPATH)/bin
|
|||||||
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
|
PROTOC_GEN_GO := $(GOPATH)/bin/protoc-gen-go
|
||||||
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
SOURCEFILES := mfer/*.go mfer/*.proto internal/*/*.go cmd/*/*.go go.mod go.sum
|
||||||
ARCH := $(shell uname -m)
|
ARCH := $(shell uname -m)
|
||||||
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh 2>/dev/null || echo unknown)
|
GITREV_BUILD := $(shell bash $(PWD)/bin/gitrev.sh)
|
||||||
APPNAME := mfer
|
APPNAME := mfer
|
||||||
VERSION := 0.1.0
|
VERSION := 0.1.0
|
||||||
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
|
export DOCKER_IMAGE_CACHE_DIR := $(HOME)/Library/Caches/Docker/$(APPNAME)-$(ARCH)
|
||||||
@@ -13,24 +13,18 @@ GOLDFLAGS += -X main.Version=$(VERSION)
|
|||||||
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
GOLDFLAGS += -X main.Gitrev=$(GITREV_BUILD)
|
||||||
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
GOFLAGS := -ldflags "$(GOLDFLAGS)"
|
||||||
|
|
||||||
.PHONY: bootstrap setup docker default run ci test check lint fmt fmt-check fmt-check-go fmt-check-md hooks fixme
|
.PHONY: docker default run ci test fixme
|
||||||
|
|
||||||
default: fmt test
|
default: fmt test
|
||||||
|
|
||||||
bootstrap:
|
run: ./mfer.cmd
|
||||||
@script/bootstrap
|
|
||||||
|
|
||||||
setup:
|
|
||||||
@script/setup
|
|
||||||
|
|
||||||
run: ./bin/mfer
|
|
||||||
./$<
|
./$<
|
||||||
./$< gen
|
./$< gen --ignore-dotfiles
|
||||||
|
|
||||||
ci: test
|
ci: test
|
||||||
|
|
||||||
test:
|
test: $(SOURCEFILES) mfer/mf.pb.go
|
||||||
@script/test
|
go test -v --timeout 3s ./...
|
||||||
|
|
||||||
$(PROTOC_GEN_GO):
|
$(PROTOC_GEN_GO):
|
||||||
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
test -e $(PROTOC_GEN_GO) || go install -v google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1
|
||||||
@@ -38,44 +32,31 @@ $(PROTOC_GEN_GO):
|
|||||||
fixme:
|
fixme:
|
||||||
@grep -nir fixme . | grep -v Makefile
|
@grep -nir fixme . | grep -v Makefile
|
||||||
|
|
||||||
check:
|
|
||||||
@script/check
|
|
||||||
|
|
||||||
fmt-check:
|
|
||||||
@script/fmt-check
|
|
||||||
|
|
||||||
# Halves of fmt-check, for environments that have only one toolchain:
|
|
||||||
# the Docker lint stage has Go but no node, the markdown stage the reverse.
|
|
||||||
fmt-check-go:
|
|
||||||
@script/fmt-check-go
|
|
||||||
|
|
||||||
fmt-check-md:
|
|
||||||
@script/prettier --check
|
|
||||||
|
|
||||||
hooks:
|
|
||||||
@script/install-precommit
|
|
||||||
|
|
||||||
devprereqs:
|
devprereqs:
|
||||||
which golangci-lint || go install -v github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
which gofumpt || go install -v mvdan.cc/gofumpt@latest
|
||||||
|
which golangci-lint || go install -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1
|
||||||
|
|
||||||
mfer/mf.pb.go: mfer/mf.proto
|
mfer/mf.pb.go: mfer/mf.proto
|
||||||
cd mfer && go generate .
|
cd mfer && go generate .
|
||||||
|
|
||||||
bin/mfer: $(SOURCEFILES) mfer/mf.pb.go
|
mfer.cmd: $(SOURCEFILES) mfer/mf.pb.go
|
||||||
protoc --version
|
protoc --version
|
||||||
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../bin/mfer $(GOFLAGS) .
|
cd cmd/mfer && go build -tags urfave_cli_no_docs -o ../../mfer.cmd $(GOFLAGS) .
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
rm -rfv mfer/*.pb.go bin/mfer cmd/mfer/mfer *.dockerimage
|
rm -rfv mfer/*.pb.go mfer.cmd cmd/mfer/mfer *.dockerimage
|
||||||
|
|
||||||
fmt:
|
fmt: mfer/mf.pb.go
|
||||||
@script/fmt
|
gofumpt -l -w mfer internal cmd
|
||||||
|
golangci-lint run --fix
|
||||||
|
-prettier -w *.json
|
||||||
|
-prettier -w *.md
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
@script/lint
|
golangci-lint run
|
||||||
|
sh -c 'test -z "$$(gofmt -l .)"'
|
||||||
|
|
||||||
docker:
|
docker: sneak-mfer.$(ARCH).tzst.dockerimage
|
||||||
@script/docker
|
|
||||||
|
|
||||||
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
|
sneak-mfer.$(ARCH).tzst.dockerimage: $(SOURCEFILES) vendor.tzst modcache.tzst
|
||||||
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
|
docker build --progress plain --build-arg GITREV=$(GITREV_BUILD) -t sneak/mfer .
|
||||||
|
|||||||
417
README.md
417
README.md
@@ -1,98 +1,60 @@
|
|||||||
# mfer
|
# mfer
|
||||||
|
|
||||||
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library and
|
[mfer](https://git.eeqj.de/sneak/mfer) is a reference implementation library
|
||||||
thin wrapper command-line utility written in [Go](https://golang.org) and first
|
and thin wrapper command-line utility written in [Go](https://golang.org)
|
||||||
published in 2022 under the [WTFPL](https://wtfpl.net) (public domain) license.
|
and first published in 2022 under the [WTFPL](https://wtfpl.net) (public
|
||||||
It specifies and generates `.mf` manifest files over a directory tree of files
|
domain) license. It specifies and generates `.mf` manifest files over a
|
||||||
to encapsulate metadata about them (such as cryptographic checksums or
|
directory tree of files to encapsulate metadata about them (such as
|
||||||
signatures over same) to aid in archiving, downloading, and streaming, or
|
cryptographic checksums or signatures over same) to aid in archiving,
|
||||||
mirroring. The manifest files' data is serialized with Google's
|
downloading, and streaming, or mirroring. The manifest files' data is
|
||||||
[protobuf serialization format](https://developers.google.com/protocol-buffers).
|
serialized with Google's [protobuf serialization
|
||||||
The structure of these files can be found
|
format](https://developers.google.com/protocol-buffers). The structure of
|
||||||
[in the format specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
these files can be found [in the format
|
||||||
which is included in the [project repository](https://git.eeqj.de/sneak/mfer).
|
specification](https://git.eeqj.de/sneak/mfer/src/branch/main/mfer/mf.proto)
|
||||||
|
which is included in the [project
|
||||||
|
repository](https://git.eeqj.de/sneak/mfer).
|
||||||
|
|
||||||
The current version is pre-1.0 and while the repo was published in 2022, there
|
The current version is pre-1.0 and while the repo was published in 2022,
|
||||||
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
there has not yet been any versioned release. [SemVer](https://semver.org)
|
||||||
used for releases.
|
will be used for releases.
|
||||||
|
|
||||||
This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
This project was started by [@sneak](https://sneak.berlin) to scratch an
|
||||||
2022 and is currently a one-person effort, though the goal is for this to emerge
|
itch in 2022 and is currently a one-person effort, though the goal is for
|
||||||
as a de-facto standard and be incorporated into other software. A compatible
|
this to emerge as a de-facto standard and be incorporated into other
|
||||||
javascript library is planned.
|
software. A compatible javascript library is planned.
|
||||||
|
|
||||||
# Build Status
|
# Build Status
|
||||||
|
|
||||||
CI runs via `script/cibuild` (`docker build .`), which executes `make check`
|
[](https://drone.datavi.be/sneak/mfer)
|
||||||
(formatting, linting, tests). The `main` branch must always be green.
|
|
||||||
|
|
||||||
# Entrypoints
|
|
||||||
|
|
||||||
This repository adheres to the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
|
||||||
development workflow, and the Makefile targets are thin shims that call them. We
|
|
||||||
provide:
|
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (Go, golangci-lint, Go module
|
|
||||||
download, and node/yarn plus the prettier version pinned in
|
|
||||||
`package.json`/`yarn.lock`), idempotently
|
|
||||||
- `script/setup` — make a fresh clone ready for development: runs
|
|
||||||
`script/bootstrap`, then `script/install-precommit`
|
|
||||||
- `script/projectname` — output the project name (`mfer`); used by other scripts
|
|
||||||
such as `script/docker`
|
|
||||||
- `script/test` — run the test suite (`go test`), regenerating the protobuf code
|
|
||||||
first if it is stale
|
|
||||||
- `script/lint` — run `golangci-lint` and verify `gofmt` cleanliness
|
|
||||||
- `script/fmt` — format all code and docs (writes): `gofumpt`,
|
|
||||||
`golangci-lint run --fix`, and `script/prettier --write`
|
|
||||||
- `script/prettier` — run prettier over the repository's canonical file set
|
|
||||||
(Markdown and JSON, minus `.prettierignore`) in the given mode, `--write` or
|
|
||||||
`--check`; the single definition of that file set, so `script/fmt` and
|
|
||||||
`script/fmt-check` cannot disagree about it
|
|
||||||
- `script/fmt-check` — check formatting without writing: `script/fmt-check-go`
|
|
||||||
plus `script/prettier --check`
|
|
||||||
- `script/fmt-check-go` — the Go half of `script/fmt-check`, on its own, for the
|
|
||||||
Docker lint stage, whose image has no node
|
|
||||||
- `script/check` — run `script/test`, `script/lint`, and `script/fmt-check`
|
|
||||||
- `script/docker` — build the Docker image tagged with the project name
|
|
||||||
- `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile runs the
|
|
||||||
checks)
|
|
||||||
- `script/precommit` — pre-commit checks: `go mod tidy` verification, then
|
|
||||||
`script/check`
|
|
||||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
|
||||||
`script/precommit`
|
|
||||||
|
|
||||||
# Participation
|
# Participation
|
||||||
|
|
||||||
The community is as yet nonexistent so there are no defined policies or norms
|
The community is as yet nonexistent so there are no defined policies or
|
||||||
yet. Primary development happens on a privately-run Gitea instance at
|
norms yet. Primary development happens on a privately-run Gitea instance at
|
||||||
[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues are
|
[https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer) and issues
|
||||||
[tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
are [tracked there](https://git.eeqj.de/sneak/mfer/issues).
|
||||||
|
|
||||||
Changes must always be formatted with a standard `go fmt`, syntactically valid,
|
Changes must always be formatted with a standard `go fmt`, syntactically
|
||||||
and must pass the linting defined in the repository (presently only the
|
valid, and must pass the linting defined in the repository (presently only
|
||||||
`golangci-lint` defaults), which can be run with a `make lint`. The `main`
|
the `golangci-lint` defaults), which can be run with a `make lint`. The
|
||||||
branch is protected and all changes must be made via
|
`main` branch is protected and all changes must be made via [pull
|
||||||
[pull requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
requests](https://git.eeqj.de/sneak/mfer/pulls) and pass CI to be merged.
|
||||||
Any changes submitted to this project must also be
|
Any changes submitted to this project must also be
|
||||||
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
[WTFPL-licensed](https://wtfpl.net) to be considered.
|
||||||
|
|
||||||
See [`REPO_POLICIES.md`](REPO_POLICIES.md) for detailed coding standards,
|
|
||||||
tooling requirements, and workflow conventions.
|
|
||||||
|
|
||||||
# Problem Statement
|
# Problem Statement
|
||||||
|
|
||||||
Given a plain URL, there is no standard way to safely and programmatically
|
Given a plain URL, there is no standard way to safely and programmatically
|
||||||
download everything "under" that URL path. `wget -r` can traverse directory
|
download everything "under" that URL path. `wget -r` can traverse directory
|
||||||
listings if they're enabled, but every server has a different format, and this
|
listings if they're enabled, but every server has a different format, and
|
||||||
does not verify cryptographic integrity of the files, or enable them to be
|
this does not verify cryptographic integrity of the files, or enable them to
|
||||||
fetched using a different protocol other than HTTP/s.
|
be fetched using a different protocol other than HTTP/s.
|
||||||
|
|
||||||
Currently, the solution that people are using are sidecar files in the format of
|
Currently, the solution that people are using are sidecar files in the
|
||||||
`SHASUMS` checksum files, as well as a `SHASUMS.asc` PGP detached signature.
|
format of `SHASUMS` checksum files, as well as a `SHASUMS.asc` PGP detached
|
||||||
This is not checksum-algorithm-agnostic and the sidecar file is not always
|
signature. This is not checksum-algorithm-agnostic and the sidecar file is
|
||||||
consistently named.
|
not always consistently named.
|
||||||
|
|
||||||
Real issues I face:
|
Real issues I face:
|
||||||
|
|
||||||
@@ -102,34 +64,35 @@ Real issues I face:
|
|||||||
- when I want to mirror an HTTP archive, I have to use special tools like
|
- when I want to mirror an HTTP archive, I have to use special tools like
|
||||||
debmirror that understand the archive format
|
debmirror that understand the archive format
|
||||||
- the debian repository metadata structure is hot garbage
|
- the debian repository metadata structure is hot garbage
|
||||||
- when I download a large file via HTTP, I have no way of knowing if the file
|
- when I download a large file via HTTP, I have no way of knowing if the
|
||||||
content is what it's supposed to be
|
file content is what it's supposed to be
|
||||||
|
|
||||||
# Proposed Solution
|
# Proposed Solution
|
||||||
|
|
||||||
A standard, a manifest file format, and a tool for generating same.
|
A standard, a manifest file format, and a tool for generating same.
|
||||||
|
|
||||||
The manifest file would be called `index.mf`, and the tool for generating such
|
The manifest file would be called `index.mf`, and the tool for generating such would be called `mfer`.
|
||||||
would be called `mfer`.
|
|
||||||
|
|
||||||
The manifest file would do several important things:
|
The manifest file would do several important things:
|
||||||
|
|
||||||
- have a standard filename, so if given `https://example.com/downloadpackage/`
|
- have a standard filename, so if given
|
||||||
one could fetch `https://example.com/downloadpackage/index.mf` to enumerate
|
`https://example.com/downloadpackage/` one could fetch
|
||||||
the full directory listing.
|
`https://example.com/downloadpackage/index.mf` to enumerate the full
|
||||||
|
directory listing.
|
||||||
- contain a version field for extensibility
|
- contain a version field for extensibility
|
||||||
- contain structured data (protobuf, json, or cbor)
|
- contain structured data (protobuf, json, or cbor)
|
||||||
- provide an inner signed container, so that the manifest file itself can embed
|
- provide an inner signed container, so that the manifest file itself can
|
||||||
a signature and a public key alongside in a single file
|
embed a signature and a public key alongside in a single file
|
||||||
- contain a list of files, each with a relative path to the manifest
|
- contain a list of files, each with a relative path to the manifest
|
||||||
- contain manifest timestamp
|
- contain manifest timestamp
|
||||||
- contain ctime/mtime information for files so that file metadata can be
|
- contain ctime/mtime information for files so that file metadata can be
|
||||||
preserved
|
preserved
|
||||||
- contain cryptographic checksums in several different algorithms for each file
|
- contain cryptographic checksums in several different algorithms for each
|
||||||
|
file
|
||||||
- probably encoded with multihash to indicate algo + hash
|
- probably encoded with multihash to indicate algo + hash
|
||||||
- sha256 at the minimum
|
- sha256 at the minimum
|
||||||
- would be nice to include an IPFS/IPLD CIDv1 root hash for each file, which
|
- would be nice to include an IPFS/IPLD CIDv1 root hash for each file,
|
||||||
likely involves doing an ipfs file object chunking
|
which likely involves doing an ipfs file object chunking
|
||||||
- maybe even including the complete IPFS/IPLD directory tree objects and
|
- maybe even including the complete IPFS/IPLD directory tree objects and
|
||||||
chunklists?
|
chunklists?
|
||||||
- this is because generating an `index.mf` does not imply publishing on
|
- this is because generating an `index.mf` does not imply publishing on
|
||||||
@@ -149,8 +112,9 @@ The manifest file would do several important things:
|
|||||||
# Non-Goals
|
# Non-Goals
|
||||||
|
|
||||||
- Manifest generation speed
|
- Manifest generation speed
|
||||||
- likely involves IPFS chunking, bittorrent chunking, and several different
|
- likely involves IPFS chunking, bittorrent chunking, and several
|
||||||
cryptographic hash functions over the entirety of each and every file
|
different cryptographic hash functions over the entirety of each and
|
||||||
|
every file
|
||||||
- Small manifest file size (within reason)
|
- Small manifest file size (within reason)
|
||||||
- 30MiB files are "small" these days, given modern storage/bandwidth
|
- 30MiB files are "small" these days, given modern storage/bandwidth
|
||||||
- metadata size should not be used as an excuse to sacrifice utility (such
|
- metadata size should not be used as an excuse to sacrifice utility (such
|
||||||
@@ -158,14 +122,13 @@ The manifest file would do several important things:
|
|||||||
|
|
||||||
# Open Questions
|
# Open Questions
|
||||||
|
|
||||||
- Should the manifest file include checksums of individual file chunks, or just
|
- Should the manifest file include checksums of individual file chunks, or just for the whole assembled file?
|
||||||
for the whole assembled file?
|
|
||||||
|
|
||||||
- If so, should the chunksize be fixed or dynamic?
|
- If so, should the chunksize be fixed or dynamic?
|
||||||
|
|
||||||
- Should the manifest signature format be GnuPG signatures, or those from
|
- Should the manifest signature format be GnuPG signatures, or those from
|
||||||
OpenBSD's signify (of which there is a good
|
OpenBSD's signify (of which there is a good [golang
|
||||||
[golang implementation](https://github.com/frankbraun/gosignify)?
|
implementation](https://github.com/frankbraun/gosignify)?
|
||||||
|
|
||||||
- Should the on-disk serialization format be proto3 or json?
|
- Should the on-disk serialization format be proto3 or json?
|
||||||
|
|
||||||
@@ -174,8 +137,8 @@ The manifest file would do several important things:
|
|||||||
- `mfer gen` / `mfer gen .`
|
- `mfer gen` / `mfer gen .`
|
||||||
- recurses under current directory and writes out an `index.mf`
|
- recurses under current directory and writes out an `index.mf`
|
||||||
- `mfer check` / `mfer check .`
|
- `mfer check` / `mfer check .`
|
||||||
- verifies checksums of all files in manifest, displaying error and exiting
|
- verifies checksums of all files in manifest, displaying error and
|
||||||
nonzero if any files are missing or corrupted
|
exiting nonzero if any files are missing or corrupted
|
||||||
- `mfer fetch https://example.com/stuff/`
|
- `mfer fetch https://example.com/stuff/`
|
||||||
- fetches `/stuff/index.mf` and downloads all files listed in manifest,
|
- fetches `/stuff/index.mf` and downloads all files listed in manifest,
|
||||||
optionally resuming any that already exist locally, and assures
|
optionally resuming any that already exist locally, and assures
|
||||||
@@ -199,28 +162,27 @@ The manifest file would do several important things:
|
|||||||
- (fetches `https://example.com/manifestdirectory/index.mf`, downloads and
|
- (fetches `https://example.com/manifestdirectory/index.mf`, downloads and
|
||||||
checksums all files, resumes any that exist locally already)
|
checksums all files, resumes any that exist locally already)
|
||||||
- `mfer fetch https://example.com/manifestdirectory/`
|
- `mfer fetch https://example.com/manifestdirectory/`
|
||||||
- a command line option to zero/omit mtime/ctime, as well as manifest timestamp,
|
- a command line option to zero/omit mtime/ctime, as well as manifest
|
||||||
and sort all directory listings so that manifest file generation is
|
timestamp, and sort all directory listings so that manifest file
|
||||||
deterministic/reproducible
|
generation is deterministic/reproducible
|
||||||
- URL format
|
- URL format `mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
|
||||||
`mfer fetch https://exmaple.com/manifestdirectory/?key=5539AD00DE4C42F3AFE11575052443F4DF2A55C2`
|
to assert in the URL which PGP signing key should be used in the manifest,
|
||||||
to assert in the URL which PGP signing key should be used in the manifest, so
|
so that shared URLs have a cryptographic trust root
|
||||||
that shared URLs have a cryptographic trust root
|
- a "well-known" key in the manifest that maps well known keys (could reuse
|
||||||
- a "well-known" key in the manifest that maps well known keys (could reuse the
|
the http spec) to specific file paths in the manifest.
|
||||||
http spec) to specific file paths in the manifest.
|
- example: a `berlin.sneak.app.slideshow` key that maps to a json
|
||||||
- example: a `berlin.sneak.app.slideshow` key that maps to a json slideshow
|
slideshow config listing what image paths to show, and for how long, and
|
||||||
config listing what image paths to show, and for how long, and in what
|
in what order
|
||||||
order
|
|
||||||
|
|
||||||
# Use Cases
|
# Use Cases
|
||||||
|
|
||||||
## Web Images
|
## Web Images
|
||||||
|
|
||||||
I'd like to be able to put a bunch of images into a directory, generate a
|
I'd like to be able to put a bunch of images into a directory, generate a
|
||||||
manifest, and then point a slideshow client (such as an ambient display, or a
|
manifest, and then point a slideshow client (such as an ambient display, or
|
||||||
react app with the target directory in a query string arg) at that statically
|
a react app with the target directory in a query string arg) at that
|
||||||
hosted directory, and have it discover the full list of images available at that
|
statically hosted directory, and have it discover the full list of images
|
||||||
URL.
|
available at that URL.
|
||||||
|
|
||||||
## Software Distribution
|
## Software Distribution
|
||||||
|
|
||||||
@@ -230,242 +192,39 @@ resumably by either HTTP or IPFS/BitTorrent without a .torrent file.
|
|||||||
## Filesystem Archive Integrity
|
## Filesystem Archive Integrity
|
||||||
|
|
||||||
I use filesystems that don't include data checksums, and I would like a
|
I use filesystems that don't include data checksums, and I would like a
|
||||||
cryptographically signed checksum file so that I can later verify that a set of
|
cryptographically signed checksum file so that I can later verify that a set
|
||||||
archive files have not been modified, none are missing, and that the checksums
|
of archive files have not been modified, none are missing, and that the
|
||||||
have not been altered in storage by a second party.
|
checksums have not been altered in storage by a second party.
|
||||||
|
|
||||||
## Filesystem-Independent Checksums
|
## Filesystem-Independent Checksums
|
||||||
|
|
||||||
I would like to be able to plug in a hard drive or flash drive and, if there is
|
I would like to be able to plug in a hard drive or flash drive and, if there
|
||||||
an `index.mf` in the root, automatically detect missing/corrupted files,
|
is an `index.mf` in the root, automatically detect missing/corrupted files,
|
||||||
regardless of filesystem format.
|
regardless of filesystem format.
|
||||||
|
|
||||||
# Collaboration
|
# Collaboration
|
||||||
|
|
||||||
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your desired
|
Please email [`sneak@sneak.berlin`](mailto:sneak@sneak.berlin) with your
|
||||||
username for an account on this Gitea instance.
|
desired username for an account on this Gitea instance.
|
||||||
|
|
||||||
# TODO: Remaining Work for 1.0
|
|
||||||
|
|
||||||
## Design Questions (Owner Decision Required)
|
|
||||||
|
|
||||||
These require @sneak's input before implementation. Answers should be added
|
|
||||||
inline below each question.
|
|
||||||
|
|
||||||
### Format Design
|
|
||||||
|
|
||||||
**1. Should `MFFileChecksum` be simplified?** Currently it's a separate message
|
|
||||||
wrapping a single `bytes multiHash` field. Since multihash already
|
|
||||||
self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath`
|
|
||||||
would be simpler and reduce per-file protobuf overhead. Is the extra message
|
|
||||||
layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**2. Should file permissions/mode be stored?** The format stores mtime/ctime but
|
|
||||||
not Unix file permissions. For archival use this may not matter, but for
|
|
||||||
software distribution or filesystem restoration it's a gap. Should we reserve a
|
|
||||||
field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**3. Should `atime` be removed from the schema?** Access time is volatile,
|
|
||||||
non-deterministic, and often disabled (`noatime`). Including it means two
|
|
||||||
manifests of the same directory at different times will differ, which conflicts
|
|
||||||
with the determinism goal. Remove it, or document it as "never set by default"?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**4. What are the path normalization rules?** The proto has `string path` with
|
|
||||||
no specification about: always forward-slash? Must be relative? No `..`
|
|
||||||
components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path
|
|
||||||
length? This is a security issue (path traversal) and a cross-platform
|
|
||||||
compatibility issue. What rules should the spec mandate?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**5. Should we add a version byte after the magic?** Currently `ZNAVSRFG` is
|
|
||||||
followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would
|
|
||||||
allow future framing changes without requiring protobuf parsing to detect the
|
|
||||||
version. `MFFileOuter.Version` serves this purpose but requires successful
|
|
||||||
deserialization to read. Worth the extra byte?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**6. Should we add a length-prefix after the magic?** Protobuf is not
|
|
||||||
self-delimiting. If we ever want to concatenate manifests or append data after
|
|
||||||
the protobuf, the current framing is insufficient. Add a varint or fixed-width
|
|
||||||
length-prefix?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
### Signature Design
|
|
||||||
|
|
||||||
**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
|
|
||||||
The code currently hashes compressed data (good for verifying before
|
|
||||||
decompression), but this should be explicitly documented. Which is the intended
|
|
||||||
behavior?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded
|
|
||||||
string?** Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex
|
|
||||||
encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or
|
|
||||||
compressed `innerMessage` directly) would be simpler. Keep the string format or
|
|
||||||
switch to raw bytes?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**9. Should we support detached signature files (`.mf.sig`)?** Embedded
|
|
||||||
signatures are better for single-file distribution. Detached `.mf.sig` files
|
|
||||||
follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP
|
|
||||||
serving. Support both modes?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**10. GPG vs pure-Go crypto for signatures?** Shelling out to `gpg` is fragile
|
|
||||||
(may not be installed, version-dependent output).
|
|
||||||
`github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could use
|
|
||||||
Ed25519/signify (simpler, no key management). Which direction?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
### Implementation Design
|
|
||||||
|
|
||||||
**11. Should manifests be deterministic by default?** This means: sort file
|
|
||||||
entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`.
|
|
||||||
Should determinism be the default, with a `--include-timestamps` flag to opt in?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**12. Should we consolidate or keep both scanner/checker implementations?**
|
|
||||||
There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go`
|
|
||||||
(typed with `FileSize`, `RelFilePath`) and `internal/scanner/` +
|
|
||||||
`internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior.
|
|
||||||
Delete the `internal/` versions?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**13. Should the `manifest` type be exported?** Currently unexported with
|
|
||||||
exported constructors (`NewManifestFromReader`, `NewManifestFromFile`).
|
|
||||||
Consumers can't declare `var m *mfer.manifest`. Export the type, or define an
|
|
||||||
interface?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
**14. What should the Go module path be for 1.0?** Currently
|
|
||||||
`sneak.berlin/go/mfer` in `go.mod` but `git.eeqj.de/sneak/mfer/mfer` in the
|
|
||||||
proto `go_package` option. Which is canonical?
|
|
||||||
|
|
||||||
> _answer:_
|
|
||||||
|
|
||||||
## Implementation Tasks
|
|
||||||
|
|
||||||
### Repo Infrastructure
|
|
||||||
|
|
||||||
- [ ] Add `.golangci.yml` (fetch from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`)
|
|
||||||
- [ ] Add `.editorconfig`
|
|
||||||
- [ ] Add `.gitea/workflows/check.yml` that runs `docker build .`
|
|
||||||
|
|
||||||
### Format & Correctness
|
|
||||||
|
|
||||||
- [ ] Resolve proto `go_package` path inconsistency
|
|
||||||
(`git.eeqj.de/sneak/mfer/mfer` vs `sneak.berlin/go/mfer`)
|
|
||||||
- [ ] Specify path invariants — add proto comments requiring UTF-8,
|
|
||||||
forward-slash, relative paths, no `..`, no leading `/`; validate in
|
|
||||||
`Builder.AddFile` and `Builder.AddFileWithHash` (pending design question
|
|
||||||
answer)
|
|
||||||
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
|
|
||||||
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file
|
|
||||||
permissions (pending design question answer)
|
|
||||||
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
|
|
||||||
(pending design question answer)
|
|
||||||
- [ ] Write format specification document — separate from README: magic, outer
|
|
||||||
structure, compression, inner structure, path invariants, signature
|
|
||||||
scheme, canonical serialization
|
|
||||||
|
|
||||||
### Library
|
|
||||||
|
|
||||||
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on
|
|
||||||
`mfer/` package versions; update CLI code (pending design question answer)
|
|
||||||
- [ ] Add deterministic file ordering — sort entries by path (lexicographic,
|
|
||||||
byte-order) in `Builder.Build()`; add test asserting byte-identical output
|
|
||||||
from two runs
|
|
||||||
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()`
|
|
||||||
with `m.pbOuter.Size` as bound
|
|
||||||
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)`
|
|
||||||
or `errors.Is(err, fs.ErrNotExist)`
|
|
||||||
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading,
|
|
||||||
return error on mismatch
|
|
||||||
- [ ] Export the `manifest` type or define a public interface (pending design
|
|
||||||
question answer) — currently consumers cannot hold a reference to a loaded
|
|
||||||
manifest in their own type declarations
|
|
||||||
- [ ] Replace GPG subprocess calls with pure-Go crypto (pending design question
|
|
||||||
answer) — current implementation shells out to `gpg` which may not be
|
|
||||||
installed
|
|
||||||
- [ ] Add timeout to any remaining subprocess calls
|
|
||||||
|
|
||||||
### CLI
|
|
||||||
|
|
||||||
- [ ] Fix flag naming — all CLI flags should use kebab-case as primary
|
|
||||||
(`--include-dotfiles`, `--follow-symlinks`)
|
|
||||||
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or
|
|
||||||
`url.JoinPath()` instead of string concatenation
|
|
||||||
- [ ] Add progress rate-limiting to Checker — throttle to once per second,
|
|
||||||
matching Scanner
|
|
||||||
- [ ] Add `--deterministic` flag or make it default — omit `createdAt`, sort
|
|
||||||
files (pending design question answer)
|
|
||||||
- [ ] Wire `--version` flag properly (currently only a `version` subcommand
|
|
||||||
exists; top-level `--version` shows urfave/cli generic output)
|
|
||||||
- [ ] Add retry logic to `fetch` — currently no retries on transient HTTP
|
|
||||||
errors; needs exponential backoff
|
|
||||||
- [ ] `fetch` command uses bare `http.Get` with no timeout — needs `http.Client`
|
|
||||||
with configurable timeout
|
|
||||||
|
|
||||||
### Testing & Robustness
|
|
||||||
|
|
||||||
- [ ] Add fuzzing tests for `NewManifestFromReader` — protobuf deserialization
|
|
||||||
of untrusted input needs fuzz coverage
|
|
||||||
- [ ] Add integration test for `freshen` CLI command — current tests only verify
|
|
||||||
setup, not the actual freshen operation end-to-end
|
|
||||||
- [ ] Add test for `fetch` CLI command end-to-end (currently only `downloadFile`
|
|
||||||
is tested)
|
|
||||||
|
|
||||||
### Documentation
|
|
||||||
|
|
||||||
- [ ] Promote `FORMAT.md` as primary spec reference; README should link to it
|
|
||||||
more prominently
|
|
||||||
- [ ] Audit and update all error messages for consistency and helpfulness
|
|
||||||
- [ ] Document the signature scheme more thoroughly (canonical string format,
|
|
||||||
verification steps)
|
|
||||||
|
|
||||||
### Release
|
|
||||||
|
|
||||||
- [ ] Finalize Go module path
|
|
||||||
- [ ] Update version constant in `mfer/constants.go`
|
|
||||||
- [ ] Add `--version` output matching SemVer
|
|
||||||
- [ ] Tag `v1.0.0`
|
|
||||||
|
|
||||||
# See Also
|
# See Also
|
||||||
|
|
||||||
## Prior Art: Metalink
|
## Prior Art: Metalink
|
||||||
|
|
||||||
- [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
|
* [Metalink - Mozilla Wiki](https://wiki.mozilla.org/Metalink)
|
||||||
- [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
|
* [Metalink - Wikipedia](https://en.wikipedia.org/wiki/Metalink)
|
||||||
- [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
|
* [RFC 5854 - The Metalink Download Description Format](https://datatracker.ietf.org/doc/html/rfc5854)
|
||||||
- [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
|
* [RFC 6249 - Metalink/HTTP: Mirrors and Hashes](https://www.rfc-editor.org/rfc/rfc6249.html)
|
||||||
|
|
||||||
## Links
|
## Links
|
||||||
|
|
||||||
- Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
* Repo: [https://git.eeqj.de/sneak/mfer](https://git.eeqj.de/sneak/mfer)
|
||||||
- Issues:
|
* Issues: [https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
||||||
[https://git.eeqj.de/sneak/mfer/issues](https://git.eeqj.de/sneak/mfer/issues)
|
|
||||||
|
|
||||||
# Authors
|
# Authors
|
||||||
|
|
||||||
- [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
|
* [@sneak <sneak@sneak.berlin>](mailto:sneak@sneak.berlin)
|
||||||
|
|
||||||
# License
|
# License
|
||||||
|
|
||||||
- [WTFPL](https://wtfpl.net)
|
* [WTFPL](https://wtfpl.net)
|
||||||
|
|||||||
408
REPO_POLICIES.md
408
REPO_POLICIES.md
@@ -1,408 +0,0 @@
|
|||||||
---
|
|
||||||
title: Repository Policies
|
|
||||||
last_modified: 2026-07-06
|
|
||||||
---
|
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
|
||||||
style conventions are in separate documents:
|
|
||||||
|
|
||||||
- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
|
|
||||||
(general, bash, Docker)
|
|
||||||
- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
|
|
||||||
- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
|
|
||||||
- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
|
|
||||||
- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
- Cross-project documentation (such as this file) must include
|
|
||||||
`last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
|
|
||||||
with the authoritative source as policies evolve.
|
|
||||||
|
|
||||||
- **ALL external references must be pinned by cryptographic hash.** This
|
|
||||||
includes Docker base images, Go modules, npm packages, GitHub Actions, and
|
|
||||||
anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
|
|
||||||
`:3.21`, etc.) are server-mutable and therefore remote code execution
|
|
||||||
vulnerabilities. The ONLY acceptable way to reference an external dependency
|
|
||||||
is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
|
|
||||||
integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
|
|
||||||
This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
|
|
||||||
etc. Instead, download a specific release archive from GitHub, verify its hash
|
|
||||||
(hardcoded in the Dockerfile or script), and only then install. Unverified
|
|
||||||
install scripts are arbitrary remote code execution. This is the single most
|
|
||||||
important rule in this document. Double-check every external reference in
|
|
||||||
every file before committing. There are zero exceptions to this rule.
|
|
||||||
|
|
||||||
- Every repo with software must have a root `Makefile` with these targets:
|
|
||||||
`make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
|
|
||||||
`make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
|
|
||||||
`make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
|
|
||||||
is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
|
|
||||||
|
|
||||||
- Repos follow the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
pattern: the implementation of each Makefile target lives in an executable
|
|
||||||
script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
|
|
||||||
`script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
|
|
||||||
`script/docker`), and the Makefile targets are thin shims that call them. The
|
|
||||||
scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
|
|
||||||
minimal containers (e.g. alpine images have no bash); locate the repo root
|
|
||||||
with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
|
|
||||||
the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
|
|
||||||
for development after a fresh clone: runs `bootstrap`, then
|
|
||||||
`install-precommit`, plus any repo-specific initialization), `test`, and
|
|
||||||
`cibuild`. `script/bootstrap` installs all dependencies idempotently and
|
|
||||||
assumes nothing is present: base tools come from nix, apt, brew, or apk
|
|
||||||
(detected in that order; apt runs noninteractive). For node it uses the
|
|
||||||
installed node if present; otherwise it installs a PINNED node version via
|
|
||||||
nvm, first installing nvm itself if missing — from a hash-verified GitHub
|
|
||||||
release archive (never `curl | sh`), with bash installed as an explicit
|
|
||||||
prerequisite since nvm requires bash. yarn is then pinned via
|
|
||||||
`corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
|
|
||||||
always exact versions. `script/cibuild` runs the CI build: it changes to the
|
|
||||||
repo root and runs `docker build .`; the Gitea workflow calls it. Four further
|
|
||||||
scripts are our own extensions to the standard: `script/check` runs
|
|
||||||
`script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
|
|
||||||
what the git pre-commit hook runs, and it calls `script/check`;
|
|
||||||
`script/install-precommit` installs the git pre-commit hook (the `make hooks`
|
|
||||||
target shims to it); and `script/projectname` (literally that filename) simply
|
|
||||||
outputs the project's name. Scripts that need the name call
|
|
||||||
`script/projectname` — e.g. `script/docker` assembles its image tag from it —
|
|
||||||
so those scripts stay byte-identical across all repos. Repo-type-specific
|
|
||||||
pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
|
|
||||||
`script/precommit`, not in the hook itself. Model scripts are at
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
|
|
||||||
must document the provided scripts in an **Entrypoints** section (see the
|
|
||||||
README requirements below).
|
|
||||||
|
|
||||||
- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
|
|
||||||
instead of invoking the underlying tools directly. The Makefile is the single
|
|
||||||
source of truth for how these operations are run.
|
|
||||||
|
|
||||||
- The Makefile is authoritative documentation for how the repo is used. Beyond
|
|
||||||
the required targets above, it should have targets for every common operation:
|
|
||||||
running a local development server (`make run`, `make dev`), re-initializing
|
|
||||||
or migrating the database (`make db-reset`, `make migrate`), building
|
|
||||||
artifacts (`make build`), generating code, seeding data, or anything else a
|
|
||||||
developer would do regularly. If someone checks out the repo and types
|
|
||||||
`make<tab>`, they should see every meaningful operation available. A new
|
|
||||||
contributor should be able to understand the entire development workflow by
|
|
||||||
reading the Makefile.
|
|
||||||
|
|
||||||
- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
|
|
||||||
as a build step so the build fails if the branch is not green. For non-server
|
|
||||||
repos, the Dockerfile should bring up a development environment and run
|
|
||||||
`make check`. For server repos, `make check` should run as an early build
|
|
||||||
stage before the final image is assembled. Dockerfiles install development
|
|
||||||
prerequisites by running `script/bootstrap` rather than duplicating installs
|
|
||||||
inline; COPY `script/` and the dependency manifests (`package.json` +
|
|
||||||
`yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
|
|
||||||
layer stays cached until dependencies change.
|
|
||||||
|
|
||||||
- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
|
|
||||||
repos use a multistage build where linting runs in an independent stage based
|
|
||||||
on the `golangci/golangci-lint` image (pinned by hash). This stage runs
|
|
||||||
`make fmt-check` and `make lint` before the full build begins. The build stage
|
|
||||||
then declares an explicit dependency on the lint stage via
|
|
||||||
`COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
|
|
||||||
linting before proceeding to compilation and tests. This ensures lint failures
|
|
||||||
surface in seconds rather than minutes, without blocking on dependency
|
|
||||||
download or compilation in the build stage.
|
|
||||||
|
|
||||||
The standard pattern for a Go repo Dockerfile is:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
# Lint stage — fast feedback on formatting and lint issues
|
|
||||||
# golangci/golangci-lint:v2.x.x, YYYY-MM-DD
|
|
||||||
FROM golangci/golangci-lint@sha256:... AS lint
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN make fmt-check
|
|
||||||
RUN make lint
|
|
||||||
|
|
||||||
# Build stage
|
|
||||||
# golang:1.x-alpine, YYYY-MM-DD
|
|
||||||
FROM golang@sha256:... AS builder
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Force BuildKit to run the lint stage before proceeding
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY . .
|
|
||||||
RUN make test
|
|
||||||
|
|
||||||
ARG VERSION=dev
|
|
||||||
RUN CGO_ENABLED=0 go build -trimpath \
|
|
||||||
-ldflags="-s -w -X main.Version=${VERSION}" \
|
|
||||||
-o /app ./cmd/app/
|
|
||||||
|
|
||||||
# Runtime stage
|
|
||||||
FROM alpine@sha256:...
|
|
||||||
COPY --from=builder /app /usr/local/bin/app
|
|
||||||
ENTRYPOINT ["app"]
|
|
||||||
```
|
|
||||||
|
|
||||||
Key points:
|
|
||||||
- The lint stage uses the `golangci/golangci-lint` image directly (it
|
|
||||||
includes both Go and the linter), so there is no need to install the
|
|
||||||
linter separately.
|
|
||||||
- `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
|
|
||||||
a stage dependency. BuildKit runs stages in parallel by default; without
|
|
||||||
this line, the build stage would not wait for lint to finish and a lint
|
|
||||||
failure might not fail the overall build.
|
|
||||||
- If the project uses `//go:embed` directives that reference build artifacts
|
|
||||||
(e.g. a web frontend compiled in a separate stage), the lint stage must
|
|
||||||
create placeholder files so the embed directives resolve. Example:
|
|
||||||
`RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
|
|
||||||
The lint stage should not depend on the actual build output — it exists to
|
|
||||||
fail fast.
|
|
||||||
- If the project requires CGO or system libraries for linting (e.g.
|
|
||||||
`vips-dev`), install them in the lint stage with `apk add`.
|
|
||||||
- The build stage runs `make test` after compilation setup. Tests run in the
|
|
||||||
build stage, not the lint stage, because they may require compiled
|
|
||||||
artifacts or heavier dependencies.
|
|
||||||
|
|
||||||
- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
|
|
||||||
runs `script/cibuild` (which runs `docker build .`) on push. Since the
|
|
||||||
Dockerfile already runs `make check`, a successful build implies all checks
|
|
||||||
pass.
|
|
||||||
|
|
||||||
- Use platform-standard formatters: `black` for Python, `prettier` for
|
|
||||||
JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
|
|
||||||
two exceptions: four-space indents (except Go), and `proseWrap: always` for
|
|
||||||
Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
|
|
||||||
HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
|
|
||||||
|
|
||||||
- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
|
|
||||||
testing is not possible in the repo, `script/precommit` may skip `script/test`
|
|
||||||
and run only `script/lint` and `script/fmt-check`. The hook is installed by
|
|
||||||
`script/install-precommit`; the Makefile must provide a `make hooks` target
|
|
||||||
that shims to it.
|
|
||||||
|
|
||||||
- All repos with software must have tests that run via the platform-standard
|
|
||||||
test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
|
|
||||||
tests exist yet, add the most minimal test possible — e.g. importing the
|
|
||||||
module under test to verify it compiles/parses. There is no excuse for
|
|
||||||
`make test` to be a no-op.
|
|
||||||
|
|
||||||
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
|
||||||
Makefile.
|
|
||||||
|
|
||||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
|
||||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
|
||||||
show full output. This keeps CI logs and `docker build` output clean on
|
|
||||||
success (just package/suite summaries) while providing full diagnostic detail
|
|
||||||
on failure (every test case, every assertion). The general shell pattern:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@<test-command> || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
<test-command-with-v>; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
Go example:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@go test -timeout 30s -race -cover ./... || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
go test -timeout 30s -race -v ./...; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
Python example:
|
|
||||||
|
|
||||||
```makefile
|
|
||||||
test:
|
|
||||||
@python -m pytest || \
|
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
|
||||||
python -m pytest -v; exit 1; }
|
|
||||||
```
|
|
||||||
|
|
||||||
The `exit 1` ensures the target always fails after a rerun — the first run
|
|
||||||
already proved the tests are broken, so the build must not pass even if a
|
|
||||||
flaky test happens to succeed on the second attempt. The rerun exists solely
|
|
||||||
for diagnostic output.
|
|
||||||
|
|
||||||
- Docker builds must complete in under 5 minutes.
|
|
||||||
|
|
||||||
- `make check` must not modify any files in the repo. Tests may use temporary
|
|
||||||
directories.
|
|
||||||
|
|
||||||
- `main` must always pass `make check`, no exceptions.
|
|
||||||
|
|
||||||
- Never commit secrets. `.env` files, credentials, API keys, and private keys
|
|
||||||
must be in `.gitignore`. No exceptions.
|
|
||||||
|
|
||||||
- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
|
|
||||||
editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
|
|
||||||
Fetch the standard `.gitignore` from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
|
|
||||||
a new repo.
|
|
||||||
|
|
||||||
- **No build artifacts in version control.** Code-derived data (compiled
|
|
||||||
bundles, minified output, generated assets) must never be committed to the
|
|
||||||
repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
|
|
||||||
should generate these at build time. Notable exception: Go protobuf generated
|
|
||||||
files (`.pb.go`) ARE committed because repos need to work with `go get`, which
|
|
||||||
downloads code but does not execute code generation.
|
|
||||||
|
|
||||||
- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
|
|
||||||
|
|
||||||
- Never force-push to `main`.
|
|
||||||
|
|
||||||
- Make all changes on a feature branch. You can do whatever you want on a
|
|
||||||
feature branch.
|
|
||||||
|
|
||||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
|
||||||
manually by the user. Fetch from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
|
||||||
with the version and date (YYYY-MM-DD).
|
|
||||||
|
|
||||||
- Use `yarn`, not `npm`.
|
|
||||||
|
|
||||||
- Write all dates as YYYY-MM-DD (ISO 8601).
|
|
||||||
|
|
||||||
- Simple projects should be configured with environment variables.
|
|
||||||
|
|
||||||
- Dockerized web services listen on port 8080 by default, overridable with
|
|
||||||
`PORT`.
|
|
||||||
|
|
||||||
- **HTTP/web services must be hardened for production internet exposure before
|
|
||||||
tagging 1.0.** This means full compliance with security best practices
|
|
||||||
including, without limitation, all of the following:
|
|
||||||
- **Security headers** on every response:
|
|
||||||
- `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
|
|
||||||
and `includeSubDomains`.
|
|
||||||
- `Content-Security-Policy` (CSP) with a restrictive default policy
|
|
||||||
(`default-src 'self'` as a baseline, tightened per-resource as
|
|
||||||
needed). Never use `unsafe-inline` or `unsafe-eval` unless
|
|
||||||
unavoidable, and document the reason.
|
|
||||||
- `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
|
|
||||||
Prefer the `frame-ancestors` CSP directive as the primary control.
|
|
||||||
- `X-Content-Type-Options: nosniff`.
|
|
||||||
- `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
|
|
||||||
- `Permissions-Policy` restricting access to browser features the
|
|
||||||
application does not use (camera, microphone, geolocation, etc.).
|
|
||||||
- **Request and response limits:**
|
|
||||||
- Maximum request body size enforced on all endpoints (e.g. Go
|
|
||||||
`http.MaxBytesReader`). Choose a sane default per-route; never accept
|
|
||||||
unbounded input.
|
|
||||||
- Maximum response body size where applicable (e.g. paginated APIs).
|
|
||||||
- `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
|
|
||||||
against slowloris attacks.
|
|
||||||
- `WriteTimeout` on the `http.Server`.
|
|
||||||
- `IdleTimeout` on the `http.Server`.
|
|
||||||
- Per-handler execution time limits via `context.WithTimeout` or
|
|
||||||
chi/stdlib `middleware.Timeout`.
|
|
||||||
- **Authentication and session security:**
|
|
||||||
- Rate limiting on password-based authentication endpoints. API keys are
|
|
||||||
high-entropy and not susceptible to brute force, so they are exempt.
|
|
||||||
- CSRF tokens on all state-mutating HTML forms. API endpoints
|
|
||||||
authenticated via `Authorization` header (Bearer token, API key) are
|
|
||||||
exempt because the browser does not attach these automatically.
|
|
||||||
- Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
|
|
||||||
MD5, or SHA.
|
|
||||||
- Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
|
|
||||||
`Strict`) attributes.
|
|
||||||
- **Reverse proxy awareness:**
|
|
||||||
- True client IP detection when behind a reverse proxy
|
|
||||||
(`X-Forwarded-For`, `X-Real-IP`). The application must accept
|
|
||||||
forwarded headers only from a configured set of trusted proxy
|
|
||||||
addresses — never trust `X-Forwarded-For` unconditionally.
|
|
||||||
- **CORS:**
|
|
||||||
- Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
|
|
||||||
an explicit allowlist of known origins. Wildcard (`*`) is acceptable
|
|
||||||
only for public, unauthenticated read-only APIs.
|
|
||||||
- **Error handling:**
|
|
||||||
- Internal errors must never leak stack traces, SQL queries, file paths,
|
|
||||||
or other implementation details to the client. Return generic error
|
|
||||||
messages in production; detailed errors only when `DEBUG` is enabled.
|
|
||||||
- **TLS:**
|
|
||||||
- Services never terminate TLS directly. They are always deployed behind
|
|
||||||
a TLS-terminating reverse proxy. The service itself listens on plain
|
|
||||||
HTTP. However, HSTS headers and `Secure` cookie flags must still be
|
|
||||||
set by the application so that the browser enforces HTTPS end-to-end.
|
|
||||||
|
|
||||||
This list is non-exhaustive. Apply defense-in-depth: if a standard security
|
|
||||||
hardening measure exists for HTTP services and is not listed here, it is
|
|
||||||
still expected. When in doubt, harden.
|
|
||||||
|
|
||||||
- `README.md` is the primary documentation. Required sections:
|
|
||||||
- **Description**: First line must include the project name, purpose,
|
|
||||||
category (web server, SPA, CLI tool, etc.), license, and author. Example:
|
|
||||||
"µPaaS is an MIT-licensed Go web application by @sneak that receives
|
|
||||||
git-frontend webhooks and deploys applications via Docker in realtime."
|
|
||||||
- **Getting Started**: Copy-pasteable install/usage code block.
|
|
||||||
- **Entrypoints**: Opens by stating that the repo adheres to the
|
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
|
||||||
standard (with that link), then documents each provided `script/`
|
|
||||||
entrypoint and its purpose.
|
|
||||||
- **Rationale**: Why does this exist?
|
|
||||||
- **Design**: How is the program structured?
|
|
||||||
- **TODO**: Update meticulously, even between commits. When planning, put
|
|
||||||
the todo list in the README so a new agent can pick up where the last one
|
|
||||||
left off.
|
|
||||||
- **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
|
|
||||||
`LICENSE` file in the repo root and a License section in the README.
|
|
||||||
- **Author**: [@sneak](https://sneak.berlin).
|
|
||||||
|
|
||||||
- First commit of a new repo should contain only `README.md`.
|
|
||||||
|
|
||||||
- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
|
|
||||||
committing.
|
|
||||||
|
|
||||||
- Use SemVer.
|
|
||||||
|
|
||||||
- Database migrations live in `internal/db/migrations/` and must be embedded in
|
|
||||||
the binary.
|
|
||||||
- `000_migration.sql` — contains ONLY the creation of the migrations
|
|
||||||
tracking table itself. Nothing else.
|
|
||||||
- `001_schema.sql` — the full application schema.
|
|
||||||
- **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
|
|
||||||
There is no installed base to migrate. Edit `001_schema.sql` directly.
|
|
||||||
- **Post-1.0.0:** add new numbered migration files for each schema change.
|
|
||||||
Never edit existing migrations after release.
|
|
||||||
|
|
||||||
- All repos should have an `.editorconfig` enforcing the project's indentation
|
|
||||||
settings.
|
|
||||||
|
|
||||||
- Avoid putting files in the repo root unless necessary. Root should contain
|
|
||||||
only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
|
|
||||||
`LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
|
|
||||||
language-specific config). Everything else goes in a subdirectory. Canonical
|
|
||||||
subdirectory names:
|
|
||||||
- `bin/` — executable scripts and tools
|
|
||||||
- `cmd/` — Go command entrypoints
|
|
||||||
- `configs/` — configuration templates and examples
|
|
||||||
- `deploy/` — deployment manifests (k8s, compose, terraform)
|
|
||||||
- `docs/` — documentation and markdown (README.md stays in root)
|
|
||||||
- `internal/` — Go internal packages
|
|
||||||
- `internal/db/migrations/` — database migrations
|
|
||||||
- `pkg/` — Go library packages
|
|
||||||
- `share/` — systemd units, data files
|
|
||||||
- `static/` — static assets (images, fonts, etc.)
|
|
||||||
- `web/` — web frontend source
|
|
||||||
|
|
||||||
- When setting up a new repo, files from the `prompts` repo may be used as
|
|
||||||
templates. Fetch them from
|
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
|
|
||||||
|
|
||||||
- New repos must contain at minimum:
|
|
||||||
- `README.md`, `.git`, `.gitignore`, `.editorconfig`
|
|
||||||
- `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
|
|
||||||
- `Makefile`
|
|
||||||
- `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
|
|
||||||
`lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
|
|
||||||
`install-precommit`)
|
|
||||||
- `Dockerfile`, `.dockerignore`
|
|
||||||
- `.gitea/workflows/check.yml`
|
|
||||||
- Go: `go.mod`, `go.sum`, `.golangci.yml`
|
|
||||||
- JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
|
|
||||||
- Python: `pyproject.toml`
|
|
||||||
213
TODO.md
213
TODO.md
@@ -1,109 +1,122 @@
|
|||||||
# Workflow
|
# TODO: mfer 1.0
|
||||||
|
|
||||||
- branch (from `main`)
|
## Design Questions
|
||||||
- do the work in Next Step
|
|
||||||
- move Next Step to the top of Completed Steps
|
|
||||||
- move the top item of Future Steps into Next Step
|
|
||||||
- commit (`TODO.md` changes in the same commit as the work)
|
|
||||||
- merge to `main` if the branch is not protected, otherwise open a PR
|
|
||||||
- push
|
|
||||||
|
|
||||||
# Status
|
*sneak: please answer inline below each question. These are preserved for posterity.*
|
||||||
|
|
||||||
pre-1.0. No git tags. README section "TODO: Remaining Work for 1.0" lists open
|
### Format Design
|
||||||
design questions and implementation tasks; policy compliance work is in flight
|
|
||||||
and unmerged.
|
|
||||||
|
|
||||||
# Next Step
|
**1. Should `MFFileChecksum` be simplified?**
|
||||||
|
Currently it's a separate message wrapping a single `bytes multiHash` field. Since multihash already self-describes the algorithm, `repeated bytes hashes` directly on `MFFilePath` would be simpler and reduce per-file protobuf overhead. Is the extra message layer intentional (e.g. planning to add per-hash metadata like `verified_at`)?
|
||||||
|
|
||||||
Work through the remaining compliance items folded from the 2026-07-02 audit
|
> *answer:*
|
||||||
(the first group under Future Steps): `.editorconfig`, `.gitignore` coverage,
|
|
||||||
gofumpt-based `fmt-check`, README "Getting Started", and the rest.
|
|
||||||
`.golangci.yml` and `TODO.md` are tracked and committed as of 2026-08-07, so the
|
|
||||||
only thing left of the `chore/align-repo-policies` branch is the list below.
|
|
||||||
|
|
||||||
# Completed Steps
|
**2. Should file permissions/mode be stored?**
|
||||||
|
The format stores mtime/ctime but not Unix file permissions. For archival use (ExFAT, filesystem-independent checksums) this may not matter, but for software distribution or filesystem restoration it's a gap. Should we reserve a field now (e.g. `optional uint32 mode = 305`) even if we don't populate it yet?
|
||||||
|
|
||||||
- 2026-08-09: added `.prettierrc`/`.prettierignore`, gave `script/fmt` and
|
> *answer:*
|
||||||
`script/fmt-check` one shared prettier file set via `script/prettier`, dropped
|
|
||||||
the `|| true` that hid prettier failures, and added a node-based Dockerfile
|
|
||||||
stage so a markdown formatting violation fails `docker build .` (#69)
|
|
||||||
- 2026-08-07: updated golangci-lint to v2.12.2 everywhere it is pinned
|
|
||||||
(`Makefile`, `Dockerfile`), added the canonical `.golangci.yml`
|
|
||||||
(`default: all`), and fixed all resulting lint findings across the codebase
|
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
|
||||||
shims, README Entrypoints section
|
|
||||||
- 2026-07-03: aligned repo tooling, docs, and config with standardized policies
|
|
||||||
(7d9a138, on chore/align-repo-policies, unmerged)
|
|
||||||
- 2026-06-28: moved to standardized repo policies (#56, on main)
|
|
||||||
- 2026-04-07: added 1.0 roadmap as README TODO section, removed old TODO.md
|
|
||||||
(#54)
|
|
||||||
- 2026-03-20: added Gitea Actions CI workflow (#53)
|
|
||||||
- 2026-03-17: added REPO_POLICIES.md, renamed CLAUDE.md to AGENTS.md (#51);
|
|
||||||
removed committed .index.mf (#52)
|
|
||||||
- 2026-03-15: split Dockerfile with pre-built golangci-lint stage for faster CI
|
|
||||||
(#45)
|
|
||||||
- 2026-03-01: 1.0 quality polish: code review, tests, bug fixes, docs (#32)
|
|
||||||
- 2026-02-20: deterministic file ordering in Builder.Build() (#28); removed
|
|
||||||
committed vendor/modcache archives (#35)
|
|
||||||
- 2026-02-08: added --seed flag for deterministic manifest UUID
|
|
||||||
|
|
||||||
# Future Steps
|
**3. Should `atime` be removed from the schema?**
|
||||||
|
Access time is volatile, non-deterministic, and often disabled (`noatime`). Including it means two manifests of the same directory at different times will differ, which conflicts with the determinism goal. Remove it, or document it as "never set by default"?
|
||||||
|
|
||||||
- Compliance (fold of TODO.md audit 2026-07-02; verify which items the in-flight
|
> *answer:*
|
||||||
branch already closes, then check off):
|
|
||||||
- Add .editorconfig (canonical copy from sneak/prompts)
|
**4. What are the path normalization rules?**
|
||||||
- Make .gitignore cover secrets (.env, _.key, _.pem), OS files (.DS_Store),
|
The proto has `string path` with no specification about: always forward-slash? Must be relative? No `..` components allowed? UTF-8 NFC vs NFD normalization (macOS vs Linux)? Max path length? This is a security issue (path traversal) and a cross-platform compatibility issue. What rules should the spec mandate?
|
||||||
and editor files (_.swp, _~)
|
|
||||||
- Make fmt-check/lint verify with gofumpt, not gofmt -l, so `make check`
|
> *answer:*
|
||||||
matches what `make fmt` writes
|
|
||||||
- Add README "Getting Started" section with copy-pasteable install/usage
|
**5. Should we add a version byte after the magic?**
|
||||||
block
|
Currently `ZNAVSRFG` is followed immediately by protobuf. Adding a version byte (`ZNAVSRFG\x01`) would allow future framing changes without requiring protobuf parsing to detect the version. `MFFileOuter.Version` serves this purpose but requires successful deserialization to read. Worth the extra byte?
|
||||||
- Move FORMAT.md from repo root to docs/ and update the AGENTS.md reference
|
|
||||||
- Pin Makefile-installed Go tools (`protoc-gen-go@v1.28.1`,
|
> *answer:*
|
||||||
`golangci-lint@v2.12.2`) by module hash, not mutable tag
|
|
||||||
- Set `make test` timeout to 30s (currently 10s)
|
**6. Should we add a length-prefix after the magic?**
|
||||||
- Add explicit README "Rationale" heading (content exists under other
|
Protobuf is not self-delimiting. If we ever want to concatenate manifests or append data after the protobuf, the current framing is insufficient. Add a varint or fixed-width length-prefix?
|
||||||
names); name the author in the README Description first line
|
|
||||||
- Reconcile root-level AGENTS.md with directory-hygiene policy (keep or
|
> *answer:*
|
||||||
relocate)
|
|
||||||
- Add a `make build` target
|
### Signature Design
|
||||||
- Rewrite `make hooks` to use printf or a heredoc instead of non-portable
|
|
||||||
`echo '...\n...'`
|
**7. What does the outer SHA-256 hash cover — compressed or uncompressed data?**
|
||||||
- Answer the 14 owner design questions in the README 1.0 roadmap:
|
The review notes it currently hashes compressed data (good for verifying before decompression), but this should be explicitly documented. Which is the intended behavior?
|
||||||
- Format: simplify MFFileChecksum; store file mode; drop atime; specify path
|
|
||||||
normalization rules; version byte after magic; length-prefix after magic
|
> *answer:*
|
||||||
- Signatures: hash covers compressed or uncompressed data; sign raw bytes vs
|
|
||||||
hex canonical string; detached .mf.sig support; GPG subprocess vs pure-Go
|
**8. Should `signatureString()` sign raw bytes instead of a hex-encoded string?**
|
||||||
crypto
|
Currently the canonical string is `MAGIC-UUID-MULTIHASH` with hex encoding, which adds a transformation layer. Signing the raw `sha256` bytes (or compressed `innerMessage` directly) would be simpler. Keep the string format or switch to raw bytes?
|
||||||
- Implementation: deterministic manifests by default; consolidate duplicate
|
|
||||||
scanner/checker implementations; export the manifest type; canonical Go
|
> *answer:*
|
||||||
module path for 1.0
|
|
||||||
- Format and correctness:
|
**9. Should we support detached signature files (`.mf.sig`)?**
|
||||||
- Resolve proto go_package vs go.mod module path inconsistency
|
Embedded signatures are better for single-file distribution. Detached `.mf.sig` files follow the familiar `SHASUMS`/`SHASUMS.asc` pattern and are simpler for HTTP serving. Support both modes?
|
||||||
- Specify and validate path invariants (UTF-8, forward-slash, relative, no
|
|
||||||
.., no leading /)
|
> *answer:*
|
||||||
- Remove or deprecate atime; reserve mode field; add version byte (all
|
|
||||||
pending design answers)
|
**10. GPG vs pure-Go crypto for signatures?**
|
||||||
- Write a standalone format specification document
|
Shelling out to `gpg` is fragile (may not be installed, version-dependent output). `github.com/ProtonMail/go-crypto` provides pure-Go OpenPGP, or we could go Ed25519/signify (simpler, no key management). Which direction?
|
||||||
- Library:
|
|
||||||
- Delete internal/scanner and internal/checker; consolidate on the mfer/
|
> *answer:*
|
||||||
package versions (pending design answer)
|
|
||||||
- Add decompression size limit via io.LimitReader in deserializeInner()
|
### Implementation Design
|
||||||
- Fix errors.Is dead code in checker; make AddFile verify totalRead == size
|
|
||||||
- Export manifest type or define a public interface (pending)
|
**11. Should manifests be deterministic by default?**
|
||||||
- Replace GPG subprocess with pure-Go crypto (pending); add timeouts to
|
This means: sort file entries by path, omit `createdAt` timestamp (or make it opt-in), no `atime`. Should determinism be the default, with a `--include-timestamps` flag to opt in?
|
||||||
remaining subprocess calls
|
|
||||||
- CLI:
|
> *answer:*
|
||||||
- Kebab-case primary flag names; fix fetch URL construction with
|
|
||||||
url.JoinPath; add http.Client timeout and retry with backoff to fetch;
|
**12. Should we consolidate or keep both scanner/checker implementations?**
|
||||||
rate-limit Checker progress output; add --deterministic flag or default;
|
There are two parallel implementations: `mfer/scanner.go` + `mfer/checker.go` (typed with `FileSize`, `RelFilePath`) and `internal/scanner/` + `internal/checker/` (raw `int64`, `string`). The `mfer/` versions are superior. Delete the `internal/` versions?
|
||||||
wire top-level --version properly
|
|
||||||
- Testing:
|
> *answer:*
|
||||||
- Fuzz NewManifestFromReader; end-to-end tests for freshen and fetch
|
|
||||||
- Documentation:
|
**13. Should the `manifest` type be exported?**
|
||||||
- Promote docs/FORMAT.md as primary spec reference; audit error messages;
|
Currently unexported with exported constructors (`New`, `NewFromPaths`, etc.). Consumers can't declare `var m *mfer.manifest`. Export the type, or define an interface?
|
||||||
document the signature scheme fully
|
|
||||||
- Release:
|
> *answer:*
|
||||||
- Finalize module path, bump version constant, SemVer --version output, tag
|
|
||||||
v1.0.0
|
**14. What should the Go module path be for 1.0?**
|
||||||
|
Currently mixed between `sneak.berlin/go/mfer` and `git.eeqj.de/sneak/mfer`. Which is canonical?
|
||||||
|
|
||||||
|
> *answer:*
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Plan
|
||||||
|
|
||||||
|
### Phase 1: Foundation (format correctness)
|
||||||
|
|
||||||
|
- [ ] Delete `internal/scanner/` and `internal/checker/` — consolidate on `mfer/` package versions; update CLI code
|
||||||
|
- [ ] Add deterministic file ordering — sort entries by path (lexicographic, byte-order) in `Builder.Build()`; add test asserting byte-identical output from two runs
|
||||||
|
- [ ] Add decompression size limit — `io.LimitReader` in `deserializeInner()` with `m.pbOuter.Size` as bound
|
||||||
|
- [ ] Fix `errors.Is` dead code in checker — replace with `os.IsNotExist(err)` or `errors.Is(err, fs.ErrNotExist)`
|
||||||
|
- [ ] Fix `AddFile` to verify size — check `totalRead == size` after reading, return error on mismatch
|
||||||
|
- [ ] Specify path invariants — add proto comments (UTF-8, forward-slash, relative, no `..`, no leading `/`); validate in `Builder.AddFile` and `Builder.AddFileWithHash`
|
||||||
|
|
||||||
|
### Phase 2: CLI polish
|
||||||
|
|
||||||
|
- [ ] Fix flag naming — all CLI flags use kebab-case as primary (`--include-dotfiles`, `--follow-symlinks`)
|
||||||
|
- [ ] Fix URL construction in fetch — use `BaseURL.JoinPath()` or `url.JoinPath()` instead of string concatenation
|
||||||
|
- [ ] Add progress rate-limiting to Checker — throttle to once per second, matching Scanner
|
||||||
|
- [ ] Add `--deterministic` flag (or make it default) — omit `createdAt`, sort files
|
||||||
|
|
||||||
|
### Phase 3: Robustness
|
||||||
|
|
||||||
|
- [ ] Replace GPG subprocess with pure-Go crypto — `github.com/ProtonMail/go-crypto` or Ed25519/signify
|
||||||
|
- [ ] Add timeout to any remaining subprocess calls
|
||||||
|
- [ ] Add fuzzing tests for `NewManifestFromReader`
|
||||||
|
- [ ] Add retry logic to fetch — exponential backoff for transient HTTP errors
|
||||||
|
|
||||||
|
### Phase 4: Format finalization
|
||||||
|
|
||||||
|
- [ ] Remove or deprecate `atime` from proto (pending design question answer)
|
||||||
|
- [ ] Reserve `optional uint32 mode = 305` in `MFFilePath` for future file permissions
|
||||||
|
- [ ] Add version byte after magic — `ZNAVSRFG\x01` for format version 1
|
||||||
|
- [ ] Write format specification document — separate from README: magic, outer structure, compression, inner structure, path invariants, signature scheme, canonical serialization
|
||||||
|
|
||||||
|
### Phase 5: Release prep
|
||||||
|
|
||||||
|
- [ ] Finalize Go module path
|
||||||
|
- [ ] Audit all error messages for consistency and helpfulness
|
||||||
|
- [ ] Add `--version` output matching SemVer
|
||||||
|
- [ ] Tag v1.0.0
|
||||||
|
|||||||
@@ -1,19 +1,13 @@
|
|||||||
// Command mfer generates and verifies file manifests.
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"sneak.berlin/go/mfer/internal/cli"
|
"git.eeqj.de/sneak/mfer/internal/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Appname is the name of this program.
|
|
||||||
const Appname = "mfer"
|
|
||||||
|
|
||||||
// Version and Gitrev are injected at build time via -ldflags.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // set via ldflags at build time
|
|
||||||
var (
|
var (
|
||||||
|
Appname string = "mfer"
|
||||||
Version string
|
Version string
|
||||||
Gitrev string
|
Gitrev string
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -6,20 +6,6 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestAppname pins the program name that main passes to cli.Run; it is
|
func TestBuild(t *testing.T) {
|
||||||
// the name that appears in usage output and in the log prefix. It also
|
assert.True(t, true)
|
||||||
// keeps this package compiled under `go test`.
|
|
||||||
func TestAppname(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(t, "mfer", Appname)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVersionDefaults documents that Version and Gitrev are empty unless
|
|
||||||
// injected at build time via -ldflags.
|
|
||||||
func TestVersionDefaults(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Empty(t, Version)
|
|
||||||
assert.Empty(t, Gitrev)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
|
|
||||||
# usage.sh - Generate and check a manifest from the repo
|
|
||||||
# Run from repo root: ./contrib/usage.sh
|
|
||||||
|
|
||||||
TMPDIR=$(mktemp -d)
|
|
||||||
MANIFEST="$TMPDIR/index.mf"
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
rm -rf "$TMPDIR"
|
|
||||||
}
|
|
||||||
trap cleanup EXIT
|
|
||||||
|
|
||||||
echo "Building mfer..."
|
|
||||||
go build -o "$TMPDIR/mfer" ./cmd/mfer
|
|
||||||
|
|
||||||
"$TMPDIR/mfer" generate -o "$MANIFEST" .
|
|
||||||
"$TMPDIR/mfer" check --base . "$MANIFEST"
|
|
||||||
19
go.mod
19
go.mod
@@ -1,14 +1,10 @@
|
|||||||
module sneak.berlin/go/mfer
|
module git.eeqj.de/sneak/mfer
|
||||||
|
|
||||||
go 1.23
|
go 1.22
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/apex/log v1.9.0
|
github.com/apex/log v1.9.0
|
||||||
github.com/davecgh/go-spew v1.1.1
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/dustin/go-humanize v1.0.1
|
|
||||||
github.com/google/uuid v1.1.2
|
|
||||||
github.com/klauspost/compress v1.18.2
|
|
||||||
github.com/multiformats/go-multihash v0.2.3
|
|
||||||
github.com/pterm/pterm v0.12.35
|
github.com/pterm/pterm v0.12.35
|
||||||
github.com/spf13/afero v1.8.0
|
github.com/spf13/afero v1.8.0
|
||||||
github.com/stretchr/testify v1.8.1
|
github.com/stretchr/testify v1.8.1
|
||||||
@@ -21,24 +17,17 @@ require (
|
|||||||
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
|
||||||
github.com/fatih/color v1.7.0 // indirect
|
github.com/fatih/color v1.7.0 // indirect
|
||||||
github.com/gookit/color v1.4.2 // indirect
|
github.com/gookit/color v1.4.2 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
|
||||||
github.com/mattn/go-colorable v0.1.2 // indirect
|
github.com/mattn/go-colorable v0.1.2 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.8 // indirect
|
github.com/mattn/go-isatty v0.0.8 // indirect
|
||||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
|
||||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
|
||||||
github.com/multiformats/go-varint v0.0.6 // indirect
|
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||||
github.com/rivo/uniseg v0.2.0 // indirect
|
github.com/rivo/uniseg v0.2.0 // indirect
|
||||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
|
||||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
|
||||||
golang.org/x/sys v0.1.0 // indirect
|
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
|
||||||
golang.org/x/text v0.3.6 // indirect
|
golang.org/x/text v0.3.4 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
lukechampine.com/blake3 v1.1.6 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
26
go.sum
26
go.sum
@@ -66,8 +66,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
|
|||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
|
||||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||||
@@ -135,7 +133,6 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe
|
|||||||
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
|
|
||||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||||
@@ -152,9 +149,6 @@ github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgb
|
|||||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
|
||||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
|
||||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||||
@@ -174,14 +168,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
|
|||||||
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
|
||||||
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||||
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
|
|
||||||
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
|
|
||||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
|
||||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
|
||||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
|
||||||
github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM=
|
|
||||||
github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY=
|
|
||||||
github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE=
|
|
||||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
@@ -208,8 +194,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
|
|||||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
||||||
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
|
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
|
||||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
|
||||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
|
||||||
github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
|
github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
|
||||||
github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
|
github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
@@ -255,8 +239,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
|
|||||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||||
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
|
|
||||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
|
||||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||||
@@ -378,9 +360,8 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c h1:taxlMj0D/1sOAuv/CbSD+MMDof2vbyPTqz5FNYKpXt8=
|
||||||
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
|
||||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
@@ -391,9 +372,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
|
||||||
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
|
||||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
||||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||||
@@ -561,8 +541,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
|
|||||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||||
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||||
lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c=
|
|
||||||
lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
|
|
||||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||||
|
|||||||
@@ -1,14 +1,15 @@
|
|||||||
// Package bork defines the sentinel errors used by the manifest
|
|
||||||
// reader and writer.
|
|
||||||
package bork
|
package bork
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// ErrMissingMagic indicates the input lacks the manifest magic bytes.
|
|
||||||
ErrMissingMagic = errors.New("missing magic bytes in file")
|
ErrMissingMagic = errors.New("missing magic bytes in file")
|
||||||
// ErrFileTruncated indicates the input ended before the expected length.
|
|
||||||
ErrFileTruncated = errors.New("file/stream is truncated abnormally")
|
ErrFileTruncated = errors.New("file/stream is truncated abnormally")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func Newf(format string, args ...interface{}) error {
|
||||||
|
return fmt.Errorf(format, args...)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,14 +1,11 @@
|
|||||||
package bork_test
|
package bork
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
|
||||||
"sneak.berlin/go/mfer/internal/bork"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuild(t *testing.T) {
|
func TestBuild(t *testing.T) {
|
||||||
t.Parallel()
|
assert.NotNil(t, ErrMissingMagic)
|
||||||
assert.Error(t, bork.ErrMissingMagic)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,338 +1,13 @@
|
|||||||
// Package cli implements the mfer command-line interface.
|
|
||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"math"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/apex/log"
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// fingerprintHexLen is the length of a full GPG key fingerprint in hex
|
func (mfa *CLIApp) checkManifestOperation(c *cli.Context) error {
|
||||||
// characters.
|
log.WithError(errors.New("unimplemented"))
|
||||||
const fingerprintHexLen = 40
|
|
||||||
|
|
||||||
var (
|
|
||||||
// errNoManifestFound indicates no manifest file was found in the
|
|
||||||
// searched directory.
|
|
||||||
errNoManifestFound = errors.New("no manifest found")
|
|
||||||
// errInvalidFingerprint indicates a malformed --require-signature
|
|
||||||
// fingerprint argument. The length is spliced in from
|
|
||||||
// fingerprintHexLen so the two cannot drift apart.
|
|
||||||
errInvalidFingerprint = errors.New(
|
|
||||||
"invalid fingerprint: must be exactly " +
|
|
||||||
strconv.Itoa(fingerprintHexLen) + " hex characters")
|
|
||||||
// errManifestNotSigned indicates a signature was required but the
|
|
||||||
// manifest is unsigned. It is wrapped mid-sentence so that the
|
|
||||||
// rendered message stays exactly as mfer has always printed it.
|
|
||||||
errManifestNotSigned = errors.New("manifest is not signed")
|
|
||||||
// errSignerMismatch indicates the embedded signing key fingerprint
|
|
||||||
// does not match the required signer. Its text is the mid-sentence
|
|
||||||
// fragment of the rendered message, which users grep for in CI and
|
|
||||||
// which must therefore not change; match it with errors.Is rather
|
|
||||||
// than by reading it.
|
|
||||||
errSignerMismatch = errors.New("does not match required")
|
|
||||||
)
|
|
||||||
|
|
||||||
// safeUint64 converts a non-negative int64 to uint64, clamping negative
|
|
||||||
// values to zero.
|
|
||||||
func safeUint64(n int64) uint64 {
|
|
||||||
if n < 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return uint64(n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// safeRateUint64 converts a bytes-per-second rate to uint64 for display.
|
|
||||||
//
|
|
||||||
// A rate is computed as bytes/elapsed, so it is +Inf when the elapsed
|
|
||||||
// time rounds to zero and NaN when zero bytes were processed in zero
|
|
||||||
// time. Neither has a defined conversion to uint64, and on amd64 +Inf
|
|
||||||
// converts to a number that renders as "8.0 EiB/s"; both display as zero
|
|
||||||
// instead.
|
|
||||||
func safeRateUint64(rate float64) uint64 {
|
|
||||||
if math.IsNaN(rate) || math.IsInf(rate, 0) || rate <= 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if rate >= math.MaxUint64 {
|
|
||||||
return math.MaxUint64
|
|
||||||
}
|
|
||||||
|
|
||||||
return uint64(rate)
|
|
||||||
}
|
|
||||||
|
|
||||||
// findManifest looks for a manifest file in the given directory.
|
|
||||||
// It checks for index.mf and .index.mf, returning the first one found.
|
|
||||||
func findManifest(fs afero.Fs, dir string) (string, error) {
|
|
||||||
candidates := []string{"index.mf", ".index.mf"}
|
|
||||||
for _, name := range candidates {
|
|
||||||
path := filepath.Join(dir, name)
|
|
||||||
|
|
||||||
exists, err := afero.Exists(fs, path)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if exists {
|
|
||||||
return path, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"%w in %s (looked for index.mf and .index.mf)", errNoManifestFound, dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchManifestToTemp downloads a manifest URL to a temporary file and
|
|
||||||
// returns the temp file path. The caller is responsible for removing it.
|
|
||||||
func (mfa *CLIApp) fetchManifestToTemp(url string) (string, error) {
|
|
||||||
rc, fetchErr := mfa.openManifestReader(url)
|
|
||||||
if fetchErr != nil {
|
|
||||||
return "", fetchErr
|
|
||||||
}
|
|
||||||
|
|
||||||
tmpFile, tmpErr := afero.TempFile(mfa.Fs, "", "mfer-manifest-*.mf")
|
|
||||||
if tmpErr != nil {
|
|
||||||
_ = rc.Close()
|
|
||||||
|
|
||||||
return "", fmt.Errorf("failed to create temp file: %w", tmpErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
tmpPath := tmpFile.Name()
|
|
||||||
_, cpErr := io.Copy(tmpFile, rc)
|
|
||||||
_ = rc.Close()
|
|
||||||
_ = tmpFile.Close()
|
|
||||||
|
|
||||||
if cpErr != nil {
|
|
||||||
_ = mfa.Fs.Remove(tmpPath)
|
|
||||||
|
|
||||||
return "", fmt.Errorf("failed to download manifest: %w", cpErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
return tmpPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifyRequiredSigner enforces the --require-signature fingerprint
|
|
||||||
// against the manifest's embedded signing key.
|
|
||||||
func verifyRequiredSigner(chk *mfer.Checker, requiredSigner string) error {
|
|
||||||
// Validate fingerprint format: must be exactly 40 hex characters
|
|
||||||
if len(requiredSigner) != fingerprintHexLen {
|
|
||||||
return fmt.Errorf("%w, got %d", errInvalidFingerprint, len(requiredSigner))
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := hex.DecodeString(requiredSigner)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid fingerprint: must be valid hex: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !chk.IsSigned() {
|
|
||||||
return fmt.Errorf("%w, but signature from %s is required",
|
|
||||||
errManifestNotSigned, requiredSigner)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Extract fingerprint from the embedded public key (not from the
|
|
||||||
// signer field). This validates the key is importable and gets its
|
|
||||||
// actual fingerprint.
|
|
||||||
embeddedFP, err := chk.ExtractEmbeddedSigningKeyFP()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"failed to extract fingerprint from embedded signing key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compare fingerprints - must be exact match (case-insensitive)
|
|
||||||
if !strings.EqualFold(embeddedFP, requiredSigner) {
|
|
||||||
return fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
|
||||||
embeddedFP, errSignerMismatch, requiredSigner)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("manifest signature verified (signer: %s)", embeddedFP)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportCheckProgress renders progress updates until the channel closes.
|
|
||||||
func reportCheckProgress(progress <-chan mfer.CheckStatus) {
|
|
||||||
for status := range progress {
|
|
||||||
if status.ETA > 0 {
|
|
||||||
log.Progressf("Checking: %d/%d files, %s/s, ETA %s, %d failures",
|
|
||||||
status.CheckedFiles,
|
|
||||||
status.TotalFiles,
|
|
||||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)),
|
|
||||||
status.ETA.Round(time.Second),
|
|
||||||
status.Failures)
|
|
||||||
} else {
|
|
||||||
log.Progressf("Checking: %d/%d files, %s/s, %d failures",
|
|
||||||
status.CheckedFiles,
|
|
||||||
status.TotalFiles,
|
|
||||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)),
|
|
||||||
status.Failures)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.ProgressDone()
|
|
||||||
}
|
|
||||||
|
|
||||||
// countCheckFailures consumes check results, counting and logging
|
|
||||||
// failures, then closes done.
|
|
||||||
func countCheckFailures(
|
|
||||||
results <-chan mfer.Result, failures *int64, done chan<- struct{},
|
|
||||||
) {
|
|
||||||
for result := range results {
|
|
||||||
if result.Status != mfer.StatusOK {
|
|
||||||
*failures++
|
|
||||||
|
|
||||||
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
|
|
||||||
} else {
|
|
||||||
log.Verbosef("%s: %s", result.Status, result.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
close(done)
|
|
||||||
}
|
|
||||||
|
|
||||||
// findExtraFiles reports files present on disk but absent from the
|
|
||||||
// manifest, counting each as a failure.
|
|
||||||
func findExtraFiles(ctx *cli.Context, chk *mfer.Checker, failures *int64) error {
|
|
||||||
extraResults := make(chan mfer.Result, 1)
|
|
||||||
extraDone := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for result := range extraResults {
|
|
||||||
*failures++
|
|
||||||
|
|
||||||
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(extraDone)
|
|
||||||
}()
|
|
||||||
|
|
||||||
err := chk.FindExtraFiles(ctx.Context, extraResults)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to check for extra files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
<-extraDone
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// runCheck runs the manifest check with progress and result reporting
|
|
||||||
// and returns the number of failures.
|
|
||||||
func runCheck(ctx *cli.Context, chk *mfer.Checker, showProgress bool) (int64, error) {
|
|
||||||
// Set up results channel
|
|
||||||
results := make(chan mfer.Result, 1)
|
|
||||||
|
|
||||||
// Set up progress channel
|
|
||||||
var progress chan mfer.CheckStatus
|
|
||||||
if showProgress {
|
|
||||||
progress = make(chan mfer.CheckStatus, 1)
|
|
||||||
|
|
||||||
go reportCheckProgress(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process results in a goroutine
|
|
||||||
var failures int64
|
|
||||||
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go countCheckFailures(results, &failures, done)
|
|
||||||
|
|
||||||
// Run check
|
|
||||||
err := chk.Check(ctx.Context, results, progress)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("check failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait for results processing to complete
|
|
||||||
<-done
|
|
||||||
|
|
||||||
// Check for extra files if requested
|
|
||||||
if ctx.Bool("no-extra-files") {
|
|
||||||
err = findExtraFiles(ctx, chk, &failures)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return failures, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
|
||||||
log.Debug("checkManifestOperation()")
|
|
||||||
|
|
||||||
manifestPath, err := mfa.resolveManifestArg(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("check: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// URL manifests need to be downloaded to a temp file for the checker
|
|
||||||
if isHTTPURL(manifestPath) {
|
|
||||||
tmpPath, tmpErr := mfa.fetchManifestToTemp(manifestPath)
|
|
||||||
if tmpErr != nil {
|
|
||||||
return fmt.Errorf("check: %w", tmpErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = mfa.Fs.Remove(tmpPath) }()
|
|
||||||
|
|
||||||
manifestPath = tmpPath
|
|
||||||
}
|
|
||||||
|
|
||||||
basePath := ctx.String("base")
|
|
||||||
showProgress := ctx.Bool("progress")
|
|
||||||
|
|
||||||
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
|
||||||
|
|
||||||
// Create checker
|
|
||||||
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to load manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check signature requirement
|
|
||||||
requiredSigner := ctx.String("require-signature")
|
|
||||||
if requiredSigner != "" {
|
|
||||||
err = verifyRequiredSigner(chk, requiredSigner)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("manifest contains %d files, %s", chk.FileCount(),
|
|
||||||
humanize.IBytes(safeUint64(int64(chk.TotalBytes()))))
|
|
||||||
|
|
||||||
failures, err := runCheck(ctx, chk, showProgress)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
elapsed := time.Since(mfa.startupTime).Seconds()
|
|
||||||
|
|
||||||
rate := float64(chk.TotalBytes()) / elapsed
|
|
||||||
if failures == 0 {
|
|
||||||
log.Infof("checked %d files (%s) in %.1fs (%s/s): all OK",
|
|
||||||
chk.FileCount(), humanize.IBytes(safeUint64(int64(chk.TotalBytes()))),
|
|
||||||
elapsed, humanize.IBytes(safeRateUint64(rate)))
|
|
||||||
} else {
|
|
||||||
log.Infof("checked %d files (%s) in %.1fs (%s/s): %d failed",
|
|
||||||
chk.FileCount(), humanize.IBytes(safeUint64(int64(chk.TotalBytes()))),
|
|
||||||
elapsed, humanize.IBytes(safeRateUint64(rate)), failures)
|
|
||||||
}
|
|
||||||
|
|
||||||
if failures > 0 {
|
|
||||||
mfa.exitCode = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,72 +1,25 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// NoColor disables colored output when set. Automatically true if the
|
var NO_COLOR bool
|
||||||
// NO_COLOR environment variable is present (per https://no-color.org/).
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // process-wide setting derived from the environment
|
|
||||||
var NoColor = noColorEnvSet()
|
|
||||||
|
|
||||||
// noColorEnvSet reports whether the NO_COLOR environment variable is
|
func init() {
|
||||||
// present.
|
NO_COLOR = false
|
||||||
func noColorEnvSet() bool {
|
if _, exists := os.LookupEnv("NO_COLOR"); exists {
|
||||||
_, exists := os.LookupEnv("NO_COLOR")
|
NO_COLOR = true
|
||||||
|
|
||||||
return exists
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunOptions contains all configuration for running the CLI application.
|
|
||||||
// Use DefaultRunOptions for standard CLI execution, or construct manually for testing.
|
|
||||||
type RunOptions struct {
|
|
||||||
Appname string // Application name displayed in help and version output
|
|
||||||
Version string // Version string (typically set at build time)
|
|
||||||
Gitrev string // Git revision hash (typically set at build time)
|
|
||||||
Args []string // Command-line arguments (typically os.Args)
|
|
||||||
Stdin io.Reader // Standard input stream
|
|
||||||
Stdout io.Writer // Standard output stream
|
|
||||||
Stderr io.Writer // Standard error stream
|
|
||||||
Fs afero.Fs // Filesystem abstraction for file operations
|
|
||||||
}
|
|
||||||
|
|
||||||
// DefaultRunOptions returns RunOptions configured for normal CLI execution.
|
|
||||||
func DefaultRunOptions(appname, version, gitrev string) *RunOptions {
|
|
||||||
return &RunOptions{
|
|
||||||
Appname: appname,
|
|
||||||
Version: version,
|
|
||||||
Gitrev: gitrev,
|
|
||||||
Args: os.Args,
|
|
||||||
Stdin: os.Stdin,
|
|
||||||
Stdout: os.Stdout,
|
|
||||||
Stderr: os.Stderr,
|
|
||||||
Fs: afero.NewOsFs(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run creates and runs the CLI application with default options.
|
func Run(Appname, Version, Gitrev string) int {
|
||||||
func Run(appname, version, gitrev string) int {
|
m := &CLIApp{}
|
||||||
return RunWithOptions(DefaultRunOptions(appname, version, gitrev))
|
m.appname = Appname
|
||||||
}
|
m.version = Version
|
||||||
|
m.gitrev = Gitrev
|
||||||
// RunWithOptions creates and runs the CLI application with the given options.
|
m.exitCode = 0
|
||||||
func RunWithOptions(opts *RunOptions) int {
|
|
||||||
m := &CLIApp{
|
|
||||||
appname: opts.Appname,
|
|
||||||
version: opts.Version,
|
|
||||||
gitrev: opts.Gitrev,
|
|
||||||
exitCode: 0,
|
|
||||||
Stdin: opts.Stdin,
|
|
||||||
Stdout: opts.Stdout,
|
|
||||||
Stderr: opts.Stderr,
|
|
||||||
Fs: opts.Fs,
|
|
||||||
}
|
|
||||||
|
|
||||||
m.run(opts.Args)
|
|
||||||
|
|
||||||
|
m.run()
|
||||||
return m.exitCode
|
return m.exitCode
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,764 +1,12 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math/rand"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
urfcli "github.com/urfave/cli/v2"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
testApp = "mfer"
|
|
||||||
testDir = "/testdir"
|
|
||||||
testFile1 = "/testdir/file1.txt"
|
|
||||||
testMF = "/testdir/test.mf"
|
|
||||||
testOutput = "/output.mf"
|
|
||||||
testOutputTmp = "/output.mf.tmp"
|
|
||||||
testManifest = "/manifest.mf"
|
|
||||||
testFlagBase = "--base"
|
|
||||||
testFlagNoExtra = "--no-extra-files"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errSimulatedWrite = errors.New("simulated write failure")
|
|
||||||
|
|
||||||
// runMu serializes CLI runs: RunWithOptions wires the process-global
|
|
||||||
// logger to the run's I/O streams, so parallel runs would cross-wire
|
|
||||||
// captured output between tests.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // guards process-global logger state in tests
|
|
||||||
var runMu sync.Mutex
|
|
||||||
|
|
||||||
// runCLI invokes RunWithOptions while holding runMu so parallel tests
|
|
||||||
// capture their own output.
|
|
||||||
func runCLI(opts *RunOptions) int {
|
|
||||||
runMu.Lock()
|
|
||||||
defer runMu.Unlock()
|
|
||||||
|
|
||||||
return RunWithOptions(opts)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
// Prevent urfave/cli from calling os.Exit during tests
|
|
||||||
urfcli.OsExiter = func(_ int) {}
|
|
||||||
|
|
||||||
os.Exit(m.Run())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuild(t *testing.T) {
|
func TestBuild(t *testing.T) {
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := &CLIApp{}
|
m := &CLIApp{}
|
||||||
assert.NotNil(t, m)
|
assert.NotNil(t, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testOpts(args []string, fs afero.Fs) *RunOptions {
|
|
||||||
return &RunOptions{
|
|
||||||
Appname: testApp,
|
|
||||||
Version: "1.0.0",
|
|
||||||
Gitrev: "abc123",
|
|
||||||
Args: args,
|
|
||||||
Stdin: &bytes.Buffer{},
|
|
||||||
Stdout: &bytes.Buffer{},
|
|
||||||
Stderr: &bytes.Buffer{},
|
|
||||||
Fs: fs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testStdout(t *testing.T, opts *RunOptions) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
buf, ok := opts.Stdout.(*bytes.Buffer)
|
|
||||||
require.True(t, ok)
|
|
||||||
|
|
||||||
return buf.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func testStderr(t *testing.T, opts *RunOptions) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
buf, ok := opts.Stderr.(*bytes.Buffer)
|
|
||||||
require.True(t, ok)
|
|
||||||
|
|
||||||
return buf.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeTestFile(t *testing.T, fs afero.Fs, path, content string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, afero.WriteFile(fs, path, []byte(content), 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVersionCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
opts := testOpts([]string{testApp, "version"}, fs)
|
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
stdout := testStdout(t, opts)
|
|
||||||
assert.Contains(t, stdout, mfer.Version)
|
|
||||||
assert.Contains(t, stdout, "abc123")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHelpCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
opts := testOpts([]string{testApp, "--help"}, fs)
|
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
stdout := testStdout(t, opts)
|
|
||||||
assert.Contains(t, stdout, cmdGenerate)
|
|
||||||
assert.Contains(t, stdout, cmdCheck)
|
|
||||||
assert.Contains(t, stdout, "fetch")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files in memory filesystem
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello world")
|
|
||||||
writeTestFile(t, fs, "/testdir/file2.txt", "test content")
|
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// Verify manifest was created
|
|
||||||
exists, err := afero.Exists(fs, testMF)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, exists)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAndCheckCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files with subdirectory
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir/subdir", 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello world")
|
|
||||||
writeTestFile(t, fs, "/testdir/subdir/file2.txt", "test content")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode, "generate failed: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// Check manifest
|
|
||||||
opts = testOpts([]string{testApp, cmdCheck, "-q", testFlagBase, testDir, testMF}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode, "check failed: %s", testStderr(t, opts))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckCommandWithMissingFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello world")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode, "generate failed: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// Delete the file
|
|
||||||
require.NoError(t, fs.Remove(testFile1))
|
|
||||||
|
|
||||||
// Check manifest - should fail
|
|
||||||
opts = testOpts([]string{testApp, cmdCheck, "-q", testFlagBase, testDir, testMF}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, "check should have failed for missing file")
|
|
||||||
}
|
|
||||||
|
|
||||||
func runCheckAfterRewrite(t *testing.T, rewritten, msg string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello world")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode, "generate failed: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
// Rewrite the file, then check the manifest - it must fail
|
|
||||||
writeTestFile(t, fs, testFile1, rewritten)
|
|
||||||
|
|
||||||
opts = testOpts([]string{testApp, cmdCheck, "-q", testFlagBase, testDir, testMF}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckCommandWithCorruptedFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Corrupt the file (change content but keep same size)
|
|
||||||
runCheckAfterRewrite(t, "HELLO WORLD",
|
|
||||||
"check should have failed for corrupted file")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckCommandWithSizeMismatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Change file size
|
|
||||||
runCheckAfterRewrite(t, "different size content here",
|
|
||||||
"check should have failed for size mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBannerOutput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Run without -q to see banner
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-o", testMF, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Banner ASCII art should be in stdout
|
|
||||||
stdout := testStdout(t, opts)
|
|
||||||
assert.Contains(t, stdout, "___")
|
|
||||||
assert.Contains(t, stdout, "\\")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnknownCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
opts := testOpts([]string{testApp, "unknown"}, fs)
|
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files including dotfiles
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
writeTestFile(t, fs, "/testdir/.hidden", "secret")
|
|
||||||
|
|
||||||
// Generate manifest without --include-dotfiles (default excludes dotfiles)
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testMF, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Check that manifest exists
|
|
||||||
exists, _ := afero.Exists(fs, testMF)
|
|
||||||
assert.True(t, exists)
|
|
||||||
|
|
||||||
// Verify manifest only has 1 file (the non-dotfile)
|
|
||||||
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 1)
|
|
||||||
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files including dotfiles
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
writeTestFile(t, fs, "/testdir/.hidden", "secret")
|
|
||||||
|
|
||||||
// Generate manifest with --include-dotfiles
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "--include-dotfiles", "-o", testMF, testDir,
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Verify manifest has 2 files (including dotfile)
|
|
||||||
manifest, err := mfer.NewManifestFromFile(fs, testMF)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipleInputPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files in multiple directories
|
|
||||||
require.NoError(t, fs.MkdirAll("/dir1", 0o755))
|
|
||||||
require.NoError(t, fs.MkdirAll("/dir2", 0o755))
|
|
||||||
writeTestFile(t, fs, "/dir1/file1.txt", "content1")
|
|
||||||
writeTestFile(t, fs, "/dir2/file2.txt", "content2")
|
|
||||||
|
|
||||||
// Generate manifest from multiple paths
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-o", testOutput, "/dir1", "/dir2",
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
|
||||||
|
|
||||||
exists, _ := afero.Exists(fs, testOutput)
|
|
||||||
assert.True(t, exists)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoExtraFilesPass(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
writeTestFile(t, fs, "/testdir/file2.txt", "world")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Check with --no-extra-files (should pass - no extra files)
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagNoExtra, testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoExtraFilesFail(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Add an extra file after manifest generation
|
|
||||||
writeTestFile(t, fs, "/testdir/extra.txt", "extra")
|
|
||||||
|
|
||||||
// Check with --no-extra-files (should fail - extra file exists)
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagNoExtra, testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, "check should fail when extra files exist")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNoExtraFilesWithSubdirectory(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files with subdirectory
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir/subdir", 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
writeTestFile(t, fs, "/testdir/subdir/file2.txt", "world")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Add extra file in subdirectory
|
|
||||||
writeTestFile(t, fs, "/testdir/subdir/extra.txt", "extra")
|
|
||||||
|
|
||||||
// Check with --no-extra-files (should fail)
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagNoExtra, testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode,
|
|
||||||
"check should fail when extra files exist in subdirectory")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckWithoutNoExtraFilesIgnoresExtra(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Add extra file
|
|
||||||
writeTestFile(t, fs, "/testdir/extra.txt", "extra")
|
|
||||||
|
|
||||||
// Check WITHOUT --no-extra-files (should pass - extra files ignored)
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode,
|
|
||||||
"check without --no-extra-files should ignore extra files")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteNoTempFileOnSuccess(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testOutput, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Verify output file exists
|
|
||||||
exists, err := afero.Exists(fs, testOutput)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, exists, "output file should exist")
|
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
|
||||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, tmpExists,
|
|
||||||
"temp file should not exist after successful generation")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteOverwriteWithForce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Create existing manifest with different content
|
|
||||||
writeTestFile(t, fs, testOutput, "old content")
|
|
||||||
|
|
||||||
// Generate manifest with --force
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-f", "-o", testOutput, testDir,
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Verify output file exists and was overwritten
|
|
||||||
content, err := afero.ReadFile(fs, testOutput)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEqual(t, "old content", string(content),
|
|
||||||
"manifest should be overwritten")
|
|
||||||
|
|
||||||
// Verify temp file does NOT exist
|
|
||||||
tmpExists, err := afero.Exists(fs, testOutputTmp)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, tmpExists,
|
|
||||||
"temp file should not exist after successful generation")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Create existing manifest
|
|
||||||
writeTestFile(t, fs, testOutput, "existing")
|
|
||||||
|
|
||||||
// Generate manifest WITHOUT --force (should fail)
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testOutput, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, "should fail when output exists without --force")
|
|
||||||
|
|
||||||
// Verify original content is preserved
|
|
||||||
content, err := afero.ReadFile(fs, testOutput)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "existing", string(content), "original file should be preserved")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteUsesTemp(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// This test verifies that generate uses a temp file by checking
|
|
||||||
// that the output file doesn't exist until generation completes.
|
|
||||||
// We do this by generating to a path and verifying the temp file
|
|
||||||
// pattern is used (output.mf.tmp -> output.mf)
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test file
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "hello")
|
|
||||||
|
|
||||||
// Generate manifest
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testOutput, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode)
|
|
||||||
|
|
||||||
// Both output file should exist and temp should not
|
|
||||||
exists, _ := afero.Exists(fs, testOutput)
|
|
||||||
assert.True(t, exists, "output file should exist")
|
|
||||||
|
|
||||||
tmpExists, _ := afero.Exists(fs, testOutputTmp)
|
|
||||||
assert.False(t, tmpExists, "temp file should be cleaned up")
|
|
||||||
|
|
||||||
// Verify manifest is valid (not empty)
|
|
||||||
content, err := afero.ReadFile(fs, testOutput)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEmpty(t, content, "manifest should not be empty")
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingWriterFs wraps a filesystem and makes writes fail after N bytes
|
|
||||||
type failingWriterFs struct {
|
|
||||||
afero.Fs
|
|
||||||
|
|
||||||
failAfter int64
|
|
||||||
written int64
|
|
||||||
}
|
|
||||||
|
|
||||||
type failingFile struct {
|
|
||||||
afero.File
|
|
||||||
|
|
||||||
fs *failingWriterFs
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *failingFile) Write(p []byte) (int, error) {
|
|
||||||
f.fs.written += int64(len(p))
|
|
||||||
if f.fs.written > f.fs.failAfter {
|
|
||||||
return 0, errSimulatedWrite
|
|
||||||
}
|
|
||||||
|
|
||||||
return f.File.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:ireturn // Create must return afero.File to satisfy afero.Fs.
|
|
||||||
func (fs *failingWriterFs) Create(name string) (afero.File, error) {
|
|
||||||
f, err := fs.Fs.Create(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &failingFile{File: f, fs: fs}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateAtomicWriteCleansUpOnError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
baseFs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
// Create test files - need enough content to trigger the write failure
|
|
||||||
require.NoError(t, baseFs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, baseFs, testFile1, "hello world this is a test file")
|
|
||||||
|
|
||||||
// Wrap with failing writer that fails after writing some bytes
|
|
||||||
fs := &failingWriterFs{Fs: baseFs, failAfter: 10}
|
|
||||||
|
|
||||||
// Generate manifest - should fail during write
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testOutput, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, "should fail due to write error")
|
|
||||||
|
|
||||||
// With atomic writes: output.mf should NOT exist (temp was cleaned up)
|
|
||||||
// With non-atomic writes: output.mf WOULD exist (partial/empty)
|
|
||||||
exists, _ := afero.Exists(baseFs, testOutput)
|
|
||||||
assert.False(t, exists,
|
|
||||||
"output file should not exist after failed generation (atomic write)")
|
|
||||||
|
|
||||||
// Temp file should also not exist
|
|
||||||
tmpExists, _ := afero.Exists(baseFs, testOutputTmp)
|
|
||||||
assert.False(t, tmpExists,
|
|
||||||
"temp file should be cleaned up after failed generation")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateValidatesInputPaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
seedValidDir := func(t *testing.T, fs afero.Fs) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, fs.MkdirAll("/validdir", 0o755))
|
|
||||||
writeTestFile(t, fs, "/validdir/file.txt", "content")
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("nonexistent path fails fast", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
seedValidDir(t, fs)
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-o", testOutput, "/nonexistent",
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode)
|
|
||||||
|
|
||||||
stderr := testStderr(t, opts)
|
|
||||||
assert.Contains(t, stderr, "path does not exist")
|
|
||||||
assert.Contains(t, stderr, "/nonexistent")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("mix of valid and invalid paths fails fast", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
seedValidDir(t, fs)
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-o", testOutput,
|
|
||||||
"/validdir", "/alsononexistent",
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode)
|
|
||||||
|
|
||||||
stderr := testStderr(t, opts)
|
|
||||||
assert.Contains(t, stderr, "path does not exist")
|
|
||||||
assert.Contains(t, stderr, "/alsononexistent")
|
|
||||||
|
|
||||||
// Output file should not have been created
|
|
||||||
exists, _ := afero.Exists(fs, testOutput)
|
|
||||||
assert.False(t, exists,
|
|
||||||
"output file should not exist when path validation fails")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid paths succeed", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
seedValidDir(t, fs)
|
|
||||||
|
|
||||||
opts := testOpts([]string{
|
|
||||||
testApp, cmdGenerate, "-q", "-o", testOutput, "/validdir",
|
|
||||||
}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
assert.Equal(t, 0, exitCode)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckDetectsManifestCorruption(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
rng := rand.New(rand.NewSource(42)) //nolint:gosec // deterministic test data
|
|
||||||
|
|
||||||
// Create many small files with random names to generate a ~1MB manifest
|
|
||||||
// Each manifest entry is roughly 50-60 bytes, so we need ~20000 files
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
|
|
||||||
numFiles := 20000
|
|
||||||
for range numFiles {
|
|
||||||
// Generate random filename
|
|
||||||
filename := fmt.Sprintf("/testdir/%08x%08x%08x.dat",
|
|
||||||
rng.Uint32(), rng.Uint32(), rng.Uint32())
|
|
||||||
// Small random content
|
|
||||||
content := make([]byte, 16+rng.Intn(48))
|
|
||||||
_, _ = rng.Read(content)
|
|
||||||
require.NoError(t, afero.WriteFile(fs, filename, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate manifest outside of testdir
|
|
||||||
opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", testManifest, testDir}, fs)
|
|
||||||
exitCode := runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode, "generate should succeed")
|
|
||||||
|
|
||||||
// Read the valid manifest and verify it's approximately 1MB
|
|
||||||
validManifest, err := afero.ReadFile(fs, testManifest)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.GreaterOrEqual(t, len(validManifest), 1024*1024,
|
|
||||||
"manifest should be at least 1MB, got %d bytes", len(validManifest))
|
|
||||||
t.Logf("manifest size: %d bytes (%d files)", len(validManifest), numFiles)
|
|
||||||
|
|
||||||
// First corruption: truncate the manifest
|
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest,
|
|
||||||
validManifest[:len(validManifest)/2], 0o644))
|
|
||||||
|
|
||||||
// Check should fail with truncated manifest
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode, "check should fail with truncated manifest")
|
|
||||||
|
|
||||||
// Verify check passes with valid manifest
|
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest, validManifest, 0o644))
|
|
||||||
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
require.Equal(t, 0, exitCode, "check should pass with valid manifest")
|
|
||||||
|
|
||||||
// Now do 500 random corruption iterations
|
|
||||||
for i := range 500 {
|
|
||||||
// Corrupt: write a random byte at a random offset
|
|
||||||
corrupted := make([]byte, len(validManifest))
|
|
||||||
copy(corrupted, validManifest)
|
|
||||||
|
|
||||||
offset := rng.Intn(len(corrupted))
|
|
||||||
originalByte := corrupted[offset]
|
|
||||||
|
|
||||||
// Make sure we actually change the byte
|
|
||||||
buf := make([]byte, 1)
|
|
||||||
|
|
||||||
newByte := originalByte
|
|
||||||
for newByte == originalByte {
|
|
||||||
_, _ = rng.Read(buf)
|
|
||||||
newByte = buf[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
corrupted[offset] = newByte
|
|
||||||
|
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest, corrupted, 0o644))
|
|
||||||
|
|
||||||
// Check should fail with corrupted manifest
|
|
||||||
opts = testOpts([]string{
|
|
||||||
testApp, cmdCheck, "-q", testFlagBase, testDir, testManifest,
|
|
||||||
}, fs)
|
|
||||||
exitCode = runCLI(opts)
|
|
||||||
assert.Equal(t, 1, exitCode,
|
|
||||||
"iteration %d: check should fail with corrupted manifest "+
|
|
||||||
"(offset %d, 0x%02x -> 0x%02x)",
|
|
||||||
i, offset, originalByte, newByte)
|
|
||||||
|
|
||||||
// Restore valid manifest for next iteration
|
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest, validManifest, 0o644))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,168 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errMsgCase is one pinned user-visible error message.
|
|
||||||
type errMsgCase struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
want string
|
|
||||||
}
|
|
||||||
|
|
||||||
const (
|
|
||||||
msgFpA = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
|
||||||
msgFpB = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
|
|
||||||
)
|
|
||||||
|
|
||||||
func checkErrMsgCases(t *testing.T, cases []errMsgCase) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
assert.Equal(t, tc.want, tc.err.Error())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestErrorMessagesVerbatim pins the exact rendered text of the CLI's
|
|
||||||
// user-visible error messages.
|
|
||||||
//
|
|
||||||
// These strings are an interface: they are grepped for in CI pipelines
|
|
||||||
// and quoted in bug reports. The messages are assembled by wrapping
|
|
||||||
// static sentinels, and it is easy to change what a user sees while
|
|
||||||
// only meaning to make an error matchable with errors.Is - which is
|
|
||||||
// precisely what happened once already. Any change to a string below is
|
|
||||||
// therefore a deliberate, separately stated change, never a side effect
|
|
||||||
// of a refactor.
|
|
||||||
func TestErrorMessagesVerbatim(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkErrMsgCases(t, []errMsgCase{
|
|
||||||
{
|
|
||||||
name: "check: no manifest found",
|
|
||||||
err: fmt.Errorf("%w in %s (looked for index.mf and .index.mf)",
|
|
||||||
errNoManifestFound, "/tmp/x"),
|
|
||||||
want: "no manifest found in /tmp/x " +
|
|
||||||
"(looked for index.mf and .index.mf)",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "check: invalid fingerprint length",
|
|
||||||
err: fmt.Errorf("%w, got %d", errInvalidFingerprint, 8),
|
|
||||||
want: "invalid fingerprint: must be exactly 40 hex characters, got 8",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "check: manifest not signed",
|
|
||||||
err: fmt.Errorf("%w, but signature from %s is required",
|
|
||||||
errManifestNotSigned, msgFpA),
|
|
||||||
want: "manifest is not signed, but signature from " + msgFpA +
|
|
||||||
" is required",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "check: signer mismatch",
|
|
||||||
err: fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
|
||||||
msgFpA, errSignerMismatch, msgFpB),
|
|
||||||
want: "embedded signing key fingerprint " + msgFpA +
|
|
||||||
" does not match required " + msgFpB,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "gen: path does not exist",
|
|
||||||
err: fmt.Errorf("%w: %s", errPathNotExist, "nope"),
|
|
||||||
want: "path does not exist: nope",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "gen: output file exists",
|
|
||||||
err: fmt.Errorf("output file %s %w", "index.mf", errOutputExists),
|
|
||||||
want: "output file index.mf already exists " +
|
|
||||||
"(use --force to overwrite)",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "mfer: unknown command",
|
|
||||||
err: fmt.Errorf("%w %q", errUnknownCommand, "bogus"),
|
|
||||||
want: `unknown command "bogus"`,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetchErrorMessagesVerbatim pins the fetch and manifest-loader
|
|
||||||
// messages; see TestErrorMessagesVerbatim for why.
|
|
||||||
func TestFetchErrorMessagesVerbatim(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkErrMsgCases(t, []errMsgCase{
|
|
||||||
{
|
|
||||||
name: "manifest_loader: http status",
|
|
||||||
err: fmt.Errorf("failed to fetch %s: %w %d",
|
|
||||||
"https://example.com/index.mf", errHTTPStatus, 404),
|
|
||||||
want: "failed to fetch https://example.com/index.mf: HTTP 404",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: manifest http status",
|
|
||||||
err: fmt.Errorf("failed to fetch manifest: %w %d",
|
|
||||||
errHTTPStatus, 404),
|
|
||||||
want: "failed to fetch manifest: HTTP 404",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: file http status",
|
|
||||||
err: fmt.Errorf("%w %d", errHTTPStatus, 500),
|
|
||||||
want: "HTTP 500",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: empty path",
|
|
||||||
err: errEmptyPath,
|
|
||||||
want: "empty path",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: absolute path",
|
|
||||||
err: fmt.Errorf("%w: %s", errAbsolutePath, "/etc/passwd"),
|
|
||||||
want: "absolute path not allowed: /etc/passwd",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: path traversal",
|
|
||||||
err: fmt.Errorf("%w: %s", errPathTraversal, "../x"),
|
|
||||||
want: "path traversal not allowed: ../x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: size mismatch",
|
|
||||||
err: fmt.Errorf("%w: expected %d bytes, got %d",
|
|
||||||
errSizeMismatch, 10, 9),
|
|
||||||
want: "size mismatch: expected 10 bytes, got 9",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: url required",
|
|
||||||
err: errURLRequired,
|
|
||||||
want: "URL argument required",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "fetch: hash mismatch",
|
|
||||||
err: errHashMismatch,
|
|
||||||
want: "hash mismatch",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSentinelsAreMatchable checks that the wrapped forms of the
|
|
||||||
// messages above remain matchable with errors.Is, which is the reason
|
|
||||||
// the sentinels exist at all.
|
|
||||||
func TestSentinelsAreMatchable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
wrapped := fmt.Errorf("embedded signing key fingerprint %s %w %s",
|
|
||||||
"a", errSignerMismatch, "b")
|
|
||||||
require.ErrorIs(t, wrapped, errSignerMismatch)
|
|
||||||
|
|
||||||
wrapped = fmt.Errorf("output file %s %w", "index.mf", errOutputExists)
|
|
||||||
require.ErrorIs(t, wrapped, errOutputExists)
|
|
||||||
|
|
||||||
wrapped = fmt.Errorf("failed to fetch manifest: %w %d", errHTTPStatus, 404)
|
|
||||||
require.ErrorIs(t, wrapped, errHTTPStatus)
|
|
||||||
|
|
||||||
assert.NotErrorIs(t, errHashMismatch, errSizeMismatch)
|
|
||||||
}
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ExportEntry represents a single file entry in the exported JSON output.
|
|
||||||
type ExportEntry struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
Size int64 `json:"size"`
|
|
||||||
Hashes []string `json:"hashes"`
|
|
||||||
Mtime *string `json:"mtime,omitempty"`
|
|
||||||
Ctime *string `json:"ctime,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) exportManifestOperation(ctx *cli.Context) error {
|
|
||||||
pathOrURL, err := mfa.resolveManifestArg(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rc, err := mfa.openManifestReader(pathOrURL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = rc.Close() }()
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(rc)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("export: failed to parse manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
entries := make([]ExportEntry, 0, len(files))
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
entry := ExportEntry{
|
|
||||||
Path: f.GetPath(),
|
|
||||||
Size: f.GetSize(),
|
|
||||||
Hashes: make([]string, 0, len(f.GetHashes())),
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, h := range f.GetHashes() {
|
|
||||||
entry.Hashes = append(entry.Hashes, hex.EncodeToString(h.GetMultiHash()))
|
|
||||||
}
|
|
||||||
|
|
||||||
if mtime, ok := entryMtime(f); ok {
|
|
||||||
t := mtime.UTC().Format(time.RFC3339Nano)
|
|
||||||
entry.Mtime = &t
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.GetCtime() != nil {
|
|
||||||
t := time.Unix(f.GetCtime().GetSeconds(), int64(f.GetCtime().GetNanos())).
|
|
||||||
UTC().Format(time.RFC3339Nano)
|
|
||||||
entry.Ctime = &t
|
|
||||||
}
|
|
||||||
|
|
||||||
entries = append(entries, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
enc := json.NewEncoder(mfa.Stdout)
|
|
||||||
enc.SetIndent("", " ")
|
|
||||||
|
|
||||||
err = enc.Encode(entries)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("export: failed to encode JSON: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
const testCmdExport = "export"
|
|
||||||
|
|
||||||
// buildTestManifest creates a manifest from in-memory files and returns its bytes.
|
|
||||||
func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
for path, content := range files {
|
|
||||||
require.NoError(t, sourceFs.MkdirAll("/", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+path, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
opts := &mfer.ScannerOptions{Fs: sourceFs}
|
|
||||||
s := mfer.NewScannerWithOptions(opts)
|
|
||||||
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
|
||||||
|
|
||||||
return buf.Bytes()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExportManifestOperation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
testFiles := map[string][]byte{
|
|
||||||
"hello.txt": []byte("Hello, World!"),
|
|
||||||
"sub/file.txt": []byte("nested content"),
|
|
||||||
}
|
|
||||||
manifestData := buildTestManifest(t, testFiles)
|
|
||||||
|
|
||||||
// Write manifest to memfs
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/test.mf", manifestData, 0o644))
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
exitCode := runCLI(&RunOptions{
|
|
||||||
Appname: testApp,
|
|
||||||
Args: []string{testApp, testCmdExport, "/test.mf"},
|
|
||||||
Stdin: &bytes.Buffer{},
|
|
||||||
Stdout: &stdout,
|
|
||||||
Stderr: &stderr,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
|
|
||||||
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
|
|
||||||
|
|
||||||
var entries []ExportEntry
|
|
||||||
require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
|
|
||||||
assert.Len(t, entries, 2)
|
|
||||||
|
|
||||||
// Verify entries have expected fields
|
|
||||||
pathSet := make(map[string]bool)
|
|
||||||
for _, e := range entries {
|
|
||||||
pathSet[e.Path] = true
|
|
||||||
assert.NotEmpty(t, e.Hashes, "entry %s should have hashes", e.Path)
|
|
||||||
assert.Positive(t, e.Size, "entry %s should have positive size", e.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.True(t, pathSet["hello.txt"])
|
|
||||||
assert.True(t, pathSet["sub/file.txt"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExportFromHTTPURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
testFiles := map[string][]byte{
|
|
||||||
"a.txt": []byte("aaa"),
|
|
||||||
}
|
|
||||||
manifestData := buildTestManifest(t, testFiles)
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write(manifestData)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
exitCode := runCLI(&RunOptions{
|
|
||||||
Appname: testApp,
|
|
||||||
Args: []string{testApp, testCmdExport, server.URL + "/index.mf"},
|
|
||||||
Stdin: &bytes.Buffer{},
|
|
||||||
Stdout: &stdout,
|
|
||||||
Stderr: &stderr,
|
|
||||||
Fs: afero.NewMemMapFs(),
|
|
||||||
})
|
|
||||||
|
|
||||||
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
|
|
||||||
|
|
||||||
var entries []ExportEntry
|
|
||||||
require.NoError(t, json.Unmarshal(stdout.Bytes(), &entries))
|
|
||||||
assert.Len(t, entries, 1)
|
|
||||||
assert.Equal(t, "a.txt", entries[0].Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestListFromHTTPURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
testFiles := map[string][]byte{
|
|
||||||
"one.txt": []byte("1"),
|
|
||||||
"two.txt": []byte("22"),
|
|
||||||
}
|
|
||||||
manifestData := buildTestManifest(t, testFiles)
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = w.Write(manifestData)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
exitCode := runCLI(&RunOptions{
|
|
||||||
Appname: testApp,
|
|
||||||
Args: []string{testApp, "list", server.URL + "/index.mf"},
|
|
||||||
Stdin: &bytes.Buffer{},
|
|
||||||
Stdout: &stdout,
|
|
||||||
Stderr: &stderr,
|
|
||||||
Fs: afero.NewMemMapFs(),
|
|
||||||
})
|
|
||||||
|
|
||||||
require.Equal(t, 0, exitCode, "stderr: %s", stderr.String())
|
|
||||||
output := stdout.String()
|
|
||||||
assert.Contains(t, output, "one.txt")
|
|
||||||
assert.Contains(t, output, "two.txt")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsHTTPURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.True(t, isHTTPURL("http://example.com/manifest.mf"))
|
|
||||||
assert.True(t, isHTTPURL("https://example.com/manifest.mf"))
|
|
||||||
assert.False(t, isHTTPURL("/local/path.mf"))
|
|
||||||
assert.False(t, isHTTPURL("relative/path.mf"))
|
|
||||||
assert.False(t, isHTTPURL("ftp://example.com/file"))
|
|
||||||
}
|
|
||||||
@@ -1,529 +1,12 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"github.com/apex/log"
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
func (mfa *CLIApp) fetchManifestOperation(c *cli.Context) error {
|
||||||
// progressChanBuffer is the buffer size of the download progress
|
log.Debugf("fetchManifestOperation()")
|
||||||
// channel.
|
panic("not implemented")
|
||||||
progressChanBuffer = 10
|
return nil //nolint
|
||||||
|
|
||||||
// bitsPerByte converts a bytes-per-second rate to bits per second.
|
|
||||||
bitsPerByte = 8
|
|
||||||
|
|
||||||
// dirPerms is the permission mode for directories created for
|
|
||||||
// downloaded files. Fetched trees are content that is normally
|
|
||||||
// published (served by a web server, read by another uid), so the
|
|
||||||
// traversal bit for group and other must stay set.
|
|
||||||
dirPerms os.FileMode = 0o755
|
|
||||||
|
|
||||||
// Bitrate unit thresholds in bits per second.
|
|
||||||
bpsPerGbps = 1e9
|
|
||||||
bpsPerMbps = 1e6
|
|
||||||
bpsPerKbps = 1e3
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
// errURLRequired indicates the fetch command was run without a URL
|
|
||||||
// argument.
|
|
||||||
errURLRequired = errors.New("URL argument required")
|
|
||||||
// errEmptyPath indicates an empty file path in the manifest.
|
|
||||||
errEmptyPath = errors.New("empty path")
|
|
||||||
// errAbsolutePath indicates an absolute file path in the manifest.
|
|
||||||
errAbsolutePath = errors.New("absolute path not allowed")
|
|
||||||
// errPathTraversal indicates a manifest path escaping the target
|
|
||||||
// directory.
|
|
||||||
errPathTraversal = errors.New("path traversal not allowed")
|
|
||||||
// errSizeMismatch indicates a downloaded file with an unexpected
|
|
||||||
// size.
|
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
|
||||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
|
||||||
// manifest hash.
|
|
||||||
errHashMismatch = errors.New("hash mismatch")
|
|
||||||
)
|
|
||||||
|
|
||||||
// DownloadProgress reports the progress of a single file download.
|
|
||||||
type DownloadProgress struct {
|
|
||||||
Path string // File path being downloaded
|
|
||||||
BytesRead int64 // Bytes downloaded so far
|
|
||||||
TotalBytes int64 // Total expected bytes (-1 if unknown)
|
|
||||||
BytesPerSec float64 // Current download rate
|
|
||||||
ETA time.Duration // Estimated time to completion
|
|
||||||
}
|
|
||||||
|
|
||||||
// httpGet issues a GET request for the given URL using the provided
|
|
||||||
// context and returns the response. The caller must close the body.
|
|
||||||
//
|
|
||||||
// Errors are returned unwrapped: this helper replaced direct http.Get
|
|
||||||
// calls, and each caller already supplies its own context string, so
|
|
||||||
// adding one here would change user-visible messages.
|
|
||||||
func httpGet(ctx context.Context, fileURL string) (*http.Response, error) {
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportDownloadProgress renders download progress until the channel
|
|
||||||
// closes, then closes done.
|
|
||||||
func reportDownloadProgress(progress <-chan DownloadProgress, done chan<- struct{}) {
|
|
||||||
defer close(done)
|
|
||||||
|
|
||||||
for p := range progress {
|
|
||||||
rate := formatBitrate(p.BytesPerSec * bitsPerByte)
|
|
||||||
if p.ETA > 0 {
|
|
||||||
log.Infof("%s: %s/%s, %s, ETA %s",
|
|
||||||
p.Path, humanize.IBytes(safeUint64(p.BytesRead)),
|
|
||||||
humanize.IBytes(safeUint64(p.TotalBytes)),
|
|
||||||
rate, p.ETA.Round(time.Second))
|
|
||||||
} else {
|
|
||||||
log.Infof("%s: %s/%s, %s",
|
|
||||||
p.Path, humanize.IBytes(safeUint64(p.BytesRead)),
|
|
||||||
humanize.IBytes(safeUint64(p.TotalBytes)), rate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestBaseURL returns the URL of the directory containing the
|
|
||||||
// manifest, with a trailing slash.
|
|
||||||
func manifestBaseURL(manifestURL string) (*url.URL, error) {
|
|
||||||
baseURL, err := url.Parse(manifestURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("fetch: invalid manifest URL: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
baseURL.Path = path.Dir(baseURL.Path)
|
|
||||||
if !strings.HasSuffix(baseURL.Path, "/") {
|
|
||||||
baseURL.Path += "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
return baseURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// downloadManifestFiles downloads every file in the manifest, reporting
|
|
||||||
// progress on the progress channel.
|
|
||||||
func downloadManifestFiles(
|
|
||||||
ctx context.Context,
|
|
||||||
baseURL *url.URL,
|
|
||||||
files []*mfer.MFFilePath,
|
|
||||||
progress chan<- DownloadProgress,
|
|
||||||
) error {
|
|
||||||
for _, f := range files {
|
|
||||||
// Sanitize the path to prevent path traversal attacks
|
|
||||||
localPath, err := sanitizePath(f.GetPath())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid path in manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fileURL := baseURL.String() + encodeFilePath(f.GetPath())
|
|
||||||
log.Infof("fetching %s", f.GetPath())
|
|
||||||
|
|
||||||
err = downloadFile(ctx, fileURL, localPath, f, progress)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to download %s: %w", f.GetPath(), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) fetchManifestOperation(ctx *cli.Context) error {
|
|
||||||
log.Debug("fetchManifestOperation()")
|
|
||||||
|
|
||||||
if ctx.Args().Len() == 0 {
|
|
||||||
return errURLRequired
|
|
||||||
}
|
|
||||||
|
|
||||||
inputURL := ctx.Args().Get(0)
|
|
||||||
|
|
||||||
manifestURL, err := resolveManifestURL(inputURL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid URL: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("fetching manifest from %s", manifestURL)
|
|
||||||
|
|
||||||
// Fetch manifest
|
|
||||||
resp, err := httpGet(ctx.Context, manifestURL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to fetch manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("failed to fetch manifest: %w %d",
|
|
||||||
errHTTPStatus, resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse manifest
|
|
||||||
manifest, err := mfer.NewManifestFromReader(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to parse manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
log.Infof("manifest contains %d files", len(files))
|
|
||||||
|
|
||||||
// Compute base URL (directory containing manifest)
|
|
||||||
baseURL, err := manifestBaseURL(manifestURL)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate total bytes to download
|
|
||||||
var totalBytes int64
|
|
||||||
for _, f := range files {
|
|
||||||
totalBytes += f.GetSize()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create progress channel and start progress reporter goroutine
|
|
||||||
progress := make(chan DownloadProgress, progressChanBuffer)
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go reportDownloadProgress(progress, done)
|
|
||||||
|
|
||||||
// Track download start time
|
|
||||||
startTime := time.Now()
|
|
||||||
|
|
||||||
// Download each file
|
|
||||||
dlErr := downloadManifestFiles(ctx.Context, baseURL, files, progress)
|
|
||||||
|
|
||||||
close(progress)
|
|
||||||
<-done
|
|
||||||
|
|
||||||
if dlErr != nil {
|
|
||||||
return dlErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print summary
|
|
||||||
elapsed := time.Since(startTime)
|
|
||||||
avgBytesPerSec := float64(totalBytes) / elapsed.Seconds()
|
|
||||||
avgRate := formatBitrate(avgBytesPerSec * bitsPerByte)
|
|
||||||
log.Infof("downloaded %d files (%s) in %.1fs (%s avg)",
|
|
||||||
len(files),
|
|
||||||
humanize.IBytes(safeUint64(totalBytes)),
|
|
||||||
elapsed.Seconds(),
|
|
||||||
avgRate)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// encodeFilePath URL-encodes each segment of a file path while preserving slashes.
|
|
||||||
func encodeFilePath(p string) string {
|
|
||||||
segments := strings.Split(p, "/")
|
|
||||||
for i, seg := range segments {
|
|
||||||
segments[i] = url.PathEscape(seg)
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(segments, "/")
|
|
||||||
}
|
|
||||||
|
|
||||||
// sanitizePath validates and sanitizes a file path from the manifest.
|
|
||||||
// It prevents path traversal attacks and rejects unsafe paths.
|
|
||||||
func sanitizePath(p string) (string, error) {
|
|
||||||
// Reject empty paths
|
|
||||||
if p == "" {
|
|
||||||
return "", errEmptyPath
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reject absolute paths
|
|
||||||
if filepath.IsAbs(p) {
|
|
||||||
return "", fmt.Errorf("%w: %s", errAbsolutePath, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean the path to resolve . and ..
|
|
||||||
cleaned := filepath.Clean(p)
|
|
||||||
|
|
||||||
// Reject paths that escape the current directory
|
|
||||||
if strings.HasPrefix(cleaned, ".."+string(filepath.Separator)) || cleaned == ".." {
|
|
||||||
return "", fmt.Errorf("%w: %s", errPathTraversal, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also check for absolute paths after cleaning (handles edge cases)
|
|
||||||
if filepath.IsAbs(cleaned) {
|
|
||||||
return "", fmt.Errorf("%w: %s", errAbsolutePath, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cleaned, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveManifestURL takes a URL and returns the manifest URL.
|
|
||||||
// If the URL already ends with .mf, it's returned as-is.
|
|
||||||
// Otherwise, index.mf is appended.
|
|
||||||
func resolveManifestURL(inputURL string) (string, error) {
|
|
||||||
parsed, err := url.Parse(inputURL)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if URL already ends with .mf
|
|
||||||
if strings.HasSuffix(parsed.Path, ".mf") {
|
|
||||||
return inputURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure path ends with /
|
|
||||||
if !strings.HasSuffix(parsed.Path, "/") {
|
|
||||||
parsed.Path += "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Append index.mf
|
|
||||||
parsed.Path += "index.mf"
|
|
||||||
|
|
||||||
return parsed.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// progressWriter wraps an io.Writer and reports progress to a channel.
|
|
||||||
type progressWriter struct {
|
|
||||||
w io.Writer
|
|
||||||
path string
|
|
||||||
total int64
|
|
||||||
written int64
|
|
||||||
startTime time.Time
|
|
||||||
progress chan<- DownloadProgress
|
|
||||||
}
|
|
||||||
|
|
||||||
func (pw *progressWriter) Write(p []byte) (int, error) {
|
|
||||||
n, err := pw.w.Write(p)
|
|
||||||
|
|
||||||
pw.written += int64(n)
|
|
||||||
if pw.progress != nil {
|
|
||||||
var (
|
|
||||||
bytesPerSec float64
|
|
||||||
eta time.Duration
|
|
||||||
)
|
|
||||||
|
|
||||||
elapsed := time.Since(pw.startTime)
|
|
||||||
if elapsed > 0 && pw.written > 0 {
|
|
||||||
bytesPerSec = float64(pw.written) / elapsed.Seconds()
|
|
||||||
if bytesPerSec > 0 && pw.total > 0 {
|
|
||||||
remainingBytes := pw.total - pw.written
|
|
||||||
eta = time.Duration(float64(remainingBytes)/bytesPerSec) * time.Second
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sendProgress(pw.progress, DownloadProgress{
|
|
||||||
Path: pw.path,
|
|
||||||
BytesRead: pw.written,
|
|
||||||
TotalBytes: pw.total,
|
|
||||||
BytesPerSec: bytesPerSec,
|
|
||||||
ETA: eta,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// formatBitrate formats a bits-per-second value with appropriate unit prefix.
|
|
||||||
func formatBitrate(bps float64) string {
|
|
||||||
switch {
|
|
||||||
case bps >= bpsPerGbps:
|
|
||||||
return fmt.Sprintf("%.1f Gbps", bps/bpsPerGbps)
|
|
||||||
case bps >= bpsPerMbps:
|
|
||||||
return fmt.Sprintf("%.1f Mbps", bps/bpsPerMbps)
|
|
||||||
case bps >= bpsPerKbps:
|
|
||||||
return fmt.Sprintf("%.1f Kbps", bps/bpsPerKbps)
|
|
||||||
default:
|
|
||||||
return fmt.Sprintf("%.0f bps", bps)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendProgress sends a progress update without blocking.
|
|
||||||
func sendProgress(ch chan<- DownloadProgress, p DownloadProgress) {
|
|
||||||
select {
|
|
||||||
case ch <- p:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tempPathFor computes the temporary download path for a local file.
|
|
||||||
// For dotfiles, just append .tmp (they're already hidden); for regular
|
|
||||||
// files, prefix with . and append .tmp.
|
|
||||||
func tempPathFor(localPath string) string {
|
|
||||||
dir := filepath.Dir(localPath)
|
|
||||||
base := filepath.Base(localPath)
|
|
||||||
|
|
||||||
var tmpName string
|
|
||||||
if strings.HasPrefix(base, ".") {
|
|
||||||
tmpName = base + ".tmp"
|
|
||||||
} else {
|
|
||||||
tmpName = "." + base + ".tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
if dir == "" || dir == "." {
|
|
||||||
return tmpName
|
|
||||||
}
|
|
||||||
|
|
||||||
return filepath.Join(dir, tmpName)
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifyDownloadedHash checks the computed sha256 digest against the
|
|
||||||
// manifest entry's hashes; at least one must match.
|
|
||||||
func verifyDownloadedHash(digest []byte, entry *mfer.MFFilePath) error {
|
|
||||||
computed, err := multihash.Encode(digest, multihash.SHA2_256)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to encode hash: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, hash := range entry.GetHashes() {
|
|
||||||
if bytes.Equal(computed, hash.GetMultiHash()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errHashMismatch
|
|
||||||
}
|
|
||||||
|
|
||||||
// downloadFile downloads a URL to a local file path with hash verification.
|
|
||||||
// It downloads to a temporary file, verifies the hash, then renames to the final path.
|
|
||||||
// Progress is reported via the progress channel.
|
|
||||||
func downloadFile(
|
|
||||||
ctx context.Context,
|
|
||||||
fileURL, localPath string,
|
|
||||||
entry *mfer.MFFilePath,
|
|
||||||
progress chan<- DownloadProgress,
|
|
||||||
) error {
|
|
||||||
// Enforce the path invariant here rather than relying on the caller,
|
|
||||||
// so every entry point to downloadFile gets the same treatment.
|
|
||||||
localPath, err := sanitizePath(localPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create parent directories if needed
|
|
||||||
dir := filepath.Dir(localPath)
|
|
||||||
if dir != "" && dir != "." {
|
|
||||||
err := os.MkdirAll(dir, dirPerms)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create directory %s: %w", dir, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tmpPath := tempPathFor(localPath)
|
|
||||||
|
|
||||||
// Fetch file
|
|
||||||
resp, err := httpGet(ctx, fileURL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("HTTP request failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("%w %d", errHTTPStatus, resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine expected size
|
|
||||||
expectedSize := entry.GetSize()
|
|
||||||
|
|
||||||
totalBytes := resp.ContentLength
|
|
||||||
if totalBytes < 0 {
|
|
||||||
totalBytes = expectedSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create temp file.
|
|
||||||
//
|
|
||||||
// G304: tmpPath is derived from localPath, which sanitizePath above
|
|
||||||
// constrains lexically to a relative path that does not escape the
|
|
||||||
// destination directory. That is a purely lexical guarantee: it does
|
|
||||||
// not resolve symlinks, so a pre-existing symlink inside the
|
|
||||||
// destination tree can still redirect this write outside of it
|
|
||||||
// (tracked in issue #86).
|
|
||||||
out, err := os.Create(tmpPath) //nolint:gosec // G304: see comment above
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create temp file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set up hash computation
|
|
||||||
h := sha256.New()
|
|
||||||
|
|
||||||
// Create progress-reporting writer that also computes hash
|
|
||||||
pw := &progressWriter{
|
|
||||||
w: io.MultiWriter(out, h),
|
|
||||||
path: localPath,
|
|
||||||
total: totalBytes,
|
|
||||||
startTime: time.Now(),
|
|
||||||
progress: progress,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Copy content while hashing and reporting progress
|
|
||||||
written, copyErr := io.Copy(pw, resp.Body)
|
|
||||||
|
|
||||||
// Close file before checking errors (to flush writes)
|
|
||||||
closeErr := out.Close()
|
|
||||||
|
|
||||||
err = finishDownload(
|
|
||||||
tmpPath, localPath, written, expectedSize, h.Sum(nil), entry,
|
|
||||||
copyErr, closeErr)
|
|
||||||
if err != nil {
|
|
||||||
_ = os.Remove(tmpPath)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishDownload validates the copy result, verifies size and hash, and
|
|
||||||
// moves the temp file into place. On error the caller removes tmpPath.
|
|
||||||
func finishDownload(
|
|
||||||
tmpPath, localPath string,
|
|
||||||
written, expectedSize int64,
|
|
||||||
digest []byte,
|
|
||||||
entry *mfer.MFFilePath,
|
|
||||||
copyErr, closeErr error,
|
|
||||||
) error {
|
|
||||||
if copyErr != nil {
|
|
||||||
return copyErr
|
|
||||||
}
|
|
||||||
|
|
||||||
if closeErr != nil {
|
|
||||||
return closeErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify size
|
|
||||||
if written != expectedSize {
|
|
||||||
return fmt.Errorf("%w: expected %d bytes, got %d",
|
|
||||||
errSizeMismatch, expectedSize, written)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify hash against manifest (at least one must match)
|
|
||||||
err := verifyDownloadedHash(digest, entry)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename temp file to final path
|
|
||||||
err = os.Rename(tmpPath, localPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,442 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testFileTxt = "file.txt"
|
|
||||||
testDirFile = "dir/file.txt"
|
|
||||||
testIndexMF = "https://example.com/path/index.mf"
|
|
||||||
|
|
||||||
// Exactly what url.Parse renders, with no wrapper of our own.
|
|
||||||
urlParseControlCharErr = `parse "http://example.com/\x7f": ` +
|
|
||||||
`net/url: invalid control character in URL`
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEncodeFilePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{testFileTxt, testFileTxt},
|
|
||||||
{testDirFile, testDirFile},
|
|
||||||
{"my file.txt", "my%20file.txt"},
|
|
||||||
{"dir/my file.txt", "dir/my%20file.txt"},
|
|
||||||
{"file#1.txt", "file%231.txt"},
|
|
||||||
{"file?v=1.txt", "file%3Fv=1.txt"},
|
|
||||||
{"path/to/file with spaces.txt", "path/to/file%20with%20spaces.txt"},
|
|
||||||
{"100%done.txt", "100%25done.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.input, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
result := encodeFilePath(tt.input)
|
|
||||||
assert.Equal(t, tt.expected, result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSanitizePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Valid paths that should be accepted
|
|
||||||
validTests := []struct {
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{testFileTxt, testFileTxt},
|
|
||||||
{testDirFile, testDirFile},
|
|
||||||
{"dir/subdir/file.txt", "dir/subdir/file.txt"},
|
|
||||||
{"./file.txt", testFileTxt},
|
|
||||||
{"./dir/file.txt", testDirFile},
|
|
||||||
{"dir/./file.txt", testDirFile},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range validTests {
|
|
||||||
t.Run("valid:"+tt.input, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
result, err := sanitizePath(tt.input)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Invalid paths that should be rejected
|
|
||||||
invalidTests := []struct {
|
|
||||||
input string
|
|
||||||
desc string
|
|
||||||
}{
|
|
||||||
{"", "empty path"},
|
|
||||||
{"..", "parent directory"},
|
|
||||||
{"../file.txt", "parent traversal"},
|
|
||||||
{"../../file.txt", "double parent traversal"},
|
|
||||||
{"dir/../../../file.txt", "traversal escaping base"},
|
|
||||||
{"/etc/passwd", "absolute path"},
|
|
||||||
{"/file.txt", "absolute path with single component"},
|
|
||||||
{"dir/../../etc/passwd", "traversal to system file"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range invalidTests {
|
|
||||||
t.Run("invalid:"+tt.desc, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := sanitizePath(tt.input)
|
|
||||||
assert.Error(t, err, "expected error for path: %s", tt.input)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveManifestURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
// Already ends with .mf - use as-is
|
|
||||||
{testIndexMF, testIndexMF},
|
|
||||||
{"https://example.com/path/custom.mf", "https://example.com/path/custom.mf"},
|
|
||||||
{"https://example.com/foo.mf", "https://example.com/foo.mf"},
|
|
||||||
|
|
||||||
// Directory with trailing slash - append index.mf
|
|
||||||
{"https://example.com/path/", testIndexMF},
|
|
||||||
{"https://example.com/", "https://example.com/index.mf"},
|
|
||||||
|
|
||||||
// Directory without trailing slash - add slash and index.mf
|
|
||||||
{"https://example.com/path", testIndexMF},
|
|
||||||
{"https://example.com", "https://example.com/index.mf"},
|
|
||||||
|
|
||||||
// With query strings
|
|
||||||
{
|
|
||||||
"https://example.com/path?foo=bar",
|
|
||||||
"https://example.com/path/index.mf?foo=bar",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.input, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
result, err := resolveManifestURL(tt.input)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, result)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The sole caller wraps this error as "invalid URL: %w", so
|
|
||||||
// resolveManifestURL must return url.Parse's error unadorned.
|
|
||||||
t.Run("invalid:control character", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := resolveManifestURL("http://example.com/\x7f")
|
|
||||||
require.ErrorContains(t, err, urlParseControlCharErr)
|
|
||||||
assert.NotContains(t, err.Error(), "failed to parse URL")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanToManifest scans sourceFs and returns the serialized manifest bytes.
|
|
||||||
func scanToManifest(t *testing.T, sourceFs afero.Fs) []byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: sourceFs})
|
|
||||||
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
|
|
||||||
|
|
||||||
var manifestBuf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, s.ToManifest(context.Background(), &manifestBuf, nil))
|
|
||||||
|
|
||||||
return manifestBuf.Bytes()
|
|
||||||
}
|
|
||||||
|
|
||||||
// chdirTemp switches the working directory to a fresh temp dir for the
|
|
||||||
// duration of the test and returns its path.
|
|
||||||
func chdirTemp(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
destDir := t.TempDir()
|
|
||||||
|
|
||||||
origDir, err := os.Getwd()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, os.Chdir(destDir))
|
|
||||||
t.Cleanup(func() { _ = os.Chdir(origDir) })
|
|
||||||
|
|
||||||
return destDir
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchTestHandler serves the manifest at /index.mf and the given files
|
|
||||||
// at their paths.
|
|
||||||
func fetchTestHandler(
|
|
||||||
manifestData []byte, testFiles map[string][]byte,
|
|
||||||
) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
path := r.URL.Path
|
|
||||||
if path == "/index.mf" {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write(manifestData)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Strip leading slash
|
|
||||||
if len(path) > 0 && path[0] == '/' {
|
|
||||||
path = path[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
content, exists := testFiles[path]
|
|
||||||
if !exists {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write(content)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestFetchFromHTTP(t *testing.T) {
|
|
||||||
// Create source filesystem with test files
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
|
|
||||||
testFiles := map[string][]byte{
|
|
||||||
"file1.txt": []byte("Hello, World!"),
|
|
||||||
"file2.txt": []byte("This is file 2 with more content."),
|
|
||||||
"subdir/file3.txt": []byte("Nested file content here."),
|
|
||||||
"subdir/deep/f.txt": []byte("Deeply nested file."),
|
|
||||||
}
|
|
||||||
|
|
||||||
for path, content := range testFiles {
|
|
||||||
fullPath := "/" + path // MemMapFs needs absolute paths
|
|
||||||
dir := filepath.Dir(fullPath)
|
|
||||||
require.NoError(t, sourceFs.MkdirAll(dir, 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, fullPath, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate manifest using scanner
|
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
|
||||||
|
|
||||||
// Create HTTP server that serves the source filesystem
|
|
||||||
server := httptest.NewServer(fetchTestHandler(manifestData, testFiles))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
// Change to a fresh destination directory for the test
|
|
||||||
destDir := chdirTemp(t)
|
|
||||||
|
|
||||||
// Parse the manifest to get file entries
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
require.Len(t, files, len(testFiles))
|
|
||||||
|
|
||||||
// Download each file using downloadFile
|
|
||||||
progress := make(chan DownloadProgress, 10)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for p := range progress {
|
|
||||||
_ = p // drain progress channel
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
baseURL := server.URL + "/"
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
localPath, err := sanitizePath(f.GetPath())
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
fileURL := baseURL + f.GetPath()
|
|
||||||
err = downloadFile(context.Background(), fileURL, localPath, f, progress)
|
|
||||||
require.NoError(t, err, "failed to download %s", f.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
close(progress)
|
|
||||||
|
|
||||||
// Verify downloaded files match originals
|
|
||||||
for path, expectedContent := range testFiles {
|
|
||||||
downloadedPath := filepath.Join(destDir, path)
|
|
||||||
//nolint:gosec // test-controlled path
|
|
||||||
downloadedContent, err := os.ReadFile(downloadedPath)
|
|
||||||
require.NoError(t, err, "failed to read downloaded file %s", path)
|
|
||||||
assert.Equal(t, expectedContent, downloadedContent,
|
|
||||||
"content mismatch for %s", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestFetchHashMismatch(t *testing.T) {
|
|
||||||
// Create source filesystem with a test file
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
originalContent := []byte("Original content")
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/file.txt", originalContent, 0o644))
|
|
||||||
|
|
||||||
// Generate and parse manifest
|
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
require.Len(t, files, 1)
|
|
||||||
|
|
||||||
// Create server that serves DIFFERENT content (to trigger hash mismatch)
|
|
||||||
tamperedContent := []byte("Tampered content!")
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write(tamperedContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
// Work in a fresh temp directory
|
|
||||||
chdirTemp(t)
|
|
||||||
|
|
||||||
// Try to download - should fail with hash mismatch
|
|
||||||
err = downloadFile(context.Background(),
|
|
||||||
server.URL+"/file.txt", testFileTxt, files[0], nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "mismatch")
|
|
||||||
|
|
||||||
// Verify temp file was cleaned up
|
|
||||||
_, err = os.Stat(".file.txt.tmp")
|
|
||||||
assert.True(t, os.IsNotExist(err),
|
|
||||||
"temp file should be cleaned up on hash mismatch")
|
|
||||||
|
|
||||||
// Verify final file was not created
|
|
||||||
_, err = os.Stat(testFileTxt)
|
|
||||||
assert.True(t, os.IsNotExist(err),
|
|
||||||
"final file should not exist on hash mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestFetchSizeMismatch(t *testing.T) {
|
|
||||||
// Create source filesystem with a test file
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
originalContent := []byte("Original content with specific size")
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/file.txt", originalContent, 0o644))
|
|
||||||
|
|
||||||
// Generate and parse manifest
|
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
require.Len(t, files, 1)
|
|
||||||
|
|
||||||
// Create server that serves content with wrong size
|
|
||||||
wrongSizeContent := []byte("Short")
|
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write(wrongSizeContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
// Work in a fresh temp directory
|
|
||||||
chdirTemp(t)
|
|
||||||
|
|
||||||
// Try to download - should fail with size mismatch
|
|
||||||
err = downloadFile(context.Background(),
|
|
||||||
server.URL+"/file.txt", testFileTxt, files[0], nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "size mismatch")
|
|
||||||
|
|
||||||
// Verify temp file was cleaned up
|
|
||||||
_, err = os.Stat(".file.txt.tmp")
|
|
||||||
assert.True(t, os.IsNotExist(err),
|
|
||||||
"temp file should be cleaned up on size mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // changes the process-global working directory
|
|
||||||
func TestFetchProgress(t *testing.T) {
|
|
||||||
// Create source filesystem with a larger test file
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
|
||||||
// Create content large enough to trigger multiple progress updates
|
|
||||||
content := bytes.Repeat([]byte("x"), 100*1024) // 100KB
|
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644))
|
|
||||||
|
|
||||||
// Generate and parse manifest
|
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
require.Len(t, files, 1)
|
|
||||||
|
|
||||||
// Create server that serves the content
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
w.Header().Set("Content-Length", "102400")
|
|
||||||
// Write in chunks to allow progress reporting
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, _ = io.Copy(w, reader)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
// Work in a fresh temp directory
|
|
||||||
chdirTemp(t)
|
|
||||||
|
|
||||||
// Set up progress channel and collect updates
|
|
||||||
progress := make(chan DownloadProgress, 100)
|
|
||||||
|
|
||||||
var progressUpdates []DownloadProgress
|
|
||||||
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for p := range progress {
|
|
||||||
progressUpdates = append(progressUpdates, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Download
|
|
||||||
err = downloadFile(context.Background(),
|
|
||||||
server.URL+"/large.txt", "large.txt", files[0], progress)
|
|
||||||
close(progress)
|
|
||||||
<-done
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Verify we got progress updates
|
|
||||||
assert.NotEmpty(t, progressUpdates, "should have received progress updates")
|
|
||||||
|
|
||||||
// Verify final progress shows complete
|
|
||||||
if len(progressUpdates) > 0 {
|
|
||||||
last := progressUpdates[len(progressUpdates)-1]
|
|
||||||
assert.Equal(t, int64(len(content)), last.BytesRead,
|
|
||||||
"final progress should show all bytes read")
|
|
||||||
assert.Equal(t, "large.txt", last.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify file was downloaded correctly
|
|
||||||
downloaded, err := os.ReadFile("large.txt")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, content, downloaded)
|
|
||||||
}
|
|
||||||
@@ -1,617 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"path/filepath"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// hashBufSize is the read buffer size used when hashing files.
|
|
||||||
hashBufSize = 64 * 1024
|
|
||||||
|
|
||||||
// scanProgressInterval is how many scanned files pass between
|
|
||||||
// progress updates.
|
|
||||||
scanProgressInterval = 100
|
|
||||||
)
|
|
||||||
|
|
||||||
// errEntryMissingMtime indicates a manifest entry that carries no
|
|
||||||
// modification time where one is required to carry it forward unchanged.
|
|
||||||
var errEntryMissingMtime = errors.New("manifest entry has no mtime")
|
|
||||||
|
|
||||||
// FreshenStatus contains progress information for the freshen operation.
|
|
||||||
type FreshenStatus struct {
|
|
||||||
Phase string // "scan" or "hash"
|
|
||||||
TotalFiles int64 // Total files to process in current phase
|
|
||||||
CurrentFiles int64 // Files processed so far
|
|
||||||
TotalBytes int64 // Total bytes to hash (hash phase only)
|
|
||||||
CurrentBytes int64 // Bytes hashed so far
|
|
||||||
BytesPerSec float64 // Current throughput rate
|
|
||||||
ETA time.Duration // Estimated time to completion
|
|
||||||
}
|
|
||||||
|
|
||||||
// freshenEntry tracks a file's status during freshen
|
|
||||||
type freshenEntry struct {
|
|
||||||
path string
|
|
||||||
size int64
|
|
||||||
mtime time.Time
|
|
||||||
needsHash bool // true if new or changed
|
|
||||||
existing *mfer.MFFilePath // existing manifest entry if unchanged
|
|
||||||
}
|
|
||||||
|
|
||||||
// freshenScanner walks the filesystem and compares it against the
|
|
||||||
// entries of an existing manifest.
|
|
||||||
type freshenScanner struct {
|
|
||||||
fs afero.Fs
|
|
||||||
absBase string
|
|
||||||
manifestBase string
|
|
||||||
includeDotfiles bool
|
|
||||||
followSymlinks bool
|
|
||||||
showProgress bool
|
|
||||||
existingByPath map[string]*mfer.MFFilePath
|
|
||||||
|
|
||||||
entries []*freshenEntry
|
|
||||||
scanCount int64
|
|
||||||
changed int64
|
|
||||||
added int64
|
|
||||||
unchanged int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveSymlink resolves a symlink to its target's FileInfo. The
|
|
||||||
// second return value is false when the entry should be skipped.
|
|
||||||
func (s *freshenScanner) resolveSymlink(path string) (fs.FileInfo, bool) {
|
|
||||||
if !s.followSymlinks {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
realPath, err := filepath.EvalSymlinks(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false // Skip broken symlinks
|
|
||||||
}
|
|
||||||
|
|
||||||
realInfo, err := s.fs.Stat(realPath)
|
|
||||||
if err != nil || realInfo.IsDir() {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return realInfo, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordEntry classifies a scanned file as changed, unchanged, or added
|
|
||||||
// relative to the existing manifest.
|
|
||||||
func (s *freshenScanner) recordEntry(relPath string, info fs.FileInfo) {
|
|
||||||
existing, inManifest := s.existingByPath[relPath]
|
|
||||||
if !inManifest {
|
|
||||||
s.added++
|
|
||||||
|
|
||||||
log.Verbosef("A %s", relPath)
|
|
||||||
s.entries = append(s.entries, &freshenEntry{
|
|
||||||
path: relPath,
|
|
||||||
size: info.Size(),
|
|
||||||
mtime: info.ModTime(),
|
|
||||||
needsHash: true,
|
|
||||||
})
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if changed (size or mtime). An entry with no recorded mtime
|
|
||||||
// cannot be compared, so it counts as changed and gets re-hashed;
|
|
||||||
// silently treating the absent mtime as the Unix epoch would classify
|
|
||||||
// every such entry as changed without saying why.
|
|
||||||
existingMtime, haveMtime := entryMtime(existing)
|
|
||||||
if !haveMtime {
|
|
||||||
log.Debugf("%s: manifest entry has no mtime, treating as changed",
|
|
||||||
relPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !haveMtime || existing.GetSize() != info.Size() ||
|
|
||||||
!existingMtime.Equal(info.ModTime()) {
|
|
||||||
s.changed++
|
|
||||||
|
|
||||||
log.Verbosef("M %s", relPath)
|
|
||||||
s.entries = append(s.entries, &freshenEntry{
|
|
||||||
path: relPath,
|
|
||||||
size: info.Size(),
|
|
||||||
mtime: info.ModTime(),
|
|
||||||
needsHash: true,
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
s.unchanged++
|
|
||||||
|
|
||||||
s.entries = append(s.entries, &freshenEntry{
|
|
||||||
path: relPath,
|
|
||||||
size: info.Size(),
|
|
||||||
mtime: info.ModTime(),
|
|
||||||
needsHash: false,
|
|
||||||
existing: existing,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
// Mark as seen
|
|
||||||
delete(s.existingByPath, relPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// walk is the afero.Walk callback for the scan phase.
|
|
||||||
func (s *freshenScanner) walk(path string, info fs.FileInfo, walkErr error) error {
|
|
||||||
if walkErr != nil {
|
|
||||||
return walkErr
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get relative path
|
|
||||||
relPath, err := filepath.Rel(s.absBase, path)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"freshen: failed to compute relative path for %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip the manifest file itself
|
|
||||||
if relPath == s.manifestBase || relPath == "."+s.manifestBase {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle dotfiles
|
|
||||||
if !s.includeDotfiles && mfer.IsHiddenPath(filepath.ToSlash(relPath)) {
|
|
||||||
if info.IsDir() {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip directories
|
|
||||||
if info.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle symlinks
|
|
||||||
if info.Mode()&fs.ModeSymlink != 0 {
|
|
||||||
realInfo, keep := s.resolveSymlink(path)
|
|
||||||
if !keep {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
info = realInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
s.scanCount++
|
|
||||||
|
|
||||||
// Check against existing manifest
|
|
||||||
s.recordEntry(relPath, info)
|
|
||||||
|
|
||||||
// Report scan progress
|
|
||||||
if s.showProgress && s.scanCount%scanProgressInterval == 0 {
|
|
||||||
log.Progressf("Scanning: %d files found", s.scanCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveFreshenManifestPath determines the manifest path from the CLI
|
|
||||||
// arguments, searching directories for a manifest where needed.
|
|
||||||
func (mfa *CLIApp) resolveFreshenManifestPath(ctx *cli.Context) (string, error) {
|
|
||||||
if ctx.Args().Len() == 0 {
|
|
||||||
return findManifest(mfa.Fs, ".")
|
|
||||||
}
|
|
||||||
|
|
||||||
arg := ctx.Args().Get(0)
|
|
||||||
|
|
||||||
info, statErr := mfa.Fs.Stat(arg)
|
|
||||||
if statErr == nil && info.IsDir() {
|
|
||||||
return findManifest(mfa.Fs, arg)
|
|
||||||
}
|
|
||||||
|
|
||||||
return arg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// freshenHasher hashes changed and added files and feeds all entries to
|
|
||||||
// a manifest builder.
|
|
||||||
type freshenHasher struct {
|
|
||||||
fs afero.Fs
|
|
||||||
absBase string
|
|
||||||
showProgress bool
|
|
||||||
totalHashBytes int64
|
|
||||||
filesToHash int64
|
|
||||||
startHash time.Time
|
|
||||||
builder *mfer.Builder
|
|
||||||
|
|
||||||
hashedFiles int64
|
|
||||||
hashedBytes int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportProgress renders hashing progress for the current byte count.
|
|
||||||
func (h *freshenHasher) reportProgress(n int64) {
|
|
||||||
if !h.showProgress {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
currentBytes := h.hashedBytes + n
|
|
||||||
elapsed := time.Since(h.startHash)
|
|
||||||
|
|
||||||
var (
|
|
||||||
rate float64
|
|
||||||
eta time.Duration
|
|
||||||
)
|
|
||||||
|
|
||||||
if elapsed > 0 && currentBytes > 0 {
|
|
||||||
rate = float64(currentBytes) / elapsed.Seconds()
|
|
||||||
|
|
||||||
remaining := h.totalHashBytes - currentBytes
|
|
||||||
if rate > 0 {
|
|
||||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if eta > 0 {
|
|
||||||
log.Progressf("Hashing: %d/%d files, %s/s, ETA %s",
|
|
||||||
h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)),
|
|
||||||
eta.Round(time.Second))
|
|
||||||
} else {
|
|
||||||
log.Progressf("Hashing: %d/%d files, %s/s",
|
|
||||||
h.hashedFiles, h.filesToHash, humanize.IBytes(safeRateUint64(rate)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// processEntry hashes the entry if needed and adds it to the builder.
|
|
||||||
func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
|
||||||
if !e.needsHash {
|
|
||||||
// Use existing entry
|
|
||||||
err := addExistingToBuilder(h.builder, e.existing)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Need to read and hash the file
|
|
||||||
absPath := filepath.Join(h.absBase, e.path)
|
|
||||||
|
|
||||||
f, err := h.fs.Open(absPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to open %s: %w", e.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
hash, bytesRead, err := hashFile(f, h.reportProgress)
|
|
||||||
_ = f.Close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to hash %s: %w", e.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.hashedBytes += bytesRead
|
|
||||||
h.hashedFiles++
|
|
||||||
|
|
||||||
// Add to builder with computed hash
|
|
||||||
err = addFileToBuilder(h.builder, e.path, e.size, e.mtime, hash)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to add %s: %w", e.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeFreshenedManifest writes the manifest atomically (write to a
|
|
||||||
// temp file, then rename over the target).
|
|
||||||
func writeFreshenedManifest(
|
|
||||||
afs afero.Fs, builder *mfer.Builder, manifestPath string,
|
|
||||||
) error {
|
|
||||||
tmpPath := manifestPath + ".tmp"
|
|
||||||
|
|
||||||
outFile, err := afs.Create(tmpPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create temp file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = builder.Build(outFile)
|
|
||||||
_ = outFile.Close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
_ = afs.Remove(tmpPath)
|
|
||||||
|
|
||||||
return fmt.Errorf("failed to write manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename temp to final
|
|
||||||
err = afs.Rename(tmpPath, manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
_ = afs.Remove(tmpPath)
|
|
||||||
|
|
||||||
return fmt.Errorf("failed to rename manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFreshenBuilder constructs the manifest builder configured from CLI
|
|
||||||
// flags.
|
|
||||||
func newFreshenBuilder(ctx *cli.Context) *mfer.Builder {
|
|
||||||
builder := mfer.NewBuilder()
|
|
||||||
if ctx.Bool("include-timestamps") {
|
|
||||||
builder.SetIncludeTimestamps(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set up signing options if sign-key is provided
|
|
||||||
if signKey := ctx.String("sign-key"); signKey != "" {
|
|
||||||
builder.SetSigningOptions(&mfer.SigningOptions{
|
|
||||||
KeyID: mfer.GPGKeyID(signKey),
|
|
||||||
})
|
|
||||||
log.Infof("signing manifest with GPG key: %s", signKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder
|
|
||||||
}
|
|
||||||
|
|
||||||
// freshenScan runs the scan phase against the loaded manifest entries
|
|
||||||
// and returns the populated scanner and the count of removed files.
|
|
||||||
func (mfa *CLIApp) freshenScan(
|
|
||||||
ctx *cli.Context, manifestPath, absBase string,
|
|
||||||
existingByPath map[string]*mfer.MFFilePath,
|
|
||||||
) (*freshenScanner, int64, error) {
|
|
||||||
log.Infof("scanning filesystem...")
|
|
||||||
|
|
||||||
startScan := time.Now()
|
|
||||||
showProgress := ctx.Bool("progress")
|
|
||||||
|
|
||||||
scanner := &freshenScanner{
|
|
||||||
fs: mfa.Fs,
|
|
||||||
absBase: absBase,
|
|
||||||
manifestBase: filepath.Base(manifestPath),
|
|
||||||
includeDotfiles: ctx.Bool("include-dotfiles"),
|
|
||||||
followSymlinks: ctx.Bool("follow-symlinks"),
|
|
||||||
showProgress: showProgress,
|
|
||||||
existingByPath: existingByPath,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := afero.Walk(mfa.Fs, absBase, scanner.walk)
|
|
||||||
|
|
||||||
if showProgress {
|
|
||||||
log.ProgressDone()
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, fmt.Errorf("failed to scan filesystem: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remaining entries in existingByPath are removed files
|
|
||||||
removed := int64(len(existingByPath))
|
|
||||||
for path := range existingByPath {
|
|
||||||
log.Verbosef("D %s", path)
|
|
||||||
}
|
|
||||||
|
|
||||||
scanDuration := time.Since(startScan)
|
|
||||||
log.Infof("scan complete in %s: %d unchanged, %d changed, %d added, %d removed",
|
|
||||||
scanDuration.Round(time.Millisecond), scanner.unchanged, scanner.changed,
|
|
||||||
scanner.added, removed)
|
|
||||||
|
|
||||||
return scanner, removed, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// hashTotals returns the total byte count and file count of entries
|
|
||||||
// that need hashing.
|
|
||||||
func hashTotals(entries []*freshenEntry) (int64, int64) {
|
|
||||||
var (
|
|
||||||
totalHashBytes int64
|
|
||||||
filesToHash int64
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, e := range entries {
|
|
||||||
if e.needsHash {
|
|
||||||
totalHashBytes += e.size
|
|
||||||
filesToHash++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return totalHashBytes, filesToHash
|
|
||||||
}
|
|
||||||
|
|
||||||
// runFreshenHash processes every entry through the hasher, aborting if
|
|
||||||
// the context is canceled.
|
|
||||||
func runFreshenHash(
|
|
||||||
ctx *cli.Context, hasher *freshenHasher, entries []*freshenEntry,
|
|
||||||
) error {
|
|
||||||
for _, e := range entries {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
err := hasher.processEntry(e)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadExistingEntries loads the manifest and indexes its file entries
|
|
||||||
// by path.
|
|
||||||
func (mfa *CLIApp) loadExistingEntries(
|
|
||||||
manifestPath string,
|
|
||||||
) (map[string]*mfer.MFFilePath, error) {
|
|
||||||
log.Infof("loading manifest from %s", manifestPath)
|
|
||||||
|
|
||||||
// Load existing manifest
|
|
||||||
manifest, err := mfer.NewManifestFromFile(mfa.Fs, manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to load manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
existingFiles := manifest.Files()
|
|
||||||
log.Infof("manifest contains %d files", len(existingFiles))
|
|
||||||
|
|
||||||
// Build map of existing entries by path
|
|
||||||
existingByPath := make(map[string]*mfer.MFFilePath, len(existingFiles))
|
|
||||||
for _, f := range existingFiles {
|
|
||||||
existingByPath[f.GetPath()] = f
|
|
||||||
}
|
|
||||||
|
|
||||||
return existingByPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) freshenManifestOperation(ctx *cli.Context) error {
|
|
||||||
log.Debug("freshenManifestOperation()")
|
|
||||||
|
|
||||||
basePath := ctx.String("base")
|
|
||||||
showProgress := ctx.Bool("progress")
|
|
||||||
|
|
||||||
// Find manifest file
|
|
||||||
manifestPath, err := mfa.resolveFreshenManifestPath(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("freshen: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
existingByPath, err := mfa.loadExistingEntries(manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
absBase, err := filepath.Abs(basePath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("freshen: invalid base path: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 1: Scan filesystem
|
|
||||||
scanner, removed, err := mfa.freshenScan(ctx, manifestPath, absBase,
|
|
||||||
existingByPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate total bytes to hash
|
|
||||||
totalHashBytes, filesToHash := hashTotals(scanner.entries)
|
|
||||||
|
|
||||||
// Phase 2: Hash changed and new files
|
|
||||||
if filesToHash > 0 {
|
|
||||||
log.Infof("hashing %d files (%s)...", filesToHash,
|
|
||||||
humanize.IBytes(safeUint64(totalHashBytes)))
|
|
||||||
}
|
|
||||||
|
|
||||||
hasher := &freshenHasher{
|
|
||||||
fs: mfa.Fs,
|
|
||||||
absBase: absBase,
|
|
||||||
showProgress: showProgress,
|
|
||||||
totalHashBytes: totalHashBytes,
|
|
||||||
filesToHash: filesToHash,
|
|
||||||
startHash: time.Now(),
|
|
||||||
builder: newFreshenBuilder(ctx),
|
|
||||||
}
|
|
||||||
|
|
||||||
err = runFreshenHash(ctx, hasher, scanner.entries)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if showProgress && filesToHash > 0 {
|
|
||||||
log.ProgressDone()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Print summary
|
|
||||||
log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
|
|
||||||
scanner.unchanged, scanner.changed, scanner.added, removed)
|
|
||||||
|
|
||||||
// Skip writing if nothing changed
|
|
||||||
if scanner.changed == 0 && scanner.added == 0 && removed == 0 {
|
|
||||||
log.Infof("manifest unchanged, skipping write")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write updated manifest atomically (write to temp, then rename)
|
|
||||||
err = writeFreshenedManifest(mfa.Fs, hasher.builder, manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
totalDuration := time.Since(mfa.startupTime)
|
|
||||||
if hasher.hashedBytes > 0 {
|
|
||||||
hashDuration := time.Since(hasher.startHash)
|
|
||||||
hashRate := float64(hasher.hashedBytes) / hashDuration.Seconds()
|
|
||||||
log.Infof("hashed %s in %.1fs (%s/s)",
|
|
||||||
humanize.IBytes(safeUint64(hasher.hashedBytes)),
|
|
||||||
totalDuration.Seconds(), humanize.IBytes(safeRateUint64(hashRate)))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("wrote %d files to %s", len(scanner.entries), manifestPath)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// hashFile reads a file and computes its SHA256 multihash.
|
|
||||||
// Progress callback is called with bytes read so far.
|
|
||||||
func hashFile(r io.Reader, progress func(int64)) ([]byte, int64, error) {
|
|
||||||
h := sha256.New()
|
|
||||||
buf := make([]byte, hashBufSize)
|
|
||||||
|
|
||||||
var total int64
|
|
||||||
|
|
||||||
for {
|
|
||||||
n, err := r.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
h.Write(buf[:n])
|
|
||||||
|
|
||||||
total += int64(n)
|
|
||||||
if progress != nil {
|
|
||||||
progress(total)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returned unwrapped: the caller renders this as
|
|
||||||
// "failed to hash <path>: <err>" and adding a second layer here
|
|
||||||
// would change that message.
|
|
||||||
if err != nil {
|
|
||||||
return nil, total, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
|
||||||
if err != nil {
|
|
||||||
return nil, total, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return mh, total, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// addFileToBuilder adds a new file entry to the builder
|
|
||||||
func addFileToBuilder(
|
|
||||||
b *mfer.Builder, path string, size int64, mtime time.Time, hash []byte,
|
|
||||||
) error {
|
|
||||||
return b.AddFileWithHash(
|
|
||||||
mfer.RelFilePath(path), mfer.FileSize(size), mfer.ModTime(mtime), hash)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addExistingToBuilder adds an existing manifest entry to the builder.
|
|
||||||
//
|
|
||||||
// Entries reach this path only when recordEntry classified them as
|
|
||||||
// unchanged, which requires a recorded mtime, so an absent mtime here is
|
|
||||||
// an error rather than something to paper over with the Unix epoch.
|
|
||||||
func addExistingToBuilder(b *mfer.Builder, entry *mfer.MFFilePath) error {
|
|
||||||
mtime, ok := entryMtime(entry)
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("%w: %s", errEntryMissingMtime, entry.GetPath())
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(entry.GetHashes()) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return b.AddFileWithHash(mfer.RelFilePath(entry.GetPath()),
|
|
||||||
mfer.FileSize(entry.GetSize()), mfer.ModTime(mtime),
|
|
||||||
entry.GetHashes()[0].GetMultiHash())
|
|
||||||
}
|
|
||||||
@@ -1,197 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// stubFileInfo is a minimal fs.FileInfo for exercising recordEntry
|
|
||||||
// without touching a filesystem.
|
|
||||||
type stubFileInfo struct {
|
|
||||||
size int64
|
|
||||||
mtime time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s stubFileInfo) Name() string { return "stub" }
|
|
||||||
func (s stubFileInfo) Size() int64 { return s.size }
|
|
||||||
func (s stubFileInfo) Mode() os.FileMode { return 0 }
|
|
||||||
func (s stubFileInfo) ModTime() time.Time { return s.mtime }
|
|
||||||
func (s stubFileInfo) IsDir() bool { return false }
|
|
||||||
func (s stubFileInfo) Sys() any { return nil }
|
|
||||||
|
|
||||||
// setupFreshenDir populates /testdir with two files, scans it, and
|
|
||||||
// writes the resulting manifest to /testdir/.index.mf.
|
|
||||||
func setupFreshenDir(t *testing.T, fs afero.Fs) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
|
||||||
writeTestFile(t, fs, testFile1, "content1")
|
|
||||||
writeTestFile(t, fs, "/testdir/file2.txt", "content2")
|
|
||||||
|
|
||||||
// Generate initial manifest
|
|
||||||
opts := &mfer.ScannerOptions{Fs: fs}
|
|
||||||
s := mfer.NewScannerWithOptions(opts)
|
|
||||||
require.NoError(t, s.EnumeratePath(testDir, nil))
|
|
||||||
|
|
||||||
var manifestBuf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, s.ToManifest(context.Background(), &manifestBuf, nil))
|
|
||||||
|
|
||||||
// Write manifest to filesystem
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/.index.mf", manifestBuf.Bytes(), 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFreshenUnchanged(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
setupFreshenDir(t, fs)
|
|
||||||
|
|
||||||
// Parse manifest to verify
|
|
||||||
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 2)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFreshenWithChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
setupFreshenDir(t, fs)
|
|
||||||
|
|
||||||
// Verify initial manifest has 2 files
|
|
||||||
manifest, err := mfer.NewManifestFromFile(fs, "/testdir/.index.mf")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 2)
|
|
||||||
|
|
||||||
// Add a new file
|
|
||||||
writeTestFile(t, fs, "/testdir/file3.txt", "content3")
|
|
||||||
|
|
||||||
// Modify file2 (change content and size)
|
|
||||||
writeTestFile(t, fs, "/testdir/file2.txt", "modified content2")
|
|
||||||
|
|
||||||
// Remove file1
|
|
||||||
require.NoError(t, fs.Remove(testFile1))
|
|
||||||
|
|
||||||
// Note: The freshen operation would need to be run here
|
|
||||||
// For now, we just verify the test setup is correct
|
|
||||||
exists, _ := afero.Exists(fs, testFile1)
|
|
||||||
assert.False(t, exists)
|
|
||||||
|
|
||||||
exists, _ = afero.Exists(fs, "/testdir/file3.txt")
|
|
||||||
assert.True(t, exists)
|
|
||||||
|
|
||||||
content, _ := afero.ReadFile(fs, "/testdir/file2.txt")
|
|
||||||
assert.Equal(t, "modified content2", string(content))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenRecordEntryMtimePresence pins the behavior of recordEntry
|
|
||||||
// with respect to MFFilePath.Mtime, which is a message pointer with
|
|
||||||
// proto3 field presence and may legitimately be absent.
|
|
||||||
//
|
|
||||||
// An absent mtime must never be read as time.Unix(0, 0): that value
|
|
||||||
// never equals a real modification time, so every entry would be
|
|
||||||
// classified as changed, re-hashed, and the manifest rewritten
|
|
||||||
// unconditionally - the exact inverse of what freshen is for, and
|
|
||||||
// silent. An entry with no mtime is therefore "changed" because it
|
|
||||||
// cannot be compared, not because it looks like it dates from 1970.
|
|
||||||
func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const relPath = "file1.txt"
|
|
||||||
|
|
||||||
mtime := time.Unix(1_700_000_000, 0)
|
|
||||||
info := stubFileInfo{size: 8, mtime: mtime}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
entry *mfer.MFFilePath
|
|
||||||
needsHash bool
|
|
||||||
changed int64
|
|
||||||
unchanged int64
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "matching mtime and size is unchanged",
|
|
||||||
entry: &mfer.MFFilePath{
|
|
||||||
Path: relPath,
|
|
||||||
Size: 8,
|
|
||||||
Mtime: &mfer.Timestamp{Seconds: mtime.Unix()},
|
|
||||||
},
|
|
||||||
needsHash: false,
|
|
||||||
changed: 0,
|
|
||||||
unchanged: 1,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "absent mtime is changed, not epoch",
|
|
||||||
entry: &mfer.MFFilePath{
|
|
||||||
Path: relPath,
|
|
||||||
Size: 8,
|
|
||||||
Mtime: nil,
|
|
||||||
},
|
|
||||||
needsHash: true,
|
|
||||||
changed: 1,
|
|
||||||
unchanged: 0,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := &freshenScanner{
|
|
||||||
existingByPath: map[string]*mfer.MFFilePath{relPath: tc.entry},
|
|
||||||
}
|
|
||||||
s.recordEntry(relPath, info)
|
|
||||||
|
|
||||||
require.Len(t, s.entries, 1)
|
|
||||||
assert.Equal(t, tc.needsHash, s.entries[0].needsHash)
|
|
||||||
assert.Equal(t, tc.changed, s.changed)
|
|
||||||
assert.Equal(t, tc.unchanged, s.unchanged)
|
|
||||||
assert.Zero(t, s.added)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshenAddExistingRejectsMissingMtime pins that an entry with no
|
|
||||||
// mtime is never carried forward into a rebuilt manifest with a
|
|
||||||
// fabricated epoch timestamp.
|
|
||||||
func TestFreshenAddExistingRejectsMissingMtime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := mfer.NewBuilder()
|
|
||||||
entry := &mfer.MFFilePath{
|
|
||||||
Path: "file1.txt",
|
|
||||||
Size: 8,
|
|
||||||
Mtime: nil,
|
|
||||||
Hashes: []*mfer.MFFileChecksum{
|
|
||||||
{MultiHash: []byte{0x12, 0x20}},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
err := addExistingToBuilder(b, entry)
|
|
||||||
require.ErrorIs(t, err, errEntryMissingMtime)
|
|
||||||
assert.Contains(t, err.Error(), "file1.txt")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntryMtime pins the presence semantics the callers depend on.
|
|
||||||
func TestEntryMtime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, ok := entryMtime(&mfer.MFFilePath{Mtime: nil})
|
|
||||||
assert.False(t, ok)
|
|
||||||
assert.True(t, got.IsZero())
|
|
||||||
|
|
||||||
got, ok = entryMtime(&mfer.MFFilePath{
|
|
||||||
Mtime: &mfer.Timestamp{Seconds: 1_700_000_000, Nanos: 500},
|
|
||||||
})
|
|
||||||
assert.True(t, ok)
|
|
||||||
assert.Equal(t, time.Unix(1_700_000_000, 500), got)
|
|
||||||
}
|
|
||||||
@@ -1,287 +1,54 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"bytes"
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
"github.com/spf13/afero"
|
"git.eeqj.de/sneak/mfer/mfer"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
// errPathNotExist indicates an input path that does not exist.
|
|
||||||
errPathNotExist = errors.New("path does not exist")
|
|
||||||
// errOutputExists indicates the output file already exists and
|
|
||||||
// --force was not given. It is wrapped mid-sentence so that the
|
|
||||||
// rendered message stays exactly as mfer has always printed it.
|
|
||||||
errOutputExists = errors.New(
|
|
||||||
"already exists (use --force to overwrite)")
|
|
||||||
)
|
|
||||||
|
|
||||||
// reportEnumProgress renders enumeration progress until the channel
|
|
||||||
// closes.
|
|
||||||
func reportEnumProgress(progress <-chan mfer.EnumerateStatus, wg *sync.WaitGroup) {
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
for status := range progress {
|
|
||||||
log.Progressf("Enumerating: %d files, %s",
|
|
||||||
status.FilesFound,
|
|
||||||
humanize.IBytes(safeUint64(int64(status.BytesFound))))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.ProgressDone()
|
|
||||||
}
|
|
||||||
|
|
||||||
// reportScanProgress renders scan progress until the channel closes.
|
|
||||||
func reportScanProgress(progress <-chan mfer.ScanStatus, wg *sync.WaitGroup) {
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
for status := range progress {
|
|
||||||
if status.ETA > 0 {
|
|
||||||
log.Progressf("Scanning: %d/%d files, %s/s, ETA %s",
|
|
||||||
status.ScannedFiles,
|
|
||||||
status.TotalFiles,
|
|
||||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)),
|
|
||||||
status.ETA.Round(time.Second))
|
|
||||||
} else {
|
|
||||||
log.Progressf("Scanning: %d/%d files, %s/s",
|
|
||||||
status.ScannedFiles,
|
|
||||||
status.TotalFiles,
|
|
||||||
humanize.IBytes(safeRateUint64(status.BytesPerSec)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.ProgressDone()
|
|
||||||
}
|
|
||||||
|
|
||||||
// collectInputPaths validates the input path arguments and returns them
|
|
||||||
// as absolute paths.
|
|
||||||
func (mfa *CLIApp) collectInputPaths(args cli.Args) ([]string, error) {
|
|
||||||
paths := make([]string, 0, args.Len())
|
|
||||||
|
|
||||||
for i := range args.Len() {
|
|
||||||
inputPath := args.Get(i)
|
|
||||||
|
|
||||||
ap, err := filepath.Abs(inputPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("generate: invalid path %q: %w", inputPath, err)
|
|
||||||
}
|
|
||||||
// Validate path exists before adding to list
|
|
||||||
if exists, _ := afero.Exists(mfa.Fs, ap); !exists {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errPathNotExist, inputPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debugf("enumerating path: %s", ap)
|
|
||||||
paths = append(paths, ap)
|
|
||||||
}
|
|
||||||
|
|
||||||
return paths, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildScannerOptions constructs scanner options from the CLI flags.
|
|
||||||
func (mfa *CLIApp) buildScannerOptions(ctx *cli.Context) *mfer.ScannerOptions {
|
|
||||||
opts := &mfer.ScannerOptions{
|
|
||||||
IncludeDotfiles: ctx.Bool("include-dotfiles"),
|
|
||||||
FollowSymLinks: ctx.Bool("follow-symlinks"),
|
|
||||||
IncludeTimestamps: ctx.Bool("include-timestamps"),
|
|
||||||
Fs: mfa.Fs,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set seed for deterministic UUID if provided
|
|
||||||
if seed := ctx.String("seed"); seed != "" {
|
|
||||||
opts.Seed = seed
|
|
||||||
|
|
||||||
log.Infof("using deterministic seed for manifest UUID")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set up signing options if sign-key is provided
|
|
||||||
if signKey := ctx.String("sign-key"); signKey != "" {
|
|
||||||
opts.SigningOptions = &mfer.SigningOptions{
|
|
||||||
KeyID: mfer.GPGKeyID(signKey),
|
|
||||||
}
|
|
||||||
log.Infof("signing manifest with GPG key: %s", signKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
return opts
|
|
||||||
}
|
|
||||||
|
|
||||||
// enumerateInputs runs the enumeration phase over the argument paths,
|
|
||||||
// or the current directory when no arguments are given.
|
|
||||||
func (mfa *CLIApp) enumerateInputs(
|
|
||||||
s *mfer.Scanner, args cli.Args, enumProgress chan mfer.EnumerateStatus,
|
|
||||||
) error {
|
|
||||||
if args.Len() == 0 {
|
|
||||||
// Default to current directory
|
|
||||||
err := s.EnumeratePath(".", enumProgress)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"generate: failed to enumerate current directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect and validate all paths first
|
|
||||||
paths, err := mfa.collectInputPaths(args)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.EnumeratePaths(enumProgress, paths...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("generate: failed to enumerate paths: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// cleanupOnSignal installs a handler that removes the temp output file
|
|
||||||
// and exits when the process is interrupted. It returns the signal
|
|
||||||
// channel so the caller can stop and close it when done.
|
|
||||||
func (mfa *CLIApp) cleanupOnSignal(outFile afero.File, tmpPath string) chan os.Signal {
|
|
||||||
sigChan := make(chan os.Signal, 1)
|
|
||||||
|
|
||||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
sig, ok := <-sigChan
|
|
||||||
if !ok || sig == nil {
|
|
||||||
return // Channel closed normally, not a signal
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = outFile.Close()
|
|
||||||
_ = mfa.Fs.Remove(tmpPath)
|
|
||||||
|
|
||||||
os.Exit(1)
|
|
||||||
}()
|
|
||||||
|
|
||||||
return sigChan
|
|
||||||
}
|
|
||||||
|
|
||||||
// runEnumeratePhase enumerates all input paths with optional progress
|
|
||||||
// reporting and logs the totals.
|
|
||||||
func (mfa *CLIApp) runEnumeratePhase(ctx *cli.Context, s *mfer.Scanner) error {
|
|
||||||
// Set up enumeration progress reporting
|
|
||||||
var (
|
|
||||||
enumProgress chan mfer.EnumerateStatus
|
|
||||||
enumWg sync.WaitGroup
|
|
||||||
)
|
|
||||||
|
|
||||||
if ctx.Bool("progress") {
|
|
||||||
enumProgress = make(chan mfer.EnumerateStatus, 1)
|
|
||||||
|
|
||||||
enumWg.Add(1)
|
|
||||||
|
|
||||||
go reportEnumProgress(enumProgress, &enumWg)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := mfa.enumerateInputs(s, ctx.Args(), enumProgress)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
enumWg.Wait()
|
|
||||||
|
|
||||||
log.Infof("enumerated %d files, %s total", s.FileCount(),
|
|
||||||
humanize.IBytes(safeUint64(int64(s.TotalBytes()))))
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
func (mfa *CLIApp) generateManifestOperation(ctx *cli.Context) error {
|
||||||
log.Debug("generateManifestOperation()")
|
log.Debug("generateManifestOperation()")
|
||||||
|
myArgs := ctx.Args()
|
||||||
|
log.Dump(myArgs)
|
||||||
|
|
||||||
s := mfer.NewScannerWithOptions(mfa.buildScannerOptions(ctx))
|
opts := &mfer.ManifestScanOptions{
|
||||||
|
IgnoreDotfiles: ctx.Bool("IgnoreDotfiles"),
|
||||||
|
FollowSymLinks: ctx.Bool("FollowSymLinks"),
|
||||||
|
}
|
||||||
|
paths := make([]string, ctx.Args().Len()-1)
|
||||||
|
for i := 0; i < ctx.Args().Len(); i++ {
|
||||||
|
ap, err := filepath.Abs(ctx.Args().Get(i))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Dump(ap)
|
||||||
|
paths = append(paths, ap)
|
||||||
|
}
|
||||||
|
mf, err := mfer.NewFromPaths(opts, paths...)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
mf.WithContext(ctx.Context)
|
||||||
|
|
||||||
// Phase 1: Enumeration - collect paths and stat files
|
log.Dump(mf)
|
||||||
err := mfa.runEnumeratePhase(ctx, s)
|
|
||||||
|
err = mf.Scan()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
showProgress := ctx.Bool("progress")
|
buf := new(bytes.Buffer)
|
||||||
|
|
||||||
// Check if output file exists
|
err = mf.WriteTo(buf)
|
||||||
outputPath := ctx.String("output")
|
|
||||||
if exists, _ := afero.Exists(mfa.Fs, outputPath); exists && !ctx.Bool("force") {
|
|
||||||
return fmt.Errorf("output file %s %w", outputPath, errOutputExists)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create temp file for atomic write
|
|
||||||
tmpPath := outputPath + ".tmp"
|
|
||||||
|
|
||||||
outFile, err := mfa.Fs.Create(tmpPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create temp file: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set up signal handler to clean up temp file on Ctrl-C
|
dat := buf.Bytes()
|
||||||
sigChan := mfa.cleanupOnSignal(outFile, tmpPath)
|
|
||||||
|
|
||||||
// Clean up temp file on any error or interruption
|
|
||||||
success := false
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
signal.Stop(sigChan)
|
|
||||||
close(sigChan)
|
|
||||||
|
|
||||||
_ = outFile.Close()
|
|
||||||
|
|
||||||
if !success {
|
|
||||||
_ = mfa.Fs.Remove(tmpPath)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Phase 2: Scan - read file contents and generate manifest
|
|
||||||
var (
|
|
||||||
scanProgress chan mfer.ScanStatus
|
|
||||||
scanWg sync.WaitGroup
|
|
||||||
)
|
|
||||||
|
|
||||||
if showProgress {
|
|
||||||
scanProgress = make(chan mfer.ScanStatus, 1)
|
|
||||||
|
|
||||||
scanWg.Add(1)
|
|
||||||
|
|
||||||
go reportScanProgress(scanProgress, &scanWg)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = s.ToManifest(ctx.Context, outFile, scanProgress)
|
|
||||||
|
|
||||||
scanWg.Wait()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to generate manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close file before rename to ensure all data is flushed
|
|
||||||
err = outFile.Close()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to close temp file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Atomic rename
|
|
||||||
err = mfa.Fs.Rename(tmpPath, outputPath)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to rename temp file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
success = true
|
|
||||||
|
|
||||||
elapsed := time.Since(mfa.startupTime).Seconds()
|
|
||||||
rate := float64(s.TotalBytes()) / elapsed
|
|
||||||
log.Infof("wrote %d files (%s) to %s in %.1fs (%s/s)", s.FileCount(),
|
|
||||||
humanize.IBytes(safeUint64(int64(s.TotalBytes()))), outputPath, elapsed,
|
|
||||||
humanize.IBytes(safeRateUint64(rate)))
|
|
||||||
|
|
||||||
|
log.Dump(dat)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (mfa *CLIApp) listManifestOperation(ctx *cli.Context) error {
|
|
||||||
// Default to ErrorLevel for clean output
|
|
||||||
log.SetLevel(log.ErrorLevel)
|
|
||||||
|
|
||||||
longFormat := ctx.Bool("long")
|
|
||||||
print0 := ctx.Bool("print0")
|
|
||||||
|
|
||||||
pathOrURL, err := mfa.resolveManifestArg(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("list: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rc, err := mfa.openManifestReader(pathOrURL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("list: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = rc.Close() }()
|
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(rc)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("list: failed to parse manifest: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
files := manifest.Files()
|
|
||||||
|
|
||||||
// Determine line ending
|
|
||||||
lineEnd := "\n"
|
|
||||||
if print0 {
|
|
||||||
lineEnd = "\x00"
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
if longFormat {
|
|
||||||
// An entry may legitimately carry no mtime; render that as
|
|
||||||
// mtimeAbsent rather than as the Unix epoch.
|
|
||||||
mtimeStr := mtimeAbsent
|
|
||||||
if mtime, ok := entryMtime(f); ok {
|
|
||||||
mtimeStr = mtime.Format(time.RFC3339)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _ = fmt.Fprintf(mfa.Stdout, "%d\t%s\t%s%s",
|
|
||||||
f.GetSize(), mtimeStr, f.GetPath(), lineEnd)
|
|
||||||
} else {
|
|
||||||
_, _ = fmt.Fprintf(mfa.Stdout, "%s%s", f.GetPath(), lineEnd)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
// manifestFetchTimeout bounds HTTP requests made to fetch a manifest.
|
|
||||||
const manifestFetchTimeout = 30 * time.Second
|
|
||||||
|
|
||||||
// errHTTPStatus indicates an HTTP response with a non-OK status code.
|
|
||||||
//
|
|
||||||
// Its text is the literal "HTTP" prefix of the rendered "HTTP <code>"
|
|
||||||
// message that mfer has always printed, so that wrapping it does not
|
|
||||||
// change any user-visible output. Match it with errors.Is; do not read
|
|
||||||
// its message.
|
|
||||||
var errHTTPStatus = errors.New("HTTP")
|
|
||||||
|
|
||||||
// isHTTPURL returns true if the string starts with http:// or https://.
|
|
||||||
func isHTTPURL(s string) bool {
|
|
||||||
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
|
|
||||||
}
|
|
||||||
|
|
||||||
// openManifestReader opens a manifest from a path or URL and returns a ReadCloser.
|
|
||||||
// The caller must close the returned reader.
|
|
||||||
func (mfa *CLIApp) openManifestReader(pathOrURL string) (io.ReadCloser, error) {
|
|
||||||
if isHTTPURL(pathOrURL) {
|
|
||||||
client := &http.Client{Timeout: manifestFetchTimeout}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, pathOrURL, nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to fetch %s: %w", pathOrURL, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("failed to fetch %s: %w %d",
|
|
||||||
pathOrURL, errHTTPStatus, resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
return resp.Body, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := mfa.Fs.Open(pathOrURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveManifestArg resolves the manifest path from CLI arguments.
|
|
||||||
// HTTP(S) URLs are returned as-is. Directories are searched for index.mf/.index.mf.
|
|
||||||
// If no argument is given, the current directory is searched.
|
|
||||||
func (mfa *CLIApp) resolveManifestArg(ctx *cli.Context) (string, error) {
|
|
||||||
if ctx.Args().Len() > 0 {
|
|
||||||
arg := ctx.Args().Get(0)
|
|
||||||
if isHTTPURL(arg) {
|
|
||||||
return arg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
info, statErr := mfa.Fs.Stat(arg)
|
|
||||||
if statErr == nil && info.IsDir() {
|
|
||||||
return findManifest(mfa.Fs, arg)
|
|
||||||
}
|
|
||||||
|
|
||||||
return arg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return findManifest(mfa.Fs, ".")
|
|
||||||
}
|
|
||||||
@@ -1,36 +1,14 @@
|
|||||||
package cli
|
package cli
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Command and flag names shared across command definitions and tests.
|
|
||||||
const (
|
|
||||||
cmdGenerate = "generate"
|
|
||||||
cmdCheck = "check"
|
|
||||||
cmdExport = "export"
|
|
||||||
|
|
||||||
flagProgress = "progress"
|
|
||||||
|
|
||||||
manifestArgsUsage = "[manifest file]"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errUnknownCommand indicates an unrecognized command argument.
|
|
||||||
var errUnknownCommand = errors.New("unknown command")
|
|
||||||
|
|
||||||
// CLIApp is the main CLI application container. It holds configuration,
|
|
||||||
// I/O streams, and filesystem abstraction to enable testing and flexibility.
|
|
||||||
//
|
|
||||||
//nolint:revive // established name used throughout the codebase and tests
|
|
||||||
type CLIApp struct {
|
type CLIApp struct {
|
||||||
appname string
|
appname string
|
||||||
version string
|
version string
|
||||||
@@ -38,325 +16,135 @@ type CLIApp struct {
|
|||||||
startupTime time.Time
|
startupTime time.Time
|
||||||
exitCode int
|
exitCode int
|
||||||
app *cli.App
|
app *cli.App
|
||||||
|
|
||||||
Stdin io.Reader // Standard input stream
|
|
||||||
Stdout io.Writer // Standard output stream for normal output
|
|
||||||
Stderr io.Writer // Standard error stream for diagnostics
|
|
||||||
Fs afero.Fs // Filesystem abstraction for all file operations
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const banner = `
|
const banner = ` ___ ___ ___ ___
|
||||||
___ ___ ___ ___
|
|
||||||
/__/\ / /\ / /\ / /\
|
/__/\ / /\ / /\ / /\
|
||||||
| |::\ / /:/_ / /:/_ / /::\
|
| |::\ / /:/_ / /:/_ / /::\
|
||||||
| |:|:\ / /:/ /\ / /:/ /\ / /:/\:\
|
| |:|:\ / /:/ /\ / /:/ /\ / /:/\:\
|
||||||
__|__|:|\:\ / /:/ /:/ / /:/ /:/_ / /:/~/:/
|
__|__|:|\:\ / /:/ /:/ / /:/ /:/_ / /:/~/:/
|
||||||
/__/::::| \:\ /__/:/ /:/ /__/:/ /:/ /\ /__/:/ /:/___
|
/__/::::| \:\ /__/:/ /:/ /__/:/ /:/ /\ /__/:/ /:/___
|
||||||
\ \:\~~\__\/ \ \:\/:/ \ \:\/:/ /:/ \ \:\/:::::/
|
\ \:\~~\__\/ \ \:\/:/ \ \:\/:/ /:/ \ \:\/:::::/
|
||||||
\ \:\ \ \::/ \ \::/ /:/ \ \::/~~~~
|
\ \:\ \ \::/ \ \::/ /:/ \ \::/~~~~
|
||||||
\ \:\ \ \:\ \ \:\/:/ \ \:\
|
\ \:\ \ \:\ \ \:\/:/ \ \:\
|
||||||
\ \:\ \ \:\ \ \::/ \ \:\
|
\ \:\ \ \:\ \ \::/ \ \:\
|
||||||
\__\/ \__\/ \__\/ \__\/`
|
\__\/ \__\/ \__\/ \__\/`
|
||||||
|
|
||||||
// VersionString returns the version and git revision formatted for display.
|
|
||||||
func (mfa *CLIApp) VersionString() string {
|
|
||||||
if mfa.gitrev != "" {
|
|
||||||
return fmt.Sprintf("%s (%s)", mfer.Version, mfa.gitrev)
|
|
||||||
}
|
|
||||||
|
|
||||||
return mfer.Version
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) printBanner() {
|
func (mfa *CLIApp) printBanner() {
|
||||||
if log.GetLevel() <= log.InfoLevel {
|
fmt.Println(banner)
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
|
||||||
_, _ = fmt.Fprintf(mfa.Stdout,
|
|
||||||
" mfer by @sneak: v%s released %s\n",
|
|
||||||
mfer.Version, mfer.ReleaseDate)
|
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, " https://sneak.berlin/go/mfer")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mfa *CLIApp) setVerbosity(c *cli.Context) {
|
func (mfa *CLIApp) VersionString() string {
|
||||||
|
return fmt.Sprintf("%s (%s)", mfa.version, mfa.gitrev)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (mfa *CLIApp) setVerbosity(v int) {
|
||||||
_, present := os.LookupEnv("MFER_DEBUG")
|
_, present := os.LookupEnv("MFER_DEBUG")
|
||||||
|
if present {
|
||||||
switch {
|
|
||||||
case present:
|
|
||||||
log.EnableDebugLogging()
|
log.EnableDebugLogging()
|
||||||
case c.Bool("quiet"):
|
} else {
|
||||||
log.SetLevel(log.ErrorLevel)
|
log.SetLevelFromVerbosity(v)
|
||||||
default:
|
|
||||||
log.SetLevelFromVerbosity(c.Count("verbose"))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// commonFlags returns the flags shared by most commands (-v, -q)
|
func (mfa *CLIApp) run() {
|
||||||
func commonFlags() []cli.Flag {
|
|
||||||
return []cli.Flag{
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "verbose",
|
|
||||||
Aliases: []string{"v"},
|
|
||||||
Usage: "Increase verbosity (-v for verbose, -vv for debug)",
|
|
||||||
Count: new(int),
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "quiet",
|
|
||||||
Aliases: []string{"q"},
|
|
||||||
Usage: "Suppress output except errors",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: cmdGenerate,
|
|
||||||
Aliases: []string{"gen"},
|
|
||||||
Usage: "Generate manifest file",
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
mfa.setVerbosity(c)
|
|
||||||
mfa.printBanner()
|
|
||||||
|
|
||||||
return mfa.generateManifestOperation(c)
|
|
||||||
},
|
|
||||||
Flags: append(commonFlags(),
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "follow-symlinks",
|
|
||||||
Aliases: []string{"L"},
|
|
||||||
Usage: "Resolve encountered symlinks",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "include-dotfiles",
|
|
||||||
Aliases: []string{"IncludeDotfiles"},
|
|
||||||
|
|
||||||
Usage: "Include dot (hidden) files (excluded by default)",
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "output",
|
|
||||||
Value: "./.index.mf",
|
|
||||||
Aliases: []string{"o"},
|
|
||||||
Usage: "Specify output filename",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "force",
|
|
||||||
Aliases: []string{"f"},
|
|
||||||
Usage: "Overwrite output file if it exists",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: flagProgress,
|
|
||||||
Aliases: []string{"P"},
|
|
||||||
Usage: "Show progress during enumeration and scanning",
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "sign-key",
|
|
||||||
Aliases: []string{"s"},
|
|
||||||
Usage: "GPG key ID to sign the manifest with",
|
|
||||||
EnvVars: []string{"MFER_SIGN_KEY"},
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "seed",
|
|
||||||
Usage: "Seed value for deterministic manifest UUID",
|
|
||||||
EnvVars: []string{"MFER_SEED"},
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "include-timestamps",
|
|
||||||
Usage: "Include createdAt timestamp in manifest " +
|
|
||||||
"(omitted by default for determinism)",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) checkCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: cmdCheck,
|
|
||||||
Usage: "Validate files using manifest file",
|
|
||||||
ArgsUsage: manifestArgsUsage,
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
mfa.setVerbosity(c)
|
|
||||||
mfa.printBanner()
|
|
||||||
|
|
||||||
return mfa.checkManifestOperation(c)
|
|
||||||
},
|
|
||||||
Flags: append(commonFlags(),
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "base",
|
|
||||||
Aliases: []string{"b"},
|
|
||||||
Value: ".",
|
|
||||||
Usage: "Base directory for resolving relative paths from manifest",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: flagProgress,
|
|
||||||
Aliases: []string{"P"},
|
|
||||||
Usage: "Show progress during checking",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "no-extra-files",
|
|
||||||
Usage: "Fail if files exist in base directory that are not in manifest",
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "require-signature",
|
|
||||||
Aliases: []string{"S"},
|
|
||||||
Usage: "Require manifest to be signed by the specified GPG key ID",
|
|
||||||
EnvVars: []string{"MFER_REQUIRE_SIGNATURE"},
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: "freshen",
|
|
||||||
Usage: "Update manifest with changed, new, and removed files",
|
|
||||||
ArgsUsage: manifestArgsUsage,
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
mfa.setVerbosity(c)
|
|
||||||
mfa.printBanner()
|
|
||||||
|
|
||||||
return mfa.freshenManifestOperation(c)
|
|
||||||
},
|
|
||||||
Flags: append(commonFlags(),
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "base",
|
|
||||||
Aliases: []string{"b"},
|
|
||||||
Value: ".",
|
|
||||||
Usage: "Base directory for resolving relative paths",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "follow-symlinks",
|
|
||||||
Aliases: []string{"L"},
|
|
||||||
Usage: "Resolve encountered symlinks",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "include-dotfiles",
|
|
||||||
Aliases: []string{"IncludeDotfiles"},
|
|
||||||
|
|
||||||
Usage: "Include dot (hidden) files (excluded by default)",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: flagProgress,
|
|
||||||
Aliases: []string{"P"},
|
|
||||||
Usage: "Show progress during scanning and hashing",
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "sign-key",
|
|
||||||
Aliases: []string{"s"},
|
|
||||||
Usage: "GPG key ID to sign the manifest with",
|
|
||||||
EnvVars: []string{"MFER_SIGN_KEY"},
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "include-timestamps",
|
|
||||||
Usage: "Include createdAt timestamp in manifest " +
|
|
||||||
"(omitted by default for determinism)",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) exportCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: cmdExport,
|
|
||||||
Usage: "Export manifest contents as JSON",
|
|
||||||
ArgsUsage: "[manifest file or URL]",
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
return mfa.exportManifestOperation(c)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) versionCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: "version",
|
|
||||||
Usage: "Show version",
|
|
||||||
Action: func(_ *cli.Context) error {
|
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, mfa.VersionString())
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) listCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: "list",
|
|
||||||
Aliases: []string{"ls"},
|
|
||||||
Usage: "List files in manifest",
|
|
||||||
ArgsUsage: manifestArgsUsage,
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
return mfa.listManifestOperation(c)
|
|
||||||
},
|
|
||||||
Flags: []cli.Flag{
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "long",
|
|
||||||
Aliases: []string{"l"},
|
|
||||||
Usage: "Show size and mtime",
|
|
||||||
},
|
|
||||||
&cli.BoolFlag{
|
|
||||||
Name: "print0",
|
|
||||||
Usage: "Separate entries with NUL character (for xargs -0)",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
|
||||||
return &cli.Command{
|
|
||||||
Name: "fetch",
|
|
||||||
Usage: "fetch manifest and referenced files",
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
mfa.setVerbosity(c)
|
|
||||||
mfa.printBanner()
|
|
||||||
|
|
||||||
return mfa.fetchManifestOperation(c)
|
|
||||||
},
|
|
||||||
Flags: commonFlags(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (mfa *CLIApp) run(args []string) {
|
|
||||||
mfa.startupTime = time.Now()
|
mfa.startupTime = time.Now()
|
||||||
|
|
||||||
if NoColor {
|
if NO_COLOR {
|
||||||
// shoutout to rob pike who thinks it's juvenile
|
// shoutout to rob pike who thinks it's juvenile
|
||||||
log.DisableStyling()
|
log.DisableStyling()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Configure log package to use our I/O streams
|
|
||||||
log.SetOutput(mfa.Stdout, mfa.Stderr)
|
|
||||||
log.Init()
|
log.Init()
|
||||||
|
|
||||||
|
var verbosity int
|
||||||
|
|
||||||
mfa.app = &cli.App{
|
mfa.app = &cli.App{
|
||||||
Name: mfa.appname,
|
Name: mfa.appname,
|
||||||
Usage: "Manifest generator",
|
Usage: "Manifest generator",
|
||||||
Version: mfa.VersionString(),
|
Version: mfa.VersionString(),
|
||||||
EnableBashCompletion: true,
|
EnableBashCompletion: true,
|
||||||
Writer: mfa.Stdout,
|
Flags: []cli.Flag{
|
||||||
ErrWriter: mfa.Stderr,
|
&cli.BoolFlag{
|
||||||
Action: func(c *cli.Context) error {
|
Name: "verbose",
|
||||||
if c.Args().Len() > 0 {
|
Usage: "Verbosity level",
|
||||||
return fmt.Errorf("%w %q", errUnknownCommand, c.Args().First())
|
Aliases: []string{"v"},
|
||||||
}
|
Count: &verbosity,
|
||||||
|
},
|
||||||
mfa.printBanner()
|
&cli.BoolFlag{
|
||||||
|
Name: "quiet",
|
||||||
return cli.ShowAppHelp(c)
|
Usage: "don't produce output except on error",
|
||||||
|
Aliases: []string{"q"},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Commands: []*cli.Command{
|
Commands: []*cli.Command{
|
||||||
mfa.generateCommand(),
|
{
|
||||||
mfa.checkCommand(),
|
Name: "generate",
|
||||||
mfa.freshenCommand(),
|
Aliases: []string{"gen"},
|
||||||
mfa.exportCommand(),
|
Usage: "Generate manifest file",
|
||||||
mfa.versionCommand(),
|
Action: func(c *cli.Context) error {
|
||||||
mfa.listCommand(),
|
if !c.Bool("quiet") {
|
||||||
mfa.fetchCommand(),
|
mfa.printBanner()
|
||||||
|
}
|
||||||
|
mfa.setVerbosity(verbosity)
|
||||||
|
return mfa.generateManifestOperation(c)
|
||||||
|
},
|
||||||
|
Flags: []cli.Flag{
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "FollowSymLinks",
|
||||||
|
Aliases: []string{"follow-symlinks"},
|
||||||
|
Usage: "Resolve encountered symlinks",
|
||||||
|
},
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "IgnoreDotfiles",
|
||||||
|
Aliases: []string{"ignore-dotfiles"},
|
||||||
|
Usage: "Ignore any dot (hidden) files encountered",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "output",
|
||||||
|
Value: "./index.mf",
|
||||||
|
Aliases: []string{"o"},
|
||||||
|
Usage: "Specify output filename",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "check",
|
||||||
|
Usage: "Validate files using manifest file",
|
||||||
|
Action: func(c *cli.Context) error {
|
||||||
|
if !c.Bool("quiet") {
|
||||||
|
mfa.printBanner()
|
||||||
|
}
|
||||||
|
mfa.setVerbosity(verbosity)
|
||||||
|
return mfa.checkManifestOperation(c)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "version",
|
||||||
|
Usage: "Show version",
|
||||||
|
Action: func(c *cli.Context) error {
|
||||||
|
fmt.Printf("%s\n", mfa.VersionString())
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "fetch",
|
||||||
|
Usage: "fetch manifest and referenced files",
|
||||||
|
Action: func(c *cli.Context) error {
|
||||||
|
if !c.Bool("quiet") {
|
||||||
|
mfa.printBanner()
|
||||||
|
}
|
||||||
|
mfa.setVerbosity(verbosity)
|
||||||
|
return mfa.fetchManifestOperation(c)
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
mfa.app.HideVersion = false
|
mfa.app.HideVersion = true
|
||||||
|
err := mfa.app.Run(os.Args)
|
||||||
err := mfa.app.Run(args)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
mfa.exitCode = 1
|
mfa.exitCode = 1
|
||||||
|
|
||||||
log.WithError(err).Debugf("exiting")
|
log.WithError(err).Debugf("exiting")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
package cli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mtimeAbsent is printed in place of a modification time when a manifest
|
|
||||||
// entry does not carry one.
|
|
||||||
const mtimeAbsent = "-"
|
|
||||||
|
|
||||||
// entryMtime returns the modification time recorded for a manifest entry.
|
|
||||||
//
|
|
||||||
// MFFilePath.Mtime is a message pointer with proto3 field presence, so an
|
|
||||||
// absent mtime is a representable, on-the-wire-valid state. It must never
|
|
||||||
// be conflated with a recorded mtime of the Unix epoch: callers that
|
|
||||||
// compare mtimes have to treat "absent" as "unknown", not as
|
|
||||||
// 1970-01-01T00:00:00Z, or every entry compares as modified. ok reports
|
|
||||||
// whether an mtime was actually recorded.
|
|
||||||
func entryMtime(entry *mfer.MFFilePath) (time.Time, bool) {
|
|
||||||
ts := entry.GetMtime()
|
|
||||||
if ts == nil {
|
|
||||||
return time.Time{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Unix(ts.GetSeconds(), int64(ts.GetNanos())), true
|
|
||||||
}
|
|
||||||
@@ -1,14 +1,8 @@
|
|||||||
// Package log provides leveled logging with progress output helpers
|
|
||||||
// on top of apex/log and pterm.
|
|
||||||
package log
|
package log
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/apex/log"
|
"github.com/apex/log"
|
||||||
acli "github.com/apex/log/handlers/cli"
|
acli "github.com/apex/log/handlers/cli"
|
||||||
@@ -16,93 +10,11 @@ import (
|
|||||||
"github.com/pterm/pterm"
|
"github.com/pterm/pterm"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Level represents log severity levels.
|
type Level = log.Level
|
||||||
// Lower values are more verbose.
|
|
||||||
type Level int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// DebugLevel is for low-level tracing and structure inspection
|
|
||||||
DebugLevel Level = iota
|
|
||||||
// VerboseLevel is for detailed operational info (file listings, etc)
|
|
||||||
VerboseLevel
|
|
||||||
// InfoLevel is for operational summaries (default)
|
|
||||||
InfoLevel
|
|
||||||
// WarnLevel is for warnings
|
|
||||||
WarnLevel
|
|
||||||
// ErrorLevel is for errors
|
|
||||||
ErrorLevel
|
|
||||||
// FatalLevel is for fatal errors
|
|
||||||
FatalLevel
|
|
||||||
)
|
|
||||||
|
|
||||||
func (l Level) String() string {
|
|
||||||
switch l {
|
|
||||||
case DebugLevel:
|
|
||||||
return "debug"
|
|
||||||
case VerboseLevel:
|
|
||||||
return "verbose"
|
|
||||||
case InfoLevel:
|
|
||||||
return "info"
|
|
||||||
case WarnLevel:
|
|
||||||
return "warn"
|
|
||||||
case ErrorLevel:
|
|
||||||
return "error"
|
|
||||||
case FatalLevel:
|
|
||||||
return "fatal"
|
|
||||||
default:
|
|
||||||
return "unknown"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// callerSkip is the runtime.Caller stack depth from the public Debug
|
|
||||||
// helpers to the caller of the log package.
|
|
||||||
const callerSkip = 2
|
|
||||||
|
|
||||||
//nolint:gochecknoglobals // package-level logger state by design
|
|
||||||
var (
|
|
||||||
// mu protects the output writers and level
|
|
||||||
mu sync.RWMutex
|
|
||||||
// stdout is the writer for progress output
|
|
||||||
stdout io.Writer = os.Stdout
|
|
||||||
// stderr is the writer for log output
|
|
||||||
stderr io.Writer = os.Stderr
|
|
||||||
// currentLevel is our log level (includes Verbose)
|
|
||||||
currentLevel = InfoLevel
|
|
||||||
)
|
|
||||||
|
|
||||||
// SetOutput configures the output writers for the log package.
|
|
||||||
// stdout is used for progress output, stderr is used for log messages.
|
|
||||||
func SetOutput(out, err io.Writer) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
|
|
||||||
stdout = out
|
|
||||||
stderr = err
|
|
||||||
|
|
||||||
pterm.SetDefaultOutput(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetStdout returns the configured stdout writer.
|
|
||||||
func GetStdout() io.Writer {
|
|
||||||
mu.RLock()
|
|
||||||
defer mu.RUnlock()
|
|
||||||
|
|
||||||
return stdout
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetStderr returns the configured stderr writer.
|
|
||||||
func GetStderr() io.Writer {
|
|
||||||
mu.RLock()
|
|
||||||
defer mu.RUnlock()
|
|
||||||
|
|
||||||
return stderr
|
|
||||||
}
|
|
||||||
|
|
||||||
// DisableStyling turns off colors and styling for terminal output.
|
|
||||||
func DisableStyling() {
|
func DisableStyling() {
|
||||||
pterm.DisableColor()
|
pterm.DisableColor()
|
||||||
pterm.DisableStyling()
|
pterm.DisableStyling()
|
||||||
|
|
||||||
pterm.Debug.Prefix.Text = ""
|
pterm.Debug.Prefix.Text = ""
|
||||||
pterm.Info.Prefix.Text = ""
|
pterm.Info.Prefix.Text = ""
|
||||||
pterm.Success.Prefix.Text = ""
|
pterm.Success.Prefix.Text = ""
|
||||||
@@ -111,183 +23,67 @@ func DisableStyling() {
|
|||||||
pterm.Fatal.Prefix.Text = ""
|
pterm.Fatal.Prefix.Text = ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// Init initializes the logger with the CLI handler and default log level.
|
|
||||||
func Init() {
|
func Init() {
|
||||||
mu.RLock()
|
log.SetHandler(acli.Default)
|
||||||
|
log.SetLevel(log.InfoLevel)
|
||||||
w := stderr
|
|
||||||
|
|
||||||
mu.RUnlock()
|
|
||||||
log.SetHandler(acli.New(w))
|
|
||||||
log.SetLevel(log.DebugLevel) // Let apex/log pass everything; we filter ourselves
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isEnabled returns true if messages at the given level should be logged.
|
func Debugf(format string, args ...interface{}) {
|
||||||
func isEnabled(l Level) bool {
|
DebugReal(fmt.Sprintf(format, args...), 2)
|
||||||
mu.RLock()
|
|
||||||
defer mu.RUnlock()
|
|
||||||
|
|
||||||
return l >= currentLevel
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fatalf logs a formatted message at fatal level.
|
|
||||||
func Fatalf(format string, args ...any) {
|
|
||||||
if isEnabled(FatalLevel) {
|
|
||||||
log.Fatalf(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fatal logs a message at fatal level.
|
|
||||||
func Fatal(arg string) {
|
|
||||||
if isEnabled(FatalLevel) {
|
|
||||||
log.Fatal(arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Errorf logs a formatted message at error level.
|
|
||||||
func Errorf(format string, args ...any) {
|
|
||||||
if isEnabled(ErrorLevel) {
|
|
||||||
log.Errorf(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error logs a message at error level.
|
|
||||||
func Error(arg string) {
|
|
||||||
if isEnabled(ErrorLevel) {
|
|
||||||
log.Error(arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Warnf logs a formatted message at warn level.
|
|
||||||
func Warnf(format string, args ...any) {
|
|
||||||
if isEnabled(WarnLevel) {
|
|
||||||
log.Warnf(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Warn logs a message at warn level.
|
|
||||||
func Warn(arg string) {
|
|
||||||
if isEnabled(WarnLevel) {
|
|
||||||
log.Warn(arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Infof logs a formatted message at info level.
|
|
||||||
func Infof(format string, args ...any) {
|
|
||||||
if isEnabled(InfoLevel) {
|
|
||||||
log.Infof(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Info logs a message at info level.
|
|
||||||
func Info(arg string) {
|
|
||||||
if isEnabled(InfoLevel) {
|
|
||||||
log.Info(arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verbosef logs a formatted message at verbose level.
|
|
||||||
func Verbosef(format string, args ...any) {
|
|
||||||
if isEnabled(VerboseLevel) {
|
|
||||||
log.Infof(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verbose logs a message at verbose level.
|
|
||||||
func Verbose(arg string) {
|
|
||||||
if isEnabled(VerboseLevel) {
|
|
||||||
log.Info(arg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Debugf logs a formatted message at debug level with caller location.
|
|
||||||
func Debugf(format string, args ...any) {
|
|
||||||
if isEnabled(DebugLevel) {
|
|
||||||
DebugReal(fmt.Sprintf(format, args...), callerSkip)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Debug logs a message at debug level with caller location.
|
|
||||||
func Debug(arg string) {
|
func Debug(arg string) {
|
||||||
if isEnabled(DebugLevel) {
|
DebugReal(arg, 2)
|
||||||
DebugReal(arg, callerSkip)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// DebugReal logs at debug level with caller info from the specified stack depth.
|
|
||||||
func DebugReal(arg string, cs int) {
|
func DebugReal(arg string, cs int) {
|
||||||
if !isEnabled(DebugLevel) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
_, callerFile, callerLine, ok := runtime.Caller(cs)
|
_, callerFile, callerLine, ok := runtime.Caller(cs)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
tag := fmt.Sprintf("%s:%d: ", callerFile, callerLine)
|
||||||
tag := fmt.Sprintf("%s:%d: ", filepath.Base(callerFile), callerLine)
|
|
||||||
log.Debug(tag + arg)
|
log.Debug(tag + arg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dump logs a spew dump of the arguments at debug level.
|
func Dump(args ...interface{}) {
|
||||||
func Dump(args ...any) {
|
DebugReal(spew.Sdump(args...), 2)
|
||||||
if isEnabled(DebugLevel) {
|
|
||||||
DebugReal(spew.Sdump(args...), callerSkip)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnableDebugLogging sets the log level to debug.
|
|
||||||
func EnableDebugLogging() {
|
func EnableDebugLogging() {
|
||||||
SetLevel(DebugLevel)
|
SetLevel(log.DebugLevel)
|
||||||
}
|
}
|
||||||
|
|
||||||
// VerbosityStepsToLogLevel converts a -v count to a log level.
|
func VerbosityStepsToLogLevel(l int) log.Level {
|
||||||
// 0 returns InfoLevel, 1 returns VerboseLevel, 2+ returns DebugLevel.
|
|
||||||
func VerbosityStepsToLogLevel(l int) Level {
|
|
||||||
switch l {
|
switch l {
|
||||||
case 0:
|
|
||||||
return InfoLevel
|
|
||||||
case 1:
|
case 1:
|
||||||
return VerboseLevel
|
return log.WarnLevel
|
||||||
default:
|
case 2:
|
||||||
return DebugLevel
|
return log.InfoLevel
|
||||||
|
case 3:
|
||||||
|
return log.DebugLevel
|
||||||
}
|
}
|
||||||
|
return log.ErrorLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetLevelFromVerbosity sets the log level based on -v flag count.
|
|
||||||
func SetLevelFromVerbosity(l int) {
|
func SetLevelFromVerbosity(l int) {
|
||||||
SetLevel(VerbosityStepsToLogLevel(l))
|
SetLevel(VerbosityStepsToLogLevel(l))
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetLevel sets the global log level.
|
func SetLevel(arg log.Level) {
|
||||||
func SetLevel(l Level) {
|
log.SetLevel(arg)
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
|
|
||||||
currentLevel = l
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetLevel returns the current log level.
|
func GetLogger() *log.Logger {
|
||||||
func GetLevel() Level {
|
if logger, ok := log.Log.(*log.Logger); ok {
|
||||||
mu.RLock()
|
return logger
|
||||||
defer mu.RUnlock()
|
}
|
||||||
|
panic("unable to get logger")
|
||||||
return currentLevel
|
}
|
||||||
|
|
||||||
|
func GetLevel() log.Level {
|
||||||
|
return GetLogger().Level
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithError returns a log entry with the error attached.
|
|
||||||
func WithError(e error) *log.Entry {
|
func WithError(e error) *log.Entry {
|
||||||
return log.Log.WithError(e)
|
return GetLogger().WithError(e)
|
||||||
}
|
|
||||||
|
|
||||||
// Progressf prints a progress message that overwrites the current line.
|
|
||||||
// Use ProgressDone() when progress is complete to move to the next line.
|
|
||||||
func Progressf(format string, args ...any) {
|
|
||||||
pterm.Printf("\r"+format, args...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProgressDone clears the progress line when progress is complete.
|
|
||||||
func ProgressDone() {
|
|
||||||
// Clear the line with spaces and return to beginning
|
|
||||||
pterm.Print("\r\033[K")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
package log_test
|
package log
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBuild(t *testing.T) {
|
func TestBuild(t *testing.T) {
|
||||||
t.Parallel()
|
Init()
|
||||||
log.Init()
|
assert.True(t, true)
|
||||||
}
|
}
|
||||||
|
|||||||
329
mfer/builder.go
329
mfer/builder.go
@@ -1,329 +0,0 @@
|
|||||||
// Package mfer implements the mfer manifest file format: building,
|
|
||||||
// serializing, verifying, and checking manifests of file trees.
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
)
|
|
||||||
|
|
||||||
// readChunkSize is the buffer size used when reading file contents for
|
|
||||||
// hashing.
|
|
||||||
const readChunkSize = 64 * 1024
|
|
||||||
|
|
||||||
// The errPath* sentinels below are worded as the trailing fragment of the
|
|
||||||
// message ValidatePath renders, because the offending path is quoted
|
|
||||||
// before them (`path %q ...`). Wrapping them mid-sentence keeps the
|
|
||||||
// rendered text exactly as mfer has always printed it. Match them with
|
|
||||||
// errors.Is rather than by reading their messages.
|
|
||||||
var (
|
|
||||||
errPathEmpty = errors.New("path cannot be empty")
|
|
||||||
errPathNotUTF8 = errors.New("is not valid UTF-8")
|
|
||||||
errPathBackslash = errors.New("contains backslash; use forward slashes only")
|
|
||||||
errPathAbsolute = errors.New("is absolute; must be relative")
|
|
||||||
errPathEmptySegment = errors.New("contains empty segment")
|
|
||||||
errPathDotDot = errors.New("contains '..' segment")
|
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
|
||||||
errNegativeSize = errors.New("size cannot be negative")
|
|
||||||
errEmptyHash = errors.New("hash cannot be nil or empty")
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidatePath checks that a file path conforms to manifest path invariants:
|
|
||||||
// - Must be valid UTF-8
|
|
||||||
// - Must use forward slashes only (no backslashes)
|
|
||||||
// - Must be relative (no leading /)
|
|
||||||
// - Must not contain ".." segments
|
|
||||||
// - Must not contain empty segments (no "//")
|
|
||||||
// - Must not be empty
|
|
||||||
func ValidatePath(p string) error {
|
|
||||||
if p == "" {
|
|
||||||
return errPathEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
if !utf8.ValidString(p) {
|
|
||||||
return fmt.Errorf("path %q %w", p, errPathNotUTF8)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.ContainsRune(p, '\\') {
|
|
||||||
return fmt.Errorf("path %q %w", p, errPathBackslash)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasPrefix(p, "/") {
|
|
||||||
return fmt.Errorf("path %q %w", p, errPathAbsolute)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, seg := range strings.Split(p, "/") {
|
|
||||||
if seg == "" {
|
|
||||||
return fmt.Errorf("path %q %w", p, errPathEmptySegment)
|
|
||||||
}
|
|
||||||
|
|
||||||
if seg == ".." {
|
|
||||||
return fmt.Errorf("path %q %w", p, errPathDotDot)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RelFilePath represents a relative file path within a manifest.
|
|
||||||
type RelFilePath string
|
|
||||||
|
|
||||||
// AbsFilePath represents an absolute file path on the filesystem.
|
|
||||||
type AbsFilePath string
|
|
||||||
|
|
||||||
// FileSize represents the size of a file in bytes.
|
|
||||||
type FileSize int64
|
|
||||||
|
|
||||||
// FileCount represents a count of files.
|
|
||||||
type FileCount int64
|
|
||||||
|
|
||||||
// ModTime represents a file's modification time.
|
|
||||||
type ModTime time.Time
|
|
||||||
|
|
||||||
// UnixSeconds represents seconds since Unix epoch.
|
|
||||||
type UnixSeconds int64
|
|
||||||
|
|
||||||
// UnixNanos represents the nanosecond component of a timestamp (0-999999999).
|
|
||||||
type UnixNanos int32
|
|
||||||
|
|
||||||
// Timestamp converts ModTime to a protobuf Timestamp.
|
|
||||||
func (m ModTime) Timestamp() *Timestamp {
|
|
||||||
return newTimestampFromTime(time.Time(m))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Multihash represents a multihash-encoded file hash (typically SHA2-256).
|
|
||||||
type Multihash []byte
|
|
||||||
|
|
||||||
// FileHashProgress reports progress during file hashing.
|
|
||||||
type FileHashProgress struct {
|
|
||||||
BytesRead FileSize // Total bytes read so far for the current file
|
|
||||||
}
|
|
||||||
|
|
||||||
// Builder constructs a manifest by adding files one at a time.
|
|
||||||
type Builder struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
files []*MFFilePath
|
|
||||||
createdAt time.Time
|
|
||||||
includeTimestamps bool
|
|
||||||
signingOptions *SigningOptions
|
|
||||||
fixedUUID []byte // if set, use this UUID instead of generating one
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewBuilder creates a new Builder.
|
|
||||||
func NewBuilder() *Builder {
|
|
||||||
return &Builder{
|
|
||||||
files: make([]*MFFilePath, 0),
|
|
||||||
createdAt: time.Now(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetSeed derives a deterministic UUID from the given seed string.
|
|
||||||
// The seed is hashed once with SHA-256 and the first 16 bytes are used
|
|
||||||
// as a fixed UUID for the manifest.
|
|
||||||
func (b *Builder) SetSeed(seed string) {
|
|
||||||
hash := sha256.Sum256([]byte(seed))
|
|
||||||
b.fixedUUID = hash[:uuidLength]
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddFile reads file content from reader, computes hashes, and adds to manifest.
|
|
||||||
// Progress updates are sent to the progress channel (if non-nil) without blocking.
|
|
||||||
// Returns the number of bytes read.
|
|
||||||
func (b *Builder) AddFile(
|
|
||||||
path RelFilePath,
|
|
||||||
size FileSize,
|
|
||||||
mtime ModTime,
|
|
||||||
reader io.Reader,
|
|
||||||
progress chan<- FileHashProgress,
|
|
||||||
) (FileSize, error) {
|
|
||||||
err := ValidatePath(string(path))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create hash writer
|
|
||||||
h := sha256.New()
|
|
||||||
|
|
||||||
// Read file in chunks, updating hash and progress
|
|
||||||
var totalRead FileSize
|
|
||||||
|
|
||||||
buf := make([]byte, readChunkSize)
|
|
||||||
|
|
||||||
for {
|
|
||||||
n, err := reader.Read(buf)
|
|
||||||
if n > 0 {
|
|
||||||
h.Write(buf[:n])
|
|
||||||
totalRead += FileSize(n)
|
|
||||||
sendFileHashProgress(progress, FileHashProgress{BytesRead: totalRead})
|
|
||||||
}
|
|
||||||
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return totalRead, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify actual bytes read matches declared size
|
|
||||||
if totalRead != size {
|
|
||||||
return totalRead, fmt.Errorf(
|
|
||||||
"%w for %q: declared %d bytes but read %d bytes",
|
|
||||||
errSizeMismatch, path, size, totalRead,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encode hash as multihash (SHA2-256)
|
|
||||||
mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
|
||||||
if err != nil {
|
|
||||||
return totalRead, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create file entry
|
|
||||||
entry := &MFFilePath{
|
|
||||||
Path: string(path),
|
|
||||||
Size: int64(size),
|
|
||||||
Hashes: []*MFFileChecksum{
|
|
||||||
{MultiHash: mh},
|
|
||||||
},
|
|
||||||
Mtime: mtime.Timestamp(),
|
|
||||||
}
|
|
||||||
|
|
||||||
b.mu.Lock()
|
|
||||||
b.files = append(b.files, entry)
|
|
||||||
b.mu.Unlock()
|
|
||||||
|
|
||||||
return totalRead, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendFileHashProgress sends a progress update without blocking.
|
|
||||||
func sendFileHashProgress(ch chan<- FileHashProgress, p FileHashProgress) {
|
|
||||||
if ch == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ch <- p:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileCount returns the number of files added to the builder.
|
|
||||||
func (b *Builder) FileCount() int {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
return len(b.files)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddFileWithHash adds a file entry with a pre-computed hash.
|
|
||||||
// This is useful when the hash is already known (e.g., from an existing manifest).
|
|
||||||
// Returns an error if path is empty, size is negative, or hash is nil/empty.
|
|
||||||
func (b *Builder) AddFileWithHash(
|
|
||||||
path RelFilePath,
|
|
||||||
size FileSize,
|
|
||||||
mtime ModTime,
|
|
||||||
hash Multihash,
|
|
||||||
) error {
|
|
||||||
err := ValidatePath(string(path))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("add file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if size < 0 {
|
|
||||||
return errNegativeSize
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(hash) == 0 {
|
|
||||||
return errEmptyHash
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := &MFFilePath{
|
|
||||||
Path: string(path),
|
|
||||||
Size: int64(size),
|
|
||||||
Hashes: []*MFFileChecksum{
|
|
||||||
{MultiHash: hash},
|
|
||||||
},
|
|
||||||
Mtime: mtime.Timestamp(),
|
|
||||||
}
|
|
||||||
|
|
||||||
b.mu.Lock()
|
|
||||||
b.files = append(b.files, entry)
|
|
||||||
b.mu.Unlock()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp.
|
|
||||||
// By default timestamps are omitted for deterministic output.
|
|
||||||
func (b *Builder) SetIncludeTimestamps(include bool) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.includeTimestamps = include
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetSigningOptions sets the GPG signing options for the manifest.
|
|
||||||
// If opts is non-nil, the manifest will be signed when Build() is called.
|
|
||||||
func (b *Builder) SetSigningOptions(opts *SigningOptions) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.signingOptions = opts
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build finalizes the manifest and writes it to the writer.
|
|
||||||
func (b *Builder) Build(w io.Writer) error {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
// Sort files by path for deterministic output
|
|
||||||
sort.Slice(b.files, func(i, j int) bool {
|
|
||||||
return b.files[i].GetPath() < b.files[j].GetPath()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Create inner manifest
|
|
||||||
inner := &MFFile{
|
|
||||||
Version: MFFile_VERSION_ONE,
|
|
||||||
Files: b.files,
|
|
||||||
}
|
|
||||||
if b.includeTimestamps {
|
|
||||||
inner.CreatedAt = newTimestampFromTime(b.createdAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create a temporary manifest to use existing serialization
|
|
||||||
m := &manifest{
|
|
||||||
pbInner: inner,
|
|
||||||
signingOptions: b.signingOptions,
|
|
||||||
fixedUUID: b.fixedUUID,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate outer wrapper
|
|
||||||
err := m.generateOuter()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("build: generate outer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate final output
|
|
||||||
err = m.generate()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("build: generate: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write to output
|
|
||||||
_, err = w.Write(m.output.Bytes())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("build: write output: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,487 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
const testFileName = "file.txt"
|
|
||||||
|
|
||||||
func TestNewBuilder(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
assert.NotNil(t, b)
|
|
||||||
assert.Equal(t, 0, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("test content")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
|
|
||||||
bytesRead, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, FileSize(len(content)), bytesRead)
|
|
||||||
assert.Equal(t, 1, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFileWithHash(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
hash := make([]byte, 34) // SHA256 multihash is 34 bytes
|
|
||||||
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 1, b.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFileWithHashValidation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
t.Run("empty path", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
hash := make([]byte, 34)
|
|
||||||
err := b.AddFileWithHash("", 100, ModTime(time.Now()), hash)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "path")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("negative size", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
hash := make([]byte, 34)
|
|
||||||
err := b.AddFileWithHash("test.txt", -1, ModTime(time.Now()), hash)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "size")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nil hash", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "hash")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("empty hash", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), []byte{})
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "hash")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid inputs", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
hash := make([]byte, 34)
|
|
||||||
err := b.AddFileWithHash("test.txt", 100, ModTime(time.Now()), hash)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 1, b.FileCount())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuild(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("test content")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
|
|
||||||
_, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Should have magic bytes
|
|
||||||
assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTimestampFromTimeExtremeDate(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Regression test: newTimestampFromTime used UnixNano() which panics
|
|
||||||
// for dates outside ~1678-2262. Now uses Nanosecond() which is safe.
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
time time.Time
|
|
||||||
}{
|
|
||||||
{"zero time", time.Time{}},
|
|
||||||
{"year 1000", time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC)},
|
|
||||||
{"year 3000", time.Date(3000, 1, 1, 0, 0, 0, 123456789, time.UTC)},
|
|
||||||
{"unix epoch", time.Unix(0, 0)},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Should not panic
|
|
||||||
ts := newTimestampFromTime(tt.time)
|
|
||||||
assert.Equal(t, tt.time.Unix(), ts.GetSeconds())
|
|
||||||
assert.Equal(t, tt.time.Nanosecond(), int(ts.GetNanos()))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderDeterministicOutput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
buildManifest := func() []byte {
|
|
||||||
b := NewBuilder()
|
|
||||||
// Use a fixed createdAt and UUID so output is reproducible
|
|
||||||
b.createdAt = time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
||||||
b.fixedUUID = make([]byte, 16) // all zeros
|
|
||||||
|
|
||||||
mtime := ModTime(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC))
|
|
||||||
|
|
||||||
// Add files in reverse order to test sorting
|
|
||||||
files := []struct {
|
|
||||||
path string
|
|
||||||
content string
|
|
||||||
}{
|
|
||||||
{"c/file.txt", "content c"},
|
|
||||||
{"a/file.txt", "content a"},
|
|
||||||
{"b/file.txt", "content b"},
|
|
||||||
}
|
|
||||||
for _, f := range files {
|
|
||||||
r := bytes.NewReader([]byte(f.content))
|
|
||||||
_, err := b.AddFile(
|
|
||||||
RelFilePath(f.path), FileSize(len(f.content)), mtime, r, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err := b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return buf.Bytes()
|
|
||||||
}
|
|
||||||
|
|
||||||
out1 := buildManifest()
|
|
||||||
out2 := buildManifest()
|
|
||||||
assert.Equal(t, out1, out2,
|
|
||||||
"two builds with same input should produce byte-identical output")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSetSeedDeterministic(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b1 := NewBuilder()
|
|
||||||
b1.SetSeed("test-seed-value")
|
|
||||||
|
|
||||||
b2 := NewBuilder()
|
|
||||||
b2.SetSeed("test-seed-value")
|
|
||||||
assert.Equal(t, b1.fixedUUID, b2.fixedUUID, "same seed should produce same UUID")
|
|
||||||
assert.Len(t, b1.fixedUUID, 16, "UUID should be 16 bytes")
|
|
||||||
|
|
||||||
b3 := NewBuilder()
|
|
||||||
b3.SetSeed("different-seed")
|
|
||||||
assert.NotEqual(t, b1.fixedUUID, b3.fixedUUID,
|
|
||||||
"different seeds should produce different UUIDs")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidatePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
valid := []string{
|
|
||||||
testFileName,
|
|
||||||
"dir/file.txt",
|
|
||||||
"a/b/c/d.txt",
|
|
||||||
"file with spaces.txt",
|
|
||||||
"日本語.txt", //nolint:gosmopolitan // deliberately tests non-ASCII UTF-8 paths
|
|
||||||
}
|
|
||||||
for _, p := range valid {
|
|
||||||
t.Run("valid:"+p, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.NoError(t, ValidatePath(p))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
invalid := []struct {
|
|
||||||
path string
|
|
||||||
desc string
|
|
||||||
}{
|
|
||||||
{"", "empty"},
|
|
||||||
{"/absolute", "absolute path"},
|
|
||||||
{"has\\backslash", "backslash"},
|
|
||||||
{"has/../traversal", "dot-dot segment"},
|
|
||||||
{"has//double", "empty segment"},
|
|
||||||
{"..", "just dot-dot"},
|
|
||||||
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
|
|
||||||
}
|
|
||||||
for _, tt := range invalid {
|
|
||||||
t.Run("invalid:"+tt.desc, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Error(t, ValidatePath(tt.path))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFileSizeMismatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("short")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
|
|
||||||
// Declare wrong size
|
|
||||||
_, err := b.AddFile("test.txt", FileSize(100), ModTime(time.Now()), reader, nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "size mismatch")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFileInvalidPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("data")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
|
|
||||||
_, err := b.AddFile("", FileSize(len(content)), ModTime(time.Now()), reader, nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
reader.Reset(content)
|
|
||||||
_, err = b.AddFile(
|
|
||||||
"/absolute", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
||||||
)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderAddFileWithProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := bytes.Repeat([]byte("x"), 1000)
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
progress := make(chan FileHashProgress, 100)
|
|
||||||
|
|
||||||
bytesRead, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, progress,
|
|
||||||
)
|
|
||||||
close(progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, FileSize(1000), bytesRead)
|
|
||||||
|
|
||||||
var updates []FileHashProgress
|
|
||||||
for p := range progress {
|
|
||||||
updates = append(updates, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotEmpty(t, updates)
|
|
||||||
// Last update should show all bytes
|
|
||||||
assert.Equal(t, FileSize(1000), updates[len(updates)-1].BytesRead)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuildRoundTrip(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Build a manifest, deserialize it, verify all fields survive round-trip
|
|
||||||
b := NewBuilder()
|
|
||||||
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
files := []struct {
|
|
||||||
path string
|
|
||||||
content []byte
|
|
||||||
}{
|
|
||||||
{"alpha.txt", []byte("alpha content")},
|
|
||||||
{"beta/gamma.txt", []byte("gamma content")},
|
|
||||||
{"beta/delta.txt", []byte("delta content")},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range files {
|
|
||||||
reader := bytes.NewReader(f.content)
|
|
||||||
_, err := b.AddFile(
|
|
||||||
RelFilePath(f.path), FileSize(len(f.content)), ModTime(now), reader, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(&buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
mfiles := m.Files()
|
|
||||||
require.Len(t, mfiles, 3)
|
|
||||||
|
|
||||||
// Verify sorted order
|
|
||||||
assert.Equal(t, "alpha.txt", mfiles[0].GetPath())
|
|
||||||
assert.Equal(t, "beta/delta.txt", mfiles[1].GetPath())
|
|
||||||
assert.Equal(t, "beta/gamma.txt", mfiles[2].GetPath())
|
|
||||||
|
|
||||||
// Verify sizes
|
|
||||||
assert.Equal(t, int64(len("alpha content")), mfiles[0].GetSize())
|
|
||||||
|
|
||||||
// Verify hashes are present
|
|
||||||
for _, f := range mfiles {
|
|
||||||
require.NotEmpty(t, f.GetHashes(), "file %s should have hashes", f.GetPath())
|
|
||||||
assert.NotEmpty(t, f.GetHashes()[0].GetMultiHash())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewManifestFromReaderInvalidMagic(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := NewManifestFromReader(bytes.NewReader([]byte("NOT_VALID")))
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "invalid file format")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewManifestFromReaderEmpty(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := NewManifestFromReader(bytes.NewReader([]byte{}))
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewManifestFromReaderTruncated(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Just the magic with nothing after
|
|
||||||
_, err := NewManifestFromReader(bytes.NewReader([]byte(MAGIC)))
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("test")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(&buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Contains(t, m.String(), "count=1")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderBuildEmpty(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err := b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Should still produce valid manifest with 0 files
|
|
||||||
assert.True(t, strings.HasPrefix(buf.String(), MAGIC))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderOmitsCreatedAtByDefault(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
content := []byte("hello")
|
|
||||||
_, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
|
||||||
bytes.NewReader(content), nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(&buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, m.pbInner.GetCreatedAt(),
|
|
||||||
"createdAt should be nil by default for deterministic output")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderIncludesCreatedAtWhenRequested(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetIncludeTimestamps(true)
|
|
||||||
|
|
||||||
content := []byte("hello")
|
|
||||||
_, err := b.AddFile(
|
|
||||||
"test.txt", FileSize(len(content)), ModTime(time.Now()),
|
|
||||||
bytes.NewReader(content), nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(&buf))
|
|
||||||
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotNil(t, m.pbInner.GetCreatedAt(),
|
|
||||||
"createdAt should be set when IncludeTimestamps is true")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderDeterministicFileOrder(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Two builds with same files in different order should produce same file ordering.
|
|
||||||
// Note: UUIDs differ per build, so we compare parsed file lists, not raw bytes.
|
|
||||||
buildAndParse := func(order []string) []*MFFilePath {
|
|
||||||
b := NewBuilder()
|
|
||||||
|
|
||||||
for _, name := range order {
|
|
||||||
content := []byte("content of " + name)
|
|
||||||
_, err := b.AddFile(
|
|
||||||
RelFilePath(name), FileSize(len(content)),
|
|
||||||
ModTime(time.Unix(1000, 0)), bytes.NewReader(content), nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, b.Build(&buf))
|
|
||||||
m, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return m.Files()
|
|
||||||
}
|
|
||||||
|
|
||||||
files1 := buildAndParse([]string{"b.txt", "a.txt"})
|
|
||||||
files2 := buildAndParse([]string{"a.txt", "b.txt"})
|
|
||||||
|
|
||||||
require.Len(t, files1, 2)
|
|
||||||
require.Len(t, files2, 2)
|
|
||||||
|
|
||||||
for i := range files1 {
|
|
||||||
assert.Equal(t, files1[i].GetPath(), files2[i].GetPath())
|
|
||||||
assert.Equal(t, files1[i].GetSize(), files2[i].GetSize())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, "a.txt", files1[0].GetPath())
|
|
||||||
assert.Equal(t, "b.txt", files1[1].GetPath())
|
|
||||||
}
|
|
||||||
390
mfer/checker.go
390
mfer/checker.go
@@ -1,390 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
)
|
|
||||||
|
|
||||||
var errNoSigningPubKey = errors.New("manifest has no signing public key")
|
|
||||||
|
|
||||||
// Result represents the outcome of checking a single file.
|
|
||||||
type Result struct {
|
|
||||||
Path RelFilePath // Relative path from manifest
|
|
||||||
Status Status // Verification result status
|
|
||||||
Message string // Human-readable description of the result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Status represents the verification status of a file.
|
|
||||||
type Status int
|
|
||||||
|
|
||||||
// Verification result statuses reported for each checked file.
|
|
||||||
const (
|
|
||||||
StatusOK Status = iota // File matches manifest (size and hash verified)
|
|
||||||
StatusMissing // File not found on disk
|
|
||||||
StatusSizeMismatch // File size differs from manifest
|
|
||||||
StatusHashMismatch // File hash differs from manifest
|
|
||||||
StatusExtra // File exists on disk but not in manifest
|
|
||||||
StatusError // Error occurred during verification
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s Status) String() string {
|
|
||||||
switch s {
|
|
||||||
case StatusOK:
|
|
||||||
return "OK"
|
|
||||||
case StatusMissing:
|
|
||||||
return "MISSING"
|
|
||||||
case StatusSizeMismatch:
|
|
||||||
return "SIZE_MISMATCH"
|
|
||||||
case StatusHashMismatch:
|
|
||||||
return "HASH_MISMATCH"
|
|
||||||
case StatusExtra:
|
|
||||||
return "EXTRA"
|
|
||||||
case StatusError:
|
|
||||||
return "ERROR"
|
|
||||||
default:
|
|
||||||
return "UNKNOWN"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CheckStatus contains progress information for the check operation.
|
|
||||||
type CheckStatus struct {
|
|
||||||
TotalFiles FileCount // Total number of files in manifest
|
|
||||||
CheckedFiles FileCount // Number of files checked so far
|
|
||||||
TotalBytes FileSize // Total bytes to verify (sum of all file sizes)
|
|
||||||
CheckedBytes FileSize // Bytes verified so far
|
|
||||||
BytesPerSec float64 // Current throughput rate
|
|
||||||
ETA time.Duration // Estimated time to completion
|
|
||||||
Failures FileCount // Number of verification failures encountered
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checker verifies files against a manifest.
|
|
||||||
type Checker struct {
|
|
||||||
basePath AbsFilePath
|
|
||||||
files []*MFFilePath
|
|
||||||
fs afero.Fs
|
|
||||||
// manifestPaths is a set of paths in the manifest for quick lookup
|
|
||||||
manifestPaths map[RelFilePath]struct{}
|
|
||||||
// manifestRelPath is the relative path of the manifest file from
|
|
||||||
// basePath (for exclusion)
|
|
||||||
manifestRelPath RelFilePath
|
|
||||||
// signature info from the manifest
|
|
||||||
signature []byte
|
|
||||||
signer []byte
|
|
||||||
signingPubKey []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
|
|
||||||
// The basePath is the directory relative to which manifest paths are resolved.
|
|
||||||
// If fs is nil, the real filesystem (OsFs) is used.
|
|
||||||
func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, error) {
|
|
||||||
if fs == nil {
|
|
||||||
fs = afero.NewOsFs()
|
|
||||||
}
|
|
||||||
|
|
||||||
m, err := NewManifestFromFile(fs, manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
abs, err := filepath.Abs(basePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
files := m.Files()
|
|
||||||
|
|
||||||
manifestPaths := make(map[RelFilePath]struct{}, len(files))
|
|
||||||
for _, f := range files {
|
|
||||||
manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute manifest's relative path from basePath for exclusion in FindExtraFiles
|
|
||||||
absManifest, err := filepath.Abs(manifestPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
manifestRel, err := filepath.Rel(abs, absManifest)
|
|
||||||
if err != nil {
|
|
||||||
manifestRel = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Checker{
|
|
||||||
basePath: AbsFilePath(abs),
|
|
||||||
files: files,
|
|
||||||
fs: fs,
|
|
||||||
manifestPaths: manifestPaths,
|
|
||||||
manifestRelPath: RelFilePath(manifestRel),
|
|
||||||
signature: m.pbOuter.GetSignature(),
|
|
||||||
signer: m.pbOuter.GetSigner(),
|
|
||||||
signingPubKey: m.pbOuter.GetSigningPubKey(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileCount returns the number of files in the manifest.
|
|
||||||
func (c *Checker) FileCount() FileCount {
|
|
||||||
return FileCount(len(c.files))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TotalBytes returns the total size of all files in the manifest.
|
|
||||||
func (c *Checker) TotalBytes() FileSize {
|
|
||||||
var total FileSize
|
|
||||||
for _, f := range c.files {
|
|
||||||
total += FileSize(f.GetSize())
|
|
||||||
}
|
|
||||||
|
|
||||||
return total
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsSigned returns true if the manifest has a signature.
|
|
||||||
func (c *Checker) IsSigned() bool {
|
|
||||||
return len(c.signature) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Signer returns the signer fingerprint if the manifest is signed, nil otherwise.
|
|
||||||
func (c *Checker) Signer() []byte {
|
|
||||||
return c.signer
|
|
||||||
}
|
|
||||||
|
|
||||||
// SigningPubKey returns the signing public key if the manifest is signed,
|
|
||||||
// nil otherwise.
|
|
||||||
func (c *Checker) SigningPubKey() []byte {
|
|
||||||
return c.signingPubKey
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExtractEmbeddedSigningKeyFP imports the manifest's embedded public key into a
|
|
||||||
// temporary keyring and extracts its fingerprint. This validates the key and
|
|
||||||
// returns its actual fingerprint from the key material itself.
|
|
||||||
func (c *Checker) ExtractEmbeddedSigningKeyFP() (string, error) {
|
|
||||||
if len(c.signingPubKey) == 0 {
|
|
||||||
return "", errNoSigningPubKey
|
|
||||||
}
|
|
||||||
|
|
||||||
return gpgExtractPubKeyFingerprint(c.signingPubKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check verifies all files against the manifest.
|
|
||||||
// Results are sent to the results channel as files are checked.
|
|
||||||
// Progress updates are sent to the progress channel approximately once per second.
|
|
||||||
// Both channels are closed when the method returns.
|
|
||||||
func (c *Checker) Check(
|
|
||||||
ctx context.Context,
|
|
||||||
results chan<- Result,
|
|
||||||
progress chan<- CheckStatus,
|
|
||||||
) error {
|
|
||||||
if results != nil {
|
|
||||||
defer close(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
if progress != nil {
|
|
||||||
defer close(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
totalFiles := FileCount(len(c.files))
|
|
||||||
totalBytes := c.TotalBytes()
|
|
||||||
|
|
||||||
var (
|
|
||||||
checkedFiles FileCount
|
|
||||||
checkedBytes FileSize
|
|
||||||
failures FileCount
|
|
||||||
)
|
|
||||||
|
|
||||||
startTime := time.Now()
|
|
||||||
lastProgressTime := time.Now()
|
|
||||||
|
|
||||||
for _, entry := range c.files {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
result := c.checkFile(entry, &checkedBytes)
|
|
||||||
if result.Status != StatusOK {
|
|
||||||
failures++
|
|
||||||
}
|
|
||||||
|
|
||||||
checkedFiles++
|
|
||||||
|
|
||||||
if results != nil {
|
|
||||||
results <- result
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send progress at most once per second (rate-limited)
|
|
||||||
if progress != nil {
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
isLast := checkedFiles == totalFiles
|
|
||||||
if isLast || now.Sub(lastProgressTime) >= time.Second {
|
|
||||||
bytesPerSec, eta := computeRateETA(
|
|
||||||
time.Since(startTime), checkedBytes, totalBytes,
|
|
||||||
)
|
|
||||||
|
|
||||||
sendCheckStatus(progress, CheckStatus{
|
|
||||||
TotalFiles: totalFiles,
|
|
||||||
CheckedFiles: checkedFiles,
|
|
||||||
TotalBytes: totalBytes,
|
|
||||||
CheckedBytes: checkedBytes,
|
|
||||||
BytesPerSec: bytesPerSec,
|
|
||||||
ETA: eta,
|
|
||||||
Failures: failures,
|
|
||||||
})
|
|
||||||
|
|
||||||
lastProgressTime = now
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FindExtraFiles walks the filesystem and reports files not in the manifest.
|
|
||||||
// Results are sent to the results channel. The channel is closed when done.
|
|
||||||
// Hidden files/directories (starting with .) are skipped, as they are excluded
|
|
||||||
// from manifests by default. The manifest file itself is also skipped.
|
|
||||||
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
|
|
||||||
if results != nil {
|
|
||||||
defer close(results)
|
|
||||||
}
|
|
||||||
|
|
||||||
walkFn := func(walkPath string, info os.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get relative path
|
|
||||||
rel, err := filepath.Rel(string(c.basePath), walkPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip hidden files and directories (dotfiles)
|
|
||||||
if IsHiddenPath(filepath.ToSlash(rel)) {
|
|
||||||
if info.IsDir() {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Skip directories
|
|
||||||
if info.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath := RelFilePath(rel)
|
|
||||||
|
|
||||||
// Skip the manifest file itself
|
|
||||||
if relPath == c.manifestRelPath {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if path is in manifest
|
|
||||||
if _, exists := c.manifestPaths[relPath]; !exists {
|
|
||||||
if results != nil {
|
|
||||||
results <- Result{
|
|
||||||
Path: relPath,
|
|
||||||
Status: StatusExtra,
|
|
||||||
Message: "not in manifest",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return afero.Walk(c.fs, string(c.basePath), walkFn)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
|
|
||||||
absPath := filepath.Join(string(c.basePath), entry.GetPath())
|
|
||||||
relPath := RelFilePath(entry.GetPath())
|
|
||||||
|
|
||||||
// Check if file exists
|
|
||||||
info, err := c.fs.Stat(absPath)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) || errors.Is(err, afero.ErrFileNotFound) {
|
|
||||||
return Result{
|
|
||||||
Path: relPath,
|
|
||||||
Status: StatusMissing,
|
|
||||||
Message: "file not found",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check size
|
|
||||||
if info.Size() != entry.GetSize() {
|
|
||||||
*checkedBytes += FileSize(info.Size())
|
|
||||||
|
|
||||||
return Result{
|
|
||||||
Path: relPath,
|
|
||||||
Status: StatusSizeMismatch,
|
|
||||||
Message: "size mismatch",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open and hash file
|
|
||||||
f, err := c.fs.Open(absPath)
|
|
||||||
if err != nil {
|
|
||||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
h := sha256.New()
|
|
||||||
|
|
||||||
n, err := io.Copy(h, f)
|
|
||||||
if err != nil {
|
|
||||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
*checkedBytes += FileSize(n)
|
|
||||||
|
|
||||||
// Encode as multihash and compare
|
|
||||||
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
|
||||||
if err != nil {
|
|
||||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check against all hashes in manifest (at least one must match)
|
|
||||||
for _, hash := range entry.GetHashes() {
|
|
||||||
if bytes.Equal(computed, hash.GetMultiHash()) {
|
|
||||||
return Result{Path: relPath, Status: StatusOK}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Result{
|
|
||||||
Path: relPath,
|
|
||||||
Status: StatusHashMismatch,
|
|
||||||
Message: "hash mismatch",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendCheckStatus sends a status update without blocking.
|
|
||||||
func sendCheckStatus(ch chan<- CheckStatus, status CheckStatus) {
|
|
||||||
if ch == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ch <- status:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,649 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
testFile1 = "file1.txt"
|
|
||||||
testFile2 = "file2.txt"
|
|
||||||
testExistsFile = "exists.txt"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStatusString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
status Status
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{StatusOK, "OK"},
|
|
||||||
{StatusMissing, "MISSING"},
|
|
||||||
{StatusSizeMismatch, "SIZE_MISMATCH"},
|
|
||||||
{StatusHashMismatch, "HASH_MISMATCH"},
|
|
||||||
{StatusExtra, "EXTRA"},
|
|
||||||
{StatusError, "ERROR"},
|
|
||||||
{Status(99), "UNKNOWN"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.expected, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(t, tt.expected, tt.status.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// createTestManifest creates a manifest file in the filesystem with the given files.
|
|
||||||
func createTestManifest(
|
|
||||||
t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
builder := NewBuilder()
|
|
||||||
|
|
||||||
for path, content := range files {
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := builder.AddFile(
|
|
||||||
RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, builder.Build(&buf))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
// createFilesOnDisk creates the given files on the filesystem under
|
|
||||||
// /data.
|
|
||||||
func createFilesOnDisk(t *testing.T, fs afero.Fs, files map[string][]byte) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
basePath := "/data"
|
|
||||||
|
|
||||||
for path, content := range files {
|
|
||||||
fullPath := basePath + "/" + path
|
|
||||||
require.NoError(t, fs.MkdirAll(basePath, 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewChecker(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
t.Run("valid manifest", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFile1: []byte("hello"),
|
|
||||||
testFile2: []byte("world"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotNil(t, chk)
|
|
||||||
assert.Equal(t, FileCount(2), chk.FileCount())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing manifest", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
_, err := NewChecker("/nonexistent.mf", "/", fs)
|
|
||||||
assert.Error(t, err)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("invalid manifest", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
|
||||||
_, err := NewChecker("/bad.mf", "/", fs)
|
|
||||||
assert.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
"small.txt": []byte("hi"),
|
|
||||||
"medium.txt": []byte("hello world"),
|
|
||||||
"large.txt": bytes.Repeat([]byte("x"), 1000),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(3), chk.FileCount())
|
|
||||||
assert.Equal(t, FileSize(2+11+1000), chk.TotalBytes())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckAllFilesOK(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFile1: []byte("content one"),
|
|
||||||
testFile2: []byte("content two"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var resultList []Result
|
|
||||||
for r := range results {
|
|
||||||
resultList = append(resultList, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Len(t, resultList, 2)
|
|
||||||
|
|
||||||
for _, r := range resultList {
|
|
||||||
assert.Equal(t, StatusOK, r.Status, "file %s should be OK", r.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMissingFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testExistsFile: []byte("I exist"),
|
|
||||||
"missing.txt": []byte("I don't exist on disk"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
// Only create one file
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testExistsFile: []byte("I exist"),
|
|
||||||
})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var okCount, missingCount int
|
|
||||||
|
|
||||||
for r := range results {
|
|
||||||
switch r.Status {
|
|
||||||
case StatusOK:
|
|
||||||
okCount++
|
|
||||||
case StatusMissing:
|
|
||||||
missingCount++
|
|
||||||
|
|
||||||
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
|
|
||||||
case StatusSizeMismatch, StatusHashMismatch, StatusExtra, StatusError:
|
|
||||||
// Not expected in this test; counted assertions below will fail.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, 1, okCount)
|
|
||||||
assert.Equal(t, 1, missingCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckSizeMismatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFileName: []byte("original content"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
// Create file with different size
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testFileName: []byte("short"),
|
|
||||||
})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
r := <-results
|
|
||||||
assert.Equal(t, StatusSizeMismatch, r.Status)
|
|
||||||
assert.Equal(t, RelFilePath(testFileName), r.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckHashMismatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
originalContent := []byte("original content")
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFileName: originalContent,
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
// Create file with same size but different content
|
|
||||||
differentContent := []byte("different contnt") // same length (16 bytes) but different
|
|
||||||
require.Len(t, differentContent, len(originalContent), "test requires same length")
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testFileName: differentContent,
|
|
||||||
})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
r := <-results
|
|
||||||
assert.Equal(t, StatusHashMismatch, r.Status)
|
|
||||||
assert.Equal(t, RelFilePath(testFileName), r.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckWithProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFile1: bytes.Repeat([]byte("a"), 100),
|
|
||||||
testFile2: bytes.Repeat([]byte("b"), 200),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
progress := make(chan CheckStatus, 10)
|
|
||||||
|
|
||||||
err = chk.Check(context.Background(), results, progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// results is fully buffered and closed; no draining needed
|
|
||||||
|
|
||||||
// Check progress was sent
|
|
||||||
var progressUpdates []CheckStatus
|
|
||||||
for p := range progress {
|
|
||||||
progressUpdates = append(progressUpdates, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotEmpty(t, progressUpdates)
|
|
||||||
// Final progress should show all files checked
|
|
||||||
final := progressUpdates[len(progressUpdates)-1]
|
|
||||||
assert.Equal(t, FileCount(2), final.TotalFiles)
|
|
||||||
assert.Equal(t, FileCount(2), final.CheckedFiles)
|
|
||||||
assert.Equal(t, FileSize(300), final.TotalBytes)
|
|
||||||
assert.Equal(t, FileSize(300), final.CheckedBytes)
|
|
||||||
assert.Equal(t, FileCount(0), final.Failures)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckContextCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
// Create many files to ensure we have time to cancel
|
|
||||||
files := make(map[string][]byte)
|
|
||||||
for i := range 100 {
|
|
||||||
files[string(rune('a'+i%26))+".txt"] = bytes.Repeat([]byte("x"), 1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel() // Cancel immediately
|
|
||||||
|
|
||||||
results := make(chan Result, 200)
|
|
||||||
err = chk.Check(ctx, results, nil)
|
|
||||||
assert.ErrorIs(t, err, context.Canceled)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
// Manifest only contains file1
|
|
||||||
manifestFiles := map[string][]byte{
|
|
||||||
testFile1: []byte("in manifest"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", manifestFiles)
|
|
||||||
|
|
||||||
// Disk has file1 and file2
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testFile1: []byte("in manifest"),
|
|
||||||
testFile2: []byte("extra file"),
|
|
||||||
})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.FindExtraFiles(context.Background(), results)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var extras []Result
|
|
||||||
for r := range results {
|
|
||||||
extras = append(extras, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Len(t, extras, 1)
|
|
||||||
assert.Equal(t, RelFilePath(testFile2), extras[0].Path)
|
|
||||||
assert.Equal(t, StatusExtra, extras[0].Status)
|
|
||||||
assert.Equal(t, "not in manifest", extras[0].Message)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFilesSkipsManifestAndDotfiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
manifestFiles := map[string][]byte{
|
|
||||||
testFile1: []byte("in manifest"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/data/.index.mf", manifestFiles)
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testFile1: []byte("in manifest"),
|
|
||||||
})
|
|
||||||
// Create dotfile and manifest that should be skipped
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("hidden"), 0o644))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/.config/settings", []byte("cfg"), 0o644))
|
|
||||||
// Create a real extra file
|
|
||||||
require.NoError(t, fs.MkdirAll("/data", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/extra.txt", []byte("extra"), 0o644))
|
|
||||||
|
|
||||||
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.FindExtraFiles(context.Background(), results)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var extras []Result
|
|
||||||
for r := range results {
|
|
||||||
extras = append(extras, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Should only report extra.txt, not .hidden, .config/settings, or .index.mf
|
|
||||||
for _, e := range extras {
|
|
||||||
t.Logf("extra: %s", e.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Len(t, extras, 1)
|
|
||||||
|
|
||||||
if len(extras) > 0 {
|
|
||||||
assert.Equal(t, RelFilePath("extra.txt"), extras[0].Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFilesContextCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel() // Cancel immediately
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.FindExtraFiles(ctx, results)
|
|
||||||
assert.ErrorIs(t, err, context.Canceled)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckNilChannels(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Should not panic with nil channels
|
|
||||||
err = chk.Check(context.Background(), nil, nil)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFilesNilChannel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{testFileName: []byte("data")}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Should not panic with nil channel
|
|
||||||
err = chk.FindExtraFiles(context.Background(), nil)
|
|
||||||
assert.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckSubdirectories(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
"dir1/file1.txt": []byte("content1"),
|
|
||||||
"dir1/dir2/file2.txt": []byte("content2"),
|
|
||||||
"dir1/dir2/dir3/deep.txt": []byte("deep content"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
|
|
||||||
// Create files with full directory structure
|
|
||||||
for path, content := range files {
|
|
||||||
fullPath := "/data/" + path
|
|
||||||
|
|
||||||
require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var okCount int
|
|
||||||
|
|
||||||
for r := range results {
|
|
||||||
assert.Equal(t, StatusOK, r.Status, "file %s should be OK", r.Path)
|
|
||||||
|
|
||||||
okCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, 3, okCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckMissingFileDetectedWithoutFallback(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Regression test: errors.Is(err, errors.New("...")) never matches because
|
|
||||||
// errors.New creates a new value each time. The fix uses os.ErrNotExist instead.
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testExistsFile: []byte("here"),
|
|
||||||
"missing.txt": []byte("not on disk"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
// Only create one file on disk
|
|
||||||
createFilesOnDisk(t, fs, map[string][]byte{
|
|
||||||
testExistsFile: []byte("here"),
|
|
||||||
})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
statusCounts := map[Status]int{}
|
|
||||||
for r := range results {
|
|
||||||
statusCounts[r.Status]++
|
|
||||||
if r.Status == StatusMissing {
|
|
||||||
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, 1, statusCounts[StatusOK], "one file should be OK")
|
|
||||||
assert.Equal(t, 1, statusCounts[StatusMissing], "one file should be MISSING")
|
|
||||||
assert.Equal(t, 0, statusCounts[StatusError], "no files should be ERROR")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFilesSkipsDotfiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Regression test for #16: FindExtraFiles should not report dotfiles
|
|
||||||
// or the manifest file itself as extra files.
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFile1: []byte("in manifest"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/data/.index.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
// Add dotfiles and manifest file on disk
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/data/.hidden", []byte("dotfile"), 0o644))
|
|
||||||
require.NoError(t, fs.MkdirAll("/data/.git", 0o755))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/data/.git/config", []byte("git config"), 0o644))
|
|
||||||
|
|
||||||
chk, err := NewChecker("/data/.index.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.FindExtraFiles(context.Background(), results)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var extras []Result
|
|
||||||
for r := range results {
|
|
||||||
extras = append(extras, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Should report NO extra files — dotfiles and manifest should be skipped
|
|
||||||
assert.Empty(t, extras,
|
|
||||||
"FindExtraFiles should not report dotfiles or manifest file as extra; got: %v",
|
|
||||||
extras)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindExtraFilesSkipsManifestFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The manifest file itself should never be reported as extra
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := map[string][]byte{
|
|
||||||
testFile1: []byte("content"),
|
|
||||||
}
|
|
||||||
createTestManifest(t, fs, "/data/index.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/data/index.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.FindExtraFiles(context.Background(), results)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var extras []Result
|
|
||||||
for r := range results {
|
|
||||||
extras = append(extras, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Empty(t, extras,
|
|
||||||
"manifest file should not be reported as extra; got: %v", extras)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckEmptyManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
// Create manifest with no files
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", map[string][]byte{})
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(0), chk.FileCount())
|
|
||||||
assert.Equal(t, FileSize(0), chk.TotalBytes())
|
|
||||||
|
|
||||||
results := make(chan Result, 10)
|
|
||||||
err = chk.Check(context.Background(), results, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var count int
|
|
||||||
for range results {
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, 0, count)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckProgressRateLimited(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Create many small files - progress should be rate-limited, not one per file.
|
|
||||||
// With rate-limiting to once per second, we should get far fewer progress
|
|
||||||
// updates than files (plus one final update).
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
files := make(map[string][]byte, 100)
|
|
||||||
|
|
||||||
for i := range 100 {
|
|
||||||
name := fmt.Sprintf("file%03d.txt", i)
|
|
||||||
files[name] = []byte("content")
|
|
||||||
}
|
|
||||||
|
|
||||||
createTestManifest(t, fs, "/manifest.mf", files)
|
|
||||||
createFilesOnDisk(t, fs, files)
|
|
||||||
|
|
||||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
results := make(chan Result, 200)
|
|
||||||
progress := make(chan CheckStatus, 200)
|
|
||||||
err = chk.Check(context.Background(), results, progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// results is fully buffered and closed; no draining needed
|
|
||||||
|
|
||||||
// Count progress updates
|
|
||||||
var progressCount int
|
|
||||||
for range progress {
|
|
||||||
progressCount++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Should be far fewer than 100 (rate-limited to once per second)
|
|
||||||
// At minimum we get the final update
|
|
||||||
assert.GreaterOrEqual(t, progressCount, 1,
|
|
||||||
"should get at least the final progress update")
|
|
||||||
assert.Less(t, progressCount, 100,
|
|
||||||
"progress should be rate-limited, not one per file")
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
const (
|
|
||||||
// Version is the current mfer release version.
|
|
||||||
Version = "0.1.0"
|
|
||||||
|
|
||||||
// ReleaseDate is the date on which Version was released.
|
|
||||||
ReleaseDate = "2025-12-17"
|
|
||||||
|
|
||||||
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
|
||||||
// data (256 MB). This prevents decompression bombs from consuming excessive
|
|
||||||
// memory.
|
|
||||||
MaxDecompressedSize int64 = 256 * 1024 * 1024
|
|
||||||
|
|
||||||
// uuidLength is the length in bytes of a binary UUID.
|
|
||||||
uuidLength = 16
|
|
||||||
)
|
|
||||||
@@ -2,187 +2,45 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"crypto/sha256"
|
"compress/gzip"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"git.eeqj.de/sneak/mfer/internal/bork"
|
||||||
"github.com/klauspost/compress/zstd"
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
"github.com/spf13/afero"
|
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
"sneak.berlin/go/mfer/internal/bork"
|
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
func (m *manifest) validateProtoOuter() error {
|
||||||
errInvalidUUIDLength = errors.New("invalid UUID length")
|
if m.pbOuter.Version != MFFileOuter_VERSION_ONE {
|
||||||
errInvalidUUIDFormat = errors.New("invalid UUID format")
|
return errors.New("unknown version")
|
||||||
errUnknownVersion = errors.New("unknown version")
|
|
||||||
errUnknownCompression = errors.New("unknown compression type")
|
|
||||||
errCompressedHashWrong = errors.New("compressed data hash mismatch")
|
|
||||||
errSignatureNoPubKey = errors.New("signature present but no public key")
|
|
||||||
errDecompressedTooLarge = errors.New("decompressed data exceeds maximum allowed size")
|
|
||||||
errUUIDMismatch = errors.New("outer and inner UUID mismatch")
|
|
||||||
errInvalidFileFormat = errors.New("invalid file format")
|
|
||||||
)
|
|
||||||
|
|
||||||
// validateUUID checks that the byte slice is a valid UUID (16 bytes, parseable).
|
|
||||||
func validateUUID(data []byte) error {
|
|
||||||
if len(data) != uuidLength {
|
|
||||||
return errInvalidUUIDLength
|
|
||||||
}
|
}
|
||||||
// Try to parse as UUID to validate format
|
if m.pbOuter.CompressionType != MFFileOuter_COMPRESSION_GZIP {
|
||||||
_, err := uuid.FromBytes(data)
|
return errors.New("unknown compression type")
|
||||||
if err != nil {
|
|
||||||
return errInvalidUUIDFormat
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
bb := bytes.NewBuffer(m.pbOuter.InnerMessage)
|
||||||
}
|
|
||||||
|
|
||||||
// validateOuterHeader checks the outer message's version, compression
|
gzr, err := gzip.NewReader(bb)
|
||||||
// type, and UUID.
|
|
||||||
func (m *manifest) validateOuterHeader() error {
|
|
||||||
if m.pbOuter.GetVersion() != MFFileOuter_VERSION_ONE {
|
|
||||||
return errUnknownVersion
|
|
||||||
}
|
|
||||||
|
|
||||||
if m.pbOuter.GetCompressionType() != MFFileOuter_COMPRESSION_ZSTD {
|
|
||||||
return errUnknownCompression
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate outer UUID before any decompression
|
|
||||||
err := validateUUID(m.pbOuter.GetUuid())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("outer UUID invalid: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifyOuterIntegrity checks the hash of the compressed payload and,
|
|
||||||
// if a signature is present, verifies it against the embedded public key.
|
|
||||||
func (m *manifest) verifyOuterIntegrity() error {
|
|
||||||
h := sha256.New()
|
|
||||||
|
|
||||||
_, err := h.Write(m.pbOuter.GetInnerMessage())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deserialize: hash write: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sha256Hash := h.Sum(nil)
|
|
||||||
if !bytes.Equal(sha256Hash, m.pbOuter.GetSha256()) {
|
|
||||||
return errCompressedHashWrong
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(m.pbOuter.GetSignature()) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(m.pbOuter.GetSigningPubKey()) == 0 {
|
|
||||||
return errSignatureNoPubKey
|
|
||||||
}
|
|
||||||
|
|
||||||
sigString, err := m.signatureString()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"failed to generate signature string for verification: %w", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = gpgVerify(
|
|
||||||
[]byte(sigString),
|
|
||||||
m.pbOuter.GetSignature(),
|
|
||||||
m.pbOuter.GetSigningPubKey(),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("signature verification failed: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("signature verified successfully")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// decompressInner decompresses the inner payload, enforcing size limits
|
|
||||||
// to prevent decompression bombs.
|
|
||||||
func (m *manifest) decompressInner() ([]byte, error) {
|
|
||||||
bb := bytes.NewBuffer(m.pbOuter.GetInnerMessage())
|
|
||||||
|
|
||||||
zr, err := zstd.NewReader(bb)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("deserialize: zstd reader: %w", err)
|
|
||||||
}
|
|
||||||
defer zr.Close()
|
|
||||||
|
|
||||||
// Limit decompressed size to prevent decompression bombs.
|
|
||||||
// Use declared size + 1 byte to detect overflow, capped at MaxDecompressedSize.
|
|
||||||
maxSize := MaxDecompressedSize
|
|
||||||
if m.pbOuter.GetSize() > 0 && m.pbOuter.GetSize() < maxSize {
|
|
||||||
maxSize = m.pbOuter.GetSize() + 1
|
|
||||||
}
|
|
||||||
|
|
||||||
limitedReader := io.LimitReader(zr, maxSize)
|
|
||||||
|
|
||||||
dat, err := io.ReadAll(limitedReader)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("deserialize: decompress: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if int64(len(dat)) >= MaxDecompressedSize {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%w of %d bytes", errDecompressedTooLarge, MaxDecompressedSize,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return dat, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *manifest) deserializeInner() error {
|
|
||||||
err := m.validateOuterHeader()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = m.verifyOuterIntegrity()
|
dat, err := io.ReadAll(gzr)
|
||||||
if err != nil {
|
defer gzr.Close()
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dat, err := m.decompressInner()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
isize := len(dat)
|
isize := len(dat)
|
||||||
if int64(isize) != m.pbOuter.GetSize() {
|
if int64(isize) != m.pbOuter.Size {
|
||||||
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.GetSize())
|
log.Debugf("truncated data, got %d expected %d", isize, m.pbOuter.Size)
|
||||||
|
|
||||||
return bork.ErrFileTruncated
|
return bork.ErrFileTruncated
|
||||||
}
|
}
|
||||||
|
log.Debugf("inner data size is %d", isize)
|
||||||
// Deserialize inner message
|
log.Dump(dat)
|
||||||
m.pbInner = new(MFFile)
|
log.Dump(m.pbOuter.Sha256)
|
||||||
|
|
||||||
err = proto.Unmarshal(dat, m.pbInner)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("deserialize: unmarshal inner: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate inner UUID
|
|
||||||
err = validateUUID(m.pbInner.GetUuid())
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("inner UUID invalid: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify UUIDs match
|
|
||||||
if !bytes.Equal(m.pbOuter.GetUuid(), m.pbInner.GetUuid()) {
|
|
||||||
return errUUIDMismatch
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Infof("loaded manifest with %d files", len(m.pbInner.GetFiles()))
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -191,26 +49,19 @@ func validateMagic(dat []byte) bool {
|
|||||||
if len(dat) < ml {
|
if len(dat) < ml {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
got := dat[0:ml]
|
got := dat[0:ml]
|
||||||
expected := []byte(MAGIC)
|
expected := []byte(MAGIC)
|
||||||
|
|
||||||
return bytes.Equal(got, expected)
|
return bytes.Equal(got, expected)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManifestFromReader reads a manifest from an io.Reader.
|
func NewFromProto(input io.Reader) (*manifest, error) {
|
||||||
//
|
m := New()
|
||||||
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
|
||||||
func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|
||||||
m := &manifest{}
|
|
||||||
|
|
||||||
dat, err := io.ReadAll(input)
|
dat, err := io.ReadAll(input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !validateMagic(dat) {
|
if !validateMagic(dat) {
|
||||||
return nil, errInvalidFileFormat
|
return nil, errors.New("invalid file format")
|
||||||
}
|
}
|
||||||
|
|
||||||
// remove magic bytes prefix:
|
// remove magic bytes prefix:
|
||||||
@@ -218,38 +69,21 @@ func NewManifestFromReader(input io.Reader) (*manifest, error) {
|
|||||||
bb := bytes.NewBuffer(dat[ml:])
|
bb := bytes.NewBuffer(dat[ml:])
|
||||||
dat = bb.Bytes()
|
dat = bb.Bytes()
|
||||||
|
|
||||||
// deserialize outer:
|
log.Dump(dat)
|
||||||
|
|
||||||
|
// deserialize:
|
||||||
m.pbOuter = new(MFFileOuter)
|
m.pbOuter = new(MFFileOuter)
|
||||||
|
|
||||||
err = proto.Unmarshal(dat, m.pbOuter)
|
err = proto.Unmarshal(dat, m.pbOuter)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// deserialize inner:
|
ve := m.validateProtoOuter()
|
||||||
err = m.deserializeInner()
|
if ve != nil {
|
||||||
if err != nil {
|
return nil, ve
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FIXME TODO deserialize inner
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManifestFromFile reads a manifest from a file path using the given filesystem.
|
|
||||||
// If fs is nil, the real filesystem (OsFs) is used.
|
|
||||||
//
|
|
||||||
//nolint:revive // unexported-return: exporting manifest is owner question 13
|
|
||||||
func NewManifestFromFile(fs afero.Fs, path string) (*manifest, error) {
|
|
||||||
if fs == nil {
|
|
||||||
fs = afero.NewOsFs()
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := fs.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
return NewManifestFromReader(f)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestValidatePathMessagesVerbatim pins the exact rendered text of every
|
|
||||||
// ValidatePath rejection.
|
|
||||||
//
|
|
||||||
// These strings are user-visible and are assembled by wrapping static
|
|
||||||
// sentinels mid-sentence, which makes them easy to reword by accident
|
|
||||||
// while refactoring for errors.Is matchability. Changing one is a
|
|
||||||
// deliberate change, not a refactoring side effect.
|
|
||||||
func TestValidatePathMessagesVerbatim(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
path string
|
|
||||||
want string
|
|
||||||
is error
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty",
|
|
||||||
path: "",
|
|
||||||
want: "path cannot be empty",
|
|
||||||
is: errPathEmpty,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "not utf8",
|
|
||||||
path: "a\xffb",
|
|
||||||
want: `path "a\xffb" is not valid UTF-8`,
|
|
||||||
is: errPathNotUTF8,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "backslash",
|
|
||||||
path: `a\b`,
|
|
||||||
want: `path "a\\b" contains backslash; ` +
|
|
||||||
"use forward slashes only",
|
|
||||||
is: errPathBackslash,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "absolute",
|
|
||||||
path: "/a/b",
|
|
||||||
want: `path "/a/b" is absolute; must be relative`,
|
|
||||||
is: errPathAbsolute,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty segment",
|
|
||||||
path: "a//b",
|
|
||||||
want: `path "a//b" contains empty segment`,
|
|
||||||
is: errPathEmptySegment,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dotdot segment",
|
|
||||||
path: "a/../b",
|
|
||||||
want: `path "a/../b" contains '..' segment`,
|
|
||||||
is: errPathDotDot,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := ValidatePath(tc.path)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Equal(t, tc.want, err.Error())
|
|
||||||
require.ErrorIs(t, err, tc.is)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSerializeInternalErrorMessagesVerbatim pins the two distinct
|
|
||||||
// "internal error" messages, which differ between generate and
|
|
||||||
// generateOuter and have always done so.
|
|
||||||
func TestSerializeInternalErrorMessagesVerbatim(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := &manifest{}
|
|
||||||
require.EqualError(t, m.generate(), "internal error: pbInner not set")
|
|
||||||
require.EqualError(t, m.generateOuter(), "internal error")
|
|
||||||
}
|
|
||||||
42
mfer/example_test.go
Normal file
42
mfer/example_test.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package mfer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAPIExample(t *testing.T) {
|
||||||
|
// read from filesystem
|
||||||
|
m, err := NewFromFS(&ManifestScanOptions{
|
||||||
|
IgnoreDotfiles: true,
|
||||||
|
}, big)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
|
||||||
|
// scan for files
|
||||||
|
m.Scan()
|
||||||
|
|
||||||
|
// serialize
|
||||||
|
var buf bytes.Buffer
|
||||||
|
m.WriteTo(&buf)
|
||||||
|
|
||||||
|
// show serialized
|
||||||
|
log.Dump(buf.Bytes())
|
||||||
|
|
||||||
|
// do it again
|
||||||
|
var buf2 bytes.Buffer
|
||||||
|
m.WriteTo(&buf2)
|
||||||
|
|
||||||
|
// should be same!
|
||||||
|
assert.True(t, bytes.Equal(buf.Bytes(), buf2.Bytes()))
|
||||||
|
|
||||||
|
// deserialize
|
||||||
|
m2, err := NewFromProto(&buf)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.NotNil(t, m2)
|
||||||
|
|
||||||
|
log.Dump(m2)
|
||||||
|
}
|
||||||
276
mfer/gpg.go
276
mfer/gpg.go
@@ -1,276 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// privateDirPerms is the permission mode for temporary GPG home
|
|
||||||
// directories.
|
|
||||||
privateDirPerms os.FileMode = 0o700
|
|
||||||
|
|
||||||
// privateFilePerms is the permission mode for temporary key,
|
|
||||||
// signature, and data files.
|
|
||||||
privateFilePerms os.FileMode = 0o600
|
|
||||||
|
|
||||||
// gpgFingerprintField is the record type tag for fingerprint lines
|
|
||||||
// in gpg --with-colons output.
|
|
||||||
gpgFingerprintField = "fpr"
|
|
||||||
|
|
||||||
// gpgFingerprintMinFields is the minimum number of colon-separated
|
|
||||||
// fields in a gpg fingerprint record (the fingerprint is field 10).
|
|
||||||
gpgFingerprintMinFields = 10
|
|
||||||
|
|
||||||
// gpg option names used from more than one call site.
|
|
||||||
gpgOptArmor = "--armor"
|
|
||||||
gpgOptHomedir = "--homedir"
|
|
||||||
gpgOptVerify = "--verify"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errGPGKeyNotFound = errors.New("gpg key not found")
|
|
||||||
errFingerprintNotFound = errors.New("fingerprint not found for key")
|
|
||||||
errImportedFPRNotFound = errors.New("fingerprint not found in imported key")
|
|
||||||
)
|
|
||||||
|
|
||||||
// GPGKeyID represents a GPG key identifier (fingerprint or key ID).
|
|
||||||
type GPGKeyID string
|
|
||||||
|
|
||||||
// SigningOptions contains options for GPG signing.
|
|
||||||
type SigningOptions struct {
|
|
||||||
KeyID GPGKeyID
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgArgs builds a gpg argument list from opts followed by positional
|
|
||||||
// arguments, separated by an explicit "--" end-of-options marker.
|
|
||||||
//
|
|
||||||
// This matters because key IDs reach gpg as bare positional arguments
|
|
||||||
// (from --sign-key / MFER_SIGN_KEY) and gpg would otherwise parse a value
|
|
||||||
// beginning with "-" as one of its own options. Callers must route every
|
|
||||||
// non-option argument through here.
|
|
||||||
func gpgArgs(opts []string, positional ...string) []string {
|
|
||||||
args := make([]string, 0, len(opts)+1+len(positional))
|
|
||||||
args = append(args, opts...)
|
|
||||||
args = append(args, "--")
|
|
||||||
args = append(args, positional...)
|
|
||||||
|
|
||||||
return args
|
|
||||||
}
|
|
||||||
|
|
||||||
// runGPG runs the gpg binary in batch mode with the given arguments and
|
|
||||||
// optional stdin, returning captured stdout and stderr.
|
|
||||||
func runGPG(stdin io.Reader, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
|
|
||||||
fullArgs := append([]string{"--batch", "--no-tty"}, args...)
|
|
||||||
|
|
||||||
// G204: the executable name is a compile-time constant. The arguments
|
|
||||||
// are not, so the guarantee that matters is placement: every
|
|
||||||
// caller-supplied value is passed either as the value of a named
|
|
||||||
// option or after the "--" end-of-options marker inserted by gpgArgs,
|
|
||||||
// and therefore cannot be reinterpreted by gpg as an option.
|
|
||||||
cmd := exec.CommandContext( //nolint:gosec // G204: see comment above
|
|
||||||
context.Background(), "gpg", fullArgs...)
|
|
||||||
cmd.Stdin = stdin
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
cmd.Stdout = &stdout
|
|
||||||
cmd.Stderr = &stderr
|
|
||||||
|
|
||||||
err := cmd.Run()
|
|
||||||
|
|
||||||
return &stdout, &stderr, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseFingerprint extracts the first fingerprint from gpg --with-colons
|
|
||||||
// output, or returns ok=false if none is present.
|
|
||||||
func parseFingerprint(colonOutput string) (string, bool) {
|
|
||||||
for _, line := range strings.Split(colonOutput, "\n") {
|
|
||||||
fields := strings.Split(line, ":")
|
|
||||||
if len(fields) >= gpgFingerprintMinFields &&
|
|
||||||
fields[0] == gpgFingerprintField {
|
|
||||||
return fields[9], true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgSign creates a detached signature of the data using the specified key.
|
|
||||||
// Returns the armored detached signature.
|
|
||||||
func gpgSign(data []byte, keyID GPGKeyID) ([]byte, error) {
|
|
||||||
stdout, stderr, err := runGPG(bytes.NewReader(data),
|
|
||||||
"--detach-sign",
|
|
||||||
gpgOptArmor,
|
|
||||||
"--local-user", string(keyID),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gpg sign failed: %w: %s", err, stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
return stdout.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgExportPublicKey exports the public key for the specified key ID.
|
|
||||||
// Returns the armored public key.
|
|
||||||
func gpgExportPublicKey(keyID GPGKeyID) ([]byte, error) {
|
|
||||||
stdout, stderr, err := runGPG(nil,
|
|
||||||
gpgArgs([]string{"--export", gpgOptArmor}, string(keyID))...,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("gpg export failed: %w: %s", err, stderr.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if stdout.Len() == 0 {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errGPGKeyNotFound, keyID)
|
|
||||||
}
|
|
||||||
|
|
||||||
return stdout.Bytes(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgGetKeyFingerprint gets the full fingerprint for a key ID.
|
|
||||||
func gpgGetKeyFingerprint(keyID GPGKeyID) ([]byte, error) {
|
|
||||||
stdout, stderr, err := runGPG(nil,
|
|
||||||
gpgArgs([]string{"--with-colons", "--fingerprint"}, string(keyID))...,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"gpg fingerprint lookup failed: %w: %s", err, stderr.String(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fpr, ok := parseFingerprint(stdout.String())
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errFingerprintNotFound, keyID)
|
|
||||||
}
|
|
||||||
|
|
||||||
return []byte(fpr), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgExtractPubKeyFingerprint imports a public key into a temporary keyring
|
|
||||||
// and extracts its fingerprint. This verifies the key is valid and returns
|
|
||||||
// the actual fingerprint from the key material.
|
|
||||||
func gpgExtractPubKeyFingerprint(pubKey []byte) (string, error) {
|
|
||||||
// Create temporary directory for GPG operations
|
|
||||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-fingerprint-*")
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to create temp dir: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
|
||||||
|
|
||||||
// Set restrictive permissions
|
|
||||||
err = os.Chmod(tmpDir, privateDirPerms)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to set temp dir permissions: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write public key to temp file
|
|
||||||
pubKeyFile := filepath.Join(tmpDir, "pubkey.asc")
|
|
||||||
|
|
||||||
err = os.WriteFile(pubKeyFile, pubKey, privateFilePerms)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to write public key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Import the public key into the temporary keyring
|
|
||||||
_, importStderr, err := runGPG(nil,
|
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"failed to import public key: %w: %s", err, importStderr.String(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// List keys to get fingerprint
|
|
||||||
listStdout, listStderr, err := runGPG(nil,
|
|
||||||
"--homedir", tmpDir,
|
|
||||||
"--with-colons",
|
|
||||||
"--fingerprint",
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf(
|
|
||||||
"failed to list keys: %w: %s", err, listStderr.String(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fpr, ok := parseFingerprint(listStdout.String())
|
|
||||||
if !ok {
|
|
||||||
return "", errImportedFPRNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
return fpr, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// gpgVerify verifies a detached signature against data using the provided public key.
|
|
||||||
// It creates a temporary keyring to import the public key for verification.
|
|
||||||
func gpgVerify(data, signature, pubKey []byte) error {
|
|
||||||
// Create temporary directory for GPG operations
|
|
||||||
tmpDir, err := os.MkdirTemp("", "mfer-gpg-verify-*")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to create temp dir: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = os.RemoveAll(tmpDir) }()
|
|
||||||
|
|
||||||
// Set restrictive permissions
|
|
||||||
err = os.Chmod(tmpDir, privateDirPerms)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to set temp dir permissions: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write public key to temp file
|
|
||||||
pubKeyFile := filepath.Join(tmpDir, "pubkey.asc")
|
|
||||||
|
|
||||||
err = os.WriteFile(pubKeyFile, pubKey, privateFilePerms)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write public key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write signature to temp file
|
|
||||||
sigFile := filepath.Join(tmpDir, "signature.asc")
|
|
||||||
|
|
||||||
err = os.WriteFile(sigFile, signature, privateFilePerms)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write signature: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write data to temp file
|
|
||||||
dataFile := filepath.Join(tmpDir, "data")
|
|
||||||
|
|
||||||
err = os.WriteFile(dataFile, data, privateFilePerms)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write data: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Import the public key into the temporary keyring
|
|
||||||
_, importStderr, err := runGPG(nil,
|
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, "--import"}, pubKeyFile)...,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"failed to import public key: %w: %s", err, importStderr.String(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify the signature
|
|
||||||
_, verifyStderr, err := runGPG(nil,
|
|
||||||
gpgArgs([]string{gpgOptHomedir, tmpDir, gpgOptVerify},
|
|
||||||
sigFile, dataFile)...,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"signature verification failed: %w: %s", err, verifyStderr.String(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
392
mfer/gpg_test.go
392
mfer/gpg_test.go
@@ -1,392 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testGPGEnv sets up a temporary GPG home directory with a test key.
|
|
||||||
// Returns the key ID and the GPG home directory; callers must point
|
|
||||||
// GNUPGHOME at the returned directory (via t.Setenv) before using the
|
|
||||||
// gpg helpers under test.
|
|
||||||
func testGPGEnv(t *testing.T) (GPGKeyID, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// Check if gpg is installed
|
|
||||||
_, err := exec.LookPath("gpg")
|
|
||||||
if err != nil {
|
|
||||||
t.Skip("gpg not installed, skipping signing test")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create temporary GPG home directory (0700 by default)
|
|
||||||
gpgHome := t.TempDir()
|
|
||||||
|
|
||||||
// Generate a test key with no passphrase
|
|
||||||
keyParams := `%no-protection
|
|
||||||
Key-Type: RSA
|
|
||||||
Key-Length: 2048
|
|
||||||
Name-Real: MFER Test Key
|
|
||||||
Name-Email: test@mfer.test
|
|
||||||
Expire-Date: 0
|
|
||||||
%commit
|
|
||||||
`
|
|
||||||
paramsFile := filepath.Join(gpgHome, "key-params")
|
|
||||||
require.NoError(t, os.WriteFile(paramsFile, []byte(keyParams), 0o600))
|
|
||||||
|
|
||||||
//nolint:gosec // paramsFile is a test-controlled path inside t.TempDir()
|
|
||||||
cmd := exec.CommandContext(context.Background(), "gpg",
|
|
||||||
"--batch", "--gen-key", paramsFile)
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
|
||||||
|
|
||||||
output, err := cmd.CombinedOutput()
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("failed to generate test GPG key: %v: %s", err, output)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get the key fingerprint
|
|
||||||
cmd = exec.CommandContext(context.Background(), "gpg",
|
|
||||||
"--list-keys", "--with-colons", "test@mfer.test")
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), "GNUPGHOME="+gpgHome)
|
|
||||||
|
|
||||||
output, err = cmd.Output()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to list test key: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse fingerprint from output
|
|
||||||
var keyID string
|
|
||||||
|
|
||||||
for _, line := range strings.Split(string(output), "\n") {
|
|
||||||
fields := strings.Split(line, ":")
|
|
||||||
if len(fields) >= gpgFingerprintMinFields &&
|
|
||||||
fields[0] == gpgFingerprintField {
|
|
||||||
keyID = fields[9]
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if keyID == "" {
|
|
||||||
t.Fatal("failed to find test key fingerprint")
|
|
||||||
}
|
|
||||||
|
|
||||||
return GPGKeyID(keyID), gpgHome
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGSign(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
data := []byte("test data to sign")
|
|
||||||
sig, err := gpgSign(data, keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEmpty(t, sig)
|
|
||||||
assert.Contains(t, string(sig), "-----BEGIN PGP SIGNATURE-----")
|
|
||||||
assert.Contains(t, string(sig), "-----END PGP SIGNATURE-----")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGExportPublicKey(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEmpty(t, pubKey)
|
|
||||||
assert.Contains(t, string(pubKey), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
|
||||||
assert.Contains(t, string(pubKey), "-----END PGP PUBLIC KEY BLOCK-----")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGGetKeyFingerprint(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
fingerprint, err := gpgGetKeyFingerprint(keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotEmpty(t, fingerprint)
|
|
||||||
// The fingerprint should be 40 hex chars
|
|
||||||
assert.Len(t, fingerprint, 40, "fingerprint should be 40 hex chars")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGPGArgsSeparatesPositionals pins that caller-supplied values are
|
|
||||||
// placed after an end-of-options marker. Key IDs arrive from --sign-key
|
|
||||||
// and MFER_SIGN_KEY as bare positional arguments, so without the marker
|
|
||||||
// a value beginning with "-" would be parsed by gpg as one of its own
|
|
||||||
// options.
|
|
||||||
func TestGPGArgsSeparatesPositionals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"--opt-a", "--opt-b", "--", "--version"},
|
|
||||||
gpgArgs([]string{"--opt-a", "--opt-b"}, "--version"))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"--opt-c", "--", "sig", "data"},
|
|
||||||
gpgArgs([]string{"--opt-c"}, "sig", "data"))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"--opt-d", "--"},
|
|
||||||
gpgArgs([]string{"--opt-d"}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGPGOptionLikeKeyIDIsNotAnOption drives real gpg with a key ID that
|
|
||||||
// looks like an option and asserts it is treated as a (nonexistent) key
|
|
||||||
// rather than executed as gpg's own --version.
|
|
||||||
func TestGPGOptionLikeKeyIDIsNotAnOption(t *testing.T) {
|
|
||||||
_, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(GPGKeyID("--version"))
|
|
||||||
require.Error(t, err)
|
|
||||||
require.ErrorIs(t, err, errGPGKeyNotFound)
|
|
||||||
assert.NotContains(t, string(pubKey), "gpg (GnuPG)")
|
|
||||||
|
|
||||||
fpr, err := gpgGetKeyFingerprint(GPGKeyID("--version"))
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.NotContains(t, string(fpr), "gpg (GnuPG)")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGSignInvalidKey(t *testing.T) {
|
|
||||||
// Set up test environment (we need GNUPGHOME set)
|
|
||||||
_, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
data := []byte("test data")
|
|
||||||
_, err := gpgSign(data, GPGKeyID("NONEXISTENT_KEY_ID_12345"))
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderWithSigning(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
// Create a builder with signing options
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetSigningOptions(&SigningOptions{
|
|
||||||
KeyID: keyID,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add a test file
|
|
||||||
content := []byte("test file content")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Build the manifest
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Parse the manifest and verify signature fields are populated
|
|
||||||
manifest, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, manifest.pbOuter)
|
|
||||||
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSignature(),
|
|
||||||
"signature should be populated")
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSigner(), "signer should be populated")
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSigningPubKey(),
|
|
||||||
"signing public key should be populated")
|
|
||||||
|
|
||||||
// Verify signature is a valid PGP signature
|
|
||||||
assert.Contains(t, string(manifest.pbOuter.GetSignature()),
|
|
||||||
"-----BEGIN PGP SIGNATURE-----")
|
|
||||||
|
|
||||||
// Verify public key is a valid PGP public key block
|
|
||||||
assert.Contains(t, string(manifest.pbOuter.GetSigningPubKey()),
|
|
||||||
"-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerWithSigning(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
// Create in-memory filesystem with test files
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/file1.txt", []byte("content1"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/file2.txt", []byte("content2"), 0o644))
|
|
||||||
|
|
||||||
// Create scanner with signing options
|
|
||||||
opts := &ScannerOptions{
|
|
||||||
Fs: fs,
|
|
||||||
SigningOptions: &SigningOptions{
|
|
||||||
KeyID: keyID,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
s := NewScannerWithOptions(opts)
|
|
||||||
|
|
||||||
// Enumerate files
|
|
||||||
require.NoError(t, s.EnumeratePath("/testdir", nil))
|
|
||||||
assert.Equal(t, FileCount(2), s.FileCount())
|
|
||||||
|
|
||||||
// Generate signed manifest
|
|
||||||
var buf bytes.Buffer
|
|
||||||
require.NoError(t, s.ToManifest(context.Background(), &buf, nil))
|
|
||||||
|
|
||||||
// Parse and verify
|
|
||||||
manifest, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSignature())
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSigner())
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSigningPubKey())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGVerify(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
data := []byte("test data to sign and verify")
|
|
||||||
sig, err := gpgSign(data, keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Verify the signature
|
|
||||||
err = gpgVerify(data, sig, pubKey)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGVerifyInvalidSignature(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
data := []byte("test data to sign")
|
|
||||||
sig, err := gpgSign(data, keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Try to verify with different data - should fail
|
|
||||||
wrongData := []byte("different data")
|
|
||||||
err = gpgVerify(wrongData, sig, pubKey)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGPGVerifyBadPublicKey(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
data := []byte("test data")
|
|
||||||
sig, err := gpgSign(data, keyID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Try to verify with invalid public key - should fail
|
|
||||||
badPubKey := []byte("not a valid public key")
|
|
||||||
err = gpgVerify(data, sig, badPubKey)
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestSignatureVerification(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
// Create a builder with signing options
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetSigningOptions(&SigningOptions{
|
|
||||||
KeyID: keyID,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Add a test file
|
|
||||||
content := []byte("test file content for verification")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Build the manifest
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Parse the manifest - signature should be verified during load
|
|
||||||
manifest, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, manifest)
|
|
||||||
|
|
||||||
// Signature should be present and valid
|
|
||||||
assert.NotEmpty(t, manifest.pbOuter.GetSignature())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestTamperedSignatureFails(t *testing.T) {
|
|
||||||
keyID, gpgHome := testGPGEnv(t)
|
|
||||||
t.Setenv("GNUPGHOME", gpgHome)
|
|
||||||
|
|
||||||
// Create a signed manifest
|
|
||||||
b := NewBuilder()
|
|
||||||
b.SetSigningOptions(&SigningOptions{
|
|
||||||
KeyID: keyID,
|
|
||||||
})
|
|
||||||
|
|
||||||
content := []byte("test file content")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Tamper with the signature by replacing some bytes
|
|
||||||
data := buf.Bytes()
|
|
||||||
// Find and modify a byte in the signature portion
|
|
||||||
for i := range data {
|
|
||||||
if i > 100 && data[i] == 'A' {
|
|
||||||
data[i] = 'B'
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to load the tampered manifest - should fail
|
|
||||||
_, err = NewManifestFromReader(bytes.NewReader(data))
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuilderWithoutSigning(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Create a builder without signing options
|
|
||||||
b := NewBuilder()
|
|
||||||
|
|
||||||
// Add a test file
|
|
||||||
content := []byte("test file content")
|
|
||||||
reader := bytes.NewReader(content)
|
|
||||||
_, err := b.AddFile("test.txt", FileSize(len(content)), ModTime{}, reader, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Build the manifest
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = b.Build(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Parse the manifest and verify signature fields are empty
|
|
||||||
manifest, err := NewManifestFromReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, manifest.pbOuter)
|
|
||||||
|
|
||||||
assert.Empty(t, manifest.pbOuter.GetSignature(),
|
|
||||||
"signature should be empty when not signing")
|
|
||||||
assert.Empty(t, manifest.pbOuter.GetSigner(),
|
|
||||||
"signer should be empty when not signing")
|
|
||||||
assert.Empty(t, manifest.pbOuter.GetSigningPubKey(),
|
|
||||||
"signing public key should be empty when not signing")
|
|
||||||
}
|
|
||||||
194
mfer/manifest.go
194
mfer/manifest.go
@@ -2,74 +2,172 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/hex"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/multiformats/go-multihash"
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
|
"github.com/spf13/afero"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
type manifestFile struct {
|
||||||
errOuterNotSet = errors.New("pbOuter not set")
|
path string
|
||||||
errUUIDNotSet = errors.New("UUID not set")
|
info fs.FileInfo
|
||||||
errSHA256NotSet = errors.New("SHA256 hash not set")
|
}
|
||||||
)
|
|
||||||
|
func (m *manifestFile) String() string {
|
||||||
|
return fmt.Sprintf("<File \"%s\">", m.path)
|
||||||
|
}
|
||||||
|
|
||||||
// manifest holds the internal representation of a manifest file.
|
|
||||||
// Use NewManifestFromFile or NewManifestFromReader to load an existing
|
|
||||||
// manifest, or use Builder to create a new one.
|
|
||||||
//
|
|
||||||
// Whether this type should be exported is an open design question owned by
|
|
||||||
// the repository owner; see README design question 13.
|
|
||||||
type manifest struct {
|
type manifest struct {
|
||||||
|
sourceFS []afero.Fs
|
||||||
|
files []*manifestFile
|
||||||
|
scanOptions *ManifestScanOptions
|
||||||
|
totalFileSize int64
|
||||||
pbInner *MFFile
|
pbInner *MFFile
|
||||||
pbOuter *MFFileOuter
|
pbOuter *MFFileOuter
|
||||||
output *bytes.Buffer
|
output *bytes.Buffer
|
||||||
signingOptions *SigningOptions
|
ctx context.Context
|
||||||
fixedUUID []byte // if set, use this UUID instead of generating one
|
errors []*error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifest) String() string {
|
func (m *manifest) String() string {
|
||||||
count := 0
|
return fmt.Sprintf("<Manifest count=%d totalSize=%d>", len(m.files), m.totalFileSize)
|
||||||
if m.pbInner != nil {
|
|
||||||
count = len(m.pbInner.GetFiles())
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("<Manifest count=%d>", count)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Files returns all file entries from a loaded manifest.
|
type ManifestScanOptions struct {
|
||||||
func (m *manifest) Files() []*MFFilePath {
|
IgnoreDotfiles bool
|
||||||
if m.pbInner == nil {
|
FollowSymLinks bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) HasError() bool {
|
||||||
|
return len(m.errors) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) AddError(e error) *manifest {
|
||||||
|
m.errors = append(m.errors, &e)
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) WithContext(c context.Context) *manifest {
|
||||||
|
m.ctx = c
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) addInputPath(inputPath string) error {
|
||||||
|
abs, err := filepath.Abs(inputPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// FIXME check to make sure inputPath/abs exists maybe
|
||||||
|
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(afero.NewOsFs(), abs))
|
||||||
|
return m.addInputFS(afs)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) addInputFS(f afero.Fs) error {
|
||||||
|
if m.sourceFS == nil {
|
||||||
|
m.sourceFS = make([]afero.Fs, 0)
|
||||||
|
}
|
||||||
|
m.sourceFS = append(m.sourceFS, f)
|
||||||
|
// FIXME do some sort of check on f here?
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func New() *manifest {
|
||||||
|
m := &manifest{}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFromPaths(options *ManifestScanOptions, inputPaths ...string) (*manifest, error) {
|
||||||
|
log.Dump(inputPaths)
|
||||||
|
m := New()
|
||||||
|
m.scanOptions = options
|
||||||
|
for _, p := range inputPaths {
|
||||||
|
err := m.addInputPath(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFromFS(options *ManifestScanOptions, fs afero.Fs) (*manifest, error) {
|
||||||
|
m := New()
|
||||||
|
m.scanOptions = options
|
||||||
|
err := m.addInputFS(fs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) GetFileCount() int64 {
|
||||||
|
return int64(len(m.files))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) GetTotalFileSize() int64 {
|
||||||
|
return m.totalFileSize
|
||||||
|
}
|
||||||
|
|
||||||
|
func pathIsHidden(p string) bool {
|
||||||
|
tp := path.Clean(p)
|
||||||
|
if strings.HasPrefix(tp, ".") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
d, f := path.Split(tp)
|
||||||
|
if strings.HasPrefix(f, ".") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if d == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
tp = d[0 : len(d)-1] // trim trailing slash from dir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) addFile(p string, fi fs.FileInfo, sfsIndex int) error {
|
||||||
|
if m.scanOptions.IgnoreDotfiles && pathIsHidden(p) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if fi != nil && fi.IsDir() {
|
||||||
return m.pbInner.GetFiles()
|
// manifests contain only files, directories are implied.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// FIXME test if 'fi' is already result of stat
|
||||||
|
fileinfo, staterr := m.sourceFS[sfsIndex].Stat(p)
|
||||||
|
if staterr != nil {
|
||||||
|
return staterr
|
||||||
|
}
|
||||||
|
cleanPath := p
|
||||||
|
if cleanPath[0:1] == "/" {
|
||||||
|
cleanPath = cleanPath[1:]
|
||||||
|
}
|
||||||
|
nf := &manifestFile{
|
||||||
|
path: cleanPath,
|
||||||
|
info: fileinfo,
|
||||||
|
}
|
||||||
|
m.files = append(m.files, nf)
|
||||||
|
m.totalFileSize = m.totalFileSize + fi.Size()
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// signatureString generates the canonical string used for signing/verification.
|
func (m *manifest) Scan() error {
|
||||||
// Format: MAGIC-UUID-MULTIHASH where UUID and multihash are hex-encoded.
|
// FIXME scan and whatever function does the hashing should take ctx
|
||||||
// Requires pbOuter to be set with Uuid and Sha256 fields.
|
for idx, sfs := range m.sourceFS {
|
||||||
func (m *manifest) signatureString() (string, error) {
|
if sfs == nil {
|
||||||
if m.pbOuter == nil {
|
return errors.New("invalid source fs")
|
||||||
return "", errOuterNotSet
|
|
||||||
}
|
}
|
||||||
|
e := afero.Walk(sfs, "/", func(p string, info fs.FileInfo, err error) error {
|
||||||
if len(m.pbOuter.GetUuid()) == 0 {
|
return m.addFile(p, info, idx)
|
||||||
return "", errUUIDNotSet
|
})
|
||||||
|
if e != nil {
|
||||||
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(m.pbOuter.GetSha256()) == 0 {
|
|
||||||
return "", errSHA256NotSet
|
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
mh, err := multihash.Encode(m.pbOuter.GetSha256(), multihash.SHA2_256)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to encode multihash: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
uuidStr := hex.EncodeToString(m.pbOuter.GetUuid())
|
|
||||||
mhStr := hex.EncodeToString(mh)
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s-%s-%s", MAGIC, uuidStr, mhStr), nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
//go:generate protoc ./mf.proto --go_out=paths=source_relative:.
|
|
||||||
658
mfer/mf.pb.go
658
mfer/mf.pb.go
@@ -1,658 +0,0 @@
|
|||||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
|
||||||
// versions:
|
|
||||||
// protoc-gen-go v1.36.11
|
|
||||||
// protoc v6.33.4
|
|
||||||
// source: mf.proto
|
|
||||||
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
|
||||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
|
||||||
reflect "reflect"
|
|
||||||
sync "sync"
|
|
||||||
unsafe "unsafe"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// Verify that this generated code is sufficiently up-to-date.
|
|
||||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
|
|
||||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
|
||||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
|
||||||
)
|
|
||||||
|
|
||||||
type MFFileOuter_Version int32
|
|
||||||
|
|
||||||
const (
|
|
||||||
MFFileOuter_VERSION_NONE MFFileOuter_Version = 0
|
|
||||||
MFFileOuter_VERSION_ONE MFFileOuter_Version = 1 // only one for now
|
|
||||||
)
|
|
||||||
|
|
||||||
// Enum value maps for MFFileOuter_Version.
|
|
||||||
var (
|
|
||||||
MFFileOuter_Version_name = map[int32]string{
|
|
||||||
0: "VERSION_NONE",
|
|
||||||
1: "VERSION_ONE",
|
|
||||||
}
|
|
||||||
MFFileOuter_Version_value = map[string]int32{
|
|
||||||
"VERSION_NONE": 0,
|
|
||||||
"VERSION_ONE": 1,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (x MFFileOuter_Version) Enum() *MFFileOuter_Version {
|
|
||||||
p := new(MFFileOuter_Version)
|
|
||||||
*p = x
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFileOuter_Version) String() string {
|
|
||||||
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFileOuter_Version) Descriptor() protoreflect.EnumDescriptor {
|
|
||||||
return file_mf_proto_enumTypes[0].Descriptor()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFileOuter_Version) Type() protoreflect.EnumType {
|
|
||||||
return &file_mf_proto_enumTypes[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFileOuter_Version) Number() protoreflect.EnumNumber {
|
|
||||||
return protoreflect.EnumNumber(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFileOuter_Version.Descriptor instead.
|
|
||||||
func (MFFileOuter_Version) EnumDescriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{1, 0}
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFileOuter_CompressionType int32
|
|
||||||
|
|
||||||
const (
|
|
||||||
MFFileOuter_COMPRESSION_NONE MFFileOuter_CompressionType = 0
|
|
||||||
MFFileOuter_COMPRESSION_ZSTD MFFileOuter_CompressionType = 1
|
|
||||||
)
|
|
||||||
|
|
||||||
// Enum value maps for MFFileOuter_CompressionType.
|
|
||||||
var (
|
|
||||||
MFFileOuter_CompressionType_name = map[int32]string{
|
|
||||||
0: "COMPRESSION_NONE",
|
|
||||||
1: "COMPRESSION_ZSTD",
|
|
||||||
}
|
|
||||||
MFFileOuter_CompressionType_value = map[string]int32{
|
|
||||||
"COMPRESSION_NONE": 0,
|
|
||||||
"COMPRESSION_ZSTD": 1,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (x MFFileOuter_CompressionType) Enum() *MFFileOuter_CompressionType {
|
|
||||||
p := new(MFFileOuter_CompressionType)
|
|
||||||
*p = x
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFileOuter_CompressionType) String() string {
|
|
||||||
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFileOuter_CompressionType) Descriptor() protoreflect.EnumDescriptor {
|
|
||||||
return file_mf_proto_enumTypes[1].Descriptor()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFileOuter_CompressionType) Type() protoreflect.EnumType {
|
|
||||||
return &file_mf_proto_enumTypes[1]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFileOuter_CompressionType) Number() protoreflect.EnumNumber {
|
|
||||||
return protoreflect.EnumNumber(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFileOuter_CompressionType.Descriptor instead.
|
|
||||||
func (MFFileOuter_CompressionType) EnumDescriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{1, 1}
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFile_Version int32
|
|
||||||
|
|
||||||
const (
|
|
||||||
MFFile_VERSION_NONE MFFile_Version = 0
|
|
||||||
MFFile_VERSION_ONE MFFile_Version = 1 // only one for now
|
|
||||||
)
|
|
||||||
|
|
||||||
// Enum value maps for MFFile_Version.
|
|
||||||
var (
|
|
||||||
MFFile_Version_name = map[int32]string{
|
|
||||||
0: "VERSION_NONE",
|
|
||||||
1: "VERSION_ONE",
|
|
||||||
}
|
|
||||||
MFFile_Version_value = map[string]int32{
|
|
||||||
"VERSION_NONE": 0,
|
|
||||||
"VERSION_ONE": 1,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
func (x MFFile_Version) Enum() *MFFile_Version {
|
|
||||||
p := new(MFFile_Version)
|
|
||||||
*p = x
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFile_Version) String() string {
|
|
||||||
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFile_Version) Descriptor() protoreflect.EnumDescriptor {
|
|
||||||
return file_mf_proto_enumTypes[2].Descriptor()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (MFFile_Version) Type() protoreflect.EnumType {
|
|
||||||
return &file_mf_proto_enumTypes[2]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x MFFile_Version) Number() protoreflect.EnumNumber {
|
|
||||||
return protoreflect.EnumNumber(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFile_Version.Descriptor instead.
|
|
||||||
func (MFFile_Version) EnumDescriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{4, 0}
|
|
||||||
}
|
|
||||||
|
|
||||||
type Timestamp struct {
|
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
|
||||||
Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"`
|
|
||||||
Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
|
||||||
sizeCache protoimpl.SizeCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *Timestamp) Reset() {
|
|
||||||
*x = Timestamp{}
|
|
||||||
mi := &file_mf_proto_msgTypes[0]
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *Timestamp) String() string {
|
|
||||||
return protoimpl.X.MessageStringOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*Timestamp) ProtoMessage() {}
|
|
||||||
|
|
||||||
func (x *Timestamp) ProtoReflect() protoreflect.Message {
|
|
||||||
mi := &file_mf_proto_msgTypes[0]
|
|
||||||
if x != nil {
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
if ms.LoadMessageInfo() == nil {
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
return ms
|
|
||||||
}
|
|
||||||
return mi.MessageOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use Timestamp.ProtoReflect.Descriptor instead.
|
|
||||||
func (*Timestamp) Descriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{0}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *Timestamp) GetSeconds() int64 {
|
|
||||||
if x != nil {
|
|
||||||
return x.Seconds
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *Timestamp) GetNanos() int32 {
|
|
||||||
if x != nil {
|
|
||||||
return x.Nanos
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFileOuter struct {
|
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
|
||||||
// required mffile root attributes 1xx
|
|
||||||
Version MFFileOuter_Version `protobuf:"varint,101,opt,name=version,proto3,enum=MFFileOuter_Version" json:"version,omitempty"`
|
|
||||||
CompressionType MFFileOuter_CompressionType `protobuf:"varint,102,opt,name=compressionType,proto3,enum=MFFileOuter_CompressionType" json:"compressionType,omitempty"`
|
|
||||||
// these are used solely to detect corruption/truncation
|
|
||||||
// and not for cryptographic integrity.
|
|
||||||
Size int64 `protobuf:"varint,103,opt,name=size,proto3" json:"size,omitempty"`
|
|
||||||
Sha256 []byte `protobuf:"bytes,104,opt,name=sha256,proto3" json:"sha256,omitempty"`
|
|
||||||
// uuid must match the uuid in the inner message
|
|
||||||
Uuid []byte `protobuf:"bytes,105,opt,name=uuid,proto3" json:"uuid,omitempty"`
|
|
||||||
InnerMessage []byte `protobuf:"bytes,199,opt,name=innerMessage,proto3" json:"innerMessage,omitempty"`
|
|
||||||
// detached signature, ascii or binary
|
|
||||||
Signature []byte `protobuf:"bytes,201,opt,name=signature,proto3,oneof" json:"signature,omitempty"`
|
|
||||||
// full GPG key id
|
|
||||||
Signer []byte `protobuf:"bytes,202,opt,name=signer,proto3,oneof" json:"signer,omitempty"`
|
|
||||||
// full GPG signing public key, ascii or binary
|
|
||||||
SigningPubKey []byte `protobuf:"bytes,203,opt,name=signingPubKey,proto3,oneof" json:"signingPubKey,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
|
||||||
sizeCache protoimpl.SizeCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) Reset() {
|
|
||||||
*x = MFFileOuter{}
|
|
||||||
mi := &file_mf_proto_msgTypes[1]
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) String() string {
|
|
||||||
return protoimpl.X.MessageStringOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*MFFileOuter) ProtoMessage() {}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) ProtoReflect() protoreflect.Message {
|
|
||||||
mi := &file_mf_proto_msgTypes[1]
|
|
||||||
if x != nil {
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
if ms.LoadMessageInfo() == nil {
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
return ms
|
|
||||||
}
|
|
||||||
return mi.MessageOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFileOuter.ProtoReflect.Descriptor instead.
|
|
||||||
func (*MFFileOuter) Descriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{1}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetVersion() MFFileOuter_Version {
|
|
||||||
if x != nil {
|
|
||||||
return x.Version
|
|
||||||
}
|
|
||||||
return MFFileOuter_VERSION_NONE
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetCompressionType() MFFileOuter_CompressionType {
|
|
||||||
if x != nil {
|
|
||||||
return x.CompressionType
|
|
||||||
}
|
|
||||||
return MFFileOuter_COMPRESSION_NONE
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetSize() int64 {
|
|
||||||
if x != nil {
|
|
||||||
return x.Size
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetSha256() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.Sha256
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetUuid() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.Uuid
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetInnerMessage() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.InnerMessage
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetSignature() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.Signature
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetSigner() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.Signer
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileOuter) GetSigningPubKey() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.SigningPubKey
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFilePath struct {
|
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
|
||||||
// required attributes:
|
|
||||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
|
||||||
// be relative (no leading /), contain no ".." segments, and no
|
|
||||||
// empty segments (no "//").
|
|
||||||
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
|
||||||
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
|
||||||
// gotta have at least one:
|
|
||||||
Hashes []*MFFileChecksum `protobuf:"bytes,3,rep,name=hashes,proto3" json:"hashes,omitempty"`
|
|
||||||
// optional per-file metadata
|
|
||||||
MimeType *string `protobuf:"bytes,301,opt,name=mimeType,proto3,oneof" json:"mimeType,omitempty"`
|
|
||||||
Mtime *Timestamp `protobuf:"bytes,302,opt,name=mtime,proto3,oneof" json:"mtime,omitempty"`
|
|
||||||
Ctime *Timestamp `protobuf:"bytes,303,opt,name=ctime,proto3,oneof" json:"ctime,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
|
||||||
sizeCache protoimpl.SizeCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) Reset() {
|
|
||||||
*x = MFFilePath{}
|
|
||||||
mi := &file_mf_proto_msgTypes[2]
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) String() string {
|
|
||||||
return protoimpl.X.MessageStringOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*MFFilePath) ProtoMessage() {}
|
|
||||||
|
|
||||||
func (x *MFFilePath) ProtoReflect() protoreflect.Message {
|
|
||||||
mi := &file_mf_proto_msgTypes[2]
|
|
||||||
if x != nil {
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
if ms.LoadMessageInfo() == nil {
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
return ms
|
|
||||||
}
|
|
||||||
return mi.MessageOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFilePath.ProtoReflect.Descriptor instead.
|
|
||||||
func (*MFFilePath) Descriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{2}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetPath() string {
|
|
||||||
if x != nil {
|
|
||||||
return x.Path
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetSize() int64 {
|
|
||||||
if x != nil {
|
|
||||||
return x.Size
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetHashes() []*MFFileChecksum {
|
|
||||||
if x != nil {
|
|
||||||
return x.Hashes
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetMimeType() string {
|
|
||||||
if x != nil && x.MimeType != nil {
|
|
||||||
return *x.MimeType
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetMtime() *Timestamp {
|
|
||||||
if x != nil {
|
|
||||||
return x.Mtime
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFilePath) GetCtime() *Timestamp {
|
|
||||||
if x != nil {
|
|
||||||
return x.Ctime
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFileChecksum struct {
|
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
|
||||||
// 1.0 golang implementation must write a multihash here
|
|
||||||
// it's ok to only ever use/verify sha256 multihash
|
|
||||||
MultiHash []byte `protobuf:"bytes,1,opt,name=multiHash,proto3" json:"multiHash,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
|
||||||
sizeCache protoimpl.SizeCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileChecksum) Reset() {
|
|
||||||
*x = MFFileChecksum{}
|
|
||||||
mi := &file_mf_proto_msgTypes[3]
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileChecksum) String() string {
|
|
||||||
return protoimpl.X.MessageStringOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*MFFileChecksum) ProtoMessage() {}
|
|
||||||
|
|
||||||
func (x *MFFileChecksum) ProtoReflect() protoreflect.Message {
|
|
||||||
mi := &file_mf_proto_msgTypes[3]
|
|
||||||
if x != nil {
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
if ms.LoadMessageInfo() == nil {
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
return ms
|
|
||||||
}
|
|
||||||
return mi.MessageOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFileChecksum.ProtoReflect.Descriptor instead.
|
|
||||||
func (*MFFileChecksum) Descriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{3}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFileChecksum) GetMultiHash() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.MultiHash
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type MFFile struct {
|
|
||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
|
||||||
Version MFFile_Version `protobuf:"varint,100,opt,name=version,proto3,enum=MFFile_Version" json:"version,omitempty"`
|
|
||||||
// required manifest attributes:
|
|
||||||
Files []*MFFilePath `protobuf:"bytes,101,rep,name=files,proto3" json:"files,omitempty"`
|
|
||||||
// uuid is a random v4 UUID generated when creating the manifest
|
|
||||||
// used as part of the signature to prevent replay attacks
|
|
||||||
Uuid []byte `protobuf:"bytes,102,opt,name=uuid,proto3" json:"uuid,omitempty"`
|
|
||||||
// optional manifest attributes 2xx:
|
|
||||||
CreatedAt *Timestamp `protobuf:"bytes,201,opt,name=createdAt,proto3,oneof" json:"createdAt,omitempty"`
|
|
||||||
unknownFields protoimpl.UnknownFields
|
|
||||||
sizeCache protoimpl.SizeCache
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) Reset() {
|
|
||||||
*x = MFFile{}
|
|
||||||
mi := &file_mf_proto_msgTypes[4]
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) String() string {
|
|
||||||
return protoimpl.X.MessageStringOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*MFFile) ProtoMessage() {}
|
|
||||||
|
|
||||||
func (x *MFFile) ProtoReflect() protoreflect.Message {
|
|
||||||
mi := &file_mf_proto_msgTypes[4]
|
|
||||||
if x != nil {
|
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
|
||||||
if ms.LoadMessageInfo() == nil {
|
|
||||||
ms.StoreMessageInfo(mi)
|
|
||||||
}
|
|
||||||
return ms
|
|
||||||
}
|
|
||||||
return mi.MessageOf(x)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deprecated: Use MFFile.ProtoReflect.Descriptor instead.
|
|
||||||
func (*MFFile) Descriptor() ([]byte, []int) {
|
|
||||||
return file_mf_proto_rawDescGZIP(), []int{4}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) GetVersion() MFFile_Version {
|
|
||||||
if x != nil {
|
|
||||||
return x.Version
|
|
||||||
}
|
|
||||||
return MFFile_VERSION_NONE
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) GetFiles() []*MFFilePath {
|
|
||||||
if x != nil {
|
|
||||||
return x.Files
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) GetUuid() []byte {
|
|
||||||
if x != nil {
|
|
||||||
return x.Uuid
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (x *MFFile) GetCreatedAt() *Timestamp {
|
|
||||||
if x != nil {
|
|
||||||
return x.CreatedAt
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var File_mf_proto protoreflect.FileDescriptor
|
|
||||||
|
|
||||||
const file_mf_proto_rawDesc = "" +
|
|
||||||
"\n" +
|
|
||||||
"\bmf.proto\";\n" +
|
|
||||||
"\tTimestamp\x12\x18\n" +
|
|
||||||
"\aseconds\x18\x01 \x01(\x03R\aseconds\x12\x14\n" +
|
|
||||||
"\x05nanos\x18\x02 \x01(\x05R\x05nanos\"\xf0\x03\n" +
|
|
||||||
"\vMFFileOuter\x12.\n" +
|
|
||||||
"\aversion\x18e \x01(\x0e2\x14.MFFileOuter.VersionR\aversion\x12F\n" +
|
|
||||||
"\x0fcompressionType\x18f \x01(\x0e2\x1c.MFFileOuter.CompressionTypeR\x0fcompressionType\x12\x12\n" +
|
|
||||||
"\x04size\x18g \x01(\x03R\x04size\x12\x16\n" +
|
|
||||||
"\x06sha256\x18h \x01(\fR\x06sha256\x12\x12\n" +
|
|
||||||
"\x04uuid\x18i \x01(\fR\x04uuid\x12#\n" +
|
|
||||||
"\finnerMessage\x18\xc7\x01 \x01(\fR\finnerMessage\x12\"\n" +
|
|
||||||
"\tsignature\x18\xc9\x01 \x01(\fH\x00R\tsignature\x88\x01\x01\x12\x1c\n" +
|
|
||||||
"\x06signer\x18\xca\x01 \x01(\fH\x01R\x06signer\x88\x01\x01\x12*\n" +
|
|
||||||
"\rsigningPubKey\x18\xcb\x01 \x01(\fH\x02R\rsigningPubKey\x88\x01\x01\",\n" +
|
|
||||||
"\aVersion\x12\x10\n" +
|
|
||||||
"\fVERSION_NONE\x10\x00\x12\x0f\n" +
|
|
||||||
"\vVERSION_ONE\x10\x01\"=\n" +
|
|
||||||
"\x0fCompressionType\x12\x14\n" +
|
|
||||||
"\x10COMPRESSION_NONE\x10\x00\x12\x14\n" +
|
|
||||||
"\x10COMPRESSION_ZSTD\x10\x01B\f\n" +
|
|
||||||
"\n" +
|
|
||||||
"_signatureB\t\n" +
|
|
||||||
"\a_signerB\x10\n" +
|
|
||||||
"\x0e_signingPubKey\"\xf0\x01\n" +
|
|
||||||
"\n" +
|
|
||||||
"MFFilePath\x12\x12\n" +
|
|
||||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x12\n" +
|
|
||||||
"\x04size\x18\x02 \x01(\x03R\x04size\x12'\n" +
|
|
||||||
"\x06hashes\x18\x03 \x03(\v2\x0f.MFFileChecksumR\x06hashes\x12 \n" +
|
|
||||||
"\bmimeType\x18\xad\x02 \x01(\tH\x00R\bmimeType\x88\x01\x01\x12&\n" +
|
|
||||||
"\x05mtime\x18\xae\x02 \x01(\v2\n" +
|
|
||||||
".TimestampH\x01R\x05mtime\x88\x01\x01\x12&\n" +
|
|
||||||
"\x05ctime\x18\xaf\x02 \x01(\v2\n" +
|
|
||||||
".TimestampH\x02R\x05ctime\x88\x01\x01B\v\n" +
|
|
||||||
"\t_mimeTypeB\b\n" +
|
|
||||||
"\x06_mtimeB\b\n" +
|
|
||||||
"\x06_ctime\".\n" +
|
|
||||||
"\x0eMFFileChecksum\x12\x1c\n" +
|
|
||||||
"\tmultiHash\x18\x01 \x01(\fR\tmultiHash\"\xd6\x01\n" +
|
|
||||||
"\x06MFFile\x12)\n" +
|
|
||||||
"\aversion\x18d \x01(\x0e2\x0f.MFFile.VersionR\aversion\x12!\n" +
|
|
||||||
"\x05files\x18e \x03(\v2\v.MFFilePathR\x05files\x12\x12\n" +
|
|
||||||
"\x04uuid\x18f \x01(\fR\x04uuid\x12.\n" +
|
|
||||||
"\tcreatedAt\x18\xc9\x01 \x01(\v2\n" +
|
|
||||||
".TimestampH\x00R\tcreatedAt\x88\x01\x01\",\n" +
|
|
||||||
"\aVersion\x12\x10\n" +
|
|
||||||
"\fVERSION_NONE\x10\x00\x12\x0f\n" +
|
|
||||||
"\vVERSION_ONE\x10\x01B\f\n" +
|
|
||||||
"\n" +
|
|
||||||
"_createdAtB\x1dZ\x1bgit.eeqj.de/sneak/mfer/mferb\x06proto3"
|
|
||||||
|
|
||||||
var (
|
|
||||||
file_mf_proto_rawDescOnce sync.Once
|
|
||||||
file_mf_proto_rawDescData []byte
|
|
||||||
)
|
|
||||||
|
|
||||||
func file_mf_proto_rawDescGZIP() []byte {
|
|
||||||
file_mf_proto_rawDescOnce.Do(func() {
|
|
||||||
file_mf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)))
|
|
||||||
})
|
|
||||||
return file_mf_proto_rawDescData
|
|
||||||
}
|
|
||||||
|
|
||||||
var file_mf_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
|
|
||||||
var file_mf_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
|
|
||||||
var file_mf_proto_goTypes = []any{
|
|
||||||
(MFFileOuter_Version)(0), // 0: MFFileOuter.Version
|
|
||||||
(MFFileOuter_CompressionType)(0), // 1: MFFileOuter.CompressionType
|
|
||||||
(MFFile_Version)(0), // 2: MFFile.Version
|
|
||||||
(*Timestamp)(nil), // 3: Timestamp
|
|
||||||
(*MFFileOuter)(nil), // 4: MFFileOuter
|
|
||||||
(*MFFilePath)(nil), // 5: MFFilePath
|
|
||||||
(*MFFileChecksum)(nil), // 6: MFFileChecksum
|
|
||||||
(*MFFile)(nil), // 7: MFFile
|
|
||||||
}
|
|
||||||
var file_mf_proto_depIdxs = []int32{
|
|
||||||
0, // 0: MFFileOuter.version:type_name -> MFFileOuter.Version
|
|
||||||
1, // 1: MFFileOuter.compressionType:type_name -> MFFileOuter.CompressionType
|
|
||||||
6, // 2: MFFilePath.hashes:type_name -> MFFileChecksum
|
|
||||||
3, // 3: MFFilePath.mtime:type_name -> Timestamp
|
|
||||||
3, // 4: MFFilePath.ctime:type_name -> Timestamp
|
|
||||||
2, // 5: MFFile.version:type_name -> MFFile.Version
|
|
||||||
5, // 6: MFFile.files:type_name -> MFFilePath
|
|
||||||
3, // 7: MFFile.createdAt:type_name -> Timestamp
|
|
||||||
8, // [8:8] is the sub-list for method output_type
|
|
||||||
8, // [8:8] is the sub-list for method input_type
|
|
||||||
8, // [8:8] is the sub-list for extension type_name
|
|
||||||
8, // [8:8] is the sub-list for extension extendee
|
|
||||||
0, // [0:8] is the sub-list for field type_name
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() { file_mf_proto_init() }
|
|
||||||
func file_mf_proto_init() {
|
|
||||||
if File_mf_proto != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
file_mf_proto_msgTypes[1].OneofWrappers = []any{}
|
|
||||||
file_mf_proto_msgTypes[2].OneofWrappers = []any{}
|
|
||||||
file_mf_proto_msgTypes[4].OneofWrappers = []any{}
|
|
||||||
type x struct{}
|
|
||||||
out := protoimpl.TypeBuilder{
|
|
||||||
File: protoimpl.DescBuilder{
|
|
||||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
|
||||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_mf_proto_rawDesc), len(file_mf_proto_rawDesc)),
|
|
||||||
NumEnums: 3,
|
|
||||||
NumMessages: 5,
|
|
||||||
NumExtensions: 0,
|
|
||||||
NumServices: 0,
|
|
||||||
},
|
|
||||||
GoTypes: file_mf_proto_goTypes,
|
|
||||||
DependencyIndexes: file_mf_proto_depIdxs,
|
|
||||||
EnumInfos: file_mf_proto_enumTypes,
|
|
||||||
MessageInfos: file_mf_proto_msgTypes,
|
|
||||||
}.Build()
|
|
||||||
File_mf_proto = out.File
|
|
||||||
file_mf_proto_goTypes = nil
|
|
||||||
file_mf_proto_depIdxs = nil
|
|
||||||
}
|
|
||||||
@@ -18,7 +18,7 @@ message MFFileOuter {
|
|||||||
|
|
||||||
enum CompressionType {
|
enum CompressionType {
|
||||||
COMPRESSION_NONE = 0;
|
COMPRESSION_NONE = 0;
|
||||||
COMPRESSION_ZSTD = 1;
|
COMPRESSION_GZIP = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
CompressionType compressionType = 102;
|
CompressionType compressionType = 102;
|
||||||
@@ -28,9 +28,6 @@ message MFFileOuter {
|
|||||||
int64 size = 103;
|
int64 size = 103;
|
||||||
bytes sha256 = 104;
|
bytes sha256 = 104;
|
||||||
|
|
||||||
// uuid must match the uuid in the inner message
|
|
||||||
bytes uuid = 105;
|
|
||||||
|
|
||||||
bytes innerMessage = 199;
|
bytes innerMessage = 199;
|
||||||
// 2xx for optional manifest root attributes
|
// 2xx for optional manifest root attributes
|
||||||
// think we might use gosignify instead of gpg:
|
// think we might use gosignify instead of gpg:
|
||||||
@@ -46,9 +43,6 @@ message MFFileOuter {
|
|||||||
|
|
||||||
message MFFilePath {
|
message MFFilePath {
|
||||||
// required attributes:
|
// required attributes:
|
||||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
|
||||||
// be relative (no leading /), contain no ".." segments, and no
|
|
||||||
// empty segments (no "//").
|
|
||||||
string path = 1;
|
string path = 1;
|
||||||
int64 size = 2;
|
int64 size = 2;
|
||||||
|
|
||||||
@@ -59,6 +53,7 @@ message MFFilePath {
|
|||||||
optional string mimeType = 301;
|
optional string mimeType = 301;
|
||||||
optional Timestamp mtime = 302;
|
optional Timestamp mtime = 302;
|
||||||
optional Timestamp ctime = 303;
|
optional Timestamp ctime = 303;
|
||||||
|
optional Timestamp atime = 304;
|
||||||
}
|
}
|
||||||
|
|
||||||
message MFFileChecksum {
|
message MFFileChecksum {
|
||||||
@@ -77,10 +72,6 @@ message MFFile {
|
|||||||
// required manifest attributes:
|
// required manifest attributes:
|
||||||
repeated MFFilePath files = 101;
|
repeated MFFilePath files = 101;
|
||||||
|
|
||||||
// uuid is a random v4 UUID generated when creating the manifest
|
|
||||||
// used as part of the signature to prevent replay attacks
|
|
||||||
bytes uuid = 102;
|
|
||||||
|
|
||||||
// optional manifest attributes 2xx:
|
// optional manifest attributes 2xx:
|
||||||
optional Timestamp createdAt = 201;
|
optional Timestamp createdAt = 201;
|
||||||
}
|
}
|
||||||
|
|||||||
74
mfer/mfer_test.go
Normal file
74
mfer/mfer_test.go
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
package mfer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/mfer/internal/log"
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Add those variables as well
|
||||||
|
var (
|
||||||
|
existingFolder = "./testdata/a-folder-that-exists"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
af *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
|
||||||
|
big *afero.Afero = &afero.Afero{Fs: afero.NewMemMapFs()}
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
log.EnableDebugLogging()
|
||||||
|
|
||||||
|
// create test files and directories
|
||||||
|
af.MkdirAll("/a/b/c", 0o755)
|
||||||
|
af.MkdirAll("/.hidden", 0o755)
|
||||||
|
af.WriteFile("/a/b/c/hello.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||||
|
af.WriteFile("/a/b/c/hello2.txt", []byte("hello world\n\n\n\n"), 0o755)
|
||||||
|
af.WriteFile("/.hidden/hello.txt", []byte("hello world\n"), 0o755)
|
||||||
|
af.WriteFile("/.hidden/hello2.txt", []byte("hello world\n"), 0o755)
|
||||||
|
|
||||||
|
big.MkdirAll("/home/user/Library", 0o755)
|
||||||
|
for i := range [25]int{} {
|
||||||
|
big.WriteFile(fmt.Sprintf("/home/user/Library/hello%d.txt", i), []byte("hello world\n"), 0o755)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPathHiddenFunc(t *testing.T) {
|
||||||
|
assert.False(t, pathIsHidden("/a/b/c/hello.txt"))
|
||||||
|
assert.True(t, pathIsHidden("/a/b/c/.hello.txt"))
|
||||||
|
assert.True(t, pathIsHidden("/a/.b/c/hello.txt"))
|
||||||
|
assert.True(t, pathIsHidden("/.a/b/c/hello.txt"))
|
||||||
|
assert.False(t, pathIsHidden("./a/b/c/hello.txt"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestGenerationOne(t *testing.T) {
|
||||||
|
m, err := NewFromFS(&ManifestScanOptions{
|
||||||
|
IgnoreDotfiles: true,
|
||||||
|
}, af)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
m.Scan()
|
||||||
|
assert.Equal(t, int64(2), m.GetFileCount())
|
||||||
|
assert.Equal(t, int64(30), m.GetTotalFileSize())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManifestGenerationTwo(t *testing.T) {
|
||||||
|
m, err := NewFromFS(&ManifestScanOptions{
|
||||||
|
IgnoreDotfiles: false,
|
||||||
|
}, af)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
assert.NotNil(t, m)
|
||||||
|
m.Scan()
|
||||||
|
assert.Equal(t, int64(4), m.GetFileCount())
|
||||||
|
assert.Equal(t, int64(54), m.GetTotalFileSize())
|
||||||
|
err = m.generate()
|
||||||
|
assert.Nil(t, err)
|
||||||
|
var buf bytes.Buffer
|
||||||
|
err = m.WriteTo(&buf)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
log.Dump(buf.Bytes())
|
||||||
|
}
|
||||||
33
mfer/output.go
Normal file
33
mfer/output.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package mfer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (m *manifest) WriteToFile(path string) error {
|
||||||
|
// FIXME refuse to overwrite without -f if file exists
|
||||||
|
|
||||||
|
f, err := os.Create(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
return m.WriteTo(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *manifest) WriteTo(output io.Writer) error {
|
||||||
|
if m.pbOuter == nil {
|
||||||
|
err := m.generate()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := output.Write(m.output.Bytes())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
590
mfer/scanner.go
590
mfer/scanner.go
@@ -1,590 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"path"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Phase 1: Enumeration
|
|
||||||
// ---------------------
|
|
||||||
// Walking directories and calling stat() on files to collect metadata.
|
|
||||||
// Builds the list of files to be scanned. Relatively fast (metadata only).
|
|
||||||
|
|
||||||
// EnumerateStatus contains progress information for the enumeration phase.
|
|
||||||
type EnumerateStatus struct {
|
|
||||||
FilesFound FileCount // Number of files discovered so far
|
|
||||||
BytesFound FileSize // Total size of discovered files (from stat)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Phase 2: Scan (ToManifest)
|
|
||||||
// --------------------------
|
|
||||||
// Reading file contents and computing hashes for manifest generation.
|
|
||||||
// This is the expensive phase that reads all file data.
|
|
||||||
|
|
||||||
// ScanStatus contains progress information for the scan phase.
|
|
||||||
type ScanStatus struct {
|
|
||||||
TotalFiles FileCount // Total number of files to scan
|
|
||||||
ScannedFiles FileCount // Number of files scanned so far
|
|
||||||
TotalBytes FileSize // Total bytes to read (sum of all file sizes)
|
|
||||||
ScannedBytes FileSize // Bytes read so far
|
|
||||||
BytesPerSec float64 // Current throughput rate
|
|
||||||
ETA time.Duration // Estimated time to completion
|
|
||||||
}
|
|
||||||
|
|
||||||
// ScannerOptions configures scanner behavior.
|
|
||||||
type ScannerOptions struct {
|
|
||||||
// IncludeDotfiles includes files and directories starting with a dot
|
|
||||||
// (default: exclude).
|
|
||||||
IncludeDotfiles bool
|
|
||||||
// FollowSymLinks resolves symlinks instead of skipping them.
|
|
||||||
FollowSymLinks bool
|
|
||||||
// IncludeTimestamps includes a createdAt timestamp in the manifest
|
|
||||||
// (default: omit for determinism).
|
|
||||||
IncludeTimestamps bool
|
|
||||||
// Fs is the filesystem to use, defaults to OsFs if nil.
|
|
||||||
Fs afero.Fs
|
|
||||||
// SigningOptions holds GPG signing options (nil = no signing).
|
|
||||||
SigningOptions *SigningOptions
|
|
||||||
// Seed, if set, derives a deterministic UUID from this seed.
|
|
||||||
Seed string
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileEntry represents a file that has been enumerated.
|
|
||||||
type FileEntry struct {
|
|
||||||
Path RelFilePath // Relative path (used in manifest)
|
|
||||||
AbsPath AbsFilePath // Absolute path (used for reading file content)
|
|
||||||
Size FileSize // File size in bytes
|
|
||||||
Mtime ModTime // Last modification time
|
|
||||||
Ctime time.Time // Creation time (platform-dependent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Scanner accumulates files and generates manifests from them.
|
|
||||||
type Scanner struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
files []*FileEntry
|
|
||||||
totalBytes FileSize // cached sum of all file sizes
|
|
||||||
options *ScannerOptions
|
|
||||||
fs afero.Fs
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewScanner creates a new Scanner with default options.
|
|
||||||
func NewScanner() *Scanner {
|
|
||||||
return NewScannerWithOptions(nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewScannerWithOptions creates a new Scanner with the given options.
|
|
||||||
func NewScannerWithOptions(opts *ScannerOptions) *Scanner {
|
|
||||||
if opts == nil {
|
|
||||||
opts = &ScannerOptions{}
|
|
||||||
}
|
|
||||||
|
|
||||||
fs := opts.Fs
|
|
||||||
if fs == nil {
|
|
||||||
fs = afero.NewOsFs()
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Scanner{
|
|
||||||
files: make([]*FileEntry, 0),
|
|
||||||
options: opts,
|
|
||||||
fs: fs,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnumerateFile adds a single file to the scanner, calling stat() to get metadata.
|
|
||||||
func (s *Scanner) EnumerateFile(filePath string) error {
|
|
||||||
abs, err := filepath.Abs(filePath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
info, err := s.fs.Stat(abs)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// For single files, use the filename as the relative path
|
|
||||||
basePath := filepath.Dir(abs)
|
|
||||||
|
|
||||||
return s.enumerateFileWithInfo(filepath.Base(abs), basePath, info, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnumeratePath walks a directory path and adds all files to the scanner.
|
|
||||||
// If progress is non-nil, status updates are sent as files are discovered.
|
|
||||||
// The progress channel is closed when the method returns.
|
|
||||||
func (s *Scanner) EnumeratePath(
|
|
||||||
inputPath string,
|
|
||||||
progress chan<- EnumerateStatus,
|
|
||||||
) error {
|
|
||||||
if progress != nil {
|
|
||||||
defer close(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
abs, err := filepath.Abs(inputPath)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
|
||||||
|
|
||||||
return s.enumerateFS(afs, abs, progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnumeratePaths walks multiple directory paths and adds all files to the scanner.
|
|
||||||
// If progress is non-nil, status updates are sent as files are discovered.
|
|
||||||
// The progress channel is closed when the method returns.
|
|
||||||
func (s *Scanner) EnumeratePaths(
|
|
||||||
progress chan<- EnumerateStatus,
|
|
||||||
inputPaths ...string,
|
|
||||||
) error {
|
|
||||||
if progress != nil {
|
|
||||||
defer close(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, p := range inputPaths {
|
|
||||||
abs, err := filepath.Abs(p)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
afs := afero.NewReadOnlyFs(afero.NewBasePathFs(s.fs, abs))
|
|
||||||
|
|
||||||
err = s.enumerateFS(afs, abs, progress)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnumerateFS walks an afero filesystem and adds all files to the scanner.
|
|
||||||
// If progress is non-nil, status updates are sent as files are discovered.
|
|
||||||
// The progress channel is closed when the method returns.
|
|
||||||
// basePath is used to compute absolute paths for file reading.
|
|
||||||
func (s *Scanner) EnumerateFS(
|
|
||||||
afs afero.Fs,
|
|
||||||
basePath string,
|
|
||||||
progress chan<- EnumerateStatus,
|
|
||||||
) error {
|
|
||||||
if progress != nil {
|
|
||||||
defer close(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.enumerateFS(afs, basePath, progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Files returns a copy of all files added to the scanner.
|
|
||||||
func (s *Scanner) Files() []*FileEntry {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
|
|
||||||
out := make([]*FileEntry, len(s.files))
|
|
||||||
copy(out, s.files)
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileCount returns the number of files in the scanner.
|
|
||||||
func (s *Scanner) FileCount() FileCount {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
|
|
||||||
return FileCount(len(s.files))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TotalBytes returns the total size of all files in the scanner.
|
|
||||||
func (s *Scanner) TotalBytes() FileSize {
|
|
||||||
s.mu.RLock()
|
|
||||||
defer s.mu.RUnlock()
|
|
||||||
|
|
||||||
return s.totalBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
// ToManifest reads all file contents, computes hashes, and generates a manifest.
|
|
||||||
// If progress is non-nil, status updates are sent approximately once per second.
|
|
||||||
// The progress channel is closed when the method returns.
|
|
||||||
// The manifest is written to the provided io.Writer.
|
|
||||||
func (s *Scanner) ToManifest(
|
|
||||||
ctx context.Context, w io.Writer, progress chan<- ScanStatus,
|
|
||||||
) error {
|
|
||||||
if progress != nil {
|
|
||||||
defer close(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.mu.RLock()
|
|
||||||
files := make([]*FileEntry, len(s.files))
|
|
||||||
copy(files, s.files)
|
|
||||||
totalFiles := FileCount(len(files))
|
|
||||||
|
|
||||||
var totalBytes FileSize
|
|
||||||
for _, f := range files {
|
|
||||||
totalBytes += f.Size
|
|
||||||
}
|
|
||||||
|
|
||||||
s.mu.RUnlock()
|
|
||||||
|
|
||||||
builder := s.configureBuilder()
|
|
||||||
|
|
||||||
var (
|
|
||||||
scannedFiles FileCount
|
|
||||||
scannedBytes FileSize
|
|
||||||
)
|
|
||||||
|
|
||||||
lastProgressTime := time.Now()
|
|
||||||
startTime := time.Now()
|
|
||||||
|
|
||||||
pt := &scanProgressTracker{
|
|
||||||
progress: progress,
|
|
||||||
totalFiles: totalFiles,
|
|
||||||
totalBytes: totalBytes,
|
|
||||||
startTime: startTime,
|
|
||||||
lastProgress: &lastProgressTime,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, entry := range files {
|
|
||||||
// Check for cancellation
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
bytesRead, err := s.scanFile(builder, pt, entry, scannedFiles, scannedBytes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
scannedFiles++
|
|
||||||
scannedBytes += bytesRead
|
|
||||||
}
|
|
||||||
|
|
||||||
// Send final progress (ETA is 0 at completion; remaining bytes are 0,
|
|
||||||
// so computeRateETA yields eta 0 and the same average rate as before)
|
|
||||||
if progress != nil {
|
|
||||||
rate, _ := computeRateETA(time.Since(startTime), scannedBytes, totalBytes)
|
|
||||||
|
|
||||||
sendScanStatus(progress, ScanStatus{
|
|
||||||
TotalFiles: totalFiles,
|
|
||||||
ScannedFiles: scannedFiles,
|
|
||||||
TotalBytes: totalBytes,
|
|
||||||
ScannedBytes: scannedBytes,
|
|
||||||
BytesPerSec: rate,
|
|
||||||
ETA: 0,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build and write manifest
|
|
||||||
//nolint:contextcheck // Build's GPG signing exec is not cancellable by design
|
|
||||||
return builder.Build(w)
|
|
||||||
}
|
|
||||||
|
|
||||||
// configureBuilder constructs a manifest builder configured from the
|
|
||||||
// scanner options.
|
|
||||||
func (s *Scanner) configureBuilder() *Builder {
|
|
||||||
builder := NewBuilder()
|
|
||||||
if s.options.IncludeTimestamps {
|
|
||||||
builder.SetIncludeTimestamps(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.options.SigningOptions != nil {
|
|
||||||
builder.SetSigningOptions(s.options.SigningOptions)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.options.Seed != "" {
|
|
||||||
builder.SetSeed(s.options.Seed)
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanFile hashes a single file into the builder, forwarding per-file
|
|
||||||
// progress updates, and returns the number of bytes read.
|
|
||||||
func (s *Scanner) scanFile(
|
|
||||||
builder *Builder,
|
|
||||||
pt *scanProgressTracker,
|
|
||||||
entry *FileEntry,
|
|
||||||
scannedFiles FileCount,
|
|
||||||
scannedBytes FileSize,
|
|
||||||
) (FileSize, error) {
|
|
||||||
// Open file
|
|
||||||
f, err := s.fs.Open(string(entry.AbsPath))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create progress channel for this file
|
|
||||||
var (
|
|
||||||
fileProgress chan FileHashProgress
|
|
||||||
wg sync.WaitGroup
|
|
||||||
)
|
|
||||||
|
|
||||||
if pt.progress != nil {
|
|
||||||
fileProgress = make(chan FileHashProgress, 1)
|
|
||||||
|
|
||||||
wg.Add(1)
|
|
||||||
|
|
||||||
go func(base FileSize, done FileCount) {
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
pt.forward(fileProgress, done, base)
|
|
||||||
}(scannedBytes, scannedFiles)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to manifest with progress channel
|
|
||||||
bytesRead, err := builder.AddFile(
|
|
||||||
entry.Path,
|
|
||||||
entry.Size,
|
|
||||||
entry.Mtime,
|
|
||||||
f,
|
|
||||||
fileProgress,
|
|
||||||
)
|
|
||||||
_ = f.Close()
|
|
||||||
|
|
||||||
// Close channel and wait for goroutine to finish
|
|
||||||
if fileProgress != nil {
|
|
||||||
close(fileProgress)
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Verbosef("+ %s (%s)", entry.Path, humanize.IBytes(sizeToUint64(bytesRead)))
|
|
||||||
|
|
||||||
return bytesRead, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// enumerateFS is the internal implementation that doesn't close the
|
|
||||||
// progress channel.
|
|
||||||
func (s *Scanner) enumerateFS(
|
|
||||||
afs afero.Fs,
|
|
||||||
basePath string,
|
|
||||||
progress chan<- EnumerateStatus,
|
|
||||||
) error {
|
|
||||||
return afero.Walk(afs, "/", func(p string, info fs.FileInfo, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !s.options.IncludeDotfiles && IsHiddenPath(p) {
|
|
||||||
if info.IsDir() {
|
|
||||||
return filepath.SkipDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.enumerateFileWithInfo(p, basePath, info, progress)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// enumerateFileWithInfo adds a file with pre-existing fs.FileInfo.
|
|
||||||
func (s *Scanner) enumerateFileWithInfo(
|
|
||||||
filePath string,
|
|
||||||
basePath string,
|
|
||||||
info fs.FileInfo,
|
|
||||||
progress chan<- EnumerateStatus,
|
|
||||||
) error {
|
|
||||||
if info.IsDir() {
|
|
||||||
// Manifests contain only files, directories are implied
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean the path - remove leading slash if present
|
|
||||||
cleanPath := filePath
|
|
||||||
if len(cleanPath) > 0 && cleanPath[0] == '/' {
|
|
||||||
cleanPath = cleanPath[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compute absolute path for file reading
|
|
||||||
absPath := filepath.Join(basePath, cleanPath)
|
|
||||||
|
|
||||||
// Handle symlinks
|
|
||||||
if info.Mode()&fs.ModeSymlink != 0 {
|
|
||||||
if !s.options.FollowSymLinks {
|
|
||||||
// Skip symlinks when not following them
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// Resolve symlink to get real file info
|
|
||||||
realPath, err := filepath.EvalSymlinks(absPath)
|
|
||||||
if err != nil {
|
|
||||||
// Skip broken symlinks
|
|
||||||
return nil //nolint:nilerr // broken symlinks are skipped by design
|
|
||||||
}
|
|
||||||
|
|
||||||
realInfo, err := s.fs.Stat(realPath)
|
|
||||||
if err != nil {
|
|
||||||
// Skip symlinks whose target cannot be stat'd
|
|
||||||
return nil //nolint:nilerr // unreadable targets are skipped by design
|
|
||||||
}
|
|
||||||
// Skip if symlink points to a directory
|
|
||||||
if realInfo.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// Use resolved path for reading, but keep original path in manifest
|
|
||||||
absPath = realPath
|
|
||||||
info = realInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
entry := &FileEntry{
|
|
||||||
Path: RelFilePath(cleanPath),
|
|
||||||
AbsPath: AbsFilePath(absPath),
|
|
||||||
Size: FileSize(info.Size()),
|
|
||||||
Mtime: ModTime(info.ModTime()),
|
|
||||||
// Note: Ctime not available from fs.FileInfo on all platforms
|
|
||||||
// Will need platform-specific code to extract it
|
|
||||||
}
|
|
||||||
|
|
||||||
s.mu.Lock()
|
|
||||||
s.files = append(s.files, entry)
|
|
||||||
s.totalBytes += entry.Size
|
|
||||||
filesFound := FileCount(len(s.files))
|
|
||||||
bytesFound := s.totalBytes
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
sendEnumerateStatus(progress, EnumerateStatus{
|
|
||||||
FilesFound: filesFound,
|
|
||||||
BytesFound: bytesFound,
|
|
||||||
})
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanProgressTracker carries the shared state needed to report rate-limited
|
|
||||||
// scan progress updates.
|
|
||||||
type scanProgressTracker struct {
|
|
||||||
progress chan<- ScanStatus
|
|
||||||
totalFiles FileCount
|
|
||||||
totalBytes FileSize
|
|
||||||
startTime time.Time
|
|
||||||
lastProgress *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// forward relays per-file hash progress to the scan progress channel,
|
|
||||||
// rate-limited to one update per second.
|
|
||||||
func (pt *scanProgressTracker) forward(
|
|
||||||
fileProgress <-chan FileHashProgress,
|
|
||||||
scannedFiles FileCount,
|
|
||||||
baseBytes FileSize,
|
|
||||||
) {
|
|
||||||
for p := range fileProgress {
|
|
||||||
// Send progress at most once per second
|
|
||||||
now := time.Now()
|
|
||||||
if now.Sub(*pt.lastProgress) < time.Second {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
currentBytes := baseBytes + p.BytesRead
|
|
||||||
rate, eta := computeRateETA(now.Sub(pt.startTime), currentBytes, pt.totalBytes)
|
|
||||||
|
|
||||||
sendScanStatus(pt.progress, ScanStatus{
|
|
||||||
TotalFiles: pt.totalFiles,
|
|
||||||
ScannedFiles: scannedFiles,
|
|
||||||
TotalBytes: pt.totalBytes,
|
|
||||||
ScannedBytes: currentBytes,
|
|
||||||
BytesPerSec: rate,
|
|
||||||
ETA: eta,
|
|
||||||
})
|
|
||||||
|
|
||||||
*pt.lastProgress = now
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// computeRateETA returns the average throughput over elapsed time and the
|
|
||||||
// estimated time to process the remaining bytes at that rate.
|
|
||||||
func computeRateETA(
|
|
||||||
elapsed time.Duration,
|
|
||||||
done FileSize,
|
|
||||||
total FileSize,
|
|
||||||
) (float64, time.Duration) {
|
|
||||||
var (
|
|
||||||
rate float64
|
|
||||||
eta time.Duration
|
|
||||||
)
|
|
||||||
|
|
||||||
if elapsed > 0 && done > 0 {
|
|
||||||
rate = float64(done) / elapsed.Seconds()
|
|
||||||
|
|
||||||
remaining := total - done
|
|
||||||
if rate > 0 {
|
|
||||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return rate, eta
|
|
||||||
}
|
|
||||||
|
|
||||||
// sizeToUint64 converts a FileSize to uint64 for display, clamping
|
|
||||||
// negative values to zero so the conversion cannot overflow.
|
|
||||||
func sizeToUint64(v FileSize) uint64 {
|
|
||||||
if v < 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return uint64(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
// IsHiddenPath returns true if the path or any of its parent directories
|
|
||||||
// start with a dot (hidden files/directories).
|
|
||||||
// The path should use forward slashes.
|
|
||||||
func IsHiddenPath(p string) bool {
|
|
||||||
tp := path.Clean(p)
|
|
||||||
if tp == "." || tp == "/" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.HasPrefix(tp, ".") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
d, f := path.Split(tp)
|
|
||||||
if strings.HasPrefix(f, ".") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if d == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
tp = d[0 : len(d)-1] // trim trailing slash from dir
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendEnumerateStatus sends a status update without blocking.
|
|
||||||
// If the channel is full, the update is dropped.
|
|
||||||
func sendEnumerateStatus(ch chan<- EnumerateStatus, status EnumerateStatus) {
|
|
||||||
if ch == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ch <- status:
|
|
||||||
default:
|
|
||||||
// Channel full, drop this update
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendScanStatus sends a status update without blocking.
|
|
||||||
// If the channel is full, the update is dropped.
|
|
||||||
func sendScanStatus(ch chan<- ScanStatus, status ScanStatus) {
|
|
||||||
if ch == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case ch <- status:
|
|
||||||
default:
|
|
||||||
// Channel full, drop this update
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNewScanner(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := NewScanner()
|
|
||||||
assert.NotNil(t, s)
|
|
||||||
assert.Equal(t, FileCount(0), s.FileCount())
|
|
||||||
assert.Equal(t, FileSize(0), s.TotalBytes())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewScannerWithOptions(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
t.Run("nil options", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(nil)
|
|
||||||
assert.NotNil(t, s)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("with options", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
opts := &ScannerOptions{
|
|
||||||
IncludeDotfiles: true,
|
|
||||||
FollowSymLinks: true,
|
|
||||||
Fs: fs,
|
|
||||||
}
|
|
||||||
s := NewScannerWithOptions(opts)
|
|
||||||
assert.NotNil(t, s)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumerateFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello world"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumerateFile("/test.txt")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(1), s.FileCount())
|
|
||||||
assert.Equal(t, FileSize(11), s.TotalBytes())
|
|
||||||
|
|
||||||
files := s.Files()
|
|
||||||
require.Len(t, files, 1)
|
|
||||||
assert.Equal(t, RelFilePath("test.txt"), files[0].Path)
|
|
||||||
assert.Equal(t, FileSize(11), files[0].Size)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumerateFileMissing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumerateFile("/nonexistent.txt")
|
|
||||||
assert.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumeratePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir/subdir", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0o644))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("two"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/subdir/file3.txt", []byte("three"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(3), s.FileCount())
|
|
||||||
assert.Equal(t, FileSize(3+3+5), s.TotalBytes())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumeratePathWithProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/testdir/file1.txt", []byte("one"), 0o644))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/testdir/file2.txt", []byte("two"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
progress := make(chan EnumerateStatus, 10)
|
|
||||||
|
|
||||||
err := s.EnumeratePath("/testdir", progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var updates []EnumerateStatus
|
|
||||||
for p := range progress {
|
|
||||||
updates = append(updates, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotEmpty(t, updates)
|
|
||||||
// Final update should show all files
|
|
||||||
final := updates[len(updates)-1]
|
|
||||||
assert.Equal(t, FileCount(2), final.FilesFound)
|
|
||||||
assert.Equal(t, FileSize(6), final.BytesFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumeratePaths(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/dir1", 0o755))
|
|
||||||
require.NoError(t, fs.MkdirAll("/dir2", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/dir1/a.txt", []byte("aaa"), 0o644))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/dir2/b.txt", []byte("bbb"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumeratePaths(nil, "/dir1", "/dir2")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(2), s.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerExcludeDotfiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir/.hidden", 0o755))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/visible.txt", []byte("visible"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/.hidden.txt", []byte("hidden"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/.hidden/inside.txt", []byte("inside"), 0o644))
|
|
||||||
|
|
||||||
t.Run("exclude by default", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: false})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(1), s.FileCount())
|
|
||||||
files := s.Files()
|
|
||||||
assert.Equal(t, RelFilePath("visible.txt"), files[0].Path)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("include when enabled", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs, IncludeDotfiles: true})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(3), s.FileCount())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerToManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/file1.txt", []byte("content one"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/file2.txt", []byte("content two"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = s.ToManifest(context.Background(), &buf, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Manifest should have magic bytes
|
|
||||||
assert.Positive(t, buf.Len())
|
|
||||||
assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerToManifestWithProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/file.txt", bytes.Repeat([]byte("x"), 1000), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
progress := make(chan ScanStatus, 10)
|
|
||||||
|
|
||||||
err = s.ToManifest(context.Background(), &buf, progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var updates []ScanStatus
|
|
||||||
for p := range progress {
|
|
||||||
updates = append(updates, p)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotEmpty(t, updates)
|
|
||||||
// Final update should show completion
|
|
||||||
final := updates[len(updates)-1]
|
|
||||||
assert.Equal(t, FileCount(1), final.TotalFiles)
|
|
||||||
assert.Equal(t, FileCount(1), final.ScannedFiles)
|
|
||||||
assert.Equal(t, FileSize(1000), final.TotalBytes)
|
|
||||||
assert.Equal(t, FileSize(1000), final.ScannedBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerToManifestContextCancellation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
// Create many files to ensure we have time to cancel
|
|
||||||
for i := range 100 {
|
|
||||||
name := string(rune('a'+i%26)) + string(rune('0'+i/26)) + ".txt"
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/"+name, bytes.Repeat([]byte("x"), 100), 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumeratePath("/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel() // Cancel immediately
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err = s.ToManifest(ctx, &buf, nil)
|
|
||||||
assert.ErrorIs(t, err, context.Canceled)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerToManifestEmptyScanner(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err := s.ToManifest(context.Background(), &buf, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Should still produce a valid manifest
|
|
||||||
assert.Positive(t, buf.Len())
|
|
||||||
assert.Equal(t, MAGIC, string(buf.Bytes()[:8]))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerFilesCopiesSlice(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("hello"), 0o644))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
require.NoError(t, s.EnumerateFile("/test.txt"))
|
|
||||||
|
|
||||||
files1 := s.Files()
|
|
||||||
files2 := s.Files()
|
|
||||||
|
|
||||||
// Should be different slices
|
|
||||||
assert.NotSame(t, &files1[0], &files2[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerEnumerateFS(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir/sub", 0o755))
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/testdir/file.txt", []byte("hello"), 0o644))
|
|
||||||
require.NoError(t,
|
|
||||||
afero.WriteFile(fs, "/testdir/sub/nested.txt", []byte("world"), 0o644))
|
|
||||||
|
|
||||||
// Create a basepath filesystem
|
|
||||||
baseFs := afero.NewBasePathFs(fs, "/testdir")
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
err := s.EnumerateFS(baseFs, "/testdir", nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(2), s.FileCount())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSendEnumerateStatusNonBlocking(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Channel with no buffer - send should not block
|
|
||||||
ch := make(chan EnumerateStatus)
|
|
||||||
|
|
||||||
// This should not block
|
|
||||||
done := make(chan bool)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
sendEnumerateStatus(ch, EnumerateStatus{FilesFound: 1})
|
|
||||||
|
|
||||||
done <- true
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
// Success - did not block
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
t.Fatal("sendEnumerateStatus blocked on full channel")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSendScanStatusNonBlocking(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Channel with no buffer - send should not block
|
|
||||||
ch := make(chan ScanStatus)
|
|
||||||
|
|
||||||
done := make(chan bool)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
sendScanStatus(ch, ScanStatus{ScannedFiles: 1})
|
|
||||||
|
|
||||||
done <- true
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
// Success - did not block
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
t.Fatal("sendScanStatus blocked on full channel")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSendStatusNilChannel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Should not panic with nil channel
|
|
||||||
sendEnumerateStatus(nil, EnumerateStatus{})
|
|
||||||
sendScanStatus(nil, ScanStatus{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerFileEntryFields(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
now := time.Now().Truncate(time.Second)
|
|
||||||
|
|
||||||
require.NoError(t, afero.WriteFile(fs, "/test.txt", []byte("content"), 0o644))
|
|
||||||
require.NoError(t, fs.Chtimes("/test.txt", now, now))
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
require.NoError(t, s.EnumerateFile("/test.txt"))
|
|
||||||
|
|
||||||
files := s.Files()
|
|
||||||
require.Len(t, files, 1)
|
|
||||||
|
|
||||||
entry := files[0]
|
|
||||||
assert.Equal(t, RelFilePath("test.txt"), entry.Path)
|
|
||||||
assert.Contains(t, string(entry.AbsPath), "test.txt")
|
|
||||||
assert.Equal(t, FileSize(7), entry.Size)
|
|
||||||
// Mtime should be set (within a second of now)
|
|
||||||
assert.WithinDuration(t, now, time.Time(entry.Mtime), 2*time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScannerLargeFileEnumeration(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
|
||||||
require.NoError(t, fs.MkdirAll("/testdir", 0o755))
|
|
||||||
|
|
||||||
// Create 100 files
|
|
||||||
for i := range 100 {
|
|
||||||
name := "/testdir/" + string(rune('a'+i%26)) + string(rune('0'+i/26%10)) + ".txt"
|
|
||||||
require.NoError(t, afero.WriteFile(fs, name, []byte("data"), 0o644))
|
|
||||||
}
|
|
||||||
|
|
||||||
s := NewScannerWithOptions(&ScannerOptions{Fs: fs})
|
|
||||||
progress := make(chan EnumerateStatus, 200)
|
|
||||||
|
|
||||||
err := s.EnumeratePath("/testdir", progress)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// progress is fully buffered and closed; no draining needed
|
|
||||||
|
|
||||||
assert.Equal(t, FileCount(100), s.FileCount())
|
|
||||||
assert.Equal(t, FileSize(400), s.TotalBytes()) // 100 * 4 bytes
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsHiddenPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
path string
|
|
||||||
hidden bool
|
|
||||||
}{
|
|
||||||
{testFileName, false},
|
|
||||||
{".hidden", true},
|
|
||||||
{"dir/file.txt", false},
|
|
||||||
{"dir/.hidden", true},
|
|
||||||
{".dir/file.txt", true},
|
|
||||||
{"/absolute/path", false},
|
|
||||||
{"/absolute/.hidden", true},
|
|
||||||
{"./relative", false}, // path.Clean removes leading ./
|
|
||||||
{"a/b/c/.d/e", true},
|
|
||||||
{".", false}, // current directory is not hidden (#14)
|
|
||||||
{"/", false}, // root is not hidden
|
|
||||||
{"./", false}, // current directory with trailing slash
|
|
||||||
{"./file.txt", false}, // file in current directory
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(t, tt.hidden, IsHiddenPath(tt.path), "IsHiddenPath(%q)", tt.path)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,173 +2,99 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/klauspost/compress/zstd"
|
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
)
|
)
|
||||||
|
|
||||||
// MAGIC is the file format magic bytes prefix (rot13 of "MANIFEST").
|
//go:generate protoc --go_out=. --go_opt=paths=source_relative mf.proto
|
||||||
|
|
||||||
|
// rot13("MANIFEST")
|
||||||
const MAGIC string = "ZNAVSRFG"
|
const MAGIC string = "ZNAVSRFG"
|
||||||
|
|
||||||
var (
|
|
||||||
// errInnerNotSet is returned by generate when the inner manifest is
|
|
||||||
// missing.
|
|
||||||
errInnerNotSet = errors.New("internal error: pbInner not set")
|
|
||||||
// errInternal is returned by generateOuter for the same condition.
|
|
||||||
// The two messages differ, and both are load-bearing for callers that
|
|
||||||
// match on text, so they are kept distinct.
|
|
||||||
errInternal = errors.New("internal error")
|
|
||||||
)
|
|
||||||
|
|
||||||
// nanosecondsInt32 converts t's nanosecond component to int32.
|
|
||||||
// time.Time.Nanosecond is documented to return a value in [0, 999999999],
|
|
||||||
// so the conversion cannot overflow. This sits directly in the manifest
|
|
||||||
// content path: silently substituting a default would zero every entry's
|
|
||||||
// mtime nanos and change the serialized bytes and their hash, so an
|
|
||||||
// out-of-contract value is a programming error and panics rather than
|
|
||||||
// being papered over.
|
|
||||||
func nanosecondsInt32(t time.Time) int32 {
|
|
||||||
n := t.Nanosecond()
|
|
||||||
if n < 0 || n > math.MaxInt32 {
|
|
||||||
panic(fmt.Sprintf(
|
|
||||||
"mfer: time.Time.Nanosecond out of contract: %d", n))
|
|
||||||
}
|
|
||||||
|
|
||||||
return int32(n)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTimestampFromTime(t time.Time) *Timestamp {
|
func newTimestampFromTime(t time.Time) *Timestamp {
|
||||||
return &Timestamp{
|
out := &Timestamp{
|
||||||
Seconds: t.Unix(),
|
Seconds: t.Unix(),
|
||||||
Nanos: nanosecondsInt32(t),
|
Nanos: int32(t.UnixNano() - (t.Unix() * 1000000000)),
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifest) generate() error {
|
func (m *manifest) generate() error {
|
||||||
if m.pbInner == nil {
|
if m.pbInner == nil {
|
||||||
return errInnerNotSet
|
e := m.generateInner()
|
||||||
|
if e != nil {
|
||||||
|
return e
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.pbOuter == nil {
|
if m.pbOuter == nil {
|
||||||
e := m.generateOuter()
|
e := m.generateOuter()
|
||||||
if e != nil {
|
if e != nil {
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
|
dat, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbOuter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("serialize: marshal outer: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
m.output = bytes.NewBuffer([]byte(MAGIC))
|
||||||
m.output = bytes.NewBufferString(MAGIC)
|
|
||||||
|
|
||||||
_, err = m.output.Write(dat)
|
_, err = m.output.Write(dat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("serialize: write output: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *manifest) generateOuter() error {
|
func (m *manifest) generateOuter() error {
|
||||||
if m.pbInner == nil {
|
if m.pbInner == nil {
|
||||||
return errInternal
|
return errors.New("internal error")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use fixed UUID if provided, otherwise generate a new one
|
|
||||||
var manifestUUID uuid.UUID
|
|
||||||
if len(m.fixedUUID) == uuidLength {
|
|
||||||
copy(manifestUUID[:], m.fixedUUID)
|
|
||||||
} else {
|
|
||||||
manifestUUID = uuid.New()
|
|
||||||
}
|
|
||||||
|
|
||||||
m.pbInner.Uuid = manifestUUID[:]
|
|
||||||
|
|
||||||
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
|
innerData, err := proto.MarshalOptions{Deterministic: true}.Marshal(m.pbInner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("serialize: marshal inner: %w", err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compress the inner data
|
|
||||||
idc := new(bytes.Buffer)
|
|
||||||
|
|
||||||
zw, err := zstd.NewWriter(idc, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("serialize: create compressor: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = zw.Write(innerData)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("serialize: compress: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = zw.Close()
|
|
||||||
|
|
||||||
compressedData := idc.Bytes()
|
|
||||||
|
|
||||||
// Hash the compressed data for integrity verification before decompression
|
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
|
h.Write(innerData)
|
||||||
|
|
||||||
_, err = h.Write(compressedData)
|
idc := new(bytes.Buffer)
|
||||||
|
gzw, err := gzip.NewWriterLevel(idc, gzip.BestCompression)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("serialize: hash write: %w", err)
|
return err
|
||||||
|
}
|
||||||
|
_, err = gzw.Write(innerData)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
sha256Hash := h.Sum(nil)
|
gzw.Close()
|
||||||
|
|
||||||
m.pbOuter = &MFFileOuter{
|
o := &MFFileOuter{
|
||||||
InnerMessage: compressedData,
|
InnerMessage: idc.Bytes(),
|
||||||
Size: int64(len(innerData)),
|
Size: int64(len(innerData)),
|
||||||
Sha256: sha256Hash,
|
Sha256: h.Sum(nil),
|
||||||
Uuid: manifestUUID[:],
|
|
||||||
Version: MFFileOuter_VERSION_ONE,
|
Version: MFFileOuter_VERSION_ONE,
|
||||||
CompressionType: MFFileOuter_COMPRESSION_ZSTD,
|
CompressionType: MFFileOuter_COMPRESSION_GZIP,
|
||||||
}
|
}
|
||||||
|
m.pbOuter = o
|
||||||
// Sign the manifest if signing options are provided
|
|
||||||
if m.signingOptions != nil && m.signingOptions.KeyID != "" {
|
|
||||||
return m.signOuter()
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// signOuter signs the outer message with the configured GPG key and
|
func (m *manifest) generateInner() error {
|
||||||
// embeds the signature, signer fingerprint, and public key.
|
m.pbInner = &MFFile{
|
||||||
func (m *manifest) signOuter() error {
|
Version: MFFile_VERSION_ONE,
|
||||||
sigString, err := m.signatureString()
|
CreatedAt: newTimestampFromTime(time.Now()),
|
||||||
if err != nil {
|
Files: []*MFFilePath{},
|
||||||
return fmt.Errorf("failed to generate signature string: %w", err)
|
|
||||||
}
|
}
|
||||||
|
for _, f := range m.files {
|
||||||
sig, err := gpgSign([]byte(sigString), m.signingOptions.KeyID)
|
nf := &MFFilePath{
|
||||||
if err != nil {
|
Path: f.path,
|
||||||
return fmt.Errorf("failed to sign manifest: %w", err)
|
// FIXME add more stuff
|
||||||
}
|
}
|
||||||
|
m.pbInner.Files = append(m.pbInner.Files, nf)
|
||||||
m.pbOuter.Signature = sig
|
|
||||||
|
|
||||||
fingerprint, err := gpgGetKeyFingerprint(m.signingOptions.KeyID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to get key fingerprint: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
m.pbOuter.Signer = fingerprint
|
|
||||||
|
|
||||||
pubKey, err := gpgExportPublicKey(m.signingOptions.KeyID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to export public key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.pbOuter.SigningPubKey = pubKey
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
59
mfer/url.go
59
mfer/url.go
@@ -1,59 +0,0 @@
|
|||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ManifestURL represents a URL pointing to a manifest file.
|
|
||||||
type ManifestURL string
|
|
||||||
|
|
||||||
// FileURL represents a URL pointing to a file to be fetched.
|
|
||||||
type FileURL string
|
|
||||||
|
|
||||||
// BaseURL represents a base URL for constructing file URLs.
|
|
||||||
type BaseURL string
|
|
||||||
|
|
||||||
// JoinPath safely joins a relative file path to a base URL.
|
|
||||||
// The path is properly URL-encoded to prevent path traversal.
|
|
||||||
func (b BaseURL) JoinPath(path RelFilePath) (FileURL, error) {
|
|
||||||
base, err := url.Parse(string(b))
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure base path ends with /
|
|
||||||
if !strings.HasSuffix(base.Path, "/") {
|
|
||||||
base.Path += "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Encode each path segment individually to preserve slashes
|
|
||||||
segments := strings.Split(string(path), "/")
|
|
||||||
for i, seg := range segments {
|
|
||||||
segments[i] = url.PathEscape(seg)
|
|
||||||
}
|
|
||||||
|
|
||||||
ref, err := url.Parse(strings.Join(segments, "/"))
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := base.ResolveReference(ref)
|
|
||||||
|
|
||||||
return FileURL(resolved.String()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// String returns the URL as a string.
|
|
||||||
func (b BaseURL) String() string {
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// String returns the URL as a string.
|
|
||||||
func (f FileURL) String() string {
|
|
||||||
return string(f)
|
|
||||||
}
|
|
||||||
|
|
||||||
// String returns the URL as a string.
|
|
||||||
func (m ManifestURL) String() string {
|
|
||||||
return string(m)
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
//nolint:testpackage // white-box tests exercise unexported internals
|
|
||||||
package mfer
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBaseURLJoinPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
base BaseURL
|
|
||||||
path RelFilePath
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{"https://example.com/dir/", testFileName, "https://example.com/dir/file.txt"},
|
|
||||||
{"https://example.com/dir", testFileName, "https://example.com/dir/file.txt"},
|
|
||||||
{"https://example.com/", "sub/file.txt", "https://example.com/sub/file.txt"},
|
|
||||||
{
|
|
||||||
"https://example.com/dir/",
|
|
||||||
"file with spaces.txt",
|
|
||||||
"https://example.com/dir/file%20with%20spaces.txt",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(string(tt.base)+"+"+string(tt.path), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
result, err := tt.base.JoinPath(tt.path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, string(result))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBaseURLString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
b := BaseURL("https://example.com/")
|
|
||||||
assert.Equal(t, "https://example.com/", b.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileURLString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := FileURL("https://example.com/file.txt")
|
|
||||||
assert.Equal(t, "https://example.com/file.txt", f.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestManifestURLString(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m := ManifestURL("https://example.com/index.mf")
|
|
||||||
assert.Equal(t, "https://example.com/index.mf", m.String())
|
|
||||||
}
|
|
||||||
BIN
modcache.tzst
Normal file
BIN
modcache.tzst
Normal file
Binary file not shown.
10
package.json
10
package.json
@@ -1,10 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "mfer",
|
|
||||||
"version": "0.1.0",
|
|
||||||
"private": true,
|
|
||||||
"description": "Development tooling for the mfer repository: prettier, used by script/fmt and script/fmt-check to format and verify Markdown and JSON.",
|
|
||||||
"license": "WTFPL",
|
|
||||||
"devDependencies": {
|
|
||||||
"prettier": "3.9.6"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
159
script/bootstrap
159
script/bootstrap
@@ -1,159 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/bootstrap: install all dependencies needed to build and develop
|
|
||||||
# this repo. Idempotent: every install is guarded by a check so already
|
|
||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
|
||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
|
||||||
# make, node, yarn, go, or python). Node is used directly if installed;
|
|
||||||
# otherwise a pinned version is installed via nvm (installing nvm
|
|
||||||
# itself first, from a hash-verified release archive, never curl | sh).
|
|
||||||
#
|
|
||||||
# Uncomment the language sections in main() that apply to this repo.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# Pinned versions, 2026-07-06. Never "latest" or "lts"; exact versions.
|
|
||||||
NODE_VERSION="22.17.0"
|
|
||||||
NVM_VERSION="0.40.3"
|
|
||||||
# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz
|
|
||||||
NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0"
|
|
||||||
YARN_VERSION="1.22.22"
|
|
||||||
|
|
||||||
PKGMGR=""
|
|
||||||
SUDO=""
|
|
||||||
|
|
||||||
detect_pkgmgr() {
|
|
||||||
[ -n "$PKGMGR" ] && return 0
|
|
||||||
if command -v nix-env >/dev/null 2>&1; then
|
|
||||||
PKGMGR="nix"
|
|
||||||
elif command -v apt-get >/dev/null 2>&1; then
|
|
||||||
PKGMGR="apt"
|
|
||||||
elif command -v brew >/dev/null 2>&1; then
|
|
||||||
PKGMGR="brew"
|
|
||||||
elif command -v apk >/dev/null 2>&1; then
|
|
||||||
PKGMGR="apk"
|
|
||||||
else
|
|
||||||
echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ "$PKGMGR" = "apt" ]; then
|
|
||||||
export DEBIAN_FRONTEND=noninteractive
|
|
||||||
if [ "$(id -u)" != "0" ]; then
|
|
||||||
SUDO="sudo"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
|
|
||||||
pkg_install() {
|
|
||||||
detect_pkgmgr
|
|
||||||
case "$PKGMGR" in
|
|
||||||
nix) nix-env -iA "nixpkgs.$1" ;;
|
|
||||||
apt) $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" ;;
|
|
||||||
brew) brew install "$3" ;;
|
|
||||||
apk) apk add --no-cache "$4" ;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
missing() {
|
|
||||||
! command -v "$1" >/dev/null 2>&1
|
|
||||||
}
|
|
||||||
|
|
||||||
# verify_sha256 <file> <expected-hash>
|
|
||||||
verify_sha256() {
|
|
||||||
if command -v sha256sum >/dev/null 2>&1; then
|
|
||||||
actual="$(sha256sum "$1" | cut -d' ' -f1)"
|
|
||||||
else
|
|
||||||
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
|
|
||||||
fi
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "bootstrap: sha256 mismatch for $1" >&2
|
|
||||||
echo " expected: $2" >&2
|
|
||||||
echo " actual: $actual" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# nvm is a bash script; run a command in a bash with nvm loaded
|
|
||||||
nvm_sh() {
|
|
||||||
bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*"
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_nvm() {
|
|
||||||
[ -s "$HOME/.nvm/nvm.sh" ] && return 0
|
|
||||||
# nvm prerequisites; nvm itself requires bash, so install it too
|
|
||||||
if missing bash; then pkg_install bash bash bash bash; fi
|
|
||||||
if missing curl; then pkg_install curl curl curl curl; fi
|
|
||||||
if missing git; then pkg_install git git git git; fi
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
curl -fsSL -o "$tmp/nvm.tar.gz" \
|
|
||||||
"https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz"
|
|
||||||
verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256"
|
|
||||||
mkdir -p "$HOME/.nvm"
|
|
||||||
tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1
|
|
||||||
rm -rf "$tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_node() {
|
|
||||||
if ! missing node; then return 0; fi
|
|
||||||
ensure_nvm
|
|
||||||
nvm_sh "nvm install $NODE_VERSION"
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_yarn() {
|
|
||||||
if ! missing yarn; then return 0; fi
|
|
||||||
if ! missing corepack; then
|
|
||||||
corepack enable
|
|
||||||
corepack prepare "yarn@$YARN_VERSION" --activate
|
|
||||||
elif [ -s "$HOME/.nvm/nvm.sh" ]; then
|
|
||||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \
|
|
||||||
corepack prepare yarn@$YARN_VERSION --activate"
|
|
||||||
else
|
|
||||||
npm install -g "yarn@$YARN_VERSION"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
install_js_deps() {
|
|
||||||
if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then
|
|
||||||
nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \
|
|
||||||
yarn install --frozen-lockfile"
|
|
||||||
else
|
|
||||||
yarn install --frozen-lockfile
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
|
|
||||||
# Base tooling (every repo)
|
|
||||||
if missing git; then pkg_install git git git git; fi
|
|
||||||
if missing make; then pkg_install gnumake make make make; fi
|
|
||||||
|
|
||||||
# ---- JS / docs repos ----
|
|
||||||
# This is a Go repo, but node and yarn are required anyway: prettier
|
|
||||||
# formats the Markdown and JSON, and script/fmt-check verifies it.
|
|
||||||
# The version is pinned by package.json/yarn.lock, whose integrity
|
|
||||||
# hashes --frozen-lockfile enforces.
|
|
||||||
ensure_node
|
|
||||||
ensure_yarn
|
|
||||||
install_js_deps
|
|
||||||
|
|
||||||
# ---- Go repos ----
|
|
||||||
if missing go; then pkg_install go golang go go; fi
|
|
||||||
# golangci-lint: packaged in nix, brew, and apk. On apt there is no
|
|
||||||
# package: download a specific release archive from GitHub and
|
|
||||||
# verify its hash (verify_sha256), never curl | sh.
|
|
||||||
if missing golangci-lint; then
|
|
||||||
pkg_install golangci-lint golangci-lint golangci-lint golangci-lint
|
|
||||||
fi
|
|
||||||
go mod download
|
|
||||||
|
|
||||||
# ---- Python repos ----
|
|
||||||
# if missing python3; then pkg_install python3 python3 python3 python3; fi
|
|
||||||
# python3 -m venv .venv
|
|
||||||
# ./.venv/bin/pip install -e '.[dev]'
|
|
||||||
|
|
||||||
echo "bootstrap complete"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
15
script/check
15
script/check
@@ -1,15 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/check: run all checks (test, lint, fmt-check). Our own
|
|
||||||
# extension to scripts-to-rule-them-all. Must not modify any files.
|
|
||||||
# Generic: usually needs no adaptation.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
"$SCRIPT_DIR/test"
|
|
||||||
"$SCRIPT_DIR/lint"
|
|
||||||
"$SCRIPT_DIR/fmt-check"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/cibuild: run the CI build. The Dockerfile runs script/check
|
|
||||||
# (via make check), so a successful build implies all checks pass.
|
|
||||||
# Generic: needs no adaptation. The Gitea workflow runs this on push.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
docker build .
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/docker: build the Docker image tagged with the project name.
|
|
||||||
# Identical in all repos; the tag comes from script/projectname.
|
|
||||||
# Generic: needs no adaptation.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
docker build -t "$("$SCRIPT_DIR/projectname")" .
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
27
script/fmt
27
script/fmt
@@ -1,27 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fmt: format all files (writes).
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
|
||||||
# (mirrors the old Makefile prerequisite; the generated file is
|
|
||||||
# committed, so this is normally a no-op).
|
|
||||||
ensure_pb() {
|
|
||||||
if [ ! -f mfer/mf.pb.go ] ||
|
|
||||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
|
||||||
(cd mfer && go generate .)
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
ensure_pb
|
|
||||||
gofumpt -l -w mfer internal cmd
|
|
||||||
golangci-lint run --fix
|
|
||||||
# Markdown and JSON, over the same file set script/fmt-check verifies.
|
|
||||||
"$SCRIPT_DIR/prettier" --write
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fmt-check: check formatting (read-only). Same scope as
|
|
||||||
# script/fmt, but fails instead of writing: Go via script/fmt-check-go,
|
|
||||||
# Markdown and JSON via script/prettier.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
"$SCRIPT_DIR/fmt-check-go"
|
|
||||||
"$SCRIPT_DIR/prettier" --check
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fmt-check-go: check Go formatting (read-only). Split out from
|
|
||||||
# script/fmt-check so the Docker lint stage, whose image has no node and
|
|
||||||
# therefore no prettier, can run the Go half on its own.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
|
||||||
# (mirrors the old Makefile prerequisite; the generated file is
|
|
||||||
# committed, so this is normally a no-op).
|
|
||||||
ensure_pb() {
|
|
||||||
if [ ! -f mfer/mf.pb.go ] ||
|
|
||||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
|
||||||
(cd mfer && go generate .)
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
ensure_pb
|
|
||||||
if [ -n "$(gofmt -l .)" ]; then
|
|
||||||
echo "gofmt: files need formatting:" >&2
|
|
||||||
gofmt -l . >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/install-precommit: install the git pre-commit hook that runs
|
|
||||||
# script/precommit. Our own extension to scripts-to-rule-them-all.
|
|
||||||
# Generic: needs no adaptation.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
hook=".git/hooks/pre-commit"
|
|
||||||
printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit
|
|
||||||
chmod +x .git/hooks/pre-commit
|
|
||||||
echo "pre-commit hook installed: runs script/precommit"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
17
script/lint
17
script/lint
@@ -1,17 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/lint: run the linter.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
golangci-lint run
|
|
||||||
if [ -n "$(gofmt -l .)" ]; then
|
|
||||||
echo "gofmt: files need formatting:" >&2
|
|
||||||
gofmt -l . >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/precommit: run by the git pre-commit hook; fails the commit if
|
|
||||||
# checks fail. Our own extension to scripts-to-rule-them-all. Go repo
|
|
||||||
# extras run first: go mod tidy and go fmt, failing the commit if they
|
|
||||||
# change go.mod or go.sum.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
go mod tidy
|
|
||||||
go fmt ./...
|
|
||||||
git diff --exit-code -- go.mod go.sum ||
|
|
||||||
{ echo "go mod tidy changed files; stage and retry" >&2; exit 1; }
|
|
||||||
"$SCRIPT_DIR/check"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/prettier: run prettier over this repo's canonical file set.
|
|
||||||
#
|
|
||||||
# Takes exactly one mode argument, --write or --check, and applies the
|
|
||||||
# same patterns in both modes. script/fmt and script/fmt-check both go
|
|
||||||
# through here, so the set of files that get formatted and the set that
|
|
||||||
# get verified cannot drift apart.
|
|
||||||
#
|
|
||||||
# Failures are never swallowed: a missing prettier is an error, not a
|
|
||||||
# silent skip. A formatter that quietly does nothing is worse than one
|
|
||||||
# that fails loudly.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
usage() {
|
|
||||||
echo "usage: script/prettier --write|--check" >&2
|
|
||||||
exit 2
|
|
||||||
}
|
|
||||||
|
|
||||||
# Prefer the version pinned by package.json/yarn.lock so that CI and
|
|
||||||
# developer machines format identically. Fall back to a prettier on PATH,
|
|
||||||
# but say so, because a different version formats differently.
|
|
||||||
find_prettier() {
|
|
||||||
if [ -x "$ROOT/node_modules/.bin/prettier" ]; then
|
|
||||||
printf '%s\n' "$ROOT/node_modules/.bin/prettier"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
if command -v prettier >/dev/null 2>&1; then
|
|
||||||
echo "prettier: node_modules/.bin/prettier is absent; using the" \
|
|
||||||
"prettier on PATH, which may be a different version than the" \
|
|
||||||
"one pinned in package.json. Run script/bootstrap to install" \
|
|
||||||
"the pinned version." >&2
|
|
||||||
command -v prettier
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
[ "$#" -eq 1 ] || usage
|
|
||||||
case "$1" in
|
|
||||||
--write | --check) mode="$1" ;;
|
|
||||||
*) usage ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
cd "$ROOT"
|
|
||||||
|
|
||||||
if ! prettier_bin="$(find_prettier)"; then
|
|
||||||
echo "prettier: not found." >&2
|
|
||||||
echo " Install it with: script/bootstrap" >&2
|
|
||||||
echo " (installs the version pinned in package.json/yarn.lock)" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Markdown and JSON, repo-wide rather than root-only, so files in
|
|
||||||
# subdirectories (docs/, once it exists) are covered too. Exclusions
|
|
||||||
# live in .prettierignore; REPO_POLICIES.md is excluded there because
|
|
||||||
# it is a verbatim copy of an upstream document.
|
|
||||||
#
|
|
||||||
# --no-error-on-unmatched-pattern is deliberately NOT used: both
|
|
||||||
# patterns always match at least one tracked file (README.md,
|
|
||||||
# package.json), so an empty match means the glob broke, and prettier
|
|
||||||
# erroring out is exactly what we want rather than a vacuous pass.
|
|
||||||
"$prettier_bin" "$mode" "**/*.md"
|
|
||||||
"$prettier_bin" "$mode" "**/*.json"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/projectname: output the name of this project. Our own
|
|
||||||
# extension to scripts-to-rule-them-all. Other scripts that need the
|
|
||||||
# name (e.g. script/docker) call this, so they can stay identical
|
|
||||||
# across all repos.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
main() {
|
|
||||||
echo "mfer"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
14
script/setup
14
script/setup
@@ -1,14 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/setup: set up the repo for development after a fresh clone:
|
|
||||||
# installs dependencies (script/bootstrap) and the git pre-commit hook.
|
|
||||||
# Add any repo-specific initialization (db init, .env template) here.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
"$SCRIPT_DIR/bootstrap"
|
|
||||||
"$SCRIPT_DIR/install-precommit"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
23
script/test
23
script/test
@@ -1,23 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/test: run the test suite.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# Regenerate mfer/mf.pb.go from mfer/mf.proto if it is missing or stale
|
|
||||||
# (mirrors the old Makefile prerequisite; the generated file is
|
|
||||||
# committed, so this is normally a no-op).
|
|
||||||
ensure_pb() {
|
|
||||||
if [ ! -f mfer/mf.pb.go ] ||
|
|
||||||
[ -n "$(find mfer/mf.proto -newer mfer/mf.pb.go 2>/dev/null)" ]; then
|
|
||||||
(cd mfer && go generate .)
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
ensure_pb
|
|
||||||
go test -v --timeout 10s ./...
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
BIN
vendor.tzst
Normal file
BIN
vendor.tzst
Normal file
Binary file not shown.
@@ -1,8 +0,0 @@
|
|||||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
|
||||||
# yarn lockfile v1
|
|
||||||
|
|
||||||
|
|
||||||
prettier@3.9.6:
|
|
||||||
version "3.9.6"
|
|
||||||
resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6"
|
|
||||||
integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==
|
|
||||||
Reference in New Issue
Block a user