Compare commits
13
Commits
813914ec63
...
v1.0.0-rc1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a3f532c8d | ||
|
|
0baf83deb6 | ||
|
|
589f1a9580 | ||
|
|
7542a68863 | ||
|
|
6540e5f285 | ||
|
|
2ac238d767 | ||
|
|
2cfea66d27 | ||
|
|
8ec3fd9a67 | ||
|
|
0057f6463b | ||
|
|
7abe9a0ab1 | ||
|
|
9199444d11 | ||
|
|
60dbcc03bc | ||
|
|
a79dd463f1 |
@@ -17,6 +17,15 @@ The current version is pre-1.0 and while the repo was published in 2022, there
|
|||||||
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
has not yet been any versioned release. [SemVer](https://semver.org) will be
|
||||||
used for releases.
|
used for releases.
|
||||||
|
|
||||||
|
A version is a git tag, and no file in the tree records a version number.
|
||||||
|
`mfer version` prints the revision the build stamped into the binary, which for
|
||||||
|
`make build` and the Docker image is the output of `git describe` on the
|
||||||
|
checkout they build. A binary built without that stamp, as
|
||||||
|
`go install sneak.berlin/go/mfer/cmd/mfer@<tag>` builds it, prints the module
|
||||||
|
version Go records in it instead. A program that imports the `mfer` package
|
||||||
|
finds the version it was built against in its own build information
|
||||||
|
(`runtime/debug.ReadBuildInfo`).
|
||||||
|
|
||||||
This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
This project was started by [@sneak](https://sneak.berlin) to scratch an itch in
|
||||||
2022 and is currently a one-person effort, though the goal is for this to emerge
|
2022 and is currently a one-person effort, though the goal is for this to emerge
|
||||||
as a de-facto standard and be incorporated into other software. A compatible
|
as a de-facto standard and be incorporated into other software. A compatible
|
||||||
@@ -289,9 +298,12 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
|||||||
else the directory holding the manifest, or the current directory for a
|
else the directory holding the manifest, or the current directory for a
|
||||||
manifest given by URL. So `mfer check /media/drive` checks a drive against
|
manifest given by URL. So `mfer check /media/drive` checks a drive against
|
||||||
the `index.mf` at its root, from any directory
|
the `index.mf` at its root, from any directory
|
||||||
- warns about each file under the base directory that the manifest does not
|
- warns about each regular file under the base directory that the manifest
|
||||||
list, hidden files included; with `--no-extra-files` each one is a failure
|
does not list, hidden files included, and each symlink there to a regular
|
||||||
instead
|
file, which gen lists only with `--follow-symlinks`; with
|
||||||
|
`--no-extra-files` each one is a failure instead. A named pipe, socket or
|
||||||
|
device, a symlink to a directory and a broken symlink are never reported,
|
||||||
|
since gen cannot list them
|
||||||
- `mfer freshen` / `mfer freshen .`
|
- `mfer freshen` / `mfer freshen .`
|
||||||
- rewrites `index.mf` to list the files now under the directory holding it,
|
- rewrites `index.mf` to list the files now under the directory holding it,
|
||||||
or under the one `--base` names, hashing only the files that are new or
|
or under the one `--base` names, hashing only the files that are new or
|
||||||
@@ -300,6 +312,9 @@ are now tracked only in the [issues](https://git.eeqj.de/sneak/mfer/issues).
|
|||||||
- leaves out hidden files unless given `--include-dotfiles`, and symlinks
|
- leaves out hidden files unless given `--include-dotfiles`, and symlinks
|
||||||
unless given `--follow-symlinks`, which lists each symlink to a regular
|
unless given `--follow-symlinks`, which lists each symlink to a regular
|
||||||
file under its own name with the contents of the file it points to
|
file under its own name with the contents of the file it points to
|
||||||
|
- refuses a signed manifest, leaving it as it was and naming the fingerprint
|
||||||
|
of the key that signed it, unless `--sign-key` or `MFER_SIGN_KEY` names a
|
||||||
|
key to sign the new one with
|
||||||
- `mfer gen --sign-key key.asc` / `mfer freshen --sign-key key.asc`
|
- `mfer gen --sign-key key.asc` / `mfer freshen --sign-key key.asc`
|
||||||
- signs the manifest with the OpenPGP secret key in `key.asc`, armored or
|
- signs the manifest with the OpenPGP secret key in `key.asc`, armored or
|
||||||
binary, as `gpg --export-secret-keys` writes it; `MFER_SIGN_KEY` names the
|
binary, as `gpg --export-secret-keys` writes it; `MFER_SIGN_KEY` names the
|
||||||
|
|||||||
+4
-6
@@ -10,14 +10,12 @@ import (
|
|||||||
// Appname is the name of this program.
|
// Appname is the name of this program.
|
||||||
const Appname = "mfer"
|
const Appname = "mfer"
|
||||||
|
|
||||||
// Version and Gitrev are injected at build time via -ldflags.
|
// Gitrev is the revision `mfer version` prints, injected at build time via
|
||||||
|
// -ldflags.
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // set via ldflags at build time
|
//nolint:gochecknoglobals // set via ldflags at build time
|
||||||
var (
|
var Gitrev string
|
||||||
Version string
|
|
||||||
Gitrev string
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
os.Exit(cli.Run(Appname, Version, Gitrev))
|
os.Exit(cli.Run(Appname, Gitrev))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,11 +15,10 @@ func TestAppname(t *testing.T) {
|
|||||||
assert.Equal(t, "mfer", Appname)
|
assert.Equal(t, "mfer", Appname)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestVersionDefaults documents that Version and Gitrev are empty unless
|
// TestGitrevDefault documents that Gitrev is empty unless injected at build
|
||||||
// injected at build time via -ldflags.
|
// time via -ldflags.
|
||||||
func TestVersionDefaults(t *testing.T) {
|
func TestGitrevDefault(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assert.Empty(t, Version)
|
|
||||||
assert.Empty(t, Gitrev)
|
assert.Empty(t, Gitrev)
|
||||||
}
|
}
|
||||||
|
|||||||
+204
-18
@@ -20,9 +20,14 @@ An `.mf` file consists of two parts, concatenated:
|
|||||||
2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message
|
2. **Outer message**: a Protocol Buffers serialized `MFFileOuter` message
|
||||||
|
|
||||||
There is no length prefix or version byte between the magic and the protobuf
|
There is no length prefix or version byte between the magic and the protobuf
|
||||||
message. The protobuf message extends to the end of the file.
|
message. The protobuf message extends to the end of the file, so a `.mf` is
|
||||||
|
always a whole file; a different framing, such as one with a length prefix,
|
||||||
|
would need a different magic. The format's version is the outer message's
|
||||||
|
`version` field.
|
||||||
|
|
||||||
See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types.
|
See [`mfer/mf.proto`](../mfer/mf.proto) for exact field numbers and types.
|
||||||
|
Readers ignore any field they do not know, in any message, so a later version
|
||||||
|
can add a field without breaking 1.0 readers.
|
||||||
|
|
||||||
## Outer Message (`MFFileOuter`)
|
## Outer Message (`MFFileOuter`)
|
||||||
|
|
||||||
@@ -34,7 +39,7 @@ The outer message contains:
|
|||||||
| `compressionType` | 102 | enum | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1) |
|
| `compressionType` | 102 | enum | Compression of `innerMessage`; must be `COMPRESSION_ZSTD` (1) |
|
||||||
| `size` | 103 | int64 | Uncompressed size of `innerMessage` (corruption detection) |
|
| `size` | 103 | int64 | Uncompressed size of `innerMessage` (corruption detection) |
|
||||||
| `sha256` | 104 | bytes | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) |
|
| `sha256` | 104 | bytes | SHA-256 hash of the **compressed** `innerMessage` (corruption detection) |
|
||||||
| `uuid` | 105 | bytes | Random v4 UUID; must match the inner message UUID |
|
| `uuid` | 105 | bytes | 16-byte UUID; must match the inner message UUID |
|
||||||
| `innerMessage` | 199 | bytes | Zstd-compressed serialized `MFFile` message |
|
| `innerMessage` | 199 | bytes | Zstd-compressed serialized `MFFile` message |
|
||||||
| `signature` | 201 | bytes (optional) | One OpenPGP signature over the canonical string, armored or binary |
|
| `signature` | 201 | bytes (optional) | One OpenPGP signature over the canonical string, armored or binary |
|
||||||
| `signer` | 202 | bytes (optional) | Fingerprint of the signing primary key, in upper-case hex |
|
| `signer` | 202 | bytes (optional) | Fingerprint of the signing primary key, in upper-case hex |
|
||||||
@@ -64,12 +69,12 @@ for the signature, the signing key and the other outer fields.
|
|||||||
After decompressing `innerMessage`, the result is a serialized `MFFile`
|
After decompressing `innerMessage`, the result is a serialized `MFFile`
|
||||||
(referred to as the manifest):
|
(referred to as the manifest):
|
||||||
|
|
||||||
| Field | Number | Type | Description |
|
| Field | Number | Type | Description |
|
||||||
| ----------- | ------ | --------------------- | ------------------------------------- |
|
| ----------- | ------ | --------------------- | ----------------------------------- |
|
||||||
| `version` | 100 | enum | Must be `VERSION_ONE` (1) |
|
| `version` | 100 | enum | Must be `VERSION_ONE` (1) |
|
||||||
| `files` | 101 | repeated `MFFilePath` | List of files in the manifest |
|
| `files` | 101 | repeated `MFFilePath` | List of files in the manifest |
|
||||||
| `uuid` | 102 | bytes | Random v4 UUID; must match outer UUID |
|
| `uuid` | 102 | bytes | 16-byte UUID; must match outer UUID |
|
||||||
| `createdAt` | 201 | Timestamp (optional) | When the manifest was created |
|
| `createdAt` | 201 | Timestamp (optional) | When the manifest was created |
|
||||||
|
|
||||||
## File Entries (`MFFilePath`)
|
## File Entries (`MFFilePath`)
|
||||||
|
|
||||||
@@ -107,6 +112,7 @@ All `path` values must satisfy these invariants:
|
|||||||
- **Forward slashes**: use `/` as the path separator (never `\`)
|
- **Forward slashes**: use `/` as the path separator (never `\`)
|
||||||
- **Relative only**: no leading `/`
|
- **Relative only**: no leading `/`
|
||||||
- **No parent traversal**: no `..` path segments
|
- **No parent traversal**: no `..` path segments
|
||||||
|
- **No current-directory segments**: no `.` path segments
|
||||||
- **No empty segments**: no `//` sequences
|
- **No empty segments**: no `//` sequences
|
||||||
- **No trailing slash**: paths refer to files, not directories
|
- **No trailing slash**: paths refer to files, not directories
|
||||||
- **Listed once**: each path appears at most once in a manifest, compared byte
|
- **Listed once**: each path appears at most once in a manifest, compared byte
|
||||||
@@ -116,6 +122,10 @@ Implementations must validate these invariants when reading and writing
|
|||||||
manifests. Paths that violate these rules must be rejected, and a reader must
|
manifests. Paths that violate these rules must be rejected, and a reader must
|
||||||
reject a manifest that lists a path more than once.
|
reject a manifest that lists a path more than once.
|
||||||
|
|
||||||
|
A path holds each name exactly as the file system gives it, with no Unicode
|
||||||
|
normalization, and readers compare paths byte for byte. So `é` stored as one
|
||||||
|
code point and `é` stored as `e` followed by a combining accent are two paths.
|
||||||
|
|
||||||
## Hash Format (`MFFileChecksum`)
|
## Hash Format (`MFFileChecksum`)
|
||||||
|
|
||||||
Each checksum is a single `bytes multiHash` field containing a
|
Each checksum is a single `bytes multiHash` field containing a
|
||||||
@@ -174,21 +184,197 @@ also refuses a manifest whose `signingPubKey` holds a DSA key or subkey, or any
|
|||||||
secret key or subkey, since checking the self-signatures of a DSA key or the
|
secret key or subkey, since checking the self-signatures of a DSA key or the
|
||||||
numbers of a secret key can take hours when those numbers are very large.
|
numbers of a secret key can take hours when those numbers are very large.
|
||||||
|
|
||||||
A signing key that has expired does not make the signature bad, because a
|
A signature is good only if the primary key, and the subkey that made the
|
||||||
manifest outlives its key. A signature past its own expiry time is not good,
|
signature, were valid at the signature's creation time under the key validity
|
||||||
whether or not the key has expired too.
|
rules of RFC 9580. The key is judged by its current self-signatures and binding
|
||||||
|
signatures: for a version 4 key the newest self-signature on its primary user
|
||||||
|
ID, for a version 6 key its newest direct-key signature, and for a subkey its
|
||||||
|
newest binding signature with the signature embedded in it. So a signature made
|
||||||
|
before the newest of those is not good, even if an older one made the key valid
|
||||||
|
at that time. The reference implementation checks this by verifying the
|
||||||
|
signature a second time with its clock set to the signature's creation time. A
|
||||||
|
key that has expired since does not make the signature bad, because a manifest
|
||||||
|
outlives its key. A signature past its own expiry time is not good, whether or
|
||||||
|
not the key has expired too.
|
||||||
|
|
||||||
## Deterministic Serialization
|
## Deterministic Serialization
|
||||||
|
|
||||||
By default, manifests are generated deterministically:
|
The reference implementation writes the same inner message for the same files,
|
||||||
|
UUID and options, unless the options include `--include-timestamps`. It
|
||||||
|
compresses the same inner message to the same bytes:
|
||||||
|
|
||||||
- File entries are sorted by `path` in **lexicographic byte order**
|
- File entries are sorted by `path` in byte order, so `B.txt` comes before
|
||||||
- `createdAt` is omitted unless explicitly requested
|
`a.txt`.
|
||||||
- `mode` is `0000` unless explicitly requested
|
- Each message is encoded by the Go protobuf library with deterministic
|
||||||
|
marshaling, which writes fields in field-number order and leaves out the
|
||||||
|
fields that are not set. A field not declared `optional` counts as not set
|
||||||
|
when it holds its default value, zero or empty.
|
||||||
|
- Each entry holds only `path`, `size`, `hashes` with exactly one hash, `mtime`
|
||||||
|
and `mode`. The hash is the SHA-256 multihash of the file's content, except
|
||||||
|
that `freshen` reads only the files that are new or whose size, `mtime` or
|
||||||
|
`mode` changed, and for every other file carries the first hash of its
|
||||||
|
existing entry over unchanged. `mtime` is the file's modification time as the
|
||||||
|
file system gives it, to the nanosecond, and `mode` is `0000` unless `gen` or
|
||||||
|
`freshen` is given `--include-permissions`.
|
||||||
|
- `createdAt` is left out unless `gen` or `freshen` is given
|
||||||
|
`--include-timestamps`. Then it holds the time the run started building the
|
||||||
|
manifest, to the nanosecond, so it differs on every run.
|
||||||
|
- A time, in `mtime` or `createdAt`, is written as `seconds`, the whole seconds
|
||||||
|
since 1970-01-01 00:00:00 UTC rounded down, and `nanos`, the rest, from 0
|
||||||
|
to 999999999. So a time before 1970 has negative `seconds` and `nanos` of 0 or
|
||||||
|
more.
|
||||||
|
- The inner message is compressed into one zstd frame, with a checksum of its
|
||||||
|
content, by the encoder of the Go library `github.com/klauspost/compress` at
|
||||||
|
its `SpeedBestCompression` level and otherwise its default settings. That
|
||||||
|
encoder gives the same bytes for the same input only within one version of the
|
||||||
|
library. Another zstd encoder gives other bytes for the same inner message,
|
||||||
|
and so another `sha256`.
|
||||||
|
- `gen` and `freshen` choose a new random version 4 UUID on every run, except
|
||||||
|
that `gen` given `--seed` takes the first 16 bytes of the SHA-256 of the seed
|
||||||
|
string instead. Those bytes are not a version 4 UUID; readers accept any 16
|
||||||
|
bytes.
|
||||||
|
|
||||||
This ensures that two independent runs over the same directory tree produce
|
So two runs of `gen` over an unchanged tree write byte-identical files only when
|
||||||
byte-identical `.mf` files (assuming file contents and metadata have not
|
both are given the same `--seed`, neither is given `--include-timestamps`, and
|
||||||
changed).
|
neither signs the manifest: a signature records the time it was made, and the
|
||||||
|
OpenPGP library adds a random salt to each one.
|
||||||
|
|
||||||
|
## Test Vector
|
||||||
|
|
||||||
|
[`mfer/testdata/test-vector.mf`](../mfer/testdata/test-vector.mf) is a signed
|
||||||
|
manifest the reference implementation made once, with the UUID
|
||||||
|
`92197fd5-32bc-4090-8aa9-f86e5518b56c` and an Ed25519 key that the OpenPGP
|
||||||
|
library made for this file and nothing else; its secret key was not kept.
|
||||||
|
`TestFormatTestVector` in `mfer/deserialize_test.go` checks that the file's
|
||||||
|
SHA-256, its canonical string and `signer` are the ones shown here, and verifies
|
||||||
|
its signature.
|
||||||
|
|
||||||
|
A reader should load the file and accept its signature. A writer given the same
|
||||||
|
files and UUID should give the same inner message; its compressed bytes, and so
|
||||||
|
its `sha256` and signature, depend on its zstd encoder (see Deterministic
|
||||||
|
Serialization).
|
||||||
|
|
||||||
|
The manifest has no `createdAt` and lists two files, each with mode `0000` and
|
||||||
|
the modification time 2026-10-13 12:00:00.5 UTC (`seconds` 1791892800, `nanos`
|
||||||
|
500000000):
|
||||||
|
|
||||||
|
| `path` | `size` | Content |
|
||||||
|
| --------------- | ------ | ----------------------- |
|
||||||
|
| `dir/world.txt` | 6 | `world` and a line feed |
|
||||||
|
| `hello.txt` | 6 | `hello` and a line feed |
|
||||||
|
|
||||||
|
The inner message, 164 bytes before compression:
|
||||||
|
|
||||||
|
```
|
||||||
|
00000000 a0 06 01 aa 06 46 0a 0d 64 69 72 2f 77 6f 72 6c |.....F..dir/worl|
|
||||||
|
00000010 64 2e 74 78 74 10 06 1a 24 0a 22 12 20 e2 58 d2 |d.txt...$.". .X.|
|
||||||
|
00000020 48 fd a9 4c 63 75 36 07 f7 c4 49 4e e0 fc be 92 |H..Lcu6...IN....|
|
||||||
|
00000030 f1 a7 6b fd ac 79 5c 9d 84 10 1e b3 17 f2 12 0c |..k..y\.........|
|
||||||
|
00000040 08 c0 ba b8 d6 06 10 80 ca b5 ee 01 aa 06 42 0a |..............B.|
|
||||||
|
00000050 09 68 65 6c 6c 6f 2e 74 78 74 10 06 1a 24 0a 22 |.hello.txt...$."|
|
||||||
|
00000060 12 20 58 91 b5 b5 22 d5 df 08 6d 0f f0 b1 10 fb |. X..."...m.....|
|
||||||
|
00000070 d9 d2 1b b4 fc 71 63 af 34 d0 82 86 a2 e8 46 f6 |.....qc.4.....F.|
|
||||||
|
00000080 be 03 f2 12 0c 08 c0 ba b8 d6 06 10 80 ca b5 ee |................|
|
||||||
|
00000090 01 b2 06 10 92 19 7f d5 32 bc 40 90 8a a9 f8 6e |........2.@....n|
|
||||||
|
000000a0 55 18 b5 6c |U..l|
|
||||||
|
```
|
||||||
|
|
||||||
|
Its compressed form is hashed into `sha256`:
|
||||||
|
|
||||||
|
```
|
||||||
|
2353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
|
||||||
|
```
|
||||||
|
|
||||||
|
So the canonical string is:
|
||||||
|
|
||||||
|
```
|
||||||
|
ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352
|
||||||
|
```
|
||||||
|
|
||||||
|
`signer` is `5E962A37590D855B467D20294947A60319611E6F`, and `signature` holds:
|
||||||
|
|
||||||
|
```
|
||||||
|
-----BEGIN PGP SIGNATURE-----
|
||||||
|
|
||||||
|
wqsEABYIAF0FgmrOejEJEElHpgMZYR5vNRQAAAAAABwAEHNhbHRAbm90YXRpb25z
|
||||||
|
Lm9wZW5wZ3Bqcy5vcmddxfykLLTqmTD7G/jHGzY+FiEEXpYqN1kNhVtGfSApSUem
|
||||||
|
AxlhHm8AAH0hAQDI6oGxvk1sGqIcu3Abvs7DM/7EalxkND614N75FCg+ywEApP6L
|
||||||
|
oIvrZUgEw8DUb9JeK7TChOi6PUAHUFY75Fp6hAE=
|
||||||
|
=nfX5
|
||||||
|
-----END PGP SIGNATURE-----
|
||||||
|
```
|
||||||
|
|
||||||
|
The whole file, 1056 bytes, whose SHA-256 is
|
||||||
|
`5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4`:
|
||||||
|
|
||||||
|
```
|
||||||
|
00000000 5a 4e 41 56 53 52 46 47 a8 06 01 b0 06 01 b8 06 |ZNAVSRFG........|
|
||||||
|
00000010 a4 01 c2 06 20 23 53 55 32 05 f0 36 a8 8c ef ee |.... #SU2..6....|
|
||||||
|
00000020 3d 0d 90 49 cc d8 e6 8b b2 40 11 a0 b4 62 b5 86 |=..I.....@...b..|
|
||||||
|
00000030 7f 1f cc 63 52 ca 06 10 92 19 7f d5 32 bc 40 90 |...cR.......2.@.|
|
||||||
|
00000040 8a a9 f8 6e 55 18 b5 6c ba 0c a1 01 28 b5 2f fd |...nU..l....(./.|
|
||||||
|
00000050 04 00 a5 04 00 94 08 a0 06 01 aa 06 46 0a 0d 64 |............F..d|
|
||||||
|
00000060 69 72 2f 77 6f 72 6c 64 2e 74 78 74 10 06 1a 24 |ir/world.txt...$|
|
||||||
|
00000070 0a 22 12 20 e2 58 d2 48 fd a9 4c 63 75 36 07 f7 |.". .X.H..Lcu6..|
|
||||||
|
00000080 c4 49 4e e0 fc be 92 f1 a7 6b fd ac 79 5c 9d 84 |.IN......k..y\..|
|
||||||
|
00000090 10 1e b3 17 f2 12 0c 08 c0 ba b8 d6 06 10 80 ca |................|
|
||||||
|
000000a0 b5 ee 01 aa 06 42 0a 09 68 65 6c 6c 6f 58 91 b5 |.....B..helloX..|
|
||||||
|
000000b0 b5 22 d5 df 08 6d 0f f0 b1 10 fb d9 d2 1b b4 fc |."...m..........|
|
||||||
|
000000c0 71 63 af 34 d0 82 86 a2 e8 46 f6 be 03 b2 06 10 |qc.4.....F......|
|
||||||
|
000000d0 92 19 7f d5 32 bc 40 90 8a a9 f8 6e 55 18 b5 6c |....2.@....nU..l|
|
||||||
|
000000e0 02 10 06 40 ba b7 10 ce 0c f8 c2 24 13 ca 0c ac |...@.......$....|
|
||||||
|
000000f0 02 2d 2d 2d 2d 2d 42 45 47 49 4e 20 50 47 50 20 |.-----BEGIN PGP |
|
||||||
|
00000100 53 49 47 4e 41 54 55 52 45 2d 2d 2d 2d 2d 0a 0a |SIGNATURE-----..|
|
||||||
|
00000110 77 71 73 45 41 42 59 49 41 46 30 46 67 6d 72 4f |wqsEABYIAF0FgmrO|
|
||||||
|
00000120 65 6a 45 4a 45 45 6c 48 70 67 4d 5a 59 52 35 76 |ejEJEElHpgMZYR5v|
|
||||||
|
00000130 4e 52 51 41 41 41 41 41 41 42 77 41 45 48 4e 68 |NRQAAAAAABwAEHNh|
|
||||||
|
00000140 62 48 52 41 62 6d 39 30 59 58 52 70 62 32 35 7a |bHRAbm90YXRpb25z|
|
||||||
|
00000150 0a 4c 6d 39 77 5a 57 35 77 5a 33 42 71 63 79 35 |.Lm9wZW5wZ3Bqcy5|
|
||||||
|
00000160 76 63 6d 64 64 78 66 79 6b 4c 4c 54 71 6d 54 44 |vcmddxfykLLTqmTD|
|
||||||
|
00000170 37 47 2f 6a 48 47 7a 59 2b 46 69 45 45 58 70 59 |7G/jHGzY+FiEEXpY|
|
||||||
|
00000180 71 4e 31 6b 4e 68 56 74 47 66 53 41 70 53 55 65 |qN1kNhVtGfSApSUe|
|
||||||
|
00000190 6d 0a 41 78 6c 68 48 6d 38 41 41 48 30 68 41 51 |m.AxlhHm8AAH0hAQ|
|
||||||
|
000001a0 44 49 36 6f 47 78 76 6b 31 73 47 71 49 63 75 33 |DI6oGxvk1sGqIcu3|
|
||||||
|
000001b0 41 62 76 73 37 44 4d 2f 37 45 61 6c 78 6b 4e 44 |Abvs7DM/7EalxkND|
|
||||||
|
000001c0 36 31 34 4e 37 35 46 43 67 2b 79 77 45 41 70 50 |614N75FCg+ywEApP|
|
||||||
|
000001d0 36 4c 0a 6f 49 76 72 5a 55 67 45 77 38 44 55 62 |6L.oIvrZUgEw8DUb|
|
||||||
|
000001e0 39 4a 65 4b 37 54 43 68 4f 69 36 50 55 41 48 55 |9JeK7TChOi6PUAHU|
|
||||||
|
000001f0 46 59 37 35 46 70 36 68 41 45 3d 0a 3d 6e 66 58 |FY75Fp6hAE=.=nfX|
|
||||||
|
00000200 35 0a 2d 2d 2d 2d 2d 45 4e 44 20 50 47 50 20 53 |5.-----END PGP S|
|
||||||
|
00000210 49 47 4e 41 54 55 52 45 2d 2d 2d 2d 2d d2 0c 28 |IGNATURE-----..(|
|
||||||
|
00000220 35 45 39 36 32 41 33 37 35 39 30 44 38 35 35 42 |5E962A37590D855B|
|
||||||
|
00000230 34 36 37 44 32 30 32 39 34 39 34 37 41 36 30 33 |467D20294947A603|
|
||||||
|
00000240 31 39 36 31 31 45 36 46 da 0c d4 03 2d 2d 2d 2d |19611E6F....----|
|
||||||
|
00000250 2d 42 45 47 49 4e 20 50 47 50 20 50 55 42 4c 49 |-BEGIN PGP PUBLI|
|
||||||
|
00000260 43 20 4b 45 59 20 42 4c 4f 43 4b 2d 2d 2d 2d 2d |C KEY BLOCK-----|
|
||||||
|
00000270 0a 0a 78 6a 4d 45 61 73 31 30 67 42 59 4a 4b 77 |..xjMEas10gBYJKw|
|
||||||
|
00000280 59 42 42 41 48 61 52 77 38 42 41 51 64 41 70 71 |YBBAHaRw8BAQdApq|
|
||||||
|
00000290 33 32 53 79 4f 72 4e 75 76 31 4d 71 53 55 66 45 |32SyOrNuv1MqSUfE|
|
||||||
|
000002a0 44 67 74 42 7a 6e 4b 59 32 39 38 2f 30 47 46 55 |DgtBznKY298/0GFU|
|
||||||
|
000002b0 76 6b 0a 6c 6f 42 71 35 69 6e 4e 4b 47 31 6d 5a |vk.loBq5inNKG1mZ|
|
||||||
|
000002c0 58 49 67 64 47 56 7a 64 43 42 32 5a 57 4e 30 62 |XIgdGVzdCB2ZWN0b|
|
||||||
|
000002d0 33 49 67 50 48 52 6c 63 33 51 74 64 6d 56 6a 64 |3IgPHRlc3QtdmVjd|
|
||||||
|
000002e0 47 39 79 51 47 31 6d 5a 58 49 75 64 47 56 7a 64 |G9yQG1mZXIudGVzd|
|
||||||
|
000002f0 44 37 43 0a 76 51 51 54 46 67 67 41 62 77 57 43 |D7C.vQQTFggAbwWC|
|
||||||
|
00000300 61 73 31 30 67 41 49 4c 42 77 6b 51 53 55 65 6d |as10gAILBwkQSUem|
|
||||||
|
00000310 41 78 6c 68 48 6d 38 31 46 41 41 41 41 41 41 41 |AxlhHm81FAAAAAAA|
|
||||||
|
00000320 48 41 41 51 63 32 46 73 64 45 42 75 62 33 52 68 |HAAQc2FsdEBub3Rh|
|
||||||
|
00000330 64 47 6c 76 0a 62 6e 4d 75 62 33 42 6c 62 6e 42 |dGlv.bnMub3BlbnB|
|
||||||
|
00000340 6e 63 47 70 7a 4c 6d 39 79 5a 30 79 62 6c 73 34 |ncGpzLm9yZ0ybls4|
|
||||||
|
00000350 71 53 43 6d 77 4a 64 45 49 45 6d 4c 72 54 55 45 |qSCmwJdEIEmLrTUE|
|
||||||
|
00000360 43 46 51 67 43 46 67 41 43 47 51 45 43 6d 77 4d |CFQgCFgACGQECmwM|
|
||||||
|
00000370 43 48 67 45 57 0a 49 51 52 65 6c 69 6f 33 57 51 |CHgEW.IQRelio3WQ|
|
||||||
|
00000380 32 46 57 30 5a 39 49 43 6c 4a 52 36 59 44 47 57 |2FW0Z9IClJR6YDGW|
|
||||||
|
00000390 45 65 62 77 41 41 31 67 73 41 2f 32 61 2f 50 58 |EebwAA1gsA/2a/PX|
|
||||||
|
000003a0 68 6c 44 49 4b 78 67 32 79 4e 30 4c 58 39 48 68 |hlDIKxg2yN0LX9Hh|
|
||||||
|
000003b0 49 44 43 54 7a 69 0a 49 32 43 53 71 44 74 2b 34 |IDCTzi.I2CSqDt+4|
|
||||||
|
000003c0 62 2f 73 7a 6d 59 64 41 51 43 66 2b 63 6a 6b 4f |b/szmYdAQCf+cjkO|
|
||||||
|
000003d0 71 6c 34 4c 47 5a 41 43 6c 2f 73 2f 2f 6d 31 71 |ql4LGZACl/s//m1q|
|
||||||
|
000003e0 62 74 6e 2f 4a 31 46 45 53 52 56 73 4e 68 65 2f |btn/J1FESRVsNhe/|
|
||||||
|
000003f0 76 6d 69 41 67 3d 3d 0a 3d 56 6b 4f 33 0a 2d 2d |vmiAg==.=VkO3.--|
|
||||||
|
00000400 2d 2d 2d 45 4e 44 20 50 47 50 20 50 55 42 4c 49 |---END PGP PUBLI|
|
||||||
|
00000410 43 20 4b 45 59 20 42 4c 4f 43 4b 2d 2d 2d 2d 2d |C KEY BLOCK-----|
|
||||||
|
```
|
||||||
|
|
||||||
## MIME Type
|
## MIME Type
|
||||||
|
|
||||||
|
|||||||
@@ -329,10 +329,10 @@ func (mfa *CLIApp) checkManifestOperation(
|
|||||||
return fmt.Errorf("load manifest: %w", err)
|
return fmt.Errorf("load manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check signature requirement
|
// Check signature requirement. An empty fingerprint is checked too,
|
||||||
requiredSigner := cmd.String(flagRequireSignature)
|
// and refused as one of the wrong length.
|
||||||
if requiredSigner != "" {
|
if cmd.IsSet(flagRequireSignature) {
|
||||||
err = verifyRequiredSigner(chk, requiredSigner)
|
err = verifyRequiredSigner(chk, cmd.String(flagRequireSignature))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,8 +26,7 @@ func noColorEnvSet() bool {
|
|||||||
// Use DefaultRunOptions for standard CLI execution, or construct manually for testing.
|
// Use DefaultRunOptions for standard CLI execution, or construct manually for testing.
|
||||||
type RunOptions struct {
|
type RunOptions struct {
|
||||||
Appname string // Application name displayed in help and version output
|
Appname string // Application name displayed in help and version output
|
||||||
Version string // Version string (typically set at build time)
|
Gitrev string // Revision stamped in at build time, or empty
|
||||||
Gitrev string // Git revision hash (typically set at build time)
|
|
||||||
Args []string // Command-line arguments (typically os.Args)
|
Args []string // Command-line arguments (typically os.Args)
|
||||||
Stdin io.Reader // Standard input stream
|
Stdin io.Reader // Standard input stream
|
||||||
Stdout io.Writer // Standard output stream
|
Stdout io.Writer // Standard output stream
|
||||||
@@ -36,10 +35,9 @@ type RunOptions struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DefaultRunOptions returns RunOptions configured for normal CLI execution.
|
// DefaultRunOptions returns RunOptions configured for normal CLI execution.
|
||||||
func DefaultRunOptions(appname, version, gitrev string) *RunOptions {
|
func DefaultRunOptions(appname, gitrev string) *RunOptions {
|
||||||
return &RunOptions{
|
return &RunOptions{
|
||||||
Appname: appname,
|
Appname: appname,
|
||||||
Version: version,
|
|
||||||
Gitrev: gitrev,
|
Gitrev: gitrev,
|
||||||
Args: os.Args,
|
Args: os.Args,
|
||||||
Stdin: os.Stdin,
|
Stdin: os.Stdin,
|
||||||
@@ -50,15 +48,14 @@ func DefaultRunOptions(appname, version, gitrev string) *RunOptions {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Run creates and runs the CLI application with default options.
|
// Run creates and runs the CLI application with default options.
|
||||||
func Run(appname, version, gitrev string) int {
|
func Run(appname, gitrev string) int {
|
||||||
return RunWithOptions(DefaultRunOptions(appname, version, gitrev))
|
return RunWithOptions(DefaultRunOptions(appname, gitrev))
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunWithOptions creates and runs the CLI application with the given options.
|
// RunWithOptions creates and runs the CLI application with the given options.
|
||||||
func RunWithOptions(opts *RunOptions) int {
|
func RunWithOptions(opts *RunOptions) int {
|
||||||
m := &CLIApp{
|
m := &CLIApp{
|
||||||
appname: opts.Appname,
|
appname: opts.Appname,
|
||||||
version: opts.Version,
|
|
||||||
gitrev: opts.Gitrev,
|
gitrev: opts.Gitrev,
|
||||||
exitCode: 0,
|
exitCode: 0,
|
||||||
maxManifestSize: mfer.MaxManifestSize,
|
maxManifestSize: mfer.MaxManifestSize,
|
||||||
|
|||||||
+190
-28
@@ -11,6 +11,7 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime/debug"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -27,6 +28,7 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
testApp = "mfer"
|
testApp = "mfer"
|
||||||
|
testGitrev = "abc123"
|
||||||
testDir = "/testdir"
|
testDir = "/testdir"
|
||||||
testFile1 = "/testdir/file1.txt"
|
testFile1 = "/testdir/file1.txt"
|
||||||
testMF = "/testdir/test.mf"
|
testMF = "/testdir/test.mf"
|
||||||
@@ -36,6 +38,8 @@ const (
|
|||||||
testFlagNoExtra = "--no-extra-files"
|
testFlagNoExtra = "--no-extra-files"
|
||||||
testFlagVersion = "--version"
|
testFlagVersion = "--version"
|
||||||
testFlagVerbose = "--verbose"
|
testFlagVerbose = "--verbose"
|
||||||
|
testFlagHelp = "--help"
|
||||||
|
testCmdHelp = "help"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errSimulatedWrite = errors.New("simulated write failure")
|
var errSimulatedWrite = errors.New("simulated write failure")
|
||||||
@@ -58,9 +62,10 @@ func runCLI(opts *RunOptions) int {
|
|||||||
// runCLIWithLevel is runCLI that also returns the log level the run left
|
// runCLIWithLevel is runCLI that also returns the log level the run left
|
||||||
// set, read while runMu still keeps other runs from changing it. Each run
|
// set, read while runMu still keeps other runs from changing it. Each run
|
||||||
// starts at the default level, as a new process does. Before releasing the
|
// starts at the default level, as a new process does. Before releasing the
|
||||||
// lock it points the process-global logger at io.Discard: other tests log
|
// lock it points the process-global logger at io.Discard: test helpers also
|
||||||
// outside the lock (manifest loads, scans), and those lines must not land in
|
// log under runMu outside any run (manifest loads, scans), and those lines
|
||||||
// this run's buffers once it has returned and its test is reading them.
|
// must not land in this run's buffers once it has returned and its test is
|
||||||
|
// reading them.
|
||||||
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
func runCLIWithLevel(opts *RunOptions) (int, log.Level) {
|
||||||
runMu.Lock()
|
runMu.Lock()
|
||||||
defer runMu.Unlock()
|
defer runMu.Unlock()
|
||||||
@@ -93,8 +98,7 @@ func TestBuild(t *testing.T) {
|
|||||||
func testOpts(args []string, fs afero.Fs) *RunOptions {
|
func testOpts(args []string, fs afero.Fs) *RunOptions {
|
||||||
return &RunOptions{
|
return &RunOptions{
|
||||||
Appname: testApp,
|
Appname: testApp,
|
||||||
Version: "1.0.0",
|
Gitrev: testGitrev,
|
||||||
Gitrev: "abc123",
|
|
||||||
Args: args,
|
Args: args,
|
||||||
Stdin: &bytes.Buffer{},
|
Stdin: &bytes.Buffer{},
|
||||||
Stdout: &bytes.Buffer{},
|
Stdout: &bytes.Buffer{},
|
||||||
@@ -136,10 +140,35 @@ func TestVersionCommand(t *testing.T) {
|
|||||||
exitCode := runCLI(opts)
|
exitCode := runCLI(opts)
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode)
|
assert.Equal(t, 0, exitCode)
|
||||||
|
assert.Equal(t, "mfer version "+testGitrev+"\n", testStdout(t, opts))
|
||||||
|
}
|
||||||
|
|
||||||
stdout := testStdout(t, opts)
|
// TestVersionFromBuildInfo covers a binary whose build stamped no Gitrev,
|
||||||
assert.Contains(t, stdout, mfer.Version)
|
// as `go install` builds it: mfer version prints the main module's version
|
||||||
assert.Contains(t, stdout, "abc123")
|
// from the build information instead.
|
||||||
|
func TestVersionFromBuildInfo(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info, ok := debug.ReadBuildInfo()
|
||||||
|
require.True(t, ok)
|
||||||
|
require.NotEmpty(t, info.Main.Version)
|
||||||
|
|
||||||
|
opts := testOpts([]string{testApp, cmdVersion}, afero.NewMemMapFs())
|
||||||
|
opts.Gitrev = ""
|
||||||
|
|
||||||
|
require.Equal(t, 0, runCLI(opts))
|
||||||
|
assert.Equal(t, "mfer version "+info.Main.Version+"\n", testStdout(t, opts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestModuleVersionEmpty covers build information that holds no module
|
||||||
|
// version, as in a build with GO111MODULE=off, and a binary with no build
|
||||||
|
// information at all. Both report "unknown": an empty version would print
|
||||||
|
// nothing after "mfer version" and drop the --version flag.
|
||||||
|
func TestModuleVersionEmpty(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(t, "unknown", moduleVersion(&debug.BuildInfo{}, true))
|
||||||
|
assert.Equal(t, "unknown", moduleVersion(nil, false))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestVFlagCollision covers the -v/--verbose vs --version flag interaction
|
// TestVFlagCollision covers the -v/--verbose vs --version flag interaction
|
||||||
@@ -165,7 +194,7 @@ func TestVFlagCollision(t *testing.T) {
|
|||||||
exitCode := runCLI(opts)
|
exitCode := runCLI(opts)
|
||||||
|
|
||||||
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
assert.Equal(t, 0, exitCode, "stderr: %s", testStderr(t, opts))
|
||||||
assert.Contains(t, testStdout(t, opts), mfer.Version)
|
assert.Contains(t, testStdout(t, opts), testGitrev)
|
||||||
assert.NotContains(t, testStderr(t, opts), "two forms of the same flag")
|
assert.NotContains(t, testStderr(t, opts), "two forms of the same flag")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -206,13 +235,13 @@ func TestVersionFlagAndCommandMatch(t *testing.T) {
|
|||||||
|
|
||||||
// TestRootVerbosityFlags asserts that -q and -v given before any subcommand
|
// TestRootVerbosityFlags asserts that -q and -v given before any subcommand
|
||||||
// name take effect: in the root itself, and in the fetch and export
|
// name take effect: in the root itself, and in the fetch and export
|
||||||
// subcommands (issue #64). It does not run in parallel: other tests log
|
// subcommands (issue #64).
|
||||||
// outside runMu (manifest loads, scans), and a line logged while one of these
|
|
||||||
// runs is in progress lands in its output.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // see above
|
|
||||||
func TestRootVerbosityFlags(t *testing.T) {
|
func TestRootVerbosityFlags(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
t.Run("quiet with no subcommand hides the banner", func(t *testing.T) {
|
t.Run("quiet with no subcommand hides the banner", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
loud := testOpts([]string{testApp}, afero.NewMemMapFs())
|
loud := testOpts([]string{testApp}, afero.NewMemMapFs())
|
||||||
require.Equal(t, 0, runCLI(loud))
|
require.Equal(t, 0, runCLI(loud))
|
||||||
assert.Contains(t, testStdout(t, loud), banner)
|
assert.Contains(t, testStdout(t, loud), banner)
|
||||||
@@ -224,6 +253,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("quiet before fetch hides the banner", func(t *testing.T) {
|
t.Run("quiet before fetch hides the banner", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, "-q", cmdFetch}, afero.NewMemMapFs())
|
opts := testOpts([]string{testApp, "-q", cmdFetch}, afero.NewMemMapFs())
|
||||||
|
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
@@ -232,6 +263,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("verbose twice before fetch enables debug logging", func(t *testing.T) {
|
t.Run("verbose twice before fetch enables debug logging", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
opts := testOpts([]string{testApp, "-v", "-v", cmdFetch}, afero.NewMemMapFs())
|
opts := testOpts([]string{testApp, "-v", "-v", cmdFetch}, afero.NewMemMapFs())
|
||||||
|
|
||||||
assert.Equal(t, 1, runCLI(opts))
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
@@ -239,6 +272,8 @@ func TestRootVerbosityFlags(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("quiet before export hides the manifest summary", func(t *testing.T) {
|
t.Run("quiet before export hides the manifest summary", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
manifest := buildTestManifest(t, map[string][]byte{"a.txt": []byte("a")})
|
manifest := buildTestManifest(t, map[string][]byte{"a.txt": []byte("a")})
|
||||||
require.NoError(t, afero.WriteFile(fs, testManifest, manifest, 0o644))
|
require.NoError(t, afero.WriteFile(fs, testManifest, manifest, 0o644))
|
||||||
@@ -321,7 +356,7 @@ func TestHelpCommand(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
opts := testOpts([]string{testApp, "--help"}, fs)
|
opts := testOpts([]string{testApp, testFlagHelp}, fs)
|
||||||
|
|
||||||
exitCode := runCLI(opts)
|
exitCode := runCLI(opts)
|
||||||
|
|
||||||
@@ -739,6 +774,9 @@ func TestBannerOutput(t *testing.T) {
|
|||||||
stdout := testStdout(t, opts)
|
stdout := testStdout(t, opts)
|
||||||
assert.Contains(t, stdout, "___")
|
assert.Contains(t, stdout, "___")
|
||||||
assert.Contains(t, stdout, "\\")
|
assert.Contains(t, stdout, "\\")
|
||||||
|
|
||||||
|
// The banner shows the version mfer version prints, and nothing after it.
|
||||||
|
assert.Contains(t, stdout, "mfer by @sneak: "+testGitrev+"\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUnknownCommand(t *testing.T) {
|
func TestUnknownCommand(t *testing.T) {
|
||||||
@@ -771,13 +809,9 @@ func TestGenerateExcludesDotfilesByDefault(t *testing.T) {
|
|||||||
assert.True(t, exists)
|
assert.True(t, exists)
|
||||||
|
|
||||||
// Verify manifest only has 1 file (the non-dotfile)
|
// Verify manifest only has 1 file (the non-dotfile)
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
paths := manifestPaths(t, fs, testMF)
|
||||||
Path: testMF,
|
require.Len(t, paths, 1)
|
||||||
Fs: fs,
|
assert.Equal(t, "file1.txt", paths[0])
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 1)
|
|
||||||
assert.Equal(t, "file1.txt", manifest.Files()[0].GetPath())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
||||||
@@ -798,12 +832,7 @@ func TestGenerateWithIncludeDotfiles(t *testing.T) {
|
|||||||
require.Equal(t, 0, exitCode)
|
require.Equal(t, 0, exitCode)
|
||||||
|
|
||||||
// Verify manifest has 2 files (including dotfile)
|
// Verify manifest has 2 files (including dotfile)
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
assert.Len(t, manifestPaths(t, fs, testMF), 2)
|
||||||
Path: testMF,
|
|
||||||
Fs: fs,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Len(t, manifest.Files(), 2)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipleInputPaths(t *testing.T) {
|
func TestMultipleInputPaths(t *testing.T) {
|
||||||
@@ -960,6 +989,53 @@ func TestCheckWarnsAboutUnlistedFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestCheckReportsUnlistedSymlinkToFileOnly makes a tree holding a symlink
|
||||||
|
// to a directory, one to a file and a broken one. gen lists the symlink to
|
||||||
|
// the file only with --follow-symlinks, and never lists the other two, so
|
||||||
|
// check --no-extra-files passes on the manifest gen --follow-symlinks writes
|
||||||
|
// and fails only on the symlink to the file on the one plain gen writes.
|
||||||
|
func TestCheckReportsUnlistedSymlinkToFileOnly(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for name, tc := range map[string]struct {
|
||||||
|
genFlags []string
|
||||||
|
exitCode int
|
||||||
|
extras []string // what check reports as not in the manifest
|
||||||
|
}{
|
||||||
|
"gen --follow-symlinks": {[]string{"--follow-symlinks"}, 0, nil},
|
||||||
|
"plain gen": {nil, 1, []string{"filelink"}},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
writeTestTree(t, fs, dir)
|
||||||
|
require.NoError(t, os.Symlink(testSubdir, filepath.Join(dir, "dirlink")))
|
||||||
|
require.NoError(t, os.Symlink(testFileTxt, filepath.Join(dir, "filelink")))
|
||||||
|
require.NoError(t,
|
||||||
|
os.Symlink("nowhere", filepath.Join(dir, "brokenlink")))
|
||||||
|
|
||||||
|
opts := testOpts(slices.Concat(
|
||||||
|
[]string{testApp, cmdGenerate, "-q"}, tc.genFlags, []string{dir},
|
||||||
|
), fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
|
|
||||||
|
opts = testOpts([]string{testApp, cmdCheck, testFlagNoExtra, dir}, fs)
|
||||||
|
exitCode := runCLI(opts)
|
||||||
|
stderr := testStderr(t, opts)
|
||||||
|
|
||||||
|
assert.Equal(t, tc.exitCode, exitCode, "stderr: %s", stderr)
|
||||||
|
assert.Equal(t, len(tc.extras),
|
||||||
|
strings.Count(stderr, "not in manifest"), stderr)
|
||||||
|
|
||||||
|
for _, extra := range tc.extras {
|
||||||
|
assert.Contains(t, stderr, "EXTRA: "+extra)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestCheckNeverReportsManifest keeps the manifest inside the checked tree,
|
// TestCheckNeverReportsManifest keeps the manifest inside the checked tree,
|
||||||
// under several names and path spellings, and names the tree both directly
|
// under several names and path spellings, and names the tree both directly
|
||||||
// and through a symlink: the manifest is never reported, even under
|
// and through a symlink: the manifest is never reported, even under
|
||||||
@@ -1224,6 +1300,11 @@ func TestGenerateFailsWithoutForceWhenOutputExists(t *testing.T) {
|
|||||||
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
func manifestPaths(t *testing.T, fs afero.Fs, path string) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger, so serialize
|
||||||
|
// with the CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||||
Path: path,
|
Path: path,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
@@ -1807,6 +1888,87 @@ func TestGenerateReadsPathAfterDoubleDash(t *testing.T) {
|
|||||||
assert.Equal(t, testFileTxt, files[0].GetPath())
|
assert.Equal(t, testFileTxt, files[0].GetPath())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestPathNamedHelp runs gen, then check with --require-signature, on a
|
||||||
|
// directory named help and on one named h, the two names of urfave/cli's
|
||||||
|
// help command. gen must write the manifest into the directory, and check
|
||||||
|
// must refuse it as unsigned.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // changes the process-global working directory
|
||||||
|
func TestPathNamedHelp(t *testing.T) {
|
||||||
|
for _, name := range []string{testCmdHelp, "h"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
chdirTemp(t)
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
require.NoError(t, fs.Mkdir(name, 0o755))
|
||||||
|
writeTestFile(t, fs, filepath.Join(name, testFileTxt), "content")
|
||||||
|
|
||||||
|
opts := testOpts([]string{testApp, cmdGenerate, "-q", name}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
|
||||||
|
files := manifestFiles(t, fs, filepath.Join(name, defaultManifestName))
|
||||||
|
require.Len(t, files, 1)
|
||||||
|
|
||||||
|
opts = testOpts([]string{
|
||||||
|
testApp, cmdCheck, "-q", name, "--" + flagRequireSignature, msgFpA,
|
||||||
|
}, fs)
|
||||||
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
|
assert.Contains(t, testStderr(t, opts), "manifest is not signed")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCommandsReadHelpAsArgument runs each command that takes a path or URL
|
||||||
|
// on help and on h where nothing by that name exists. Each must fail on the
|
||||||
|
// missing path or bad URL instead of printing its help.
|
||||||
|
func TestCommandsReadHelpAsArgument(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, command := range []string{
|
||||||
|
cmdGenerate, cmdCheck, cmdFreshen, cmdList, cmdExport, cmdFetch,
|
||||||
|
} {
|
||||||
|
for _, name := range []string{testCmdHelp, "h"} {
|
||||||
|
t.Run(command+" "+name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
opts := testOpts([]string{testApp, command, name}, afero.NewMemMapFs())
|
||||||
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
|
assert.NotContains(t, testStdout(t, opts), "USAGE:")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHelpBesideArgument runs each command that takes a path or URL with
|
||||||
|
// --help or -h before and after a directory, and runs help on that command.
|
||||||
|
// Each must print what the command's own --help prints.
|
||||||
|
func TestHelpBesideArgument(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, command := range []string{
|
||||||
|
cmdGenerate, cmdCheck, cmdFreshen, cmdList, cmdExport, cmdFetch,
|
||||||
|
} {
|
||||||
|
want := testOpts([]string{testApp, command, testFlagHelp}, afero.NewMemMapFs())
|
||||||
|
require.Equal(t, 0, runCLI(want))
|
||||||
|
require.Contains(t, testStdout(t, want), testApp+" "+command)
|
||||||
|
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{command, testDir, testFlagHelp},
|
||||||
|
{command, testFlagHelp, testDir},
|
||||||
|
{command, testDir, "-h"},
|
||||||
|
{testCmdHelp, command},
|
||||||
|
} {
|
||||||
|
t.Run(strings.Join(args, " "), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
opts := testOpts(slices.Concat([]string{testApp}, args), afero.NewMemMapFs())
|
||||||
|
assert.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
assert.Equal(t, testStdout(t, want), testStdout(t, opts))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCheckDetectsManifestCorruption(t *testing.T) {
|
func TestCheckDetectsManifestCorruption(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -54,6 +54,11 @@ func runLocked(fn func() error) error {
|
|||||||
func unsignedChecker(t *testing.T) *mfer.Checker {
|
func unsignedChecker(t *testing.T) *mfer.Checker {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// The scan and the manifest load log to the process-global logger, so
|
||||||
|
// serialize with the CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
require.NoError(t, fs.MkdirAll("/d", 0o755))
|
||||||
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/d/f.txt", []byte("hi"), 0o644))
|
||||||
@@ -183,6 +188,11 @@ func signedManifest(t *testing.T, files map[string][]byte) []byte {
|
|||||||
func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
|
func signedChecker(t *testing.T, manifest []byte) *mfer.Checker {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger, so serialize
|
||||||
|
// with the CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
fs := afero.NewMemMapFs()
|
fs := afero.NewMemMapFs()
|
||||||
require.NoError(t, afero.WriteFile(fs, "/index.mf", manifest, 0o644))
|
require.NoError(t, afero.WriteFile(fs, "/index.mf", manifest, 0o644))
|
||||||
|
|
||||||
@@ -473,6 +483,49 @@ func TestFreshenReadErrorMessage(t *testing.T) {
|
|||||||
"read "+filepath.Join(root, "sub")+": is a directory")
|
"read "+filepath.Join(root, "sub")+": is a directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFreshenSizeChangedMessage has freshen hash a file that has grown or
|
||||||
|
// shrunk since the scan recorded its size.
|
||||||
|
func TestFreshenSizeChangedMessage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
writeTestFile(t, fs, "/d/log.txt", "hello")
|
||||||
|
|
||||||
|
t.Run("grown", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hasher := &freshenHasher{
|
||||||
|
fs: fs,
|
||||||
|
absBase: "/d",
|
||||||
|
builder: mfer.NewBuilder(),
|
||||||
|
}
|
||||||
|
|
||||||
|
err := hasher.processEntry(
|
||||||
|
&freshenEntry{path: "log.txt", size: 3, needsHash: true})
|
||||||
|
require.ErrorIs(t, err, errSizeMismatch)
|
||||||
|
assert.EqualError(t, err, `size mismatch for "log.txt": `+
|
||||||
|
`3 bytes when scanned but 5 bytes when read; `+
|
||||||
|
`the file changed while freshen ran`)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("shrunk", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
hasher := &freshenHasher{
|
||||||
|
fs: fs,
|
||||||
|
absBase: "/d",
|
||||||
|
builder: mfer.NewBuilder(),
|
||||||
|
}
|
||||||
|
|
||||||
|
err := hasher.processEntry(
|
||||||
|
&freshenEntry{path: "log.txt", size: 8, needsHash: true})
|
||||||
|
require.ErrorIs(t, err, errSizeMismatch)
|
||||||
|
assert.EqualError(t, err, `size mismatch for "log.txt": `+
|
||||||
|
`8 bytes when scanned but 5 bytes when read; `+
|
||||||
|
`the file changed while freshen ran`)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestURLRequiredMessage(t *testing.T) {
|
func TestURLRequiredMessage(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,11 @@ const testCmdExport = "export"
|
|||||||
func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
|
func buildTestManifest(t *testing.T, files map[string][]byte) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// The scan logs to the process-global logger, so serialize with the
|
||||||
|
// CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
sourceFs := afero.NewMemMapFs()
|
sourceFs := afero.NewMemMapFs()
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
require.NoError(t, sourceFs.MkdirAll("/", 0o755))
|
require.NoError(t, sourceFs.MkdirAll("/", 0o755))
|
||||||
|
|||||||
+43
-20
@@ -86,7 +86,8 @@ var (
|
|||||||
// symlink that already exists in the target directory.
|
// symlink that already exists in the target directory.
|
||||||
errSymlinkInPath = errors.New("symlink in path not allowed")
|
errSymlinkInPath = errors.New("symlink in path not allowed")
|
||||||
// errSizeMismatch indicates a downloaded file with an unexpected
|
// errSizeMismatch indicates a downloaded file with an unexpected
|
||||||
// size.
|
// size, or a file freshen read at a size other than the one its scan
|
||||||
|
// recorded.
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
errSizeMismatch = errors.New("size mismatch")
|
||||||
// errHashMismatch indicates a downloaded file whose hash matches no
|
// errHashMismatch indicates a downloaded file whose hash matches no
|
||||||
// manifest hash.
|
// manifest hash.
|
||||||
@@ -125,8 +126,9 @@ type retryingClient struct {
|
|||||||
// to a limit that doubles each time, unless the server's Retry-After
|
// to a limit that doubles each time, unless the server's Retry-After
|
||||||
// header says how long to wait. Any other failure, an error from use
|
// header says how long to wait. Any other failure, an error from use
|
||||||
// included, is returned at once and unwrapped; a non-OK status is
|
// included, is returned at once and unwrapped; a non-OK status is
|
||||||
// returned as errHTTPStatus. use must start over each time it is called,
|
// returned as errHTTPStatus. If ctx ends before a retry, the error wraps
|
||||||
// since a retry calls it again with a new response.
|
// both ctx's error and the last try's error. use must start over each
|
||||||
|
// time it is called, since a retry calls it again with a new response.
|
||||||
func (c retryingClient) get(
|
func (c retryingClient) get(
|
||||||
ctx context.Context, rawURL string, use func(*http.Response) error,
|
ctx context.Context, rawURL string, use func(*http.Response) error,
|
||||||
) error {
|
) error {
|
||||||
@@ -168,15 +170,19 @@ func (c retryingClient) get(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !retry || attempt == fetchAttempts || ctx.Err() != nil {
|
if !retry || attempt == fetchAttempts {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||||
|
}
|
||||||
|
|
||||||
log.Warnf("%s: %s, retrying in %s", rawURL, err, wait.Round(time.Millisecond))
|
log.Warnf("%s: %s, retrying in %s", rawURL, err, wait.Round(time.Millisecond))
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return ctx.Err()
|
return fmt.Errorf("%w: %w", ctx.Err(), err)
|
||||||
case <-time.After(wait):
|
case <-time.After(wait):
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -429,6 +435,14 @@ func (mfa *CLIApp) fetchManifestOperation(
|
|||||||
func (mfa *CLIApp) fetchManifest(
|
func (mfa *CLIApp) fetchManifest(
|
||||||
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
ctx context.Context, cmd *cli.Command, client retryingClient, manifestURL string,
|
||||||
) ([]byte, []*mfer.MFFilePath, error) {
|
) ([]byte, []*mfer.MFFilePath, error) {
|
||||||
|
// An empty fingerprint is refused before anything is requested, with
|
||||||
|
// the message check gives for it; any other is checked against the
|
||||||
|
// manifest once it is downloaded.
|
||||||
|
requiredSigner := cmd.String(flagRequireSignature)
|
||||||
|
if cmd.IsSet(flagRequireSignature) && requiredSigner == "" {
|
||||||
|
return nil, nil, fmt.Errorf("%w, got 0", errInvalidFingerprint)
|
||||||
|
}
|
||||||
|
|
||||||
log.Infof("fetching manifest from %s", manifestURL)
|
log.Infof("fetching manifest from %s", manifestURL)
|
||||||
|
|
||||||
// Read the whole manifest before parsing it, so that a connection
|
// Read the whole manifest before parsing it, so that a connection
|
||||||
@@ -460,7 +474,6 @@ func (mfa *CLIApp) fetchManifest(
|
|||||||
return nil, nil, fmt.Errorf("parse manifest: %w", err)
|
return nil, nil, fmt.Errorf("parse manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
requiredSigner := cmd.String(flagRequireSignature)
|
|
||||||
if requiredSigner != "" {
|
if requiredSigner != "" {
|
||||||
err = verifyFetchedSigner(manifestData, requiredSigner)
|
err = verifyFetchedSigner(manifestData, requiredSigner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -679,21 +692,26 @@ func resolveManifestURL(inputURL string) (string, error) {
|
|||||||
return parsed.JoinPath(defaultManifestName).String(), nil
|
return parsed.JoinPath(defaultManifestName).String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// progressWriter wraps an io.Writer and reports progress to a channel.
|
// progressWriter wraps an io.Writer and reports progress to a channel,
|
||||||
|
// at most once per second and once more when the last byte is written.
|
||||||
type progressWriter struct {
|
type progressWriter struct {
|
||||||
w io.Writer
|
w io.Writer
|
||||||
path string
|
path string
|
||||||
total int64
|
total int64
|
||||||
written int64
|
written int64
|
||||||
startTime time.Time
|
startTime time.Time
|
||||||
progress chan<- DownloadProgress
|
lastProgress time.Time
|
||||||
|
progress chan<- DownloadProgress
|
||||||
}
|
}
|
||||||
|
|
||||||
func (pw *progressWriter) Write(p []byte) (int, error) {
|
func (pw *progressWriter) Write(p []byte) (int, error) {
|
||||||
n, err := pw.w.Write(p)
|
n, err := pw.w.Write(p)
|
||||||
|
|
||||||
pw.written += int64(n)
|
pw.written += int64(n)
|
||||||
if pw.progress != nil {
|
|
||||||
|
isLast := pw.written == pw.total
|
||||||
|
if pw.progress != nil &&
|
||||||
|
(isLast || time.Since(pw.lastProgress) >= time.Second) {
|
||||||
var (
|
var (
|
||||||
bytesPerSec float64
|
bytesPerSec float64
|
||||||
eta time.Duration
|
eta time.Duration
|
||||||
@@ -715,6 +733,8 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
|
|||||||
BytesPerSec: bytesPerSec,
|
BytesPerSec: bytesPerSec,
|
||||||
ETA: eta,
|
ETA: eta,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
pw.lastProgress = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
@@ -899,13 +919,16 @@ func saveResponse(
|
|||||||
// Set up hash computation
|
// Set up hash computation
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
|
|
||||||
// Create progress-reporting writer that also computes hash
|
// Create progress-reporting writer that also computes hash. The first
|
||||||
|
// update waits a second, as gen's and check's do.
|
||||||
|
start := time.Now()
|
||||||
pw := &progressWriter{
|
pw := &progressWriter{
|
||||||
w: io.MultiWriter(out, h),
|
w: io.MultiWriter(out, h),
|
||||||
path: localPath,
|
path: localPath,
|
||||||
total: totalBytes,
|
total: totalBytes,
|
||||||
startTime: time.Now(),
|
startTime: start,
|
||||||
progress: progress,
|
lastProgress: start,
|
||||||
|
progress: progress,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy content while hashing and reporting progress. One byte past
|
// Copy content while hashing and reporting progress. One byte past
|
||||||
|
|||||||
+103
-34
@@ -171,6 +171,11 @@ func TestResolveManifestURL(t *testing.T) {
|
|||||||
func scanToManifest(t *testing.T, sourceFs afero.Fs) []byte {
|
func scanToManifest(t *testing.T, sourceFs afero.Fs) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// The scan logs to the process-global logger, so serialize with the
|
||||||
|
// CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: sourceFs})
|
s := mfer.NewScannerWithOptions(&mfer.ScannerOptions{Fs: sourceFs})
|
||||||
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
|
require.NoError(t, s.EnumerateFS(sourceFs, "/", nil))
|
||||||
|
|
||||||
@@ -314,8 +319,11 @@ func TestFetchFromHTTP(t *testing.T) {
|
|||||||
// Change to a fresh destination directory for the test
|
// Change to a fresh destination directory for the test
|
||||||
destDir := chdirTemp(t)
|
destDir := chdirTemp(t)
|
||||||
|
|
||||||
// Parse the manifest to get file entries
|
// Parse the manifest to get file entries. Loading a manifest logs to
|
||||||
|
// the process-global logger.
|
||||||
|
runMu.Lock()
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
runMu.Unlock()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
files := manifest.Files()
|
files := manifest.Files()
|
||||||
@@ -365,7 +373,10 @@ func TestFetchHashMismatch(t *testing.T) {
|
|||||||
// Generate and parse manifest
|
// Generate and parse manifest
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
manifestData := scanToManifest(t, sourceFs)
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger.
|
||||||
|
runMu.Lock()
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
runMu.Unlock()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
files := manifest.Files()
|
files := manifest.Files()
|
||||||
@@ -411,7 +422,10 @@ func TestFetchSizeMismatch(t *testing.T) {
|
|||||||
// Generate and parse manifest
|
// Generate and parse manifest
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
manifestData := scanToManifest(t, sourceFs)
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger.
|
||||||
|
runMu.Lock()
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
runMu.Unlock()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
files := manifest.Files()
|
files := manifest.Files()
|
||||||
@@ -516,14 +530,17 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
|
|||||||
func TestFetchProgress(t *testing.T) {
|
func TestFetchProgress(t *testing.T) {
|
||||||
// Create source filesystem with a larger test file
|
// Create source filesystem with a larger test file
|
||||||
sourceFs := afero.NewMemMapFs()
|
sourceFs := afero.NewMemMapFs()
|
||||||
// Create content large enough to trigger multiple progress updates
|
// Content large enough that io.Copy writes it in at least four chunks
|
||||||
content := bytes.Repeat([]byte("x"), 100*1024) // 100KB
|
content := bytes.Repeat([]byte("x"), 100*1024) // 100KB
|
||||||
require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644))
|
require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644))
|
||||||
|
|
||||||
// Generate and parse manifest
|
// Generate and parse manifest
|
||||||
manifestData := scanToManifest(t, sourceFs)
|
manifestData := scanToManifest(t, sourceFs)
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger.
|
||||||
|
runMu.Lock()
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
runMu.Unlock()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
files := manifest.Files()
|
files := manifest.Files()
|
||||||
@@ -559,8 +576,11 @@ func TestFetchProgress(t *testing.T) {
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
// Download
|
// Download
|
||||||
|
start := time.Now()
|
||||||
err = downloadFile(context.Background(), testClient(),
|
err = downloadFile(context.Background(), testClient(),
|
||||||
server.URL+"/large.txt", ".", "large.txt", files[0], progress)
|
server.URL+"/large.txt", ".", "large.txt", files[0], progress)
|
||||||
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
close(progress)
|
close(progress)
|
||||||
<-done
|
<-done
|
||||||
|
|
||||||
@@ -569,6 +589,12 @@ func TestFetchProgress(t *testing.T) {
|
|||||||
// Verify we got progress updates
|
// Verify we got progress updates
|
||||||
assert.NotEmpty(t, progressUpdates, "should have received progress updates")
|
assert.NotEmpty(t, progressUpdates, "should have received progress updates")
|
||||||
|
|
||||||
|
// At most one update per second, plus the one reporting the file
|
||||||
|
// complete: a download that takes under a second sends two at most,
|
||||||
|
// however many chunks it writes.
|
||||||
|
assert.LessOrEqual(t, len(progressUpdates), 2+int(elapsed/time.Second),
|
||||||
|
"one progress update per second at most, plus the last")
|
||||||
|
|
||||||
// Verify final progress shows complete
|
// Verify final progress shows complete
|
||||||
if len(progressUpdates) > 0 {
|
if len(progressUpdates) > 0 {
|
||||||
last := progressUpdates[len(progressUpdates)-1]
|
last := progressUpdates[len(progressUpdates)-1]
|
||||||
@@ -871,23 +897,59 @@ func TestGetHonorsRetryAfter(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestGetStopsWaitingWhenCanceled gives get a server that always answers
|
// instant503Transport answers every request with a 503 at once, so the
|
||||||
// 503 and an hour's wait before each retry, then lets the context expire
|
// 503 is in hand before any deadline can pass.
|
||||||
// during that wait. get returning at all shows it stopped waiting.
|
type instant503Transport struct{}
|
||||||
|
|
||||||
|
func (instant503Transport) RoundTrip(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusServiceUnavailable,
|
||||||
|
Body: http.NoBody,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetStopsWaitingWhenCanceled answers every request with a 503 and
|
||||||
|
// sets an hour's wait before each retry, then lets the context expire
|
||||||
|
// during that wait. get returning at all shows it stopped waiting; it must
|
||||||
|
// return the deadline error and keep the 503 with it.
|
||||||
func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
server := httptest.NewServer(
|
|
||||||
http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
client := testClient()
|
client := testClient()
|
||||||
|
client.client.Transport = instant503Transport{}
|
||||||
client.firstDelay = time.Hour
|
client.firstDelay = time.Hour
|
||||||
|
|
||||||
err := getNothing(client, server.URL, 50*time.Millisecond)
|
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
|
require.ErrorIs(t, err, errHTTPStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
// late503Transport answers every request with a 503, but only once the
|
||||||
|
// request's context has ended. A real server cannot stand in for it: the
|
||||||
|
// client drops a response still on its way when the context ends.
|
||||||
|
type late503Transport struct{}
|
||||||
|
|
||||||
|
func (late503Transport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
<-req.Context().Done()
|
||||||
|
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusServiceUnavailable,
|
||||||
|
Body: http.NoBody,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGetStopsWhenContextEndsDuringRequest lets the context expire while
|
||||||
|
// the first request is still waiting for its 503. get must return the
|
||||||
|
// deadline error and keep the 503 with it.
|
||||||
|
func TestGetStopsWhenContextEndsDuringRequest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
client := testClient()
|
||||||
|
client.client.Transport = late503Transport{}
|
||||||
|
|
||||||
|
err := getNothing(client, "http://example.invalid/", 50*time.Millisecond)
|
||||||
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
|
require.ErrorIs(t, err, errHTTPStatus)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDownloadFileRetriesToSuccess serves a file that fails twice before
|
// TestDownloadFileRetriesToSuccess serves a file that fails twice before
|
||||||
@@ -897,9 +959,12 @@ func TestGetStopsWaitingWhenCanceled(t *testing.T) {
|
|||||||
//nolint:paralleltest // changes the process-global working directory
|
//nolint:paralleltest // changes the process-global working directory
|
||||||
func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
||||||
content := []byte("the whole file, every byte of it")
|
content := []byte("the whole file, every byte of it")
|
||||||
|
manifestData := manifestOf(t, map[string][]byte{testFileTxt: content})
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(
|
// Loading a manifest logs to the process-global logger.
|
||||||
manifestOf(t, map[string][]byte{testFileTxt: content})))
|
runMu.Lock()
|
||||||
|
manifest, err := mfer.NewManifestFromReader(bytes.NewReader(manifestData))
|
||||||
|
runMu.Unlock()
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
var requests atomic.Int32
|
var requests atomic.Int32
|
||||||
@@ -922,8 +987,11 @@ func TestDownloadFileRetriesToSuccess(t *testing.T) {
|
|||||||
|
|
||||||
chdirTemp(t)
|
chdirTemp(t)
|
||||||
|
|
||||||
err = downloadFile(context.Background(), testClient(),
|
// downloadFile logs each retry to the process-global logger.
|
||||||
server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
|
err = runLocked(func() error {
|
||||||
|
return downloadFile(context.Background(), testClient(),
|
||||||
|
server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
|
||||||
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, int32(3), requests.Load())
|
assert.Equal(t, int32(3), requests.Load())
|
||||||
|
|
||||||
@@ -1243,10 +1311,10 @@ func TestFetchRequireSignature(t *testing.T) {
|
|||||||
|
|
||||||
// TestFetchRefusesListedManifestName fetches manifests that list, at the
|
// TestFetchRefusesListedManifestName fetches manifests that list, at the
|
||||||
// top of the tree, the name fetch saves the manifest under or that name's
|
// top of the tree, the name fetch saves the manifest under or that name's
|
||||||
// temp file: as a file, as a directory, in capitals, and with a leading
|
// temp file: as a file, as a directory and in capitals. Saving the
|
||||||
// "./". Saving the manifest would replace or remove what is listed there,
|
// manifest would replace or remove what is listed there, so fetch must
|
||||||
// so fetch must refuse the manifest before it creates the destination or
|
// refuse the manifest before it creates the destination or requests any
|
||||||
// requests any file.
|
// file.
|
||||||
func TestFetchRefusesListedManifestName(t *testing.T) {
|
func TestFetchRefusesListedManifestName(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1255,7 +1323,6 @@ func TestFetchRefusesListedManifestName(t *testing.T) {
|
|||||||
tempPathFor(defaultManifestName),
|
tempPathFor(defaultManifestName),
|
||||||
defaultManifestName + "/" + testFileTxt,
|
defaultManifestName + "/" + testFileTxt,
|
||||||
"INDEX.MF",
|
"INDEX.MF",
|
||||||
"./" + defaultManifestName,
|
|
||||||
} {
|
} {
|
||||||
t.Run(listed, func(t *testing.T) {
|
t.Run(listed, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1331,9 +1398,9 @@ func TestFetchRefusesListedTempName(t *testing.T) {
|
|||||||
// directory, one of them under a spelling the manifest does not list,
|
// directory, one of them under a spelling the manifest does not list,
|
||||||
// and check reports that file as not in the manifest. So fetch must
|
// and check reports that file as not in the manifest. So fetch must
|
||||||
// refuse each on every filesystem before it creates the destination or
|
// refuse each on every filesystem before it creates the destination or
|
||||||
// requests any file. A file and a directory with the same name, and a
|
// requests any file. A file and a directory with the same name are
|
||||||
// file listed twice, are refused the same way. A manifest whose names
|
// refused the same way. A manifest whose names differ in more than
|
||||||
// differ in more than letter case is fetched in full.
|
// letter case is fetched in full.
|
||||||
func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1343,7 +1410,6 @@ func TestFetchRefusesNamesEqualIgnoringCase(t *testing.T) {
|
|||||||
{"Dir", "dir/x", "dir/x (the file Dir)"},
|
{"Dir", "dir/x", "dir/x (the file Dir)"},
|
||||||
{"Dir/a.txt", "dir/b.txt", "dir/b.txt (the directory Dir)"},
|
{"Dir/a.txt", "dir/b.txt", "dir/b.txt (the directory Dir)"},
|
||||||
{"dir", "dir/x", "dir/x (the file dir)"},
|
{"dir", "dir/x", "dir/x (the file dir)"},
|
||||||
{"./x.txt", "x.txt", "x.txt (the file ./x.txt)"},
|
|
||||||
} {
|
} {
|
||||||
t.Run(tc.first+" and "+tc.second, func(t *testing.T) {
|
t.Run(tc.first+" and "+tc.second, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1404,14 +1470,18 @@ func TestFetchSetsRecordedMode(t *testing.T) {
|
|||||||
require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
|
require.NoError(t, afero.WriteFile(sourceFs, "/"+p, content, modes[p]))
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
|
||||||
Fs: sourceFs,
|
|
||||||
IncludePermissions: true,
|
|
||||||
})
|
|
||||||
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
|
||||||
|
|
||||||
var manifest bytes.Buffer
|
var manifest bytes.Buffer
|
||||||
require.NoError(t, scanner.ToManifest(context.Background(), &manifest, nil))
|
|
||||||
|
// The scan logs to the process-global logger.
|
||||||
|
require.NoError(t, runLocked(func() error {
|
||||||
|
scanner := mfer.NewScannerWithOptions(&mfer.ScannerOptions{
|
||||||
|
Fs: sourceFs,
|
||||||
|
IncludePermissions: true,
|
||||||
|
})
|
||||||
|
require.NoError(t, scanner.EnumerateFS(sourceFs, "/", nil))
|
||||||
|
|
||||||
|
return scanner.ToManifest(context.Background(), &manifest, nil)
|
||||||
|
}))
|
||||||
|
|
||||||
tree := fetchTestHandler(manifest.Bytes(), files)
|
tree := fetchTestHandler(manifest.Bytes(), files)
|
||||||
|
|
||||||
@@ -1573,8 +1643,7 @@ func manifestWithMode(t *testing.T, path string, content []byte, mode uint32) []
|
|||||||
}
|
}
|
||||||
|
|
||||||
// builtManifest returns a manifest of files, built directly rather than
|
// builtManifest returns a manifest of files, built directly rather than
|
||||||
// scanned, since a scan lists no hidden files and never a path starting
|
// scanned, since a scan lists no hidden files.
|
||||||
// with "./".
|
|
||||||
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
func builtManifest(t *testing.T, files map[string][]byte) []byte {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
+68
-19
@@ -38,6 +38,10 @@ var (
|
|||||||
errEntryMissingHash = errors.New("manifest entry has no hash")
|
errEntryMissingHash = errors.New("manifest entry has no hash")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// errSignKeyNeeded indicates a signed manifest given to freshen with no
|
||||||
|
// key to sign the rewritten one.
|
||||||
|
var errSignKeyNeeded = errors.New("give --sign-key to keep it signed")
|
||||||
|
|
||||||
// FreshenStatus contains progress information for the freshen operation.
|
// FreshenStatus contains progress information for the freshen operation.
|
||||||
type FreshenStatus struct {
|
type FreshenStatus struct {
|
||||||
Phase string // "scan" or "hash"
|
Phase string // "scan" or "hash"
|
||||||
@@ -311,6 +315,16 @@ func (h *freshenHasher) processEntry(e *freshenEntry) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The scan recorded the size; a file that grew or shrank since would
|
||||||
|
// get an entry whose size and hash describe different contents.
|
||||||
|
if bytesRead != e.size {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w for %q: %d bytes when scanned but %d bytes when read; "+
|
||||||
|
"the file changed while freshen ran",
|
||||||
|
errSizeMismatch, e.path, e.size, bytesRead,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
h.hashedBytes += bytesRead
|
h.hashedBytes += bytesRead
|
||||||
h.hashedFiles++
|
h.hashedFiles++
|
||||||
|
|
||||||
@@ -351,24 +365,39 @@ func writeFreshenedManifest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newFreshenBuilder constructs the manifest builder configured from CLI
|
// newFreshenBuilder constructs the manifest builder configured from CLI
|
||||||
// flags.
|
// flags, and returns the fingerprint of the --sign-key key, or "" when
|
||||||
func (mfa *CLIApp) newFreshenBuilder(cmd *cli.Command) (*mfer.Builder, error) {
|
// none is given. A --sign-key given with an empty value is refused.
|
||||||
|
func (mfa *CLIApp) newFreshenBuilder(
|
||||||
|
cmd *cli.Command,
|
||||||
|
) (*mfer.Builder, string, error) {
|
||||||
builder := mfer.NewBuilder()
|
builder := mfer.NewBuilder()
|
||||||
if cmd.Bool("include-timestamps") {
|
if cmd.Bool("include-timestamps") {
|
||||||
builder.SetIncludeTimestamps(true)
|
builder.SetIncludeTimestamps(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var signKeyFingerprint string
|
||||||
|
|
||||||
// Set up signing options if sign-key is provided
|
// Set up signing options if sign-key is provided
|
||||||
if signKey := cmd.String("sign-key"); signKey != "" {
|
if cmd.IsSet("sign-key") {
|
||||||
|
signKey := cmd.String("sign-key")
|
||||||
|
if signKey == "" {
|
||||||
|
return nil, "", errEmptySignKey
|
||||||
|
}
|
||||||
|
|
||||||
signing, err := mfa.signingOptions(signKey)
|
signing, err := mfa.signingOptions(signKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
signKeyFingerprint, err = mfer.SecretKeyFingerprint(signing.SecretKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("%s: %w", signKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
builder.SetSigningOptions(signing)
|
builder.SetSigningOptions(signing)
|
||||||
}
|
}
|
||||||
|
|
||||||
return builder, nil
|
return builder, signKeyFingerprint, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// freshenScan runs the scan phase against the loaded manifest entries
|
// freshenScan runs the scan phase against the loaded manifest entries
|
||||||
@@ -480,11 +509,19 @@ func runFreshenHash(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadExistingEntries loads the manifest and indexes its file entries
|
// existingManifest is what freshen uses of the manifest it rewrites.
|
||||||
// by path.
|
type existingManifest struct {
|
||||||
func (mfa *CLIApp) loadExistingEntries(
|
entriesByPath map[string]*mfer.MFFilePath
|
||||||
manifestPath string,
|
signer string // fingerprint of the key it is signed with, "" if unsigned
|
||||||
) (map[string]*mfer.MFFilePath, error) {
|
hasCreatedAt bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadExistingManifest loads the manifest and indexes its file entries
|
||||||
|
// by path. It refuses a signed manifest when no --sign-key is given, as
|
||||||
|
// the rewritten manifest would be unsigned.
|
||||||
|
func (mfa *CLIApp) loadExistingManifest(
|
||||||
|
cmd *cli.Command, manifestPath string,
|
||||||
|
) (*existingManifest, error) {
|
||||||
log.Infof("loading manifest from %s", manifestPath)
|
log.Infof("loading manifest from %s", manifestPath)
|
||||||
|
|
||||||
// Load existing manifest
|
// Load existing manifest
|
||||||
@@ -496,6 +533,10 @@ func (mfa *CLIApp) loadExistingEntries(
|
|||||||
return nil, fmt.Errorf("load manifest: %w", err)
|
return nil, fmt.Errorf("load manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if signer := manifest.Signer(); signer != nil && cmd.String("sign-key") == "" {
|
||||||
|
return nil, fmt.Errorf("manifest is signed by %s: %w", signer, errSignKeyNeeded)
|
||||||
|
}
|
||||||
|
|
||||||
existingFiles := manifest.Files()
|
existingFiles := manifest.Files()
|
||||||
log.Infof("manifest contains %d files", len(existingFiles))
|
log.Infof("manifest contains %d files", len(existingFiles))
|
||||||
|
|
||||||
@@ -505,7 +546,11 @@ func (mfa *CLIApp) loadExistingEntries(
|
|||||||
existingByPath[f.GetPath()] = f
|
existingByPath[f.GetPath()] = f
|
||||||
}
|
}
|
||||||
|
|
||||||
return existingByPath, nil
|
return &existingManifest{
|
||||||
|
entriesByPath: existingByPath,
|
||||||
|
signer: string(manifest.Signer()),
|
||||||
|
hasCreatedAt: manifest.CreatedAt() != nil,
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (mfa *CLIApp) freshenManifestOperation(
|
func (mfa *CLIApp) freshenManifestOperation(
|
||||||
@@ -513,20 +558,18 @@ func (mfa *CLIApp) freshenManifestOperation(
|
|||||||
) error {
|
) error {
|
||||||
log.Debug("freshenManifestOperation()")
|
log.Debug("freshenManifestOperation()")
|
||||||
|
|
||||||
showProgress := cmd.Bool("progress")
|
|
||||||
|
|
||||||
// Find manifest file
|
// Find manifest file
|
||||||
manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
|
manifestPath, err := mfa.resolveFreshenManifestPath(cmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
builder, err := mfa.newFreshenBuilder(cmd)
|
builder, signKeyFingerprint, err := mfa.newFreshenBuilder(cmd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
existingByPath, err := mfa.loadExistingEntries(manifestPath)
|
existing, err := mfa.loadExistingManifest(cmd, manifestPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -538,7 +581,7 @@ func (mfa *CLIApp) freshenManifestOperation(
|
|||||||
|
|
||||||
// Phase 1: Scan filesystem
|
// Phase 1: Scan filesystem
|
||||||
scanner, removed, err := mfa.freshenScan(cmd, manifestPath, absBase,
|
scanner, removed, err := mfa.freshenScan(cmd, manifestPath, absBase,
|
||||||
existingByPath)
|
existing.entriesByPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -555,7 +598,7 @@ func (mfa *CLIApp) freshenManifestOperation(
|
|||||||
hasher := &freshenHasher{
|
hasher := &freshenHasher{
|
||||||
fs: mfa.Fs,
|
fs: mfa.Fs,
|
||||||
absBase: absBase,
|
absBase: absBase,
|
||||||
showProgress: showProgress,
|
showProgress: cmd.Bool("progress"),
|
||||||
totalHashBytes: totalHashBytes,
|
totalHashBytes: totalHashBytes,
|
||||||
filesToHash: filesToHash,
|
filesToHash: filesToHash,
|
||||||
startHash: time.Now(),
|
startHash: time.Now(),
|
||||||
@@ -571,8 +614,14 @@ func (mfa *CLIApp) freshenManifestOperation(
|
|||||||
log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
|
log.Infof("freshen complete: %d unchanged, %d changed, %d added, %d removed",
|
||||||
scanner.unchanged, scanner.changed, scanner.added, removed)
|
scanner.unchanged, scanner.changed, scanner.added, removed)
|
||||||
|
|
||||||
// Skip writing if nothing changed
|
// Skip writing only when the manifest would come out the same: nothing
|
||||||
if scanner.changed == 0 && scanner.added == 0 && removed == 0 {
|
// in the tree changed, and the manifest already holds what --sign-key,
|
||||||
|
// --include-timestamps and --include-permissions ask for. The scan
|
||||||
|
// counts a file whose recorded mode is not what --include-permissions
|
||||||
|
// asks for as changed.
|
||||||
|
if scanner.changed == 0 && scanner.added == 0 && removed == 0 &&
|
||||||
|
existing.signer == signKeyFingerprint &&
|
||||||
|
existing.hasCreatedAt == cmd.Bool("include-timestamps") {
|
||||||
log.Infof("manifest unchanged, skipping write")
|
log.Infof("manifest unchanged, skipping write")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -70,6 +70,11 @@ func runFreshen(t *testing.T, fs afero.Fs, root, manifestPath string) {
|
|||||||
func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
|
func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
// Loading a manifest logs to the process-global logger, so serialize
|
||||||
|
// with the CLI runs.
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||||
Path: path,
|
Path: path,
|
||||||
Fs: fs,
|
Fs: fs,
|
||||||
@@ -79,6 +84,23 @@ func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
|
|||||||
return manifest.Files()
|
return manifest.Files()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// manifestCreatedAt returns the createdAt timestamp of the manifest at
|
||||||
|
// path, or nil if it has none.
|
||||||
|
func manifestCreatedAt(t *testing.T, fs afero.Fs, path string) *mfer.Timestamp {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
runMu.Lock()
|
||||||
|
defer runMu.Unlock()
|
||||||
|
|
||||||
|
manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
|
||||||
|
Path: path,
|
||||||
|
Fs: fs,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return manifest.CreatedAt()
|
||||||
|
}
|
||||||
|
|
||||||
// TestFreshenUnchanged freshens a tree that has not changed since gen
|
// TestFreshenUnchanged freshens a tree that has not changed since gen
|
||||||
// made its manifest: the manifest must list the same entries as before.
|
// made its manifest: the manifest must list the same entries as before.
|
||||||
func TestFreshenUnchanged(t *testing.T) {
|
func TestFreshenUnchanged(t *testing.T) {
|
||||||
@@ -136,6 +158,41 @@ func TestFreshenRecordsModeOnlyWhenAsked(t *testing.T) {
|
|||||||
recordedModes())
|
recordedModes())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFreshenIncludeTimestampsOnUnchangedTree runs freshen
|
||||||
|
// --include-timestamps on a tree that has not changed since gen made its
|
||||||
|
// manifest, which has no createdAt timestamp. freshen must rewrite the
|
||||||
|
// manifest with one. Run again with the flag it must leave the manifest as
|
||||||
|
// it is; run without it, it must rewrite the manifest without one.
|
||||||
|
func TestFreshenIncludeTimestampsOnUnchangedTree(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
root, manifestPath := setupFreshenDir(t, fs, map[string]string{testFileTxt: "hello"})
|
||||||
|
|
||||||
|
freshenWithTimestamps := func() {
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdFreshen, "-q", "--include-timestamps",
|
||||||
|
testFlagBase, root, manifestPath,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
|
}
|
||||||
|
|
||||||
|
freshenWithTimestamps()
|
||||||
|
assert.NotNil(t, manifestCreatedAt(t, fs, manifestPath))
|
||||||
|
|
||||||
|
before, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
freshenWithTimestamps()
|
||||||
|
|
||||||
|
after, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, before, after, "manifest that has a createdAt was rewritten")
|
||||||
|
|
||||||
|
runFreshen(t, fs, root, manifestPath)
|
||||||
|
assert.Nil(t, manifestCreatedAt(t, fs, manifestPath))
|
||||||
|
}
|
||||||
|
|
||||||
// assertManifestLists asserts that the manifest at manifestPath lists
|
// assertManifestLists asserts that the manifest at manifestPath lists
|
||||||
// exactly the files in want, each with the size and SHA-256 hash of its
|
// exactly the files in want, each with the size and SHA-256 hash of its
|
||||||
// content in want and the mtime of the file of that name under root.
|
// content in want and the mtime of the file of that name under root.
|
||||||
@@ -243,6 +300,58 @@ func TestFreshenWithChanges(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// growingFileFs appends to one file each time it is opened, as a program
|
||||||
|
// still writing to the file would between freshen's scan and its hash.
|
||||||
|
type growingFileFs struct {
|
||||||
|
afero.Fs
|
||||||
|
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
//nolint:ireturn // Open must return afero.File to satisfy afero.Fs.
|
||||||
|
func (f growingFileFs) Open(name string) (afero.File, error) {
|
||||||
|
if name == f.path {
|
||||||
|
file, err := f.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = file.WriteString(" world")
|
||||||
|
_ = file.Close()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.Fs.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFreshenFileChangedSinceScan has a new file grow after freshen's scan
|
||||||
|
// recorded its size and before freshen hashes it: freshen must fail and
|
||||||
|
// leave the manifest as it was.
|
||||||
|
func TestFreshenFileChangedSinceScan(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
osFs := afero.NewOsFs()
|
||||||
|
root, manifestPath := setupFreshenDir(t, osFs,
|
||||||
|
map[string]string{testFileTxt: "content"})
|
||||||
|
grown := filepath.Join(root, "log.txt")
|
||||||
|
writeTestFile(t, osFs, grown, "hello")
|
||||||
|
|
||||||
|
before, err := afero.ReadFile(osFs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdFreshen, "-q", testFlagBase, root, manifestPath,
|
||||||
|
}, growingFileFs{Fs: osFs, path: grown})
|
||||||
|
assert.Equal(t, 1, runCLI(opts), "stderr: %s", testStderr(t, opts))
|
||||||
|
|
||||||
|
after, err := afero.ReadFile(osFs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, before, after, "manifest changed")
|
||||||
|
}
|
||||||
|
|
||||||
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
// TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a
|
||||||
// subdirectory of the tree it lists: the manifest is not listed, while an
|
// subdirectory of the tree it lists: the manifest is not listed, while an
|
||||||
// ordinary file of the same name at the top of the tree is. The manifest
|
// ordinary file of the same name at the top of the tree is. The manifest
|
||||||
@@ -436,7 +545,11 @@ func TestFreshenRecordEntryMtimePresence(t *testing.T) {
|
|||||||
s := &freshenScanner{
|
s := &freshenScanner{
|
||||||
existingByPath: map[string]*mfer.MFFilePath{relPath: tc.entry},
|
existingByPath: map[string]*mfer.MFFilePath{relPath: tc.entry},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recordEntry logs to the process-global logger.
|
||||||
|
runMu.Lock()
|
||||||
s.recordEntry(relPath, info)
|
s.recordEntry(relPath, info)
|
||||||
|
runMu.Unlock()
|
||||||
|
|
||||||
require.Len(t, s.entries, 1)
|
require.Len(t, s.entries, 1)
|
||||||
assert.Equal(t, tc.needsHash, s.entries[0].needsHash)
|
assert.Equal(t, tc.needsHash, s.entries[0].needsHash)
|
||||||
|
|||||||
+10
-2
@@ -29,6 +29,8 @@ var (
|
|||||||
"already exists (use --force to overwrite)")
|
"already exists (use --force to overwrite)")
|
||||||
// errEmptyOutput indicates --output given with an empty value.
|
// errEmptyOutput indicates --output given with an empty value.
|
||||||
errEmptyOutput = errors.New("--output must not be empty")
|
errEmptyOutput = errors.New("--output must not be empty")
|
||||||
|
// errEmptySignKey indicates --sign-key given with an empty value.
|
||||||
|
errEmptySignKey = errors.New("--sign-key must not be empty")
|
||||||
)
|
)
|
||||||
|
|
||||||
// reportEnumProgress renders enumeration progress until the channel
|
// reportEnumProgress renders enumeration progress until the channel
|
||||||
@@ -121,7 +123,8 @@ func (mfa *CLIApp) outputPath(cmd *cli.Command) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// buildScannerOptions constructs scanner options from the CLI flags and
|
// buildScannerOptions constructs scanner options from the CLI flags and
|
||||||
// the path the manifest is written to.
|
// the path the manifest is written to. A --sign-key given with an empty
|
||||||
|
// value is refused.
|
||||||
func (mfa *CLIApp) buildScannerOptions(
|
func (mfa *CLIApp) buildScannerOptions(
|
||||||
cmd *cli.Command, output string,
|
cmd *cli.Command, output string,
|
||||||
) (*mfer.ScannerOptions, error) {
|
) (*mfer.ScannerOptions, error) {
|
||||||
@@ -144,7 +147,12 @@ func (mfa *CLIApp) buildScannerOptions(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Set up signing options if sign-key is provided
|
// Set up signing options if sign-key is provided
|
||||||
if signKey := cmd.String("sign-key"); signKey != "" {
|
if cmd.IsSet("sign-key") {
|
||||||
|
signKey := cmd.String("sign-key")
|
||||||
|
if signKey == "" {
|
||||||
|
return nil, errEmptySignKey
|
||||||
|
}
|
||||||
|
|
||||||
signing, err := mfa.signingOptions(signKey)
|
signing, err := mfa.signingOptions(signKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
+67
-33
@@ -6,12 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"runtime/debug"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
"github.com/urfave/cli/v3"
|
"github.com/urfave/cli/v3"
|
||||||
"sneak.berlin/go/mfer/internal/log"
|
"sneak.berlin/go/mfer/internal/log"
|
||||||
"sneak.berlin/go/mfer/mfer"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Command and flag names shared across command definitions and tests.
|
// Command and flag names shared across command definitions and tests.
|
||||||
@@ -58,7 +58,6 @@ var errUnexpectedArgument = errors.New("unexpected argument")
|
|||||||
//nolint:revive // established name used throughout the codebase and tests
|
//nolint:revive // established name used throughout the codebase and tests
|
||||||
type CLIApp struct {
|
type CLIApp struct {
|
||||||
appname string
|
appname string
|
||||||
version string
|
|
||||||
gitrev string
|
gitrev string
|
||||||
startupTime time.Time
|
startupTime time.Time
|
||||||
exitCode int
|
exitCode int
|
||||||
@@ -87,13 +86,26 @@ const banner = `
|
|||||||
\ \:\ \ \:\ \ \::/ \ \:\
|
\ \:\ \ \:\ \ \::/ \ \:\
|
||||||
\__\/ \__\/ \__\/ \__\/`
|
\__\/ \__\/ \__\/ \__\/`
|
||||||
|
|
||||||
// VersionString returns the version and git revision formatted for display.
|
// VersionString returns the version mfer reports: the revision the build
|
||||||
|
// stamped into main.Gitrev or, when it stamped none, as with `go install`,
|
||||||
|
// the main module's version from the binary's build information.
|
||||||
func (mfa *CLIApp) VersionString() string {
|
func (mfa *CLIApp) VersionString() string {
|
||||||
if mfa.gitrev != "" {
|
if mfa.gitrev != "" {
|
||||||
return fmt.Sprintf("%s (%s)", mfer.Version, mfa.gitrev)
|
return mfa.gitrev
|
||||||
}
|
}
|
||||||
|
|
||||||
return mfer.Version
|
return moduleVersion(debug.ReadBuildInfo())
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleVersion returns the main module's version from the binary's build
|
||||||
|
// information, or "unknown" when the binary has none or it holds no version,
|
||||||
|
// as in a build with GO111MODULE=off.
|
||||||
|
func moduleVersion(info *debug.BuildInfo, ok bool) string {
|
||||||
|
if !ok || info.Main.Version == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
|
||||||
|
return info.Main.Version
|
||||||
}
|
}
|
||||||
|
|
||||||
// printVersion writes the version line shared by the --version flag and the
|
// printVersion writes the version line shared by the --version flag and the
|
||||||
@@ -106,8 +118,7 @@ func (mfa *CLIApp) printBanner() {
|
|||||||
if log.GetLevel() <= log.InfoLevel {
|
if log.GetLevel() <= log.InfoLevel {
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
_, _ = fmt.Fprintln(mfa.Stdout, banner)
|
||||||
_, _ = fmt.Fprintf(mfa.Stdout,
|
_, _ = fmt.Fprintf(mfa.Stdout,
|
||||||
" mfer by @sneak: v%s released %s\n",
|
" mfer by @sneak: %s\n", mfa.VersionString())
|
||||||
mfer.Version, mfer.ReleaseDate)
|
|
||||||
_, _ = fmt.Fprintln(mfa.Stdout, " https://sneak.berlin/go/mfer")
|
_, _ = fmt.Fprintln(mfa.Stdout, " https://sneak.berlin/go/mfer")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -168,6 +179,17 @@ func refuseExtraArgs(_ context.Context, cmd *cli.Command) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// showCommandHelp is the CommandNotFound of the commands that take a path or
|
||||||
|
// URL. Those commands set HideHelpCommand, so that a path or URL named help or
|
||||||
|
// h is an argument rather than urfave/cli's help command; each sets it itself,
|
||||||
|
// because on the root it would also remove `mfer help`. With --help,
|
||||||
|
// urfave/cli looks the command's first argument up as a help topic, so
|
||||||
|
// `check DIR --help` asks for a topic DIR and finds none; this prints the
|
||||||
|
// command's own help instead, as `check --help` does.
|
||||||
|
func showCommandHelp(ctx context.Context, cmd *cli.Command, _ string) {
|
||||||
|
_ = cli.ShowCommandHelp(ctx, cmd.Root(), cmd.Name)
|
||||||
|
}
|
||||||
|
|
||||||
// requireSignatureFlag returns the --require-signature flag taken by the
|
// requireSignatureFlag returns the --require-signature flag taken by the
|
||||||
// check and fetch subcommands.
|
// check and fetch subcommands.
|
||||||
func requireSignatureFlag() *cli.StringFlag {
|
func requireSignatureFlag() *cli.StringFlag {
|
||||||
@@ -191,10 +213,12 @@ func includePermissionsFlag() *cli.BoolFlag {
|
|||||||
|
|
||||||
func (mfa *CLIApp) generateCommand() *cli.Command {
|
func (mfa *CLIApp) generateCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdGenerate,
|
Name: cmdGenerate,
|
||||||
Aliases: []string{"gen"},
|
Aliases: []string{"gen"},
|
||||||
Usage: "Generate manifest file",
|
Usage: "Generate manifest file",
|
||||||
ArgsUsage: "[path ...]",
|
ArgsUsage: "[path ...]",
|
||||||
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||||
mfa.setVerbosity(cmd)
|
mfa.setVerbosity(cmd)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
@@ -253,10 +277,12 @@ func (mfa *CLIApp) generateCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) checkCommand() *cli.Command {
|
func (mfa *CLIApp) checkCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdCheck,
|
Name: cmdCheck,
|
||||||
Usage: "Validate files using manifest file",
|
Usage: "Validate files using manifest file",
|
||||||
ArgsUsage: manifestArgsUsage,
|
ArgsUsage: manifestArgsUsage,
|
||||||
ArgValidator: refuseExtraArgs,
|
ArgValidator: refuseExtraArgs,
|
||||||
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||||
mfa.setVerbosity(cmd)
|
mfa.setVerbosity(cmd)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
@@ -287,10 +313,12 @@ func (mfa *CLIApp) checkCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
func (mfa *CLIApp) freshenCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdFreshen,
|
Name: cmdFreshen,
|
||||||
Usage: "Update manifest with changed, new, and removed files",
|
Usage: "Update manifest with changed, new, and removed files",
|
||||||
ArgsUsage: manifestArgsUsage,
|
ArgsUsage: manifestArgsUsage,
|
||||||
ArgValidator: refuseExtraArgs,
|
ArgValidator: refuseExtraArgs,
|
||||||
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||||
mfa.setVerbosity(cmd)
|
mfa.setVerbosity(cmd)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
@@ -338,10 +366,12 @@ func (mfa *CLIApp) freshenCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) exportCommand() *cli.Command {
|
func (mfa *CLIApp) exportCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdExport,
|
Name: cmdExport,
|
||||||
Usage: "Export manifest contents as JSON",
|
Usage: "Export manifest contents as JSON",
|
||||||
ArgsUsage: "[manifest file or URL]",
|
ArgsUsage: "[manifest file or URL]",
|
||||||
ArgValidator: refuseExtraArgs,
|
ArgValidator: refuseExtraArgs,
|
||||||
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||||
mfa.setVerbosity(cmd)
|
mfa.setVerbosity(cmd)
|
||||||
|
|
||||||
@@ -364,12 +394,14 @@ func (mfa *CLIApp) versionCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) listCommand() *cli.Command {
|
func (mfa *CLIApp) listCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdList,
|
Name: cmdList,
|
||||||
Aliases: []string{"ls"},
|
Aliases: []string{"ls"},
|
||||||
Usage: "List files in manifest",
|
Usage: "List files in manifest",
|
||||||
ArgsUsage: manifestArgsUsage,
|
ArgsUsage: manifestArgsUsage,
|
||||||
ArgValidator: refuseExtraArgs,
|
ArgValidator: refuseExtraArgs,
|
||||||
Action: mfa.listManifestOperation,
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
|
Action: mfa.listManifestOperation,
|
||||||
Flags: []cli.Flag{
|
Flags: []cli.Flag{
|
||||||
&cli.BoolFlag{
|
&cli.BoolFlag{
|
||||||
Name: "long",
|
Name: "long",
|
||||||
@@ -386,10 +418,12 @@ func (mfa *CLIApp) listCommand() *cli.Command {
|
|||||||
|
|
||||||
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
func (mfa *CLIApp) fetchCommand() *cli.Command {
|
||||||
return &cli.Command{
|
return &cli.Command{
|
||||||
Name: cmdFetch,
|
Name: cmdFetch,
|
||||||
Usage: "fetch manifest and referenced files",
|
Usage: "fetch manifest and referenced files",
|
||||||
ArgsUsage: "URL",
|
ArgsUsage: "URL",
|
||||||
ArgValidator: refuseExtraArgs,
|
ArgValidator: refuseExtraArgs,
|
||||||
|
HideHelpCommand: true,
|
||||||
|
CommandNotFound: showCommandHelp,
|
||||||
Action: func(ctx context.Context, cmd *cli.Command) error {
|
Action: func(ctx context.Context, cmd *cli.Command) error {
|
||||||
mfa.setVerbosity(cmd)
|
mfa.setVerbosity(cmd)
|
||||||
mfa.printBanner()
|
mfa.printBanner()
|
||||||
|
|||||||
@@ -4,8 +4,12 @@ package cli
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"io"
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -73,6 +77,103 @@ func TestGenAndFreshenSignWithKeyFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFreshenSignedManifestNeedsSignKey runs freshen on a signed manifest
|
||||||
|
// after a file is added. Without --sign-key it must fail, naming the
|
||||||
|
// signer and --sign-key, and leave the manifest as it was; with --sign-key
|
||||||
|
// it must rewrite the manifest, signed.
|
||||||
|
func TestFreshenSignedManifestNeedsSignKey(t *testing.T) {
|
||||||
|
t.Setenv(envSignKeyPassphrase, "")
|
||||||
|
|
||||||
|
secretKey, fingerprint := testSecretKey(t, nil, nil)
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
keyFile := filepath.Join(t.TempDir(), "key.asc")
|
||||||
|
root := t.TempDir()
|
||||||
|
manifestPath := filepath.Join(root, defaultManifestName)
|
||||||
|
|
||||||
|
require.NoError(t, afero.WriteFile(fs, keyFile, secretKey, 0o600))
|
||||||
|
writeTestFile(t, fs, filepath.Join(root, testFileTxt), "hello")
|
||||||
|
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdGenerate, "-q", testFlagSignKey, keyFile,
|
||||||
|
"-o", manifestPath, root,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
|
||||||
|
before, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
writeTestFile(t, fs, filepath.Join(root, "added.txt"), "added")
|
||||||
|
|
||||||
|
opts = testOpts([]string{testApp, cmdFreshen, "-q", manifestPath}, fs)
|
||||||
|
assert.Equal(t, 1, runCLI(opts))
|
||||||
|
assert.Contains(t, testStderr(t, opts), fingerprint)
|
||||||
|
assert.Contains(t, testStderr(t, opts), testFlagSignKey)
|
||||||
|
|
||||||
|
after, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, before, after)
|
||||||
|
|
||||||
|
opts = testOpts([]string{
|
||||||
|
testApp, cmdFreshen, "-q", testFlagSignKey, keyFile, manifestPath,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
|
||||||
|
opts = testOpts([]string{
|
||||||
|
testApp, cmdCheck, "-q",
|
||||||
|
"--" + flagRequireSignature, fingerprint, manifestPath,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
assert.Len(t, manifestFiles(t, fs, manifestPath), 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFreshenSignsUnchangedTree runs freshen with --sign-key on a tree that
|
||||||
|
// has not changed since gen made its unsigned manifest. freshen must
|
||||||
|
// rewrite the manifest signed by that key. Run again with the same key it
|
||||||
|
// must leave the manifest as it is; with another key it must rewrite it
|
||||||
|
// signed by that one.
|
||||||
|
func TestFreshenSignsUnchangedTree(t *testing.T) {
|
||||||
|
t.Setenv(envSignKeyPassphrase, "")
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
_, manifestPath := setupFreshenDir(t, fs, map[string]string{testFileTxt: "hello"})
|
||||||
|
|
||||||
|
newKeyFile := func() (string, string) {
|
||||||
|
secretKey, fingerprint := testSecretKey(t, nil, nil)
|
||||||
|
keyFile := filepath.Join(t.TempDir(), "key.asc")
|
||||||
|
require.NoError(t, afero.WriteFile(fs, keyFile, secretKey, 0o600))
|
||||||
|
|
||||||
|
return keyFile, fingerprint
|
||||||
|
}
|
||||||
|
|
||||||
|
freshenAndCheckSignedBy := func(keyFile, fingerprint string) {
|
||||||
|
opts := testOpts([]string{
|
||||||
|
testApp, cmdFreshen, "-q", testFlagSignKey, keyFile, manifestPath,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
|
||||||
|
opts = testOpts([]string{
|
||||||
|
testApp, cmdCheck, "-q",
|
||||||
|
"--" + flagRequireSignature, fingerprint, manifestPath,
|
||||||
|
}, fs)
|
||||||
|
require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
|
||||||
|
}
|
||||||
|
|
||||||
|
keyFile, fingerprint := newKeyFile()
|
||||||
|
freshenAndCheckSignedBy(keyFile, fingerprint)
|
||||||
|
|
||||||
|
before, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
freshenAndCheckSignedBy(keyFile, fingerprint)
|
||||||
|
|
||||||
|
after, err := afero.ReadFile(fs, manifestPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, before, after, "manifest already signed by the key was rewritten")
|
||||||
|
|
||||||
|
freshenAndCheckSignedBy(newKeyFile())
|
||||||
|
}
|
||||||
|
|
||||||
// TestSignWithProtectedKeyNeedsPassphrase runs gen with a protected key,
|
// TestSignWithProtectedKeyNeedsPassphrase runs gen with a protected key,
|
||||||
// with MFER_SIGN_KEY_PASSPHRASE empty and no terminal to ask on. gen must
|
// with MFER_SIGN_KEY_PASSPHRASE empty and no terminal to ask on. gen must
|
||||||
// fail, naming the variable, and write no manifest.
|
// fail, naming the variable, and write no manifest.
|
||||||
@@ -207,3 +308,105 @@ func TestGenAsksForPassphraseOnTerminal(t *testing.T) {
|
|||||||
}, fs)
|
}, fs)
|
||||||
require.Equal(t, 0, runCLI(check), testStderr(t, check))
|
require.Equal(t, 0, runCLI(check), testStderr(t, check))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// assertEmptySignKeyRefused runs gen and freshen, each with flags, on a
|
||||||
|
// directory holding a file and an unsigned manifest. Each must fail,
|
||||||
|
// naming --sign-key, and leave the manifest as it was.
|
||||||
|
func assertEmptySignKeyRefused(t *testing.T, flags ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
manifest := manifestOf(t, map[string][]byte{testFileTxt: []byte("listed")})
|
||||||
|
|
||||||
|
for command, rest := range map[string][]string{
|
||||||
|
cmdGenerate: {"-q", "-f", "-o", testMF, testDir},
|
||||||
|
cmdFreshen: {"-q", testMF},
|
||||||
|
} {
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||||
|
writeTestFile(t, fs, testFile1, "hello")
|
||||||
|
require.NoError(t, afero.WriteFile(fs, testMF, manifest, 0o644))
|
||||||
|
|
||||||
|
opts := testOpts(slices.Concat([]string{testApp, command}, flags, rest), fs)
|
||||||
|
assert.Equal(t, 1, runCLI(opts), command)
|
||||||
|
assert.Contains(t, testStderr(t, opts), errEmptySignKey.Error(), command)
|
||||||
|
|
||||||
|
content, err := afero.ReadFile(fs, testMF)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, manifest, content, command)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGenAndFreshenRefuseEmptySignKey runs gen and freshen with --sign-key
|
||||||
|
// given an empty value, as an unset shell variable gives it.
|
||||||
|
func TestGenAndFreshenRefuseEmptySignKey(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertEmptySignKeyRefused(t, testFlagSignKey, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestGenAndFreshenRefuseEmptySignKeyFromEnvironment runs gen and freshen
|
||||||
|
// with MFER_SIGN_KEY set to an empty string.
|
||||||
|
func TestGenAndFreshenRefuseEmptySignKeyFromEnvironment(t *testing.T) {
|
||||||
|
t.Setenv("MFER_SIGN_KEY", "")
|
||||||
|
|
||||||
|
assertEmptySignKeyRefused(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertEmptyFingerprintRefused runs check on an unsigned manifest, and
|
||||||
|
// fetch of that manifest from a server, each with flags. Each must refuse
|
||||||
|
// the empty fingerprint as malformed, and fetch must request nothing.
|
||||||
|
func assertEmptyFingerprintRefused(t *testing.T, flags ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
const want = "invalid fingerprint: must be exactly 40 hex characters, got 0"
|
||||||
|
|
||||||
|
files := map[string][]byte{testFileTxt: []byte("listed")}
|
||||||
|
manifest := manifestOf(t, files)
|
||||||
|
|
||||||
|
fs := afero.NewMemMapFs()
|
||||||
|
require.NoError(t, fs.MkdirAll(testDir, 0o755))
|
||||||
|
writeTestFile(t, fs, filepath.Join(testDir, testFileTxt), "listed")
|
||||||
|
require.NoError(t, afero.WriteFile(fs, testMF, manifest, 0o644))
|
||||||
|
|
||||||
|
opts := testOpts(slices.Concat(
|
||||||
|
[]string{testApp, cmdCheck, "-q"}, flags, []string{testMF},
|
||||||
|
), fs)
|
||||||
|
assert.Equal(t, 1, runCLI(opts), cmdCheck)
|
||||||
|
assert.Contains(t, testStderr(t, opts), want, cmdCheck)
|
||||||
|
|
||||||
|
var requests atomic.Int32
|
||||||
|
|
||||||
|
tree := fetchTestHandler(manifest, files)
|
||||||
|
|
||||||
|
server := httptest.NewServer(
|
||||||
|
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
requests.Add(1)
|
||||||
|
tree.ServeHTTP(w, r)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
opts = testOpts(slices.Concat(
|
||||||
|
[]string{testApp, cmdFetch, "-q", "--" + flagDest, t.TempDir()},
|
||||||
|
flags, []string{server.URL},
|
||||||
|
), afero.NewOsFs())
|
||||||
|
assert.Equal(t, 1, runCLI(opts), cmdFetch)
|
||||||
|
assert.Contains(t, testStderr(t, opts), want, cmdFetch)
|
||||||
|
assert.Zero(t, requests.Load(), "fetch made a request")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCheckAndFetchRefuseEmptyRequireSignature runs check and fetch with
|
||||||
|
// --require-signature given an empty value, as an unset shell variable
|
||||||
|
// gives it.
|
||||||
|
func TestCheckAndFetchRefuseEmptyRequireSignature(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertEmptyFingerprintRefused(t, "--"+flagRequireSignature, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCheckAndFetchRefuseEmptyRequireSignatureFromEnvironment runs check
|
||||||
|
// and fetch with MFER_REQUIRE_SIGNATURE set to an empty string.
|
||||||
|
func TestCheckAndFetchRefuseEmptyRequireSignatureFromEnvironment(t *testing.T) {
|
||||||
|
t.Setenv("MFER_REQUIRE_SIGNATURE", "")
|
||||||
|
|
||||||
|
assertEmptyFingerprintRefused(t)
|
||||||
|
}
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ var (
|
|||||||
errPathAbsolute = errors.New("is absolute; must be relative")
|
errPathAbsolute = errors.New("is absolute; must be relative")
|
||||||
errPathEmptySegment = errors.New("contains empty segment")
|
errPathEmptySegment = errors.New("contains empty segment")
|
||||||
errPathDotDot = errors.New("contains '..' segment")
|
errPathDotDot = errors.New("contains '..' segment")
|
||||||
|
errPathDot = errors.New("contains '.' segment")
|
||||||
errSizeMismatch = errors.New("size mismatch")
|
errSizeMismatch = errors.New("size mismatch")
|
||||||
errNegativeSize = errors.New("size cannot be negative")
|
errNegativeSize = errors.New("size cannot be negative")
|
||||||
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
errHashNotMultihash = errors.New("hash is not a valid multihash")
|
||||||
@@ -46,6 +47,7 @@ var (
|
|||||||
// - Must use forward slashes only (no backslashes)
|
// - Must use forward slashes only (no backslashes)
|
||||||
// - Must be relative (no leading /)
|
// - Must be relative (no leading /)
|
||||||
// - Must not contain ".." segments
|
// - Must not contain ".." segments
|
||||||
|
// - Must not contain "." segments
|
||||||
// - Must not contain empty segments (no "//")
|
// - Must not contain empty segments (no "//")
|
||||||
// - Must not be empty
|
// - Must not be empty
|
||||||
func ValidatePath(p string) error {
|
func ValidatePath(p string) error {
|
||||||
@@ -73,6 +75,10 @@ func ValidatePath(p string) error {
|
|||||||
if seg == ".." {
|
if seg == ".." {
|
||||||
return fmt.Errorf("path %q %w", p, errPathDotDot)
|
return fmt.Errorf("path %q %w", p, errPathDotDot)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if seg == "." {
|
||||||
|
return fmt.Errorf("path %q %w", p, errPathDot)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -286,6 +286,9 @@ func TestValidatePath(t *testing.T) {
|
|||||||
{"has/../traversal", "dot-dot segment"},
|
{"has/../traversal", "dot-dot segment"},
|
||||||
{"has//double", "empty segment"},
|
{"has//double", "empty segment"},
|
||||||
{"..", "just dot-dot"},
|
{"..", "just dot-dot"},
|
||||||
|
{"./a.txt", "leading dot segment"},
|
||||||
|
{"has/./dot", "interior dot segment"},
|
||||||
|
{".", "just dot"},
|
||||||
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
|
{string([]byte{0xff, 0xfe}), "invalid UTF-8"},
|
||||||
}
|
}
|
||||||
for _, tt := range invalid {
|
for _, tt := range invalid {
|
||||||
|
|||||||
+13
-10
@@ -264,11 +264,12 @@ func (c *Checker) Check(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FindExtraFiles walks the filesystem and reports files not in the manifest,
|
// FindExtraFiles walks the filesystem and reports the regular files, and
|
||||||
// hidden files and directories included. The manifest file itself is not
|
// symlinks to them, that are not in the manifest, hidden files and
|
||||||
// reported. Anything the search cannot read, such as a directory that cannot
|
// directories included. The manifest file itself is not reported. Anything
|
||||||
// be listed, is reported with StatusError and the search goes on. Results are
|
// the search cannot read, such as a directory that cannot be listed, is
|
||||||
// sent to the results channel. The channel is closed when done.
|
// reported with StatusError and the search goes on. Results are sent to the
|
||||||
|
// results channel. The channel is closed when done.
|
||||||
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
|
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
|
||||||
if results != nil {
|
if results != nil {
|
||||||
defer close(results)
|
defer close(results)
|
||||||
@@ -318,11 +319,6 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// gen never lists a named pipe, a socket or a device.
|
|
||||||
if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// A symlink is compared by what it points to, so a manifest reached
|
// A symlink is compared by what it points to, so a manifest reached
|
||||||
// through one is not reported either.
|
// through one is not reported either.
|
||||||
if info.Mode()&os.ModeSymlink != 0 {
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
@@ -332,6 +328,13 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// gen lists regular files and symlinks to them only, so a named
|
||||||
|
// pipe, a socket, a device, a symlink to a directory and a broken
|
||||||
|
// symlink are never reported.
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Skip the manifest file itself, however its path is spelled
|
// Skip the manifest file itself, however its path is spelled
|
||||||
if os.SameFile(info, c.manifestInfo) {
|
if os.SameFile(info, c.manifestInfo) {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
package mfer
|
package mfer
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// Version is the current mfer release version.
|
|
||||||
Version = "0.1.0"
|
|
||||||
|
|
||||||
// ReleaseDate is the date on which Version was released.
|
|
||||||
ReleaseDate = "2025-12-17"
|
|
||||||
|
|
||||||
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
// MaxDecompressedSize is the maximum allowed size of decompressed manifest
|
||||||
// data (256 MiB). This prevents decompression bombs from consuming excessive
|
// data (256 MiB). This prevents decompression bombs from consuming excessive
|
||||||
// memory.
|
// memory.
|
||||||
|
|||||||
@@ -80,6 +80,8 @@ func TestDeserializeRejectsInvalidEntryPaths(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"parent traversal", "../escape"},
|
{"parent traversal", "../escape"},
|
||||||
{"interior traversal", "a/../../escape"},
|
{"interior traversal", "a/../../escape"},
|
||||||
|
{"leading dot segment", "./a.txt"},
|
||||||
|
{"interior dot segment", "dir/./a.txt"},
|
||||||
{"absolute path", "/etc/passwd"},
|
{"absolute path", "/etc/passwd"},
|
||||||
{"backslash path", `a\b`},
|
{"backslash path", `a\b`},
|
||||||
{"double slash", "a//b"},
|
{"double slash", "a//b"},
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ package mfer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
"uuid"
|
"uuid"
|
||||||
|
|
||||||
@@ -52,3 +55,38 @@ func TestReadAtMost(t *testing.T) {
|
|||||||
"file exceeds maximum allowed size of 65536 bytes")
|
"file exceeds maximum allowed size of 65536 bytes")
|
||||||
assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
|
assert.Equal(t, maxSize-1, input.Len(), "bytes left unread")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFormatTestVector checks the test vector that docs/FORMAT.md shows: the
|
||||||
|
// SHA-256 of the whole file, so that any change to the file fails, then loads
|
||||||
|
// it, verifies its signature, and checks the canonical string and the signer
|
||||||
|
// the document gives for it.
|
||||||
|
func TestFormatTestVector(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
path = "testdata/test-vector.mf"
|
||||||
|
fileSHA256 = "5864187d671b2a55f8e015f858248f92c51956b549cf548b6bb54477c2407fd4"
|
||||||
|
signer = "5E962A37590D855B467D20294947A60319611E6F"
|
||||||
|
)
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
require.Equal(t, fileSHA256, hex.EncodeToString(sum[:]))
|
||||||
|
|
||||||
|
m, err := NewManifestFromFile(&ManifestFromFileOptions{Path: path})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
canonicalString, err := m.signatureString()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "ZNAVSRFG-92197fd532bc40908aa9f86e5518b56c-"+
|
||||||
|
"12202353553205f036a88cefee3d0d9049ccd8e68bb24011a0b462b5867f1fcc6352",
|
||||||
|
canonicalString)
|
||||||
|
|
||||||
|
keyFingerprint, err := verifySignature([]byte(canonicalString),
|
||||||
|
m.pbOuter.GetSignature(), m.pbOuter.GetSigningPubKey())
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, signer, keyFingerprint)
|
||||||
|
assert.Equal(t, signer, string(m.pbOuter.GetSigner()))
|
||||||
|
}
|
||||||
|
|||||||
@@ -62,6 +62,12 @@ func TestValidatePathMessagesVerbatim(t *testing.T) {
|
|||||||
want: `path "a/../b" contains '..' segment`,
|
want: `path "a/../b" contains '..' segment`,
|
||||||
is: errPathDotDot,
|
is: errPathDotDot,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "dot segment",
|
||||||
|
path: "a/./b",
|
||||||
|
want: `path "a/./b" contains '.' segment`,
|
||||||
|
is: errPathDot,
|
||||||
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
@@ -47,6 +47,23 @@ func (m *manifest) Files() []*MFFilePath {
|
|||||||
return m.pbInner.GetFiles()
|
return m.pbInner.GetFiles()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Signer returns the fingerprint of the key that made a loaded manifest's
|
||||||
|
// signature, which loading the manifest checked, or nil if the manifest is
|
||||||
|
// not signed.
|
||||||
|
func (m *manifest) Signer() []byte {
|
||||||
|
if len(m.pbOuter.GetSignature()) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return m.pbOuter.GetSigner()
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreatedAt returns the createdAt timestamp of a loaded manifest, or nil
|
||||||
|
// if the manifest has none.
|
||||||
|
func (m *manifest) CreatedAt() *Timestamp {
|
||||||
|
return m.pbInner.GetCreatedAt()
|
||||||
|
}
|
||||||
|
|
||||||
// signatureString generates the canonical string used for signing/verification.
|
// signatureString generates the canonical string used for signing/verification.
|
||||||
// Format: MAGIC-UUID-MULTIHASH where UUID and multihash are hex-encoded.
|
// Format: MAGIC-UUID-MULTIHASH where UUID and multihash are hex-encoded.
|
||||||
// Requires pbOuter to be set with Uuid and Sha256 fields.
|
// Requires pbOuter to be set with Uuid and Sha256 fields.
|
||||||
|
|||||||
+2
-2
@@ -333,8 +333,8 @@ type MFFilePath struct {
|
|||||||
state protoimpl.MessageState `protogen:"open.v1"`
|
state protoimpl.MessageState `protogen:"open.v1"`
|
||||||
// required attributes:
|
// required attributes:
|
||||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
// Path invariants: must be valid UTF-8, use forward slashes only,
|
||||||
// be relative (no leading /), contain no ".." segments, and no
|
// be relative (no leading /), contain no ".." or "." segments, and
|
||||||
// empty segments (no "//").
|
// no empty segments (no "//").
|
||||||
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
|
||||||
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||||
// gotta have at least one:
|
// gotta have at least one:
|
||||||
|
|||||||
+2
-2
@@ -49,8 +49,8 @@ message MFFileOuter {
|
|||||||
message MFFilePath {
|
message MFFilePath {
|
||||||
// required attributes:
|
// required attributes:
|
||||||
// Path invariants: must be valid UTF-8, use forward slashes only,
|
// Path invariants: must be valid UTF-8, use forward slashes only,
|
||||||
// be relative (no leading /), contain no ".." segments, and no
|
// be relative (no leading /), contain no ".." or "." segments, and
|
||||||
// empty segments (no "//").
|
// no empty segments (no "//").
|
||||||
string path = 1;
|
string path = 1;
|
||||||
int64 size = 2;
|
int64 size = 2;
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
4f8ea5276a8e66590e71ed154a9176b0f68ffa9ab69d8de5aff0e0d9b731164d mf.proto
|
1668b2f6b8639a9213873474e58dbacb3a4ef77cc27c42253cad8dd076e5f206 mf.proto
|
||||||
|
|||||||
+39
-3
@@ -57,7 +57,9 @@ var (
|
|||||||
"signature must hold exactly one signature")
|
"signature must hold exactly one signature")
|
||||||
errNotOneArmoredBlock = errors.New(
|
errNotOneArmoredBlock = errors.New(
|
||||||
"must be exactly one armored block and nothing else")
|
"must be exactly one armored block and nothing else")
|
||||||
errMalformedArmor = errors.New("armor is malformed")
|
errMalformedArmor = errors.New("armor is malformed")
|
||||||
|
errKeyNotValidWhenSigned = errors.New(
|
||||||
|
"signature was made when its key was not valid")
|
||||||
)
|
)
|
||||||
|
|
||||||
// SigningOptions holds the key a manifest is signed with.
|
// SigningOptions holds the key a manifest is signed with.
|
||||||
@@ -81,6 +83,19 @@ func SecretKeyIsProtected(secretKey []byte) (bool, error) {
|
|||||||
return isProtected(key), nil
|
return isProtected(key), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SecretKeyFingerprint returns the fingerprint of the OpenPGP secret key
|
||||||
|
// secretKey, armored or binary, as a manifest signed with it records its
|
||||||
|
// signer. It fails unless secretKey holds one version 4 primary key with
|
||||||
|
// its secret key.
|
||||||
|
func SecretKeyFingerprint(secretKey []byte) (string, error) {
|
||||||
|
key, err := readSecretKey(secretKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return fingerprint(key), nil
|
||||||
|
}
|
||||||
|
|
||||||
// CheckSigningKey fails unless opts can sign now: opts.SecretKey must hold
|
// CheckSigningKey fails unless opts can sign now: opts.SecretKey must hold
|
||||||
// one version 4 primary key with a secret key that may sign and has not
|
// one version 4 primary key with a secret key that may sign and has not
|
||||||
// expired or been revoked, and opts.Passphrase must unlock it when it is
|
// expired or been revoked, and opts.Passphrase must unlock it when it is
|
||||||
@@ -245,8 +260,9 @@ func fingerprint(key *openpgp.Entity) string {
|
|||||||
|
|
||||||
// verifySignature checks that signature is one good, unexpired OpenPGP
|
// verifySignature checks that signature is one good, unexpired OpenPGP
|
||||||
// signature over data, made by the one primary key in pubKey or one of its
|
// signature over data, made by the one primary key in pubKey or one of its
|
||||||
// subkeys, and returns that primary key's fingerprint. signature and pubKey
|
// subkeys while that key was valid, and returns that primary key's
|
||||||
// may each be armored or binary.
|
// fingerprint. The key may have expired since. signature and pubKey may
|
||||||
|
// each be armored or binary.
|
||||||
func verifySignature(data, signature, pubKey []byte) (string, error) {
|
func verifySignature(data, signature, pubKey []byte) (string, error) {
|
||||||
key, err := readOneKey(pubKey, "embedded public key block", true)
|
key, err := readOneKey(pubKey, "embedded public key block", true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -281,6 +297,26 @@ func verifySignature(data, signature, pubKey []byte) (string, error) {
|
|||||||
return "", fmt.Errorf("verify signature: %w", err)
|
return "", fmt.Errorf("verify signature: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The check above judges the key as of now. Checking the signature a
|
||||||
|
// second time with the library's clock set to its creation time
|
||||||
|
// requires the primary key, and the subkey that made the signature, to
|
||||||
|
// have been valid then under the key validity rules of RFC 9580. The
|
||||||
|
// key is judged by its current self-signatures and binding signatures:
|
||||||
|
// for a version 4 key the newest self-signature on its primary user
|
||||||
|
// ID, for a version 6 key its newest direct-key signature, and for a
|
||||||
|
// subkey its newest binding signature with the signature embedded in
|
||||||
|
// it. So a signature made before the newest of those is refused.
|
||||||
|
signedAt := &packet.Config{
|
||||||
|
Time: func() time.Time { return sig.CreationTime },
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err = openpgp.VerifyDetachedSignature(openpgp.EntityList{key},
|
||||||
|
bytes.NewReader(data), bytes.NewReader(sigData), signedAt)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("verify signature: %w: %w",
|
||||||
|
errKeyNotValidWhenSigned, err)
|
||||||
|
}
|
||||||
|
|
||||||
return fingerprint(key), nil
|
return fingerprint(key), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+160
-2
@@ -4,6 +4,7 @@ package mfer
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"crypto"
|
||||||
"crypto/dsa" //nolint:staticcheck // SA1019: tests need a DSA key to refuse
|
"crypto/dsa" //nolint:staticcheck // SA1019: tests need a DSA key to refuse
|
||||||
"io"
|
"io"
|
||||||
"math/big"
|
"math/big"
|
||||||
@@ -176,6 +177,25 @@ func TestBuilderWithSigning(t *testing.T) {
|
|||||||
"-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
"-----BEGIN PGP PUBLIC KEY BLOCK-----")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestManifestSigner checks that Signer gives the signing key's
|
||||||
|
// fingerprint for a signed manifest, and nil once the signature is taken
|
||||||
|
// out, although the signer field is still there.
|
||||||
|
func TestManifestSigner(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
key := newTestKey(t, nil)
|
||||||
|
data := signedTestManifest(t, &SigningOptions{SecretKey: armoredSecretKeys(t, key)})
|
||||||
|
|
||||||
|
signed, err := NewManifestFromReader(bytes.NewReader(data))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, fingerprint(key), string(signed.Signer()))
|
||||||
|
|
||||||
|
unsigned, err := NewManifestFromReader(bytes.NewReader(
|
||||||
|
rewriteOuter(t, data, func(outer *MFFileOuter) { outer.Signature = nil })))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, unsigned.Signer())
|
||||||
|
}
|
||||||
|
|
||||||
func TestScannerWithSigning(t *testing.T) {
|
func TestScannerWithSigning(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -261,6 +281,21 @@ func TestSecretKeyIsProtectedWithoutPassphrase(t *testing.T) {
|
|||||||
assert.False(t, protected)
|
assert.False(t, protected)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSecretKeyFingerprint checks that SecretKeyFingerprint gives the
|
||||||
|
// signer that a manifest signed with the key records.
|
||||||
|
func TestSecretKeyFingerprint(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
opts := testSigningOptions(t)
|
||||||
|
|
||||||
|
got, err := SecretKeyFingerprint(opts.SecretKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
m, err := NewManifestFromReader(bytes.NewReader(signedTestManifest(t, opts)))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, string(m.Signer()), got)
|
||||||
|
}
|
||||||
|
|
||||||
// TestSigningKeyFileWithTwoKeys signs with a key file that holds two keys,
|
// TestSigningKeyFileWithTwoKeys signs with a key file that holds two keys,
|
||||||
// as gpg writes it for a user ID that two keys have. It names no one key
|
// as gpg writes it for a user ID that two keys have. It names no one key
|
||||||
// to sign with, so signing must fail.
|
// to sign with, so signing must fail.
|
||||||
@@ -419,8 +454,9 @@ func TestVerifySignature(t *testing.T) {
|
|||||||
|
|
||||||
// TestVerifySignatureKeyExpiredSince verifies two signatures made in 2020
|
// TestVerifySignatureKeyExpiredSince verifies two signatures made in 2020
|
||||||
// by a key that expired a day after it was made. A signature with no
|
// by a key that expired a day after it was made. A signature with no
|
||||||
// expiry time is still good. One that expired an hour after it was made is
|
// expiry time is still good, as is one made then by a signing subkey that
|
||||||
// refused, although the key has expired too.
|
// expired a day after it was made. One that expired an hour after it was
|
||||||
|
// made is refused, although the key has expired too.
|
||||||
func TestVerifySignatureKeyExpiredSince(t *testing.T) {
|
func TestVerifySignatureKeyExpiredSince(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -448,10 +484,132 @@ func TestVerifySignatureKeyExpiredSince(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, fingerprint(key), signer)
|
assert.Equal(t, fingerprint(key), signer)
|
||||||
|
|
||||||
|
withSubkey := newTestKey(t, &packet.Config{Time: config.Time})
|
||||||
|
require.NoError(t, withSubkey.AddSigningSubkey(&packet.Config{
|
||||||
|
Algorithm: packet.PubKeyAlgoEdDSA, Time: config.Time,
|
||||||
|
KeyLifetimeSecs: config.KeyLifetimeSecs,
|
||||||
|
}))
|
||||||
|
|
||||||
|
var subkeySig bytes.Buffer
|
||||||
|
|
||||||
|
require.NoError(t, openpgp.ArmoredDetachSign(&subkeySig, withSubkey,
|
||||||
|
bytes.NewReader(data), config))
|
||||||
|
|
||||||
|
subkeyPubKey, err := armoredPublicKey(withSubkey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = verifySignature(data, subkeySig.Bytes(), subkeyPubKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
_, err = verifySignature(data, expiringSig.Bytes(), pubKey)
|
_, err = verifySignature(data, expiringSig.Bytes(), pubKey)
|
||||||
require.ErrorIs(t, err, pgperrors.ErrSignatureExpired)
|
require.ErrorIs(t, err, pgperrors.ErrSignatureExpired)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// signDated returns a detached signature over data by signer, dated when.
|
||||||
|
// openpgp.DetachSign signs only with a key that is valid at the time its
|
||||||
|
// config gives, so this makes the signature packet without that check.
|
||||||
|
func signDated(t *testing.T, signer *packet.PrivateKey, data []byte,
|
||||||
|
when time.Time,
|
||||||
|
) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
sig := &packet.Signature{
|
||||||
|
SigType: packet.SigTypeBinary,
|
||||||
|
PubKeyAlgo: signer.PubKeyAlgo,
|
||||||
|
Hash: crypto.SHA256,
|
||||||
|
CreationTime: when,
|
||||||
|
IssuerKeyId: &signer.KeyId,
|
||||||
|
}
|
||||||
|
|
||||||
|
h, err := sig.PrepareSign(nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = h.Write(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, sig.Sign(h, signer, nil))
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
require.NoError(t, sig.Serialize(&buf))
|
||||||
|
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestVerifySignatureRefusesKeyNotValidWhenSigned verifies signatures
|
||||||
|
// dated when the key or subkey that made them was not valid: after the key
|
||||||
|
// expired; after the key's self-signature, or the signing subkey's binding
|
||||||
|
// signature, expired, the key or subkey having expired since as well; and
|
||||||
|
// before the key was made, also by a key made in 2100. Each must be
|
||||||
|
// refused.
|
||||||
|
func TestVerifySignatureRefusesKeyNotValidWhenSigned(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
made := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
twoDaysLater := made.Add(48 * time.Hour)
|
||||||
|
day := uint32((24 * time.Hour).Seconds())
|
||||||
|
clock := func(when time.Time) func() time.Time {
|
||||||
|
return func() time.Time { return when }
|
||||||
|
}
|
||||||
|
|
||||||
|
expiredAfterADay := newTestKey(t, &packet.Config{
|
||||||
|
Time: clock(made), KeyLifetimeSecs: day,
|
||||||
|
})
|
||||||
|
selfSignatureExpiredAfterADay := newTestKey(t, &packet.Config{
|
||||||
|
Time: clock(made), KeyLifetimeSecs: 30 * day, SigLifetimeSecs: day,
|
||||||
|
})
|
||||||
|
|
||||||
|
withSubkey := newTestKey(t, &packet.Config{Time: clock(made)})
|
||||||
|
require.NoError(t, withSubkey.AddSigningSubkey(&packet.Config{
|
||||||
|
Algorithm: packet.PubKeyAlgoEdDSA, Time: clock(made),
|
||||||
|
KeyLifetimeSecs: 30 * day, SigLifetimeSecs: day,
|
||||||
|
}))
|
||||||
|
subkey := withSubkey.Subkeys[len(withSubkey.Subkeys)-1].PrivateKey
|
||||||
|
|
||||||
|
madeADayLater := newTestKey(t, &packet.Config{
|
||||||
|
Time: clock(made.Add(24 * time.Hour)),
|
||||||
|
})
|
||||||
|
madeIn2100 := newTestKey(t, &packet.Config{
|
||||||
|
Time: clock(time.Date(2100, 1, 1, 0, 0, 0, 0, time.UTC)),
|
||||||
|
})
|
||||||
|
|
||||||
|
for name, test := range map[string]struct {
|
||||||
|
key *openpgp.Entity
|
||||||
|
signer *packet.PrivateKey
|
||||||
|
when time.Time
|
||||||
|
}{
|
||||||
|
"after the key expired": {
|
||||||
|
expiredAfterADay, expiredAfterADay.PrivateKey, twoDaysLater,
|
||||||
|
},
|
||||||
|
"after the self-signature expired": {
|
||||||
|
selfSignatureExpiredAfterADay,
|
||||||
|
selfSignatureExpiredAfterADay.PrivateKey, twoDaysLater,
|
||||||
|
},
|
||||||
|
"after the binding signature expired": {
|
||||||
|
withSubkey, subkey, twoDaysLater,
|
||||||
|
},
|
||||||
|
"before the key was made": {
|
||||||
|
madeADayLater, madeADayLater.PrivateKey, made,
|
||||||
|
},
|
||||||
|
"before a key made in 2100": {
|
||||||
|
madeIn2100, madeIn2100.PrivateKey, made,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
data := []byte("signed in 2020")
|
||||||
|
|
||||||
|
pubKey, err := armoredPublicKey(test.key)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
_, err = verifySignature(data,
|
||||||
|
signDated(t, test.signer, data, test.when), pubKey)
|
||||||
|
require.ErrorIs(t, err, errKeyNotValidWhenSigned)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManifestSignatureVerification(t *testing.T) {
|
func TestManifestSignatureVerification(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
Vendored
BIN
Binary file not shown.
Reference in New Issue
Block a user