fetch: destination directory, skip files already present, save the manifest, require a signer (closes #101)

fetch takes --dest (default .) and writes every file there through the existing symlink and hard-link guards, which now work relative to that directory. A file already there with the listed size and hash is skipped; a leftover temp file is still replaced. Once every file verifies, the manifest is saved as index.mf through the same temp file and rename, so check runs on the result. --require-signature is shared with check and enforced through verifyRequiredSigner, on a Checker over the manifest held in memory, before anything is downloaded or written. Model: opus-5-5
2026-10-04 15:20:55 +00:00
parent 400a2f8f63
commit 438b73eddf
6 changed files with 486 additions and 134 deletions
@@ -338,7 +338,7 @@ func TestFetchFromHTTP(t *testing.T) {

 		fileURL := baseURL + f.GetPath()
 		err = downloadFile(context.Background(), testClient(),
-			fileURL, localPath, f, progress)
+			fileURL, ".", localPath, f, progress)
 		require.NoError(t, err, "failed to download %s", f.GetPath())
 	}

@@ -386,7 +386,7 @@ func TestFetchHashMismatch(t *testing.T) {

 	// Try to download - should fail with hash mismatch
 	err = downloadFile(context.Background(), testClient(),
-		server.URL+"/file.txt", testFileTxt, files[0], nil)
+		server.URL+"/file.txt", ".", testFileTxt, files[0], nil)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mismatch")

@@ -432,7 +432,7 @@ func TestFetchSizeMismatch(t *testing.T) {

 	// Try to download - should fail with size mismatch
 	err = downloadFile(context.Background(), testClient(),
-		server.URL+"/file.txt", testFileTxt, files[0], nil)
+		server.URL+"/file.txt", ".", testFileTxt, files[0], nil)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "size mismatch")

@@ -490,7 +490,7 @@ func TestFetchProgress(t *testing.T) {

 	// Download
 	err = downloadFile(context.Background(), testClient(),
-		server.URL+"/large.txt", "large.txt", files[0], progress)
+		server.URL+"/large.txt", ".", "large.txt", files[0], progress)
 	close(progress)
 	<-done

@@ -787,7 +787,7 @@ func TestDownloadFileRetriesToSuccess(t *testing.T) {
 	chdirTemp(t)

 	err = downloadFile(context.Background(), testClient(),
-		server.URL+"/"+testFileTxt, testFileTxt, manifest.Files()[0], nil)
+		server.URL+"/"+testFileTxt, ".", testFileTxt, manifest.Files()[0], nil)
 	require.NoError(t, err)
 	assert.Equal(t, int32(3), requests.Load())

@@ -878,7 +878,7 @@ func TestFetchEscapedPaths(t *testing.T) {

 // TestFetchTree runs fetch on a tree with nested directories. Every file
 // the manifest lists must land under its own path with its own content,
-// and nothing else may be left in the destination.
+// beside the manifest, and nothing else may be left in the destination.
 //
 //nolint:paralleltest // changes the process-global working directory
 func TestFetchTree(t *testing.T) {
@@ -889,7 +889,9 @@ func TestFetchTree(t *testing.T) {
 		"other/deep/est.txt": []byte("in a second directory"),
 	}

-	server := httptest.NewServer(fetchTestHandler(manifestOf(t, files), files))
+	manifest := manifestOf(t, files)
+
+	server := httptest.NewServer(fetchTestHandler(manifest, files))
 	defer server.Close()

 	dest := chdirTemp(t)
@@ -897,7 +899,9 @@ func TestFetchTree(t *testing.T) {
 	opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
 	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))

-	assert.Equal(t, files, filesUnder(t, dest))
+	want := maps.Clone(files)
+	want[defaultManifestName] = manifest
+	assert.Equal(t, want, filesUnder(t, dest))
 }

 // TestFetchFailsOnHashMismatch runs fetch against a server that serves a
@@ -922,9 +926,10 @@ func TestFetchFailsOnHashMismatch(t *testing.T) {
 // TestFetchIntoPartlyFilledDestination runs fetch where an interrupted
 // fetch of an older version of the tree left one file current, one file
 // out of date and one half written to its temp file, beside a file the
-// manifest does not list. fetch downloads every file the manifest lists,
-// those already present included, and replaces what is there; the file
-// the manifest does not list is left alone.
+// manifest does not list. fetch skips the current file and downloads the
+// other two. The out-of-date file has the same size as the new version,
+// so only its hash shows that it must be replaced. The file the manifest
+// does not list is left alone, and the manifest is saved beside the files.
 //
 //nolint:paralleltest // changes the process-global working directory
 func TestFetchIntoPartlyFilledDestination(t *testing.T) {
@@ -935,7 +940,8 @@ func TestFetchIntoPartlyFilledDestination(t *testing.T) {
 	}
 	unlisted := []byte("not in the manifest")

-	tree := fetchTestHandler(manifestOf(t, files), files)
+	manifest := manifestOf(t, files)
+	tree := fetchTestHandler(manifest, files)

 	var (
 		mu        sync.Mutex
@@ -963,21 +969,170 @@ func TestFetchIntoPartlyFilledDestination(t *testing.T) {
 		os.WriteFile(tempPathFor("sub/partial.txt"), []byte("cut off"), 0o600))
 	require.NoError(t, os.WriteFile("unlisted.txt", unlisted, 0o600))

-	opts := testOpts([]string{testApp, cmdFetch, "-q", server.URL}, afero.NewOsFs())
+	opts := testOpts([]string{testApp, cmdFetch, server.URL}, afero.NewOsFs())
 	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+	assert.Contains(t, testStderr(t, opts), "skipping current.txt: already present")

 	want := maps.Clone(files)
 	want["unlisted.txt"] = unlisted
+	want[defaultManifestName] = manifest
 	assert.Equal(t, want, filesUnder(t, dest))

 	mu.Lock()
 	defer mu.Unlock()

 	assert.ElementsMatch(t, []string{
-		"/" + defaultManifestName, "/current.txt", "/sub/changed.txt", "/sub/partial.txt",
+		"/" + defaultManifestName, "/sub/changed.txt", "/sub/partial.txt",
 	}, requested)
 }

+// TestFetchIntoDest fetches a tree with --dest into a directory that does
+// not exist yet. The files and the manifest must land there and nowhere
+// else, and check must pass on the result with no extra files. A second
+// fetch into the same directory must download nothing but the manifest.
+//
+//nolint:paralleltest // changes the process-global working directory
+func TestFetchIntoDest(t *testing.T) {
+	files := map[string][]byte{
+		"top.txt":     []byte("at the top"),
+		"sub/one.txt": []byte("one level down"),
+	}
+
+	manifest := manifestOf(t, files)
+	tree := fetchTestHandler(manifest, files)
+
+	var (
+		mu        sync.Mutex
+		requested []string
+	)
+
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			mu.Lock()
+
+			requested = append(requested, r.URL.Path)
+
+			mu.Unlock()
+
+			tree.ServeHTTP(w, r)
+		}))
+	defer server.Close()
+
+	cwd := chdirTemp(t)
+	dest := filepath.Join(t.TempDir(), "mirror")
+	fetch := []string{testApp, cmdFetch, "-q", "--" + flagDest, dest, server.URL}
+
+	opts := testOpts(fetch, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+
+	want := maps.Clone(files)
+	want[defaultManifestName] = manifest
+	assert.Equal(t, want, filesUnder(t, dest))
+	assert.Empty(t, filesUnder(t, cwd), "fetch wrote outside --dest")
+
+	check := testOpts([]string{
+		testApp, cmdCheck, "-q", testFlagBase, dest, testFlagNoExtra,
+		filepath.Join(dest, defaultManifestName),
+	}, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(check), testStderr(t, check))
+
+	mu.Lock()
+	requested = nil
+	mu.Unlock()
+
+	opts = testOpts(fetch, afero.NewOsFs())
+	require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+	assert.Equal(t, want, filesUnder(t, dest))
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.Equal(t, []string{"/" + defaultManifestName}, requested)
+}
+
+// TestFetchRequireSignature runs fetch with --require-signature. A
+// manifest that is unsigned, or signed by another key, must stop fetch
+// with check's message before it downloads or writes anything; the
+// required key lets it through. The signed cases need gpg and are skipped
+// without it, as the other signing tests are.
+//
+//nolint:paralleltest // signedManifest calls t.Setenv, which bars t.Parallel
+func TestFetchRequireSignature(t *testing.T) {
+	files := map[string][]byte{testFileTxt: []byte("signed file")}
+
+	t.Run("unsigned", func(t *testing.T) {
+		assertFetchRefused(t, manifestOf(t, files), files, msgFpA,
+			"manifest is not signed, but signature from "+msgFpA+" is required")
+	})
+
+	t.Run("signed", func(t *testing.T) {
+		manifest := signedManifest(t, files)
+
+		signer, err := signedChecker(t, manifest).
+			ExtractEmbeddedSigningKeyFP(context.Background())
+		require.NoError(t, err)
+
+		assertFetchRefused(t, manifest, files, msgFpB,
+			"embedded signing key fingerprint "+signer+" does not match required "+msgFpB)
+
+		server := httptest.NewServer(fetchTestHandler(manifest, files))
+		defer server.Close()
+
+		dest := t.TempDir()
+
+		opts := testOpts([]string{
+			testApp, cmdFetch, "-q", "--" + flagDest, dest,
+			"--" + flagRequireSignature, signer, server.URL,
+		}, afero.NewOsFs())
+		require.Equal(t, 0, runCLI(opts), testStderr(t, opts))
+		assert.Equal(t, files[testFileTxt], filesUnder(t, dest)[testFileTxt])
+	})
+}
+
+// assertFetchRefused serves manifest, a manifest of files, and fetches it
+// with --require-signature signer into a directory that does not exist
+// yet. fetch must fail with message after requesting only the manifest,
+// and must not create the directory.
+func assertFetchRefused(
+	t *testing.T, manifest []byte, files map[string][]byte, signer, message string,
+) {
+	t.Helper()
+
+	tree := fetchTestHandler(manifest, files)
+
+	var (
+		mu        sync.Mutex
+		requested []string
+	)
+
+	server := httptest.NewServer(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			mu.Lock()
+
+			requested = append(requested, r.URL.Path)
+
+			mu.Unlock()
+
+			tree.ServeHTTP(w, r)
+		}))
+	defer server.Close()
+
+	dest := filepath.Join(t.TempDir(), "mirror")
+
+	opts := testOpts([]string{
+		testApp, cmdFetch, "-q", "--" + flagDest, dest,
+		"--" + flagRequireSignature, signer, server.URL,
+	}, afero.NewOsFs())
+	assert.Equal(t, 1, runCLI(opts))
+	assert.Contains(t, testStderr(t, opts), message)
+	assert.NoDirExists(t, dest, "fetch wrote before checking the signer")
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.Equal(t, []string{"/" + defaultManifestName}, requested)
+}
+
 // TestFetchTimeoutFlag runs fetch with --timeout against a server that
 // never answers. Without the flag's limit the request would wait forever;
 // once fetch gives up on it, the server cancels fetch's context so that