End-to-end tests for freshen and fetch (closes #66)

The freshen tests built a fixture and never ran freshen. They now run
gen and freshen through the command entry point on a temp dir: an
unchanged tree leaves the manifest's entries as they were, and a
modified file (also one edited without changing its size), a new or a
deleted file gives a manifest that lists exactly the tree, with sizes,
hashes and mtimes from disk, on which check passes.

New fetch tests run the command against an httptest server: a nested
tree lands complete, a hash mismatch exits non-zero and leaves no file,
and a partly filled destination gets every listed file downloaded again
and replaced, while a file the manifest does not list is left alone.

Model: opus-5-5

internal/cli/freshen_test.go

@@ -2,11 +2,11 @@
 package cli
 
 import (
-	"bytes"
-	"context"
 	"crypto/sha256"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 
@@ -14,6 +14,7 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
 	"sneak.berlin/go/mfer/mfer"
 )
 
@@ -31,77 +32,178 @@ func (s stubFileInfo) ModTime() time.Time { return s.mtime }
 func (s stubFileInfo) IsDir() bool        { return false }
 func (s stubFileInfo) Sys() any           { return nil }
 
-// setupFreshenDir populates /testdir with two files, scans it, and
-// writes the resulting manifest to /testdir/index.mf.
-func setupFreshenDir(t *testing.T, fs afero.Fs) {
+// setupFreshenDir writes files, by path and content, into a fresh temp
+// dir and runs gen on it. It returns the temp dir and the path of the
+// manifest gen wrote there.
+func setupFreshenDir(
+	t *testing.T, fs afero.Fs, files map[string]string,
+) (string, string) {
 	t.Helper()
 
-	require.NoError(t, fs.MkdirAll(testDir, 0o755))
-	writeTestFile(t, fs, testFile1, "content1")
-	writeTestFile(t, fs, "/testdir/file2.txt", "content2")
+	root := t.TempDir()
+	manifestPath := filepath.Join(root, defaultManifestName)
 
-	// Generate initial manifest
-	opts := &mfer.ScannerOptions{Fs: fs}
-	s := mfer.NewScannerWithOptions(opts)
-	require.NoError(t, s.EnumeratePath(testDir, nil))
+	for path, content := range files {
+		require.NoError(t, fs.MkdirAll(filepath.Dir(filepath.Join(root, path)), 0o750))
+		writeTestFile(t, fs, filepath.Join(root, path), content)
+	}
 
-	var manifestBuf bytes.Buffer
+	opts := testOpts([]string{testApp, cmdGenerate, "-q", "-o", manifestPath, root}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
 
-	require.NoError(t, s.ToManifest(context.Background(), &manifestBuf, nil))
-
-	// Write manifest to filesystem
-	require.NoError(t,
-		afero.WriteFile(fs, "/testdir/index.mf", manifestBuf.Bytes(), 0o644))
+	return root, manifestPath
 }
 
+// runFreshen runs freshen on the manifest at manifestPath for the tree
+// at root and requires it to succeed.
+func runFreshen(t *testing.T, fs afero.Fs, root, manifestPath string) {
+	t.Helper()
+
+	opts := testOpts([]string{
+		testApp, cmdFreshen, "-q", testFlagBase, root, manifestPath,
+	}, fs)
+	require.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+}
+
+// manifestFiles returns the file entries of the manifest at path.
+func manifestFiles(t *testing.T, fs afero.Fs, path string) []*mfer.MFFilePath {
+	t.Helper()
+
+	manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
+		Path: path,
+		Fs:   fs,
+	})
+	require.NoError(t, err)
+
+	return manifest.Files()
+}
+
+// TestFreshenUnchanged freshens a tree that has not changed since gen
+// made its manifest: the manifest must list the same entries as before.
 func TestFreshenUnchanged(t *testing.T) {
 	t.Parallel()
 
-	fs := afero.NewMemMapFs()
-	setupFreshenDir(t, fs)
+	fs := afero.NewOsFs()
+	root, manifestPath := setupFreshenDir(t, fs,
+		map[string]string{testFileTxt: "content1", testDirFile: "content2"})
+	before := manifestFiles(t, fs, manifestPath)
 
-	// Parse manifest to verify
-	manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
-		Path: "/testdir/index.mf",
-		Fs:   fs,
-	})
-	require.NoError(t, err)
-	assert.Len(t, manifest.Files(), 2)
+	runFreshen(t, fs, root, manifestPath)
+
+	after := manifestFiles(t, fs, manifestPath)
+	require.Len(t, after, len(before))
+
+	for i := range before {
+		assert.True(t, proto.Equal(before[i], after[i]),
+			"entry for %s changed", before[i].GetPath())
+	}
 }
 
+// assertManifestLists asserts that the manifest at manifestPath lists
+// exactly the files in want, each with the size and SHA-256 hash of its
+// content in want and the mtime of the file of that name under root.
+func assertManifestLists(
+	t *testing.T, fs afero.Fs, root, manifestPath string, want map[string]string,
+) {
+	t.Helper()
+
+	files := manifestFiles(t, fs, manifestPath)
+	listed := make([]string, 0, len(files))
+
+	for _, f := range files {
+		listed = append(listed, f.GetPath())
+
+		content, ok := want[f.GetPath()]
+		if !ok {
+			continue // reported by the ElementsMatch below
+		}
+
+		digest := sha256.Sum256([]byte(content))
+		hash, err := multihash.Encode(digest[:], multihash.SHA2_256)
+		require.NoError(t, err)
+
+		assert.Equal(t, int64(len(content)), f.GetSize(), f.GetPath())
+		require.NotEmpty(t, f.GetHashes(), f.GetPath())
+		assert.Equal(t, hash, f.GetHashes()[0].GetMultiHash(), f.GetPath())
+
+		info, err := fs.Stat(filepath.Join(root, f.GetPath()))
+		require.NoError(t, err)
+
+		mtime, ok := entryMtime(f)
+		assert.True(t, ok && mtime.Equal(info.ModTime()),
+			"%s: manifest has mtime %v, file has %v",
+			f.GetPath(), mtime, info.ModTime())
+	}
+
+	assert.ElementsMatch(t, slices.Collect(maps.Keys(want)), listed)
+}
+
+// TestFreshenWithChanges makes one change to a tree after gen made its
+// manifest, then freshens the manifest. The rewritten manifest must list
+// exactly the files now in the tree, and check must pass on the tree.
 func TestFreshenWithChanges(t *testing.T) {
 	t.Parallel()
 
-	fs := afero.NewMemMapFs()
-	setupFreshenDir(t, fs)
+	tree := map[string]string{testFileTxt: "content1", testDirFile: "content2"}
 
-	// Verify initial manifest has 2 files
-	manifest, err := mfer.NewManifestFromFile(&mfer.ManifestFromFileOptions{
-		Path: "/testdir/index.mf",
-		Fs:   fs,
-	})
-	require.NoError(t, err)
-	assert.Len(t, manifest.Files(), 2)
+	// Every file a case writes gets this mtime, which differs from the one
+	// gen recorded, so an edit that keeps the size is told apart by its
+	// mtime whatever the filesystem's clock resolution.
+	writtenMtime := time.Unix(1_700_000_000, 0)
 
-	// Add a new file
-	writeTestFile(t, fs, "/testdir/file3.txt", "content3")
+	for _, tc := range []struct {
+		name   string
+		write  map[string]string // files to write, by path and content
+		remove string            // file to delete, if any
+	}{
+		{
+			name:  "modified file",
+			write: map[string]string{testDirFile: "modified content2"},
+		},
+		{
+			name:  "modified file, same size",
+			write: map[string]string{testDirFile: "CONTENT2"},
+		},
+		{
+			name:  "new file",
+			write: map[string]string{"dir/new.txt": "content3"},
+		},
+		{
+			name:   "deleted file",
+			remove: testFileTxt,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
 
-	// Modify file2 (change content and size)
-	writeTestFile(t, fs, "/testdir/file2.txt", "modified content2")
+			fs := afero.NewOsFs()
+			root, manifestPath := setupFreshenDir(t, fs, tree)
 
-	// Remove file1
-	require.NoError(t, fs.Remove(testFile1))
+			// want is the tree as it is after the change.
+			want := maps.Clone(tree)
 
-	// Note: The freshen operation would need to be run here
-	// For now, we just verify the test setup is correct
-	exists, _ := afero.Exists(fs, testFile1)
-	assert.False(t, exists)
+			for path, content := range tc.write {
+				writeTestFile(t, fs, filepath.Join(root, path), content)
+				require.NoError(t, fs.Chtimes(
+					filepath.Join(root, path), writtenMtime, writtenMtime))
+				want[path] = content
+			}
 
-	exists, _ = afero.Exists(fs, "/testdir/file3.txt")
-	assert.True(t, exists)
+			if tc.remove != "" {
+				require.NoError(t, fs.Remove(filepath.Join(root, tc.remove)))
+				delete(want, tc.remove)
+			}
 
-	content, _ := afero.ReadFile(fs, "/testdir/file2.txt")
-	assert.Equal(t, "modified content2", string(content))
+			runFreshen(t, fs, root, manifestPath)
+
+			assertManifestLists(t, fs, root, manifestPath, want)
+
+			opts := testOpts([]string{
+				testApp, cmdCheck, "-q", testFlagNoExtra, testFlagBase, root, manifestPath,
+			}, fs)
+			assert.Equal(t, 0, runCLI(opts), "stderr: %s", testStderr(t, opts))
+		})
+	}
 }
 
 // TestFreshenLeavesManifestOutOfListing freshens a manifest kept in a