List a file:// destination through a symlinked directory (closes #296)

List and ListStream walked the destination with afero.Walk, which uses
Lstat and so reported a symlinked metadata/ or blobs/ directory as one
file, while Put, Get and Stat follow the link. The store then listed as
empty, and prune removed the local snapshot records and deleted every
blob. The listings, and DeletePartialUploads, which remote nuke uses,
now walk a view of the filesystem without its Lstat method, so afero
falls back to Stat and enters the link. A prefix lists as empty only
when it is truly missing: when it or a directory above it is a symlink
whose target is missing, such as a link to an unmounted volume, the
listing fails.

Judgement call: a symlink loop is not detected; it is followed until the
system refuses the path, and the listing fails.

Model: opus-5-5

internal/storage/file_backend_test.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"errors"
 	"io/fs"
+	"os"
 	"path/filepath"
+	"reflect"
+	"strings"
 	"testing"
 
 	"sneak.berlin/go/vaultik/internal/storage"
@@ -60,3 +63,232 @@ func TestFileStorerListMissingDestination(t *testing.T) {
 		t.Errorf("ListStream error = %v, want a not-exist error", streamErr)
 	}
 }
+
+// The two directories at the top of a destination, either of which the
+// tests below replace with a symlink.
+const (
+	metadataDir = "metadata"
+	blobsDir    = "blobs"
+)
+
+// moveBehindSymlink moves base/dir to a directory elsewhere and puts a
+// symlink to it in its place.
+func moveBehindSymlink(t *testing.T, base, dir string) {
+	t.Helper()
+
+	linkPath := filepath.Join(base, dir)
+	movedPath := filepath.Join(t.TempDir(), dir)
+
+	err := os.Rename(linkPath, movedPath)
+	if err != nil {
+		t.Fatalf("moving %s away: %v", dir, err)
+	}
+
+	err = os.Symlink(movedPath, linkPath)
+	if err != nil {
+		t.Fatalf("linking %s: %v", dir, err)
+	}
+}
+
+// TestFileStorerListsThroughSymlinkedDirectory checks that List and
+// ListStream read through a symlinked metadata/ or blobs/ directory, as Get
+// and Stat do, instead of listing the link as one key. Listing the whole
+// destination meets the link part-way through the walk; listing its prefix
+// starts the walk at the link. A prefix missing behind the link lists as
+// empty.
+func TestFileStorerListsThroughSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+			key := dir + "/aa/bb/object"
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			err = s.Put(context.Background(), key, strings.NewReader("data"))
+			if err != nil {
+				t.Fatalf("Put: %v", err)
+			}
+
+			moveBehindSymlink(t, base, dir)
+
+			want := []string{key}
+
+			for _, prefix := range []string{"", dir + "/"} {
+				got := listSorted(t, s, prefix)
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("List(%q) = %v, want %v", prefix, got, want)
+				}
+
+				got = listStreamKeys(t, s, prefix)
+				if !reflect.DeepEqual(got, want) {
+					t.Errorf("ListStream(%q) = %v, want %v", prefix, got, want)
+				}
+			}
+
+			absent := dir + "/absent/"
+
+			if got := listSorted(t, s, absent); len(got) != 0 {
+				t.Errorf("List(%q) = %v, want empty", absent, got)
+			}
+
+			if got := listStreamKeys(t, s, absent); len(got) != 0 {
+				t.Errorf("ListStream(%q) = %v, want empty", absent, got)
+			}
+		})
+	}
+}
+
+// TestFileStorerListFailsOnSymlinkWithMissingTarget checks that List and
+// ListStream fail on a symlinked metadata/ or blobs/ directory whose target
+// is missing, such as a link to a volume that is not mounted, as they do on
+// a missing destination directory, instead of listing it as empty. That
+// holds for a prefix below the link too.
+func TestFileStorerListFailsOnSymlinkWithMissingTarget(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			unmounted := filepath.Join(t.TempDir(), "unmounted")
+
+			err = os.Symlink(unmounted, filepath.Join(base, dir))
+			if err != nil {
+				t.Fatalf("Symlink: %v", err)
+			}
+
+			for _, prefix := range []string{"", dir + "/", dir + "/aa/"} {
+				keys, err := s.List(ctx, prefix)
+				if !errors.Is(err, fs.ErrNotExist) {
+					t.Errorf("List(%q) = %v, %v; want a not-exist error",
+						prefix, keys, err)
+				}
+
+				var streamErr error
+
+				for object := range s.ListStream(ctx, prefix) {
+					if object.Err != nil {
+						streamErr = object.Err
+					}
+				}
+
+				if !errors.Is(streamErr, fs.ErrNotExist) {
+					t.Errorf("ListStream(%q) error = %v, want a not-exist error",
+						prefix, streamErr)
+				}
+			}
+		})
+	}
+}
+
+// TestFileStorerListFailsOnSymlinkLoop checks that a symlink pointing back
+// at the destination makes List and ListStream fail instead of walking it
+// forever.
+func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	base := t.TempDir()
+
+	s, err := storage.NewFileStorer(base)
+	if err != nil {
+		t.Fatalf("NewFileStorer: %v", err)
+	}
+
+	err = s.Put(ctx, testBlobKey, strings.NewReader("data"))
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	err = os.Symlink(base, filepath.Join(base, "blobs", "loop"))
+	if err != nil {
+		t.Fatalf("Symlink: %v", err)
+	}
+
+	keys, err := s.List(ctx, "blobs/")
+	if err == nil {
+		t.Errorf("List = %d keys and no error, want an error", len(keys))
+	}
+
+	var streamErr error
+
+	for object := range s.ListStream(ctx, "blobs/") {
+		if object.Err != nil {
+			streamErr = object.Err
+		}
+	}
+
+	if streamErr == nil {
+		t.Error("ListStream sent no error, want one")
+	}
+}
+
+// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that
+// DeletePartialUploads, which remote nuke uses, removes a leftover temp
+// file behind a symlinked metadata/ or blobs/ directory and keeps the
+// object at the real key.
+func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+
+	for _, dir := range []string{metadataDir, blobsDir} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			base := t.TempDir()
+			key := dir + "/aa/bb/object"
+
+			s, err := storage.NewFileStorer(base)
+			if err != nil {
+				t.Fatalf("NewFileStorer: %v", err)
+			}
+
+			err = s.Put(ctx, key, strings.NewReader("data"))
+			if err != nil {
+				t.Fatalf("Put: %v", err)
+			}
+
+			leftover := filepath.Join(base, key+"-123456.partial")
+
+			err = os.WriteFile(leftover, []byte("half"), 0o600)
+			if err != nil {
+				t.Fatalf("writing leftover temp file: %v", err)
+			}
+
+			moveBehindSymlink(t, base, dir)
+
+			err = s.DeletePartialUploads(ctx, dir+"/")
+			if err != nil {
+				t.Fatalf("DeletePartialUploads: %v", err)
+			}
+
+			// leftover runs through the link.
+			_, err = os.Stat(leftover)
+			if !os.IsNotExist(err) {
+				t.Errorf("leftover temp file was not removed: %v", err)
+			}
+
+			_, err = s.Stat(ctx, key)
+			if err != nil {
+				t.Errorf("Stat of the real key: %v", err)
+			}
+		})
+	}
+}