vaultik/internal/snapshot/skip_errors_test.go

package snapshot_test

import (
	"context"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/spf13/afero"
	"sneak.berlin/go/vaultik/internal/database"
	"sneak.berlin/go/vaultik/internal/snapshot"
	"sneak.berlin/go/vaultik/internal/ui"
)

// errSimTempFail is the one-time temp-file creation failure blobTempFailFs
// injects, mirroring a full temp filesystem.
var errSimTempFail = errors.New("simulated temp-file creation failure")

// errSimRead is the read failure readFailFile injects for a file that opens
// but cannot be read.
var errSimRead = errors.New("simulated read failure")

// blobTempFailFs fails the first temp-file creation for a packer blob, then
// behaves normally, simulating a one-time failure to start a new blob.
type blobTempFailFs struct {
	afero.Fs

	mu     sync.Mutex
	failed bool
}

//nolint:ireturn // afero.Fs.OpenFile is defined to return the interface.
func (f *blobTempFailFs) OpenFile(
	name string, flag int, perm os.FileMode,
) (afero.File, error) {
	if strings.Contains(name, "vaultik-blob-") {
		f.mu.Lock()
		firstTime := !f.failed
		f.failed = true
		f.mu.Unlock()

		if firstTime {
			return nil, errSimTempFail
		}
	}

	return f.Fs.OpenFile(name, flag, perm)
}

// readFailFile wraps an afero.File whose Read always fails.
type readFailFile struct {
	afero.File
}

func (readFailFile) Read([]byte) (int, error) {
	return 0, errSimRead
}

// readFailFs fails reads of one target path after a successful open.
type readFailFs struct {
	afero.Fs

	target string
}

//nolint:ireturn // afero.Fs.Open is defined to return the interface.
func (f *readFailFs) Open(name string) (afero.File, error) {
	file, err := f.Fs.Open(name)
	if err != nil {
		return nil, err
	}

	if name == f.target {
		return readFailFile{File: file}, nil
	}

	return file, nil
}

// cancelOnOpenFs cancels the run when the scanner opens the target file to
// back it up, as Ctrl-C partway through processing would, and records each
// file opened after that.
type cancelOnOpenFs struct {
	afero.Fs

	target            string
	cancel            context.CancelFunc
	cancelled         bool
	openedAfterCancel []string
}

//nolint:ireturn // afero.Fs.Open is defined to return the interface.
func (f *cancelOnOpenFs) Open(name string) (afero.File, error) {
	if f.cancelled {
		f.openedAfterCancel = append(f.openedAfterCancel, name)
	}

	if name == f.target {
		f.cancel()
		f.cancelled = true
	}

	return f.Fs.Open(name)
}

// linkRemovedAfterLstatFs is the real filesystem, except that the symlink at
// target is removed right after the walk lstats it, as happens when a link is
// deleted during a backup. The scanner's readlink of it then fails.
type linkRemovedAfterLstatFs struct {
	afero.OsFs

	t      *testing.T
	target string
}

func (f *linkRemovedAfterLstatFs) LstatIfPossible(
	name string,
) (os.FileInfo, bool, error) {
	info, lstatCalled, err := f.OsFs.LstatIfPossible(name)
	if err == nil && name == f.target {
		rmErr := os.Remove(name)
		if rmErr != nil {
			f.t.Errorf("removing %s: %v", name, rmErr)
		}
	}

	return info, lstatCalled, err
}

// writeSkipErrorTestFile writes one file into fs with a fixed mtime.
func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) {
	t.Helper()

	err := fs.MkdirAll(filepath.Dir(path), 0755)
	if err != nil {
		t.Fatalf("mkdir: %v", err)
	}

	err = afero.WriteFile(fs, path, []byte(content), 0644)
	if err != nil {
		t.Fatalf("write %s: %v", path, err)
	}

	when := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)

	err = fs.Chtimes(path, when, when)
	if err != nil {
		t.Fatalf("chtimes %s: %v", path, err)
	}
}

// runSkipErrorScan scans source on fs under ctx with the given skip-errors
// setting, printing user-facing messages to uiw (nil discards them), and
// returns the repositories (for inspection) and the scan error.
func runSkipErrorScan(
	ctx context.Context, t *testing.T, fs afero.Fs, source string,
	skipErrors bool, uiw *ui.Writer,
) (*database.Repositories, error) {
	t.Helper()

	db, err := database.New(ctx, ":memory:")
	if err != nil {
		t.Fatalf("create test db: %v", err)
	}

	t.Cleanup(func() {
		cerr := db.Close()
		if cerr != nil {
			t.Errorf("close db: %v", cerr)
		}
	})

	repos := database.NewRepositories(db)

	scanner := snapshot.NewScanner(snapshot.ScannerConfig{
		FS:               fs,
		ChunkSize:        int64(1024 * 16),
		Repositories:     repos,
		MaxBlobSize:      int64(1024 * 1024),
		CompressionLevel: 3,
		AgeRecipients:    []string{testAgePublicKey},
		UI:               uiw,
		SkipErrors:       skipErrors,
	})

	snapshotID := "test-snapshot-skip-errors"
	createTestSnapshotRecord(ctx, t, repos, snapshotID)

	_, err = scanner.Scan(ctx, source, snapshotID)

	return repos, err
}

// TestScannerPackingFailureAbortsUnderSkipErrors checks that a failure to start
// a new blob aborts the run even with --skip-errors. Otherwise the file would
// be skipped while its chunk had already been registered as pending, letting a
// later blob record that chunk in the chunks table with no blob to back it —
// a snapshot that completes with a file that cannot be restored.
func TestScannerPackingFailureAbortsUnderSkipErrors(t *testing.T) {
	t.Parallel()

	// Two files with distinct content so each yields a distinct chunk: the
	// first fails to start a blob, and without the fix the second's blob would
	// commit the first's orphaned chunk row.
	fs := &blobTempFailFs{Fs: afero.NewMemMapFs()}
	writeSkipErrorTestFile(t, fs, "/source/file1.txt", "first file content")
	writeSkipErrorTestFile(t, fs, "/source/file2.txt", "second file content")

	repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
	if err == nil {
		t.Fatal("expected scan to abort on the packer error, got nil")
	}

	// ListUnpacked returns chunks recorded with no blob_chunks row: exactly the
	// unrestorable state this fix prevents.
	unpacked, err := repos.Chunks.ListUnpacked(context.Background(), 10)
	if err != nil {
		t.Fatalf("listing unpacked chunks: %v", err)
	}

	if len(unpacked) != 0 {
		t.Fatalf("expected no chunk recorded without a blob, got %d", len(unpacked))
	}
}

// TestScannerReadErrorAbortsWithoutSkipErrors checks that a file read error
// aborts the run when --skip-errors is not set.
func TestScannerReadErrorAbortsWithoutSkipErrors(t *testing.T) {
	t.Parallel()

	const target = "/source/unreadable.txt"

	fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
	writeSkipErrorTestFile(t, fs, target, "content that cannot be read")

	_, err := runSkipErrorScan(context.Background(), t, fs, "/source", false, nil)
	if err == nil {
		t.Fatal("expected scan to fail on the read error, got nil")
	}
}

// TestScannerReadErrorSkippedWithSkipErrors checks that a file read error is
// skipped and the run completes when --skip-errors is set.
func TestScannerReadErrorSkippedWithSkipErrors(t *testing.T) {
	t.Parallel()

	const target = "/source/unreadable.txt"

	fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
	writeSkipErrorTestFile(t, fs, target, "content that cannot be read")

	repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
	if err != nil {
		t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
	}

	chunks, err := repos.FileChunks.GetByFile(context.Background(), target)
	if err != nil {
		t.Fatalf("getting file chunks: %v", err)
	}

	if len(chunks) != 0 {
		t.Fatalf("expected unreadable file skipped, got %d chunks", len(chunks))
	}
}

// writeSymlinkSource creates a source directory on disk holding one symlink
// and returns the directory and the symlink's path.
func writeSymlinkSource(t *testing.T) (string, string) {
	t.Helper()

	sourceDir := t.TempDir()
	linkPath := filepath.Join(sourceDir, "link")

	err := os.Symlink("target.txt", linkPath)
	if err != nil {
		t.Fatalf("creating symlink: %v", err)
	}

	return sourceDir, linkPath
}

// TestScannerUnreadableSymlinkAbortsWithoutSkipErrors checks that a symlink
// whose target cannot be read aborts the run when --skip-errors is not set.
func TestScannerUnreadableSymlinkAbortsWithoutSkipErrors(t *testing.T) {
	t.Parallel()

	sourceDir, linkPath := writeSymlinkSource(t)
	fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}

	_, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, false, nil)
	if !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("expected scan to fail on the removed symlink, got %v", err)
	}
}

// TestScannerUnreadableSymlinkSkippedWithSkipErrors checks that a symlink
// whose target cannot be read is skipped with an error line, and the run
// completes, when --skip-errors is set.
func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
	t.Parallel()

	sourceDir, linkPath := writeSymlinkSource(t)
	fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
	uiw := ui.NewWithColor(io.Discard, false)

	repos, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, true, uiw)
	if err != nil {
		t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
	}

	if uiw.ErrorCount() != 1 {
		t.Fatalf("expected one error line for the symlink, got %d",
			uiw.ErrorCount())
	}

	file, err := repos.Files.GetByPath(context.Background(), linkPath)
	if err != nil {
		t.Fatalf("getting %s: %v", linkPath, err)
	}

	if file != nil {
		t.Fatalf("expected %s not to be recorded", linkPath)
	}
}

// TestScannerCancelStopsSkipErrorsRun checks that a --skip-errors backup
// cancelled partway through processing returns the cancellation error,
// opens no further file, and reports no file as failed.
func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name          string
		targetContent string
	}{
		// The cancellation lands while the target is being read.
		{name: "while reading a file", targetContent: "first file content"},
		// An empty target has no chunks, so the cancellation goes unnoticed
		// until the run moves on to the next file.
		{name: "between files", targetContent: ""},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			const target = "/source/a.txt"

			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()

			fs := &cancelOnOpenFs{
				Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
			}
			writeSkipErrorTestFile(t, fs, target, tt.targetContent)
			writeSkipErrorTestFile(t, fs, "/source/b.txt", "second file content")
			writeSkipErrorTestFile(t, fs, "/source/c.txt", "third file content")

			uiw := ui.NewWithColor(io.Discard, false)

			_, err := runSkipErrorScan(ctx, t, fs, "/source", true, uiw)
			if !errors.Is(err, context.Canceled) {
				t.Fatalf("expected the cancellation error, got %v", err)
			}

			if len(fs.openedAfterCancel) != 0 {
				t.Fatalf("expected no file opened after the cancellation, got %v",
					fs.openedAfterCancel)
			}

			if uiw.ErrorCount() != 0 {
				t.Fatalf("expected no file reported as failed, got %d error lines",
					uiw.ErrorCount())
			}
		})
	}
}

// TestScannerRemovesBlobTempFileWhenRunStops checks that a run stopped
// part-way, by a file it cannot read or by a cancellation, removes the
// temp file of the blob it was packing.
func TestScannerRemovesBlobTempFileWhenRunStops(t *testing.T) {
	t.Parallel()

	// a.txt is backed up first and starts a blob; the run stops at b.txt.
	const target = "/source/b.txt"

	tests := []struct {
		name  string
		newFs func(cancel context.CancelFunc) afero.Fs
	}{
		{
			name: "file cannot be read",
			newFs: func(context.CancelFunc) afero.Fs {
				return &readFailFs{Fs: afero.NewMemMapFs(), target: target}
			},
		},
		{
			name: "cancelled",
			newFs: func(cancel context.CancelFunc) afero.Fs {
				return &cancelOnOpenFs{
					Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
				}
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()

			fs := tt.newFs(cancel)
			writeSkipErrorTestFile(t, fs, "/source/a.txt", "first file content")
			writeSkipErrorTestFile(t, fs, target, "second file content")

			_, err := runSkipErrorScan(ctx, t, fs, "/source", false, nil)
			if err == nil {
				t.Fatal("expected the scan to stop with an error, got nil")
			}

			left, err := afero.Glob(fs, filepath.Join(os.TempDir(), "vaultik-blob-*"))
			if err != nil {
				t.Fatalf("listing blob temp files: %v", err)
			}

			if len(left) != 0 {
				t.Fatalf("expected no blob temp file left, got %v", left)
			}
		})
	}
}