vaultik/internal/vaultik/prune_grace_period_test.go

package vaultik_test

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/spf13/afero"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/vaultik/internal/config"
	"sneak.berlin/go/vaultik/internal/database"
	"sneak.berlin/go/vaultik/internal/log"
	"sneak.berlin/go/vaultik/internal/storage"
	"sneak.berlin/go/vaultik/internal/storage/faultstore"
	"sneak.berlin/go/vaultik/internal/ui"
	"sneak.berlin/go/vaultik/internal/vaultik"
)

// TestPruneDuringBackupKeepsItsBlobs runs a prune from another host
// sharing the destination store at the moment a backup has uploaded its
// blobs and its db.zst.age but not yet its manifest, so no manifest lists
// those blobs. The prune must leave the blobs and the db.zst.age, and the
// backup must then restore.
func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	fs := afero.NewOsFs()
	tempDir := t.TempDir()
	dataDir := filepath.Join(tempDir, "src")
	storeDir := t.TempDir()
	restoreDir := filepath.Join(tempDir, "restored")
	dbPath := filepath.Join(tempDir, "index.sqlite")

	ctx := context.Background()
	cfg := faultTestConfig()
	testFiles := writeFaultSourceTree(t, fs, dataDir)

	inner, err := storage.NewFileStorer(storeDir)
	require.NoError(t, err)

	otherHostCfg := faultTestConfig()
	otherHostCfg.PruneGracePeriod = time.Hour
	otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs)

	var (
		pruned   bool
		pruneErr error
	)

	store := faultstore.New(inner)
	store.OnPut = func(key string) faultstore.PutAction {
		if strings.HasSuffix(key, "/manifest.json.zst") {
			pruned = true
			pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true})
		}

		return faultstore.PutNormal
	}

	db, err := database.New(ctx, dbPath)
	require.NoError(t, err)

	repos := database.NewRepositories(db)

	id := fullFaultBackup(ctx, t, fs, store, cfg, repos, dataDir, dbPath, "shared")
	require.NoError(t, db.Close())

	require.True(t, pruned, "the prune must run before the manifest upload")
	require.NoError(t, pruneErr)

	v := newReaderVaultik(ctx, cfg, inner, nil, fs)
	require.NoError(t, v.Restore(&vaultik.RestoreOptions{
		SnapshotID: id,
		TargetDir:  restoreDir,
		Verify:     true,
	}), "a prune during the backup must not delete the backup's blobs "+
		"or its db.zst.age")

	assertRestoredTree(t, fs, restoreDir, testFiles)
}

// TestPruneKeepsUnreferencedBlobWithNoTime checks that a blob no
// manifest lists, whose listing gives no modification time (as an S3
// listing entry without LastModified does), is kept and counted with
// the blobs too new to delete: its age is unknown.
func TestPruneKeepsUnreferencedBlobWithNoTime(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	env := newListEnv(t)

	// Written into the mock store's map directly rather than through
	// Put, so its listing gives no time.
	blobKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] +
		"/" + testBlobHashA
	env.store.data[blobKey] = []byte("blob")

	require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))

	assert.True(t, env.store.hasKey(blobKey),
		"a blob of unknown age must be kept")

	var result vaultik.PruneBlobsResult

	require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result))
	assert.Equal(t, 0, result.BlobsDeleted)
	assert.Equal(t, 1, result.BlobsTooNew)
}

// TestPruneDeletesUnreferencedBlobOlderThanGracePeriod checks that the
// grace period only delays deletion: a blob no manifest lists is deleted
// once it is older than the grace period, and kept and counted until
// then.
func TestPruneDeletesUnreferencedBlobOlderThanGracePeriod(t *testing.T) {
	log.Initialize(log.Config{})
	t.Parallel()

	ctx := context.Background()
	storeDir := t.TempDir()

	store, err := storage.NewFileStorer(storeDir)
	require.NoError(t, err)

	oldKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] +
		"/" + testBlobHashA
	newKey := "blobs/" + testBlobHashB[:2] + "/" + testBlobHashB[2:4] +
		"/" + testBlobHashB

	for _, key := range []string{oldKey, newKey} {
		require.NoError(t, store.Put(ctx, key, strings.NewReader("blob")))
	}

	gracePeriod := time.Hour
	beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute)
	require.NoError(t, os.Chtimes(filepath.Join(storeDir, oldKey),
		beforeGracePeriod, beforeGracePeriod))

	stdout := &bytes.Buffer{}
	v := &vaultik.Vaultik{
		Config:  &config.Config{PruneGracePeriod: gracePeriod},
		Storage: store,
		Stdout:  stdout,
		UI:      ui.NewWithColor(stdout, false),
	}
	v.SetContext(ctx)

	require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true}))

	_, err = store.Stat(ctx, oldKey)
	require.ErrorIs(t, err, storage.ErrNotFound,
		"a blob older than the grace period must be deleted")

	_, err = store.Stat(ctx, newKey)
	require.NoError(t, err, "a blob newer than the grace period must be kept")

	var result vaultik.PruneBlobsResult

	require.NoError(t, json.Unmarshal(stdout.Bytes(), &result))
	assert.Equal(t, 1, result.BlobsDeleted)
	assert.Equal(t, 1, result.BlobsTooNew)
}