From 63c824220cfba24ccc2e93a9b92dfbb5bb9e5ea3 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 08:04:05 +0000 Subject: [PATCH] Keep unreferenced blobs newer than prune_grace_period (closes #387) A backup uploads its blobs before the manifest that lists them, and the process lock covers one user on one machine, so a prune from another host or user sharing the destination deleted a running backup's blobs. Prune now deletes an unreferenced blob only once its listed time is older than the new config key prune_grace_period (default 24h; a negative value is refused). A newer blob, or one listed with no time, is kept and counted (blobs_too_new). A backup uploads a blob already on the destination again instead of skipping it, so it gets a fresh time. The rclone backend takes each object's time from the listing (use_server_modtime). Cost: after database delete, the next backup uploads all its data again. Side effect: a rebuilt blob replaces the stored copy; if compression_level or the recipients changed since, older snapshots listing it fail verify. Model: opus-5-5 --- README.md | 47 +++- config.example.yml | 9 + docs/DATAMODEL.md | 7 +- docs/REPOSTRUCTURE.md | 4 +- internal/cli/config.go | 7 + internal/cli/database.go | 6 +- internal/cli/entry_prune_quiet_test.go | 8 +- internal/cli/prune.go | 5 +- internal/cli/snapshot.go | 4 +- internal/config/config.go | 17 ++ internal/config/config_test.go | 38 +++ internal/database/blobs.go | 4 +- internal/s3/client.go | 11 +- internal/snapshot/scanner.go | 65 ++--- internal/storage/conformance_test.go | 38 ++- internal/storage/file.go | 5 +- internal/storage/rclone.go | 13 +- internal/storage/rclone_test.go | 75 ++++++ internal/storage/s3.go | 7 +- internal/storage/storer.go | 9 +- internal/vaultik/info.go | 8 +- internal/vaultik/prune.go | 76 ++++-- internal/vaultik/prune_grace_period_test.go | 260 ++++++++++++++++++++ internal/vaultik/remove_snapshot_test.go | 12 +- internal/vaultik/reused_blob_verify_test.go | 18 +- 25 files changed, 639 insertions(+), 114 deletions(-) create mode 100644 internal/vaultik/prune_grace_period_test.go diff --git a/README.md b/README.md index 32a0c81..55fda7c 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,12 @@ vaultik snapshot restore ~/vaultik-restore # 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w ``` +On a destination shared between hosts or users, the prune in that cron line +relies on `prune_grace_period` (default `24h`) to leave the blobs of a backup +another one is still running, so it must be longer than the longest backup +run; otherwise give each host its own prefix or directory. See +[locking](#locking). + ## restoring on another machine Restoring on a host that never ran the backup — a replacement machine @@ -192,7 +198,17 @@ process-wide lock at `$XDG_DATA_HOME/vaultik/vaultik.pid` (`~/.local/share/vaultik/vaultik.pid` on Linux) for the whole run. Only one of them runs at a time: a second one exits immediately with an "already running" error rather than waiting, so two writers can never -corrupt the local index or the destination store. +corrupt the local index. + +The lock covers one user on one machine. It does not stop another user, +or another host, from writing to the same destination store at the same +time. A backup uploads its blobs before the manifest that lists them, so a +prune from elsewhere at that moment finds blobs no snapshot lists. It +deletes such a blob only once it is older than `prune_grace_period` +(default `24h`), which therefore must be longer than the longest backup run +of any host or user sharing the destination. `remote nuke` deletes +everything regardless. To not depend on the grace period, give each host +or user its own prefix (or directory) on the destination. Read-only commands — `info`, `snapshot list`, `snapshot verify`, and `remote info` — do not take the lock and are never blocked, so they run @@ -296,7 +312,8 @@ value where a list belongs or a word where a number belongs), is refused. * `--cron`: Silent on total success; warnings and errors are still printed (for crontab) * `--prune`: After backup, drop older snapshots of each backed-up name and - remove orphaned blobs from remote storage. By default keeps only the latest + remove orphaned blobs older than `prune_grace_period` from remote storage, + as `prune` does. By default keeps only the latest snapshot per name; use `--keep-newer-than` for a rolling window. When `--skip-errors` skipped a file or directory it could not read or list, the prune does not run and every older snapshot is kept, since one of them may @@ -423,9 +440,12 @@ local snapshot whose remote metadata is missing is dropped from the local index; (2) delete orphaned local rows (files, chunks, blobs no longer referenced by any completed snapshot); (3) list every remote manifest on the destination store to compute the still-referenced blob -set and delete any blob not in that set. Step (3) reads all remote -manifests — network cost scales with the number of snapshots. `snapshot -create --prune` runs the same cleanup automatically; this is the +set and delete any blob not in that set that is older than +`prune_grace_period` (default `24h`). A newer one is kept and counted: it +may belong to a backup still running, on this host or another sharing the +destination store, that has not yet uploaded its manifest. Step (3) reads +all remote manifests — network cost scales with the number of snapshots. +`snapshot create --prune` runs the same cleanup automatically; this is the manual entry point for the same work. If the destination store cannot be listed, or the local index holds snapshots and the destination has no `vaultik-destination.txt` (an unplugged volume's empty mount point), @@ -436,6 +456,9 @@ exits non-zero. * `--json`: Output stats as JSON. Also skips the confirmation prompt, as `--force` does. A blob that cannot be deleted is counted in `blobs_failed`, and the document is printed before the non-zero exit. + A blob kept for being newer than `prune_grace_period`, or because the + listing of the destination gives no time for it, is counted in + `blobs_too_new` and not in `blobs_found`. **`info`**: Display system configuration, storage settings, encryption recipients, and local database statistics. @@ -470,8 +493,8 @@ deletes what is left. **`database delete`**: Delete the local SQLite state database file entirely. Remote storage is unaffected; the next backup will do a full -scan and re-deduplicate against existing remote blobs, and the local -index will re-bind to the currently configured storage destination. +scan and upload all the data again, and the local index will re-bind to +the currently configured storage destination. Use this after changing `storage_url`, or `s3.endpoint`, `s3.bucket` or `s3.prefix`, to a different destination. * `--force`: Skip confirmation prompt @@ -510,6 +533,12 @@ being written, a killed upload can leave a truncated object under its name, which a later backup takes for complete. A leftover `.partial` file is ignored and can be deleted. +A destination shared between hosts or users, each with its own local index, +relies on `prune_grace_period` to keep one's prune from deleting the blobs of +another's backup still running, so it must be longer than the longest backup +run; otherwise give each host its own prefix (or directory). See +[locking](#locking). + Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is still supported for backward compatibility. `storage_url` takes precedence if both are set. @@ -596,7 +625,8 @@ format does and does not protect. 1. List all snapshot manifests 2. Build set of all referenced blob hashes 3. List all blobs in storage -4. Delete any blob not in the referenced set +4. Delete any blob not in the referenced set that is older than + `prune_grace_period` ### chunking and deduplication @@ -641,6 +671,7 @@ complete annotated example also lives in | `chunk_size` | `10MB` | Average chunk size for content-defined chunking | | `blob_size_limit` | `10GB` | Maximum blob size before splitting. Must be at least four times `chunk_size` (the largest chunk the chunker can emit), otherwise a single-chunk blob could exceed the limit. A backup needs free temporary space, because each blob is written whole to a temporary file before it is uploaded (up to about `blob_size_limit`; an rclone destination that cannot stream uploads needs about twice that) and the metadata export writes copies of the local index. Temporary files go to `$TMPDIR` (default `/tmp`); with `TMPDIR` unset, SQLite writes one of those copies to `/var/tmp` | | `compression_level` | `3` | zstd compression level (1-19) | +| `prune_grace_period` | `24h` | How old a blob that no snapshot lists must be before `prune` deletes it. A backup uploads its blobs before the manifest that lists them, so on a destination shared between hosts or users this must be longer than the longest backup run (see [locking](#locking)). A duration such as `36h` or `90m`; `0s` deletes such blobs at once | | `hostname` | system hostname | Hostname used in snapshot IDs | | `index_path` | platform data dir | Local SQLite index path | diff --git a/config.example.yml b/config.example.yml index 55dae8e..cd951f0 100644 --- a/config.example.yml +++ b/config.example.yml @@ -328,6 +328,15 @@ storage_url: "rclone://myremote/path/to/backups" # Higher = better compression but slower # Default: 3 compression_level: 5 + +# How old a blob that no snapshot lists must be before prune deletes it +# A backup uploads its blobs before the manifest that lists them, so on a +# destination shared between hosts or users this must be longer than the +# longest backup run +# Supports: 36h, 90m, etc.; 0s deletes them at once +# Default: 24h +#prune_grace_period: 24h + # Hostname to use in backup metadata # Default: system hostname #hostname: myserver diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index 9f94fde..eacc68f 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -90,7 +90,7 @@ Stores information about packed, compressed, and encrypted blob files. - `created_ts` (INTEGER NOT NULL) - Creation timestamp - `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress) - `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression -- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage: when the blob was already stored, under another compression level or other recipients, this can differ from the size of the local copy +- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage - `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded) **Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs. @@ -185,7 +185,7 @@ Tracks blob upload metrics. - `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing 4. **Upload** - - `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (the upload is skipped when the blob is already stored) + - `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (a blob already stored is uploaded again, so that a prune does not take it for an unreferenced blob older than `prune_grace_period`) - `INSERT INTO uploads` - Record upload metrics with snapshot_id - `INSERT INTO snapshot_blobs` - Associate blob with snapshot @@ -228,7 +228,8 @@ The restore process doesn't use the local database. Instead: 1. **Identify Unreferenced Blobs** - Query blobs not referenced by any remaining snapshot - - Delete from S3 and local database + - Delete from S3, once older than `prune_grace_period`, and from the local + database ### 6. Incomplete Snapshot Cleanup diff --git a/docs/REPOSTRUCTURE.md b/docs/REPOSTRUCTURE.md index 6a0661a..ae93a3b 100644 --- a/docs/REPOSTRUCTURE.md +++ b/docs/REPOSTRUCTURE.md @@ -150,7 +150,7 @@ These are known, deliberate properties of the format and the tooling, recorded s 3. **Chunking uses no secret.** The FastCDC parameters are fixed and public. The default 10 MB average yields chunks between 2.5 MB and 40 MB, and any file of 2.5 MB or less is a single chunk. At the default 10 GB `blob_size_limit` a blob holds hundreds of chunks, so individual chunk lengths are not visible in the blob's size; lowering the limit toward the chunk size begins to expose them. 4. **Decrypted data on local disk.** Several commands stage plaintext under `$TMPDIR`: `snapshot restore` writes decrypted blobs under `vaultik-blobcache-*/` (no size cap) and the decrypted metadata database at `vaultik-restore-*/snapshot.db`; `verify --deep` writes that database at `vaultik-verify-*/snapshot.db`; `snapshot create` keeps a plaintext copy of the index at `vaultik-snapshot-*/snapshot.db`. These files are created `0600` and removed on success, but a `kill -9` or a power loss leaves them behind — delete any leftover `vaultik-*` directory under `$TMPDIR` by hand. `$TMPDIR` should be trusted to the same degree as the restore target. 5. **Store permissions differ per command.** The backed-up host needs only PUT to run `snapshot create`: it writes blobs and metadata and neither reads nor deletes them. Other commands need more — `snapshot verify`, `snapshot restore`, and `prune` list and read; `prune`, `snapshot purge`, `snapshot remove`, and `remote nuke` also delete. The recommended cron line uses `--prune`, which runs `prune` on the backed-up host, so granting that host `--prune` gives it credentials that can delete its own backups. To keep the source host to PUT only, prune from a separate trusted host instead. -6. **Changing recipients does not re-encrypt existing data.** Deduplicated chunks and same-named blobs already on the destination stay encrypted to the recipients in force when they were written. A new snapshot that reuses them cannot be restored with a newly added recipient's key alone, because those reused objects were never encrypted to it. To make everything readable by a new key, run `vaultik database delete` and take a full backup to a fresh destination or prefix. +6. **Changing recipients does not re-encrypt existing data.** Deduplicated chunks already on the destination stay encrypted to the recipients in force when they were written. A new snapshot that reuses them cannot be restored with a newly added recipient's key alone, because those reused objects were never encrypted to it. To make everything readable by a new key, run `vaultik database delete` and take a full backup to a fresh destination or prefix. 7. **X25519 recipients only.** vaultik rejects age ssh and plugin recipients. Long-lived ciphertext held by a third party (the destination operator) has no fallback if X25519 is ever broken: there is no second recipient type and no post-quantum option. ## Consistency Guarantees @@ -166,7 +166,7 @@ The prune operation is safe because: 1. It keeps every blob listed in any snapshot's manifest and deletes only blobs that no manifest references 2. Manifests are unencrypted and can be read without keys 3. If any manifest cannot be downloaded or decoded, prune deletes nothing and exits with an error, rather than treating that snapshot's blobs as unreferenced -4. Prune requires exclusive access to the destination: running it during a concurrent backup can race a snapshot whose manifest is not yet written, so do not prune while a backup is in progress +4. It deletes an unreferenced blob only once the blob is older than `prune_grace_period` (default `24h`). A backup uploads its blobs before the manifest that lists them, so a prune running meanwhile, on this host or another sharing the destination, leaves them as long as no backup run takes longer than the grace period. `remote nuke` has no such check ## Restoration Requirements diff --git a/internal/cli/config.go b/internal/cli/config.go index 468d031..f4dff25 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -239,6 +239,13 @@ storage_url: "" # Default: 3 # compression_level: 3 +# How old a blob that no snapshot lists must be before prune deletes it. +# A backup uploads its blobs before the manifest that lists them, so on a +# destination shared between hosts or users this must be longer than the +# longest backup run. Accepts: 36h, 90m, etc. 0s deletes them at once. +# Default: 24h +# prune_grace_period: 24h + # Hostname used in snapshot IDs. Default: system hostname. # hostname: myserver diff --git a/internal/cli/database.go b/internal/cli/database.go index 1ae477b..ec2f4bd 100644 --- a/internal/cli/database.go +++ b/internal/cli/database.go @@ -48,9 +48,9 @@ This will erase all local tracking of: - The storage-binding record The remote storage is NOT affected. After deletion, the next backup -will perform a full scan and re-deduplicate against existing remote -blobs, and the local index will re-bind to the currently configured -storage destination on that run. +will perform a full scan and upload all the data again, and the local +index will re-bind to the currently configured storage destination on +that run. Use --force to skip the confirmation prompt.`, Args: cobra.NoArgs, diff --git a/internal/cli/entry_prune_quiet_test.go b/internal/cli/entry_prune_quiet_test.go index 92e7a3b..30e1a33 100644 --- a/internal/cli/entry_prune_quiet_test.go +++ b/internal/cli/entry_prune_quiet_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,8 +15,8 @@ import ( const flagCron = "--cron" // plantUnreferencedBlob writes a blob that no snapshot references to the -// destination store at storeDir, so that a prune deletes it. Returns its -// path. +// destination store at storeDir, dated long before any prune grace period, +// so that a prune deletes it. Returns its path. func plantUnreferencedBlob(t *testing.T, storeDir string) string { t.Helper() @@ -25,6 +26,9 @@ func plantUnreferencedBlob(t *testing.T, storeDir string) string { require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) require.NoError(t, os.WriteFile(path, []byte("unreferenced"), 0o600)) + longAgo := time.Unix(0, 0) + require.NoError(t, os.Chtimes(path, longAgo, longAgo)) + return path } diff --git a/internal/cli/prune.go b/internal/cli/prune.go index 707617b..c6e551d 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -20,7 +20,10 @@ Local cleanup drops incomplete snapshots and any files, chunks, or blobs no longer referenced by a completed snapshot. Remote cleanup scans every snapshot manifest in the destination store, builds the set of still-referenced blob hashes, and deletes any blob not in that -set. +set that is older than prune_grace_period (default 24h). A newer one +is kept: it may belong to a backup still running, on this host or +another sharing the destination store, that has not yet uploaded its +manifest. Snapshot create --prune runs the same cleanup automatically; this command is the manual entry point for the same work (e.g. after a diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 7464681..efb176b 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -132,8 +132,8 @@ on Linux), then /etc/vaultik/config.yml.`, "Run in cron mode (silent unless warning or error)") cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove "+ - "orphaned blobs (not after a backup that skipped unreadable "+ - "files)") + "orphaned blobs older than prune_grace_period (not after a "+ + "backup that skipped unreadable files)") cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration "+ "(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+ diff --git a/internal/config/config.go b/internal/config/config.go index 64778c5..a2b86b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" "filippo.io/age" "git.eeqj.de/sneak/smartconfig" @@ -38,6 +39,7 @@ const ( defaultChunkSize = Size(10 * 1000 * 1000) // 10MB defaultS3PartSize = Size(5 * 1024 * 1024) // 5MiB defaultCompressionLevel = 3 + defaultPruneGracePeriod = 24 * time.Hour minChunkSize = 1000 * 1000 // 1MB minCompressionLevel = 1 maxCompressionLevel = 19 @@ -67,6 +69,8 @@ var ( errFileURLNotAbsolute = errors.New( "a file:// storage_url needs an absolute path, " + "as in file:///path/to/backup") + errNegativePruneGracePeriod = errors.New( + "prune_grace_period must not be negative") errStorageNotConfigured = errors.New( "storage not configured; set storage_url or provide s3.endpoint + " + "s3.bucket + credentials") @@ -187,6 +191,13 @@ type Config struct { Snapshots map[string]SnapshotConfig `yaml:"snapshots"` CompressionLevel int `yaml:"compression_level"` + // PruneGracePeriod is how old a blob no manifest lists must be before + // a prune deletes it. A backup lists its blobs in a manifest only + // once it has uploaded them all, so a younger one may belong to a + // backup still running, on this host or another sharing the + // destination store. + PruneGracePeriod time.Duration `yaml:"prune_grace_period"` + // StorageURL specifies the storage backend using a URL format. // Takes precedence over S3Config if set. // Supported formats: @@ -288,6 +299,7 @@ func Parse(data []byte) (*Config, error) { ChunkSize: defaultChunkSize, IndexPath: filepath.Join(xdg.DataHome, appName, "index.sqlite"), CompressionLevel: defaultCompressionLevel, + PruneGracePeriod: defaultPruneGracePeriod, S3: S3Config{PartSize: defaultS3PartSize}, } @@ -356,6 +368,7 @@ func Parse(data []byte) (*Config, error) { // exceeds the configured limit // - Compression level must be between 1 and 19 // - S3 part size must be between 5MiB and 5GiB, the part sizes S3 accepts +// - Prune grace period must not be negative // // Returns an error describing the first validation failure encountered. func (c *Config) Validate() error { @@ -405,6 +418,10 @@ func (c *Config) Validate() error { return errBadS3PartSize } + if c.PruneGracePeriod < 0 { + return errNegativePruneGracePeriod + } + return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 897b7e7..ae8ea85 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -412,6 +412,44 @@ func TestDocumentedSizeExamples(t *testing.T) { } } +// TestValidatePruneGracePeriod checks that a negative prune_grace_period +// is refused rather than acting like 0s, which deletes unreferenced blobs +// at once, and that 0s and the documented examples are accepted. +func TestValidatePruneGracePeriod(t *testing.T) { + t.Parallel() + + const withoutGracePeriod = "snapshots:\n" + + " test:\n" + + " paths: [/tmp/vaultik-test-source]\n" + + "storage_url: file:///tmp/vaultik-test-storage\n" + + tests := []struct { + setting string + wantErr error + }{ + {"prune_grace_period: 0s", nil}, + {"prune_grace_period: 90m", nil}, + {"prune_grace_period: 36h", nil}, + {"prune_grace_period: -1h", errNegativePruneGracePeriod}, + } + + for _, tt := range tests { + t.Run(tt.setting, func(t *testing.T) { + t.Parallel() + + cfg, err := Parse([]byte(withoutGracePeriod + tt.setting + "\n")) + if err != nil { + t.Fatalf("Parse() unexpected error: %v", err) + } + + err = cfg.Validate() + if !errors.Is(err, tt.wantErr) { + t.Fatalf("Validate() error = %v, want %v", err, tt.wantErr) + } + }) + } +} + // TestLoadRejectsRelativeFileURL checks that a file:// storage_url with a // relative path fails at load. Accepted, it would resolve against the // working directory, so a backup run from another directory would write diff --git a/internal/database/blobs.go b/internal/database/blobs.go index aec4dca..c61ace6 100644 --- a/internal/database/blobs.go +++ b/internal/database/blobs.go @@ -155,9 +155,7 @@ func (r *BlobRepository) UpdateFinished( } // UpdateUploaded marks a blob as uploaded and records the size of the -// stored object as its compressed size. That size can differ from the -// local copy's when an identical blob was already stored with another -// compression level or other recipients. +// stored object as its compressed size. func (r *BlobRepository) UpdateUploaded( ctx context.Context, tx *sql.Tx, id string, storedSize int64, ) error { diff --git a/internal/s3/client.go b/internal/s3/client.go index 0867a8a..fe74a82 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -8,6 +8,7 @@ import ( "io" "strings" "sync/atomic" + "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -302,7 +303,10 @@ func IsNotFound(err error) bool { type ObjectInfo struct { Key string Size int64 - Err error + // ModTime is the object's LastModified time. Only ListObjectsStream + // sets it. + ModTime time.Time + Err error } // ListObjectsStream lists objects with the given prefix and returns a channel. @@ -343,8 +347,9 @@ func (c *Client) ListObjectsStream( } ch <- ObjectInfo{ - Key: key, - Size: *obj.Size, + Key: key, + Size: *obj.Size, + ModTime: aws.ToTime(obj.LastModified), } } } diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index 47a79fd..d16b38c 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -1641,8 +1641,7 @@ func (s *Scanner) handleBlobReady( blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) - storedSize, blobExists, err := s.uploadBlobIfNeeded( - ctx, blobPath, blobWithReader, startTime, result) + err := s.uploadBlob(ctx, blobPath, blobWithReader, startTime, result) if err != nil { cleanupErr := s.cleanupBlobTempFile(blobWithReader) @@ -1650,8 +1649,7 @@ func (s *Scanner) handleBlobReady( fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err), cleanupErr) } - err = s.recordBlobMetadata( - ctx, finishedBlob, storedSize, blobExists, startTime) + err = s.recordBlobMetadata(ctx, finishedBlob, startTime) if err != nil { cleanupErr := s.cleanupBlobTempFile(blobWithReader) @@ -1675,46 +1673,33 @@ func (s *Scanner) handleBlobReady( return nil } -// uploadBlobIfNeeded uploads the blob to storage if it doesn't already -// exist. It returns the size of the object in storage, which for an -// existing object can differ from the local copy's, and whether it -// already existed. -func (s *Scanner) uploadBlobIfNeeded( +// uploadBlob uploads the blob to storage. A blob already there, left by +// an earlier run stopped before its manifest, is uploaded again rather +// than skipped: keeping its old time would let a prune elsewhere delete +// it as an unreferenced blob older than prune_grace_period before this +// backup's manifest lists it. +func (s *Scanner) uploadBlob( ctx context.Context, blobPath string, blobWithReader *blob.WithReader, startTime time.Time, result *ScanResult, -) (int64, bool, error) { +) error { finishedBlob := blobWithReader.FinishedBlob - - // Check if blob already exists (deduplication after restart) destination := s.storage.Info().Location - stored, err := s.storage.Stat(ctx, blobPath) - if err == nil { - log.Info("Blob already exists in storage, skipping upload", - "hash", finishedBlob.Hash, - "size", humanize.Bytes(safeUint64(finishedBlob.Compressed))) - s.ui.Infof("Blob %s (%s) already exists at %s. Skipping upload.", - s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), - s.ui.Path(destination)) - - return stored.Size, true, nil - } - s.ui.Beginf("Uploading blob %s (%s) to %s.", s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination)) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime) - err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, + err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback) if err != nil { log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) - return 0, false, fmt.Errorf("uploading blob to storage: %w", err) + return fmt.Errorf("uploading blob to storage: %w", err) } uploadDuration := time.Since(startTime) @@ -1745,7 +1730,7 @@ func (s *Scanner) uploadBlobIfNeeded( stats.BytesUploaded.Add(finishedBlob.Compressed) } - return finishedBlob.Compressed, false, nil + return nil } // makeUploadProgressCallback creates a progress callback for blob uploads. @@ -1815,8 +1800,6 @@ func (s *Scanner) makeUploadProgressCallback( func (s *Scanner) recordBlobMetadata( ctx context.Context, finishedBlob *blob.FinishedBlob, - storedSize int64, - blobExists bool, startTime time.Time, ) error { finishedBlobID, err := types.ParseBlobID(finishedBlob.ID) @@ -1828,7 +1811,7 @@ func (s *Scanner) recordBlobMetadata( return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { err := s.repos.Blobs.UpdateUploaded( - txCtx, tx, finishedBlob.ID, storedSize) + txCtx, tx, finishedBlob.ID, finishedBlob.Compressed) if err != nil { return fmt.Errorf("updating blob upload timestamp: %w", err) } @@ -1839,19 +1822,17 @@ func (s *Scanner) recordBlobMetadata( return fmt.Errorf("adding blob to snapshot: %w", err) } - if !blobExists { - upload := &database.Upload{ - BlobHash: finishedBlob.Hash, - SnapshotID: s.snapshotID, - UploadedAt: startTime, - Size: finishedBlob.Compressed, - DurationMs: uploadDuration.Milliseconds(), - } + upload := &database.Upload{ + BlobHash: finishedBlob.Hash, + SnapshotID: s.snapshotID, + UploadedAt: startTime, + Size: finishedBlob.Compressed, + DurationMs: uploadDuration.Milliseconds(), + } - err := s.repos.Uploads.Create(txCtx, tx, upload) - if err != nil { - return fmt.Errorf("recording upload metrics: %w", err) - } + err = s.repos.Uploads.Create(txCtx, tx, upload) + if err != nil { + return fmt.Errorf("recording upload metrics: %w", err) } return nil diff --git a/internal/storage/conformance_test.go b/internal/storage/conformance_test.go index e5c35f9..8e8ebbe 100644 --- a/internal/storage/conformance_test.go +++ b/internal/storage/conformance_test.go @@ -8,6 +8,7 @@ import ( "reflect" "sort" "testing" + "time" "sneak.berlin/go/vaultik/internal/storage" ) @@ -19,15 +20,17 @@ import ( // constructor, so the contract is defined once. // // It exercises the public Storer interface: round-trip, stat, list with -// prefix filtering, overwrite, delete, delete-of-missing, and not-found on -// Get and Stat. Each section takes its own fresh backend instance, so the -// order of sections never matters and no section sees another's objects. +// prefix filtering, the modification time ListStream reports, overwrite, +// delete, delete-of-missing, and not-found on Get and Stat. Each section +// takes its own fresh backend instance, so the order of sections never +// matters and no section sees another's objects. func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) { t.Helper() conformanceRoundTrip(t, newStorer(t)) conformanceOverwrite(t, newStorer(t)) conformanceList(t, newStorer(t)) + conformanceListStreamModTime(t, newStorer(t)) conformanceDelete(t, newStorer(t)) conformanceNotFound(t, newStorer(t)) } @@ -118,6 +121,35 @@ func conformanceList(t *testing.T, s storage.Storer) { } } +// conformanceListStreamModTime checks that ListStream gives an object +// just written a modification time of about now, which prune compares +// with its grace period. +func conformanceListStreamModTime(t *testing.T, s storage.Storer) { + t.Helper() + + ctx := context.Background() + + err := s.Put(ctx, "blobs/dd/written-now", bytes.NewReader([]byte("data"))) + if err != nil { + t.Fatalf("Put: %v", err) + } + + var listed []storage.ObjectInfo + + for info := range s.ListStream(ctx, "blobs/") { + listed = append(listed, info) + } + + if len(listed) != 1 || listed[0].Err != nil { + t.Fatalf("ListStream(\"blobs/\") = %v, want the one object", listed) + } + + if time.Since(listed[0].ModTime).Abs() > time.Minute { + t.Errorf("ListStream ModTime = %v, want within a minute of now", + listed[0].ModTime) + } +} + // conformanceDelete checks that Delete removes an object and that deleting // a missing key is not an error. func conformanceDelete(t *testing.T, s storage.Storer) { diff --git a/internal/storage/file.go b/internal/storage/file.go index af97ef7..734a748 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -235,8 +235,9 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec // Normalize path separators relPath = strings.ReplaceAll(relPath, string(filepath.Separator), "/") ch <- ObjectInfo{ - Key: relPath, - Size: info.Size(), + Key: relPath, + Size: info.Size(), + ModTime: info.ModTime(), } } diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 4ca2be1..39413cb 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -53,6 +53,14 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e rclonePath += path } + // On S3, Swift and some other remotes rclone keeps a time of its own in + // each object's metadata, which costs one request per object to read. + // rclone's use_server_modtime setting makes ModTime the time the + // listing gives instead: when the object was stored, which is what + // prune compares with prune_grace_period. + ctx, ci := fs.AddConfig(ctx) + ci.UseServerModTime = true + // Create the rclone filesystem fsys, err := fs.NewFs(ctx, rclonePath) if err != nil { @@ -211,8 +219,9 @@ func (r *RcloneStorer) ListStream( if prefix == "" || strings.HasPrefix(key, prefix) { ch <- ObjectInfo{ - Key: key, - Size: obj.Size(), + Key: key, + Size: obj.Size(), + ModTime: obj.ModTime(ctx), } } }) diff --git a/internal/storage/rclone_test.go b/internal/storage/rclone_test.go index 1f2d9ee..32ac7a2 100644 --- a/internal/storage/rclone_test.go +++ b/internal/storage/rclone_test.go @@ -5,11 +5,17 @@ import ( "context" "errors" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" + "sync/atomic" "testing" + "time" + "github.com/johannesboyne/gofakes3" + "github.com/johannesboyne/gofakes3/backend/s3mem" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" @@ -429,3 +435,72 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) { t.Errorf("object is not in the remote's directory: %v", err) } } + +// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an +// rclone S3 remote takes each object's time from the listing itself. By +// default rclone reads the time it keeps in each object's metadata, one +// request per object, and every command that lists the destination store +// would slow down as it grows. +// +//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config +func TestRcloneListStreamOnS3MakesNoRequestPerObject(t *testing.T) { + backend := s3mem.New() + + err := backend.CreateBucket(s3TestBucket) + if err != nil { + t.Fatalf("create bucket: %v", err) + } + + const body = "blob-bytes" + + _, err = backend.PutObject(s3TestBucket, testBlobKey, nil, + strings.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("seed object: %v", err) + } + + var headRequests atomic.Int64 + + fakeS3 := gofakes3.New(backend).Server() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + headRequests.Add(1) + } + + fakeS3.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + + remote := ":s3,provider=Other" + + ",access_key_id=" + s3TestAccessKeyID + + ",secret_access_key=" + s3TestSecretAccessKey + + ",endpoint='" + srv.URL + "'" + ctx := context.Background() + + s, err := storage.NewRcloneStorer(ctx, remote, s3TestBucket) + if err != nil { + t.Fatalf("NewRcloneStorer: %v", err) + } + + headRequests.Store(0) + + var listed []storage.ObjectInfo + + for info := range s.ListStream(ctx, "blobs/") { + listed = append(listed, info) + } + + if len(listed) != 1 || listed[0].Err != nil { + t.Fatalf("ListStream(\"blobs/\") = %v, want the one object", listed) + } + + if time.Since(listed[0].ModTime).Abs() > time.Minute { + t.Errorf("ListStream ModTime = %v, want within a minute of now", + listed[0].ModTime) + } + + if n := headRequests.Load(); n != 0 { + t.Errorf("ListStream made %d HEAD request(s), want none", n) + } +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 4f2b53b..bbe133b 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -89,9 +89,10 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI for info := range s.client.ListObjectsStream(ctx, prefix, false) { ch <- ObjectInfo{ - Key: info.Key, - Size: info.Size, - Err: info.Err, + Key: info.Key, + Size: info.Size, + ModTime: info.ModTime, + Err: info.Err, } } }() diff --git a/internal/storage/storer.go b/internal/storage/storer.go index 85081f3..3417cb7 100644 --- a/internal/storage/storer.go +++ b/internal/storage/storer.go @@ -14,6 +14,7 @@ import ( "context" "errors" "io" + "time" ) // ErrNotFound is returned when an object does not exist. @@ -27,7 +28,10 @@ type ProgressCallback func(bytesTransferred int64) error type ObjectInfo struct { Key string // Object key/path Size int64 // Size in bytes - Err error // Error for streaming results (nil on success) + // ModTime is when the object was written, as the store reports it. + // Only ListStream sets it; Stat leaves it zero. + ModTime time.Time + Err error // Error for streaming results (nil on success) } // Info provides human-readable storage configuration. @@ -67,7 +71,8 @@ type Storer interface { // For large result sets, prefer ListStream. List(ctx context.Context, prefix string) ([]string, error) - // ListStream returns a channel of ObjectInfo for large result sets. + // ListStream returns a channel of ObjectInfo, with each object's key, + // size and modification time, for large result sets. // The channel is closed when listing completes. // If an error occurs during listing, the final item will have Err set. ListStream(ctx context.Context, prefix string) <-chan ObjectInfo diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index 64656d8..9cd034f 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -457,9 +457,9 @@ func (v *Vaultik) scanRemoteBlobStorage( return err } - for _, size := range allBlobs { + for _, blob := range allBlobs { result.TotalBlobCount++ - result.TotalBlobSize += size + result.TotalBlobSize += blob.Size } // A blob named only by a manifest that could not be read, or by one @@ -474,10 +474,10 @@ func (v *Vaultik) scanRemoteBlobStorage( orphanedSize int64 ) - for hash, size := range allBlobs { + for hash, blob := range allBlobs { if _, referenced := referencedBlobs[hash]; !referenced { orphanedCount++ - orphanedSize += size + orphanedSize += blob.Size } } diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index b7148e9..56619d7 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "strings" + "time" "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" ) // PruneOptions contains options for the prune command @@ -163,6 +165,10 @@ type PruneBlobsResult struct { BlobsDeleted int `json:"blobs_deleted"` BlobsFailed int `json:"blobs_failed,omitempty"` BytesFreed int64 `json:"bytes_freed"` + // BlobsTooNew counts the unreferenced blobs kept because they are + // newer than the prune grace period, or of unknown age. BlobsFound + // does not include them. + BlobsTooNew int `json:"blobs_too_new"` } // Prune removes orphaned data from the local index database AND @@ -208,9 +214,15 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return err } - unreferencedBlobs, totalSize := v.findUnreferencedBlobs(allBlobs, allBlobsReferenced) + unreferencedBlobs, totalSize, tooNew := v.findUnreferencedBlobs( + allBlobs, allBlobsReferenced) - result := &PruneBlobsResult{BlobsFound: len(unreferencedBlobs)} + result := &PruneBlobsResult{ + BlobsFound: len(unreferencedBlobs), + BlobsTooNew: tooNew, + } + + v.reportBlobsTooNew(tooNew, opts) if len(unreferencedBlobs) == 0 { log.Info("No unreferenced blobs found") @@ -344,11 +356,12 @@ func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { return snapshotIDs, nil } -// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage -func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { +// listAllRemoteBlobs returns a map of all blob hashes in remote storage +// to their listing entries. +func (v *Vaultik) listAllRemoteBlobs() (map[string]storage.ObjectInfo, error) { log.Info("Listing all blobs in storage") - allBlobs := make(map[string]int64) + allBlobs := make(map[string]storage.ObjectInfo) blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") for object := range blobObjectCh { @@ -372,7 +385,7 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { continue } - allBlobs[parts[3]] = object.Size + allBlobs[parts[3]] = object } log.Info("Found blobs in storage", "count", len(allBlobs)) @@ -380,30 +393,59 @@ func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { return allBlobs, nil } -// findUnreferencedBlobs returns blob hashes not referenced by any -// manifest and their total size. +// findUnreferencedBlobs returns the hashes of the blobs no manifest +// references that are older than the prune grace period, and their total +// size. It also returns how many unreferenced blobs it left out for +// being newer than that, or of unknown age because the listing gave no +// time for them. func (v *Vaultik) findUnreferencedBlobs( - allBlobs map[string]int64, referenced map[string]bool, -) ([]string, int64) { + allBlobs map[string]storage.ObjectInfo, referenced map[string]bool, +) ([]string, int64, int) { var ( unreferenced []string totalSize int64 + tooNew int ) - for hash, size := range allBlobs { - if !referenced[hash] { - unreferenced = append(unreferenced, hash) - totalSize += size + cutoff := time.Now().UTC().Add(-v.Config.PruneGracePeriod) + + for hash, blob := range allBlobs { + if referenced[hash] { + continue } + + if blob.ModTime.IsZero() || blob.ModTime.After(cutoff) { + tooNew++ + + continue + } + + unreferenced = append(unreferenced, hash) + totalSize += blob.Size } - return unreferenced, totalSize + return unreferenced, totalSize, tooNew +} + +// reportBlobsTooNew says how many unreferenced blobs prune kept for being +// newer than the grace period or of unknown age. Under --json the +// document carries the count instead. +func (v *Vaultik) reportBlobsTooNew(tooNew int, opts *PruneOptions) { + if tooNew == 0 || opts.JSON { + return + } + + v.UI.Infof("Kept %d unreferenced blob(s) newer than "+ + "prune_grace_period (%s), or of unknown age: a backup still "+ + "running may need them.", + tooNew, v.Config.PruneGracePeriod) } // deleteUnreferencedBlobs deletes the given blobs from storage and // populates the result. func (v *Vaultik) deleteUnreferencedBlobs( - unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult, + unreferencedBlobs []string, allBlobs map[string]storage.ObjectInfo, + result *PruneBlobsResult, ) { log.Info("Deleting unreferenced blobs") @@ -418,7 +460,7 @@ func (v *Vaultik) deleteUnreferencedBlobs( } result.BlobsDeleted++ - result.BytesFreed += allBlobs[hash] + result.BytesFreed += allBlobs[hash].Size if (i+1)%progressLogEvery == 0 || i == len(unreferencedBlobs)-1 { log.Info("Deletion progress", diff --git a/internal/vaultik/prune_grace_period_test.go b/internal/vaultik/prune_grace_period_test.go new file mode 100644 index 0000000..8dc2fbe --- /dev/null +++ b/internal/vaultik/prune_grace_period_test.go @@ -0,0 +1,260 @@ +package vaultik_test + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/config" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/storage/faultstore" + "sneak.berlin/go/vaultik/internal/ui" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// TestPruneDuringBackupKeepsItsBlobs runs a prune from another host +// sharing the destination store at the moment a backup has uploaded its +// blobs but not yet its manifest, so no manifest lists those blobs. The +// prune must leave them, and the backup must then restore. +func TestPruneDuringBackupKeepsItsBlobs(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + storeDir := t.TempDir() + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + ctx := context.Background() + cfg := faultTestConfig() + testFiles := writeFaultSourceTree(t, fs, dataDir) + + inner, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + otherHostCfg := faultTestConfig() + otherHostCfg.PruneGracePeriod = time.Hour + otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs) + + var ( + pruned bool + pruneErr error + ) + + store := faultstore.New(inner) + store.OnPut = func(key string) faultstore.PutAction { + if strings.HasSuffix(key, "/manifest.json.zst") { + pruned = true + pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true}) + } + + return faultstore.PutNormal + } + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + repos := database.NewRepositories(db) + + id := fullFaultBackup(ctx, t, fs, store, cfg, repos, dataDir, dbPath, "shared") + require.NoError(t, db.Close()) + + require.True(t, pruned, "the prune must run before the manifest upload") + require.NoError(t, pruneErr) + + v := newReaderVaultik(ctx, cfg, inner, nil, fs) + require.NoError(t, v.Restore(&vaultik.RestoreOptions{ + SnapshotID: id, + TargetDir: restoreDir, + Verify: true, + }), "a prune during the backup must not delete the backup's blobs") + + assertRestoredTree(t, fs, restoreDir, testFiles) +} + +// TestPruneDuringBackupKeepsBlobsOfKilledRun covers a backup that builds +// the blobs an earlier run, stopped before its manifest upload, left on +// the destination store, older than the grace period by the time of the +// backup. A prune from another host runs before the backup's manifest +// upload, and the backup must then restore. +func TestPruneDuringBackupKeepsBlobsOfKilledRun(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + storeDir := t.TempDir() + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + + ctx := context.Background() + testFiles := writeFaultSourceTree(t, fs, dataDir) + + cfg := faultTestConfig() + cfg.IndexPath = dbPath + cfg.ChunkSize = config.Size(faultChunkSize) + cfg.Snapshots = map[string]config.SnapshotConfig{ + "data": {Paths: []string{dataDir}}, + } + + inner, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + otherHostCfg := faultTestConfig() + otherHostCfg.PruneGracePeriod = time.Hour + otherHost := newReaderVaultik(ctx, otherHostCfg, inner, nil, fs) + + var ( + killedRun = true + pruned bool + pruneErr error + ) + + store := faultstore.New(inner) + store.OnPut = func(key string) faultstore.PutAction { + if !strings.HasSuffix(key, "/manifest.json.zst") { + return faultstore.PutNormal + } + + if killedRun { + return faultstore.PutFail + } + + pruned = true + pruneErr = otherHost.PruneBlobs(&vaultik.PruneOptions{Force: true}) + + return faultstore.PutNormal + } + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + repos := database.NewRepositories(db) + v := newBackupVaultik(ctx, cfg, store, repos, db, fs) + opts := &vaultik.SnapshotCreateOptions{Cron: true} + + require.Error(t, v.CreateSnapshot(opts), + "the first run must stop at its manifest upload") + + beforeGracePeriod := time.Now().Add(-2 * otherHostCfg.PruneGracePeriod) + + for object := range inner.ListStream(ctx, "blobs/") { + require.NoError(t, object.Err) + require.NoError(t, os.Chtimes(filepath.Join(storeDir, object.Key), + beforeGracePeriod, beforeGracePeriod)) + } + + killedRun = false + + require.NoError(t, v.CreateSnapshot(opts)) + require.True(t, pruned, "the prune must run before the manifest upload") + require.NoError(t, pruneErr) + + local, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit) + require.NoError(t, err) + require.Len(t, local, 1) + require.NoError(t, db.Close()) + + reader := newReaderVaultik(ctx, cfg, inner, nil, fs) + require.NoError(t, reader.Restore(&vaultik.RestoreOptions{ + SnapshotID: local[0].ID.String(), + TargetDir: restoreDir, + Verify: true, + }), "a prune during the backup must not delete the blobs it rebuilt") + + assertRestoredTree(t, fs, restoreDir, testFiles) +} + +// TestPruneKeepsUnreferencedBlobWithNoTime checks that a blob no +// manifest lists, whose listing gives no modification time (as an S3 +// listing entry without LastModified does), is kept and counted with +// the blobs too new to delete: its age is unknown. +func TestPruneKeepsUnreferencedBlobWithNoTime(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + env := newListEnv(t) + + // Written into the mock store's map directly rather than through + // Put, so its listing gives no time. + blobKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] + + "/" + testBlobHashA + env.store.data[blobKey] = []byte("blob") + + require.NoError(t, env.v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) + + assert.True(t, env.store.hasKey(blobKey), + "a blob of unknown age must be kept") + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &result)) + assert.Equal(t, 0, result.BlobsDeleted) + assert.Equal(t, 1, result.BlobsTooNew) +} + +// TestPruneDeletesUnreferencedBlobOlderThanGracePeriod checks that the +// grace period only delays deletion: a blob no manifest lists is deleted +// once it is older than the grace period, and kept and counted until +// then. +func TestPruneDeletesUnreferencedBlobOlderThanGracePeriod(t *testing.T) { + log.Initialize(log.Config{}) + t.Parallel() + + ctx := context.Background() + storeDir := t.TempDir() + + store, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + oldKey := "blobs/" + testBlobHashA[:2] + "/" + testBlobHashA[2:4] + + "/" + testBlobHashA + newKey := "blobs/" + testBlobHashB[:2] + "/" + testBlobHashB[2:4] + + "/" + testBlobHashB + + for _, key := range []string{oldKey, newKey} { + require.NoError(t, store.Put(ctx, key, strings.NewReader("blob"))) + } + + gracePeriod := time.Hour + beforeGracePeriod := time.Now().Add(-gracePeriod - time.Minute) + require.NoError(t, os.Chtimes(filepath.Join(storeDir, oldKey), + beforeGracePeriod, beforeGracePeriod)) + + stdout := &bytes.Buffer{} + v := &vaultik.Vaultik{ + Config: &config.Config{PruneGracePeriod: gracePeriod}, + Storage: store, + Stdout: stdout, + UI: ui.NewWithColor(stdout, false), + } + v.SetContext(ctx) + + require.NoError(t, v.PruneBlobs(&vaultik.PruneOptions{JSON: true})) + + _, err = store.Stat(ctx, oldKey) + require.ErrorIs(t, err, storage.ErrNotFound, + "a blob older than the grace period must be deleted") + + _, err = store.Stat(ctx, newKey) + require.NoError(t, err, "a blob newer than the grace period must be kept") + + var result vaultik.PruneBlobsResult + + require.NoError(t, json.Unmarshal(stdout.Bytes(), &result)) + assert.Equal(t, 1, result.BlobsDeleted) + assert.Equal(t, 1, result.BlobsTooNew) +} diff --git a/internal/vaultik/remove_snapshot_test.go b/internal/vaultik/remove_snapshot_test.go index c9cc7fe..9d43aec 100644 --- a/internal/vaultik/remove_snapshot_test.go +++ b/internal/vaultik/remove_snapshot_test.go @@ -26,11 +26,14 @@ import ( type testStorer struct { mu sync.Mutex data map[string][]byte + // putTimes holds when Put stored each object; ListStream reports it. + putTimes map[string]time.Time } func newTestStorer() *testStorer { return &testStorer{ - data: make(map[string][]byte), + data: make(map[string][]byte), + putTimes: make(map[string]time.Time), } } @@ -44,6 +47,7 @@ func (s *testStorer) Put(_ context.Context, key string, reader io.Reader) error } s.data[key] = data + s.putTimes[key] = time.Now().UTC() return nil } @@ -87,6 +91,7 @@ func (s *testStorer) Delete(_ context.Context, key string) error { defer s.mu.Unlock() delete(s.data, key) + delete(s.putTimes, key) return nil } @@ -120,8 +125,9 @@ func (s *testStorer) ListStream( for key, data := range s.data { if prefix == "" || strings.HasPrefix(key, prefix) { ch <- storage.ObjectInfo{ - Key: key, - Size: int64(len(data)), + Key: key, + Size: int64(len(data)), + ModTime: s.putTimes[key], } } } diff --git a/internal/vaultik/reused_blob_verify_test.go b/internal/vaultik/reused_blob_verify_test.go index f499e0e..c0e30c8 100644 --- a/internal/vaultik/reused_blob_verify_test.go +++ b/internal/vaultik/reused_blob_verify_test.go @@ -14,13 +14,13 @@ import ( "sneak.berlin/go/vaultik/internal/vaultik" ) -// TestVerifyPassesWhenBackupReusesStoredBlob backs up a file, then backs it -// up again from an empty local index, as after `database delete`, with an -// extra age recipient. The second backup builds the same blob, finds it -// already stored and skips the upload, but its own copy is larger, because -// age writes a header entry for each recipient. The second snapshot must -// record the size of the stored object, so shallow and deep verify pass. -func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) { +// TestVerifyPassesWhenBackupRebuildsStoredBlob backs up a file, then backs +// it up again from an empty local index, as after `database delete`, with +// an extra age recipient. The second backup builds the same blob, already +// stored, and uploads it again; its copy is larger, because age writes a +// header entry for each recipient. The second snapshot must record the +// size of the object now stored, so shallow and deep verify pass. +func TestVerifyPassesWhenBackupRebuildsStoredBlob(t *testing.T) { log.Initialize(log.Config{}) t.Parallel() @@ -58,8 +58,8 @@ func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) { snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(), env.cfg.Hostname, "second", dataDir, secondDBPath) require.Positive(t, result.BlobsCreated) - require.Zero(t, result.BlobsUploaded, - "the second backup should find every blob already stored") + require.Equal(t, result.BlobsCreated, result.BlobsUploaded, + "the second backup should upload every blob again") require.NoError(t, env.db.Close())