Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac489af3d7 |
@@ -22,34 +22,6 @@ the tag exists and is exercised; what is left is merging `next` to
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-10-13: Recorded the size of the stored object for a blob a backup
|
||||
found already stored
|
||||
([issue #300](https://git.eeqj.de/sneak/vaultik/issues/300)). When the
|
||||
upload was skipped, the blob row and the manifest kept the size of the
|
||||
local copy, which differs from the stored object's when
|
||||
`compression_level` or the recipients changed since it was written.
|
||||
`snapshot verify` then failed on a good snapshot, shallow and deep,
|
||||
and the listed sizes were wrong. Marking the blob uploaded now records
|
||||
the size of the object in storage.
|
||||
|
||||
- 2026-10-12: Rejected a retention duration longer than about 292 years
|
||||
([issue #289](https://git.eeqj.de/sneak/vaultik/issues/289)). The
|
||||
number given to `--older-than` or `--keep-newer-than` was multiplied
|
||||
by the length of its unit with no limit, so from `293y`, `15251w`,
|
||||
`3559mo` or `106752d`, or a sum of parts such as `200y200y`, the
|
||||
result wrapped to a negative duration. The cutoff then lay in the
|
||||
future, every snapshot of the name was deleted, the one just made
|
||||
included, and the command exited 0. Such a duration is now rejected
|
||||
as invalid and nothing is deleted.
|
||||
|
||||
- 2026-10-08: Made a backup cancelled under `--skip-errors` stop at once
|
||||
([issue #286](https://git.eeqj.de/sneak/vaultik/issues/286)). After
|
||||
Ctrl-C or SIGTERM, phase 2 treated the cancellation error from each
|
||||
remaining file like an unreadable file: it opened the file, printed an
|
||||
error line for it, counted it as failed and went on to the next one.
|
||||
The run now stops at the first file after the cancellation, and no
|
||||
file is counted as failed because of it.
|
||||
|
||||
- 2026-10-08: Made `remote nuke` delete the `.partial` files that
|
||||
uploads cut off part-way leave on the destination store
|
||||
([issue #281](https://git.eeqj.de/sneak/vaultik/issues/281)). The
|
||||
@@ -57,15 +29,6 @@ the tag exists and is exercised; what is left is merging `next` to
|
||||
place and still reported the store empty. It now removes them under
|
||||
`metadata/` and `blobs/` as its last step.
|
||||
|
||||
- 2026-10-08: Made a local index error while recording a directory or
|
||||
symlink stop a backup under `--skip-errors`
|
||||
([issue #284](https://git.eeqj.de/sneak/vaultik/issues/284)). Phase 2
|
||||
only records such an entry in the local index, since it has no data
|
||||
to open or read, but an error doing so was skipped like an unreadable
|
||||
file. The snapshot then completed without the entry, and a restore
|
||||
did not recreate it. The error now stops the backup, as it does
|
||||
without the flag.
|
||||
|
||||
- 2026-10-08: Counted a file that a backup could not store as failed
|
||||
([issue #280](https://git.eeqj.de/sneak/vaultik/issues/280)). A file
|
||||
that phase 1 counted and phase 2 could not open, because it was
|
||||
|
||||
+2
-2
@@ -90,7 +90,7 @@ Stores information about packed, compressed, and encrypted blob files.
|
||||
- `created_ts` (INTEGER NOT NULL) - Creation timestamp
|
||||
- `finished_ts` (INTEGER) - Finalization timestamp (NULL if in progress)
|
||||
- `uncompressed_size` (INTEGER NOT NULL DEFAULT 0) - Total size of chunks before compression
|
||||
- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption. Once uploaded, the size of the object in storage: when the blob was already stored, under another compression level or other recipients, this differs from the size of the local copy
|
||||
- `compressed_size` (INTEGER NOT NULL DEFAULT 0) - Size after compression and encryption
|
||||
- `uploaded_ts` (INTEGER) - Upload completion timestamp (NULL if not uploaded)
|
||||
|
||||
**Purpose:** Tracks blob lifecycle from creation through upload. The UUID primary key allows immediate association of chunks with blobs.
|
||||
@@ -183,7 +183,7 @@ Tracks blob upload metrics.
|
||||
- `UPDATE blobs SET blob_hash = ?, finished_ts = ?` - Finalize blob after packing
|
||||
|
||||
4. **Upload**
|
||||
- `UPDATE blobs SET uploaded_ts = ?, compressed_size = ?` - Mark blob as uploaded, with the size of the object in storage (the upload is skipped when the blob is already stored)
|
||||
- `UPDATE blobs SET uploaded_ts = ?` - Mark blob as uploaded
|
||||
- `INSERT INTO uploads` - Record upload metrics with snapshot_id
|
||||
- `INSERT INTO snapshot_blobs` - Associate blob with snapshot
|
||||
|
||||
|
||||
@@ -159,16 +159,13 @@ func (r *BlobRepository) UpdateFinished(
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateUploaded marks a blob as uploaded and records the size of the
|
||||
// stored object as its compressed size. That size can differ from the
|
||||
// local copy's when an identical blob was already stored with another
|
||||
// compression level or other recipients.
|
||||
// UpdateUploaded marks a blob as uploaded
|
||||
func (r *BlobRepository) UpdateUploaded(
|
||||
ctx context.Context, tx *sql.Tx, id string, storedSize int64,
|
||||
ctx context.Context, tx *sql.Tx, id string,
|
||||
) error {
|
||||
query := `
|
||||
UPDATE blobs
|
||||
SET uploaded_ts = ?, compressed_size = ?
|
||||
UPDATE blobs
|
||||
SET uploaded_ts = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
@@ -176,9 +173,9 @@ func (r *BlobRepository) UpdateUploaded(
|
||||
|
||||
var err error
|
||||
if tx != nil {
|
||||
_, err = tx.ExecContext(ctx, query, now, storedSize, id)
|
||||
_, err = tx.ExecContext(ctx, query, now, id)
|
||||
} else {
|
||||
_, err = r.db.ExecWithLog(ctx, query, now, storedSize, id)
|
||||
_, err = r.db.ExecWithLog(ctx, query, now, id)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -122,8 +122,8 @@ func TestBlobRepositoryUpdates(t *testing.T) {
|
||||
t.Errorf("expected compressed size 500, got %d", updated.CompressedSize)
|
||||
}
|
||||
|
||||
// Test UpdateUploaded, with a stored object larger than the local copy
|
||||
err = repo.UpdateUploaded(ctx, nil, blob.ID.String(), 600)
|
||||
// Test UpdateUploaded
|
||||
err = repo.UpdateUploaded(ctx, nil, blob.ID.String())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to update blob as uploaded: %v", err)
|
||||
}
|
||||
@@ -141,10 +141,6 @@ func TestBlobRepositoryUpdates(t *testing.T) {
|
||||
if uploaded.UploadedTS.Before(now.Add(-1 * time.Second)) {
|
||||
t.Error("uploaded timestamp should be around test time")
|
||||
}
|
||||
|
||||
if uploaded.CompressedSize != 600 {
|
||||
t.Errorf("expected the stored size 600, got %d", uploaded.CompressedSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobRepositoryDuplicate(t *testing.T) {
|
||||
|
||||
@@ -1312,13 +1312,6 @@ func (s *Scanner) processPhase(
|
||||
|
||||
// Process each file
|
||||
for _, fileToProcess := range filesToProcess {
|
||||
// Check context cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
// Update progress
|
||||
if s.progress != nil {
|
||||
s.progress.GetStats().CurrentFile.Store(fileToProcess.Path)
|
||||
@@ -1355,32 +1348,13 @@ func (s *Scanner) processPhase(
|
||||
return s.finalizeProcessPhase(ctx, result)
|
||||
}
|
||||
|
||||
// processFileWithErrorHandling records a directory or symlink, or wraps
|
||||
// processFileStreaming for a regular file with error recovery for deleted
|
||||
// files and skip-errors mode. Returns (skipped, error).
|
||||
// processFileWithErrorHandling wraps processFileStreaming with error recovery for
|
||||
// deleted files and skip-errors mode. Returns (skipped, error).
|
||||
func (s *Scanner) processFileWithErrorHandling(
|
||||
ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
|
||||
) (bool, error) {
|
||||
// A directory or symlink has no data to open or read; it is only
|
||||
// recorded in the local index. An error recording it stops the run
|
||||
// even under --skip-errors, or the snapshot would complete without it.
|
||||
mode := os.FileMode(fileToProcess.File.Mode)
|
||||
if mode&os.ModeSymlink != 0 || mode.IsDir() {
|
||||
err := s.recordNonRegularFile(ctx, fileToProcess)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
err := s.processFileStreaming(ctx, fileToProcess, result)
|
||||
if err != nil {
|
||||
// A cancelled run stops here even under --skip-errors, rather than
|
||||
// counting this file as failed and going on to the next one.
|
||||
if ctx.Err() != nil {
|
||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||
}
|
||||
// A packer/database/encryption/upload failure means the chunk's data
|
||||
// may not have been stored. Skipping the file would let the snapshot
|
||||
// record a file whose chunk is in no blob and cannot be restored, so
|
||||
@@ -1418,7 +1392,12 @@ func (s *Scanner) processFileWithErrorHandling(
|
||||
|
||||
// countFailedFile counts a file that phase 2 could not store as failed
|
||||
// and takes its size back out of BytesScanned, where phase 1 put it.
|
||||
// Phase 1 counts no directories, so a directory is not counted here.
|
||||
func countFailedFile(fileToProcess *FileToProcess, result *ScanResult) {
|
||||
if fileToProcess.FileInfo.IsDir() {
|
||||
return
|
||||
}
|
||||
|
||||
result.FilesFailed++
|
||||
result.BytesScanned -= fileToProcess.FileInfo.Size()
|
||||
}
|
||||
@@ -1509,8 +1488,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
||||
// snapshot references has uploaded_ts set, so deduplication and
|
||||
// interrupted-run repair treat these blobs as trustworthy.
|
||||
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||
err := s.repos.Blobs.UpdateUploaded(
|
||||
ctx, tx, b.ID, b.Compressed)
|
||||
err := s.repos.Blobs.UpdateUploaded(ctx, tx, b.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marking blob uploaded: %w", err)
|
||||
}
|
||||
@@ -1546,7 +1524,7 @@ func (s *Scanner) handleBlobReady(
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
|
||||
finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
|
||||
|
||||
storedSize, blobExists, err := s.uploadBlobIfNeeded(
|
||||
blobExists, err := s.uploadBlobIfNeeded(
|
||||
ctx, blobPath, blobWithReader, startTime, result)
|
||||
if err != nil {
|
||||
s.cleanupBlobTempFile(blobWithReader)
|
||||
@@ -1554,8 +1532,7 @@ func (s *Scanner) handleBlobReady(
|
||||
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
|
||||
}
|
||||
|
||||
err = s.recordBlobMetadata(
|
||||
ctx, finishedBlob, storedSize, blobExists, startTime)
|
||||
err = s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime)
|
||||
if err != nil {
|
||||
s.cleanupBlobTempFile(blobWithReader)
|
||||
|
||||
@@ -1577,22 +1554,20 @@ func (s *Scanner) handleBlobReady(
|
||||
}
|
||||
|
||||
// uploadBlobIfNeeded uploads the blob to storage if it doesn't already
|
||||
// exist. It returns the size of the object in storage, which for an
|
||||
// existing object can differ from the local copy's, and whether it
|
||||
// already existed.
|
||||
// exist, returns whether it existed
|
||||
func (s *Scanner) uploadBlobIfNeeded(
|
||||
ctx context.Context,
|
||||
blobPath string,
|
||||
blobWithReader *blob.WithReader,
|
||||
startTime time.Time,
|
||||
result *ScanResult,
|
||||
) (int64, bool, error) {
|
||||
) (bool, error) {
|
||||
finishedBlob := blobWithReader.FinishedBlob
|
||||
|
||||
// Check if blob already exists (deduplication after restart)
|
||||
destination := s.storage.Info().Location
|
||||
|
||||
stored, err := s.storage.Stat(ctx, blobPath)
|
||||
_, err := s.storage.Stat(ctx, blobPath)
|
||||
if err == nil {
|
||||
log.Info("Blob already exists in storage, skipping upload",
|
||||
"hash", finishedBlob.Hash,
|
||||
@@ -1601,7 +1576,7 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
|
||||
s.ui.Path(destination))
|
||||
|
||||
return stored.Size, true, nil
|
||||
return true, nil
|
||||
}
|
||||
|
||||
s.ui.Beginf("Uploading blob %s (%s) to %s.",
|
||||
@@ -1615,7 +1590,7 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
if err != nil {
|
||||
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||
|
||||
return 0, false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||
return false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||
}
|
||||
|
||||
uploadDuration := time.Since(startTime)
|
||||
@@ -1646,7 +1621,7 @@ func (s *Scanner) uploadBlobIfNeeded(
|
||||
stats.BytesUploaded.Add(finishedBlob.Compressed)
|
||||
}
|
||||
|
||||
return finishedBlob.Compressed, false, nil
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// makeUploadProgressCallback creates a progress callback for blob uploads.
|
||||
@@ -1716,7 +1691,6 @@ func (s *Scanner) makeUploadProgressCallback(
|
||||
func (s *Scanner) recordBlobMetadata(
|
||||
ctx context.Context,
|
||||
finishedBlob *blob.FinishedBlob,
|
||||
storedSize int64,
|
||||
blobExists bool,
|
||||
startTime time.Time,
|
||||
) error {
|
||||
@@ -1728,8 +1702,7 @@ func (s *Scanner) recordBlobMetadata(
|
||||
uploadDuration := time.Since(startTime)
|
||||
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
err := s.repos.Blobs.UpdateUploaded(
|
||||
txCtx, tx, finishedBlob.ID, storedSize)
|
||||
err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("updating blob upload timestamp: %w", err)
|
||||
}
|
||||
@@ -1797,11 +1770,16 @@ func (e *packerError) Error() string { return e.err.Error() }
|
||||
|
||||
func (e *packerError) Unwrap() error { return e.err }
|
||||
|
||||
// processFileStreaming processes a regular file by streaming chunks directly
|
||||
// to the packer
|
||||
// processFileStreaming processes a file by streaming chunks directly to the packer
|
||||
func (s *Scanner) processFileStreaming(
|
||||
ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
|
||||
) error {
|
||||
// Symlinks and directories have no data to chunk — just record them in the DB.
|
||||
mode := os.FileMode(fileToProcess.File.Mode)
|
||||
if mode&os.ModeSymlink != 0 || mode.IsDir() {
|
||||
return s.recordNonRegularFile(ctx, fileToProcess)
|
||||
}
|
||||
|
||||
file, err := s.fs.Open(fileToProcess.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening file: %w", wrapPermissionError(fileToProcess.Path, err))
|
||||
|
||||
@@ -82,32 +82,6 @@ func (f *readFailFs) Open(name string) (afero.File, error) {
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// cancelOnOpenFs cancels the run when the scanner opens the target file to
|
||||
// back it up, as Ctrl-C partway through processing would, and records each
|
||||
// file opened after that.
|
||||
type cancelOnOpenFs struct {
|
||||
afero.Fs
|
||||
|
||||
target string
|
||||
cancel context.CancelFunc
|
||||
cancelled bool
|
||||
openedAfterCancel []string
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Open is defined to return the interface.
|
||||
func (f *cancelOnOpenFs) Open(name string) (afero.File, error) {
|
||||
if f.cancelled {
|
||||
f.openedAfterCancel = append(f.openedAfterCancel, name)
|
||||
}
|
||||
|
||||
if name == f.target {
|
||||
f.cancel()
|
||||
f.cancelled = true
|
||||
}
|
||||
|
||||
return f.Fs.Open(name)
|
||||
}
|
||||
|
||||
// linkRemovedAfterLstatFs is the real filesystem, except that the symlink at
|
||||
// target is removed right after the walk lstats it, as happens when a link is
|
||||
// deleted during a backup. The scanner's readlink of it then fails.
|
||||
@@ -154,16 +128,15 @@ func writeSkipErrorTestFile(t *testing.T, fs afero.Fs, path, content string) {
|
||||
}
|
||||
}
|
||||
|
||||
// runSkipErrorScan scans source on fs under ctx with the given skip-errors
|
||||
// setting, printing user-facing messages to uiw (nil discards them), and
|
||||
// returns the repositories (for inspection) and the scan error.
|
||||
// runSkipErrorScan scans source on fs with the given skip-errors setting,
|
||||
// printing user-facing messages to uiw (nil discards them), and returns the
|
||||
// repositories (for inspection) and the scan error.
|
||||
func runSkipErrorScan(
|
||||
ctx context.Context, t *testing.T, fs afero.Fs, source string,
|
||||
skipErrors bool, uiw *ui.Writer,
|
||||
t *testing.T, fs afero.Fs, source string, skipErrors bool, uiw *ui.Writer,
|
||||
) (*database.Repositories, error) {
|
||||
t.Helper()
|
||||
|
||||
db, err := database.New(ctx, ":memory:")
|
||||
db, err := database.NewTestDB()
|
||||
if err != nil {
|
||||
t.Fatalf("create test db: %v", err)
|
||||
}
|
||||
@@ -188,6 +161,7 @@ func runSkipErrorScan(
|
||||
SkipErrors: skipErrors,
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
snapshotID := "test-snapshot-skip-errors"
|
||||
createTestSnapshotRecord(ctx, t, repos, snapshotID)
|
||||
|
||||
@@ -211,7 +185,7 @@ func TestScannerPackingFailureAbortsUnderSkipErrors(t *testing.T) {
|
||||
writeSkipErrorTestFile(t, fs, "/source/file1.txt", "first file content")
|
||||
writeSkipErrorTestFile(t, fs, "/source/file2.txt", "second file content")
|
||||
|
||||
repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
|
||||
repos, err := runSkipErrorScan(t, fs, "/source", true, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected scan to abort on the packer error, got nil")
|
||||
}
|
||||
@@ -238,7 +212,7 @@ func TestScannerReadErrorAbortsWithoutSkipErrors(t *testing.T) {
|
||||
fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||
writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
|
||||
|
||||
_, err := runSkipErrorScan(context.Background(), t, fs, "/source", false, nil)
|
||||
_, err := runSkipErrorScan(t, fs, "/source", false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected scan to fail on the read error, got nil")
|
||||
}
|
||||
@@ -254,7 +228,7 @@ func TestScannerReadErrorSkippedWithSkipErrors(t *testing.T) {
|
||||
fs := &readFailFs{Fs: afero.NewMemMapFs(), target: target}
|
||||
writeSkipErrorTestFile(t, fs, target, "content that cannot be read")
|
||||
|
||||
repos, err := runSkipErrorScan(context.Background(), t, fs, "/source", true, nil)
|
||||
repos, err := runSkipErrorScan(t, fs, "/source", true, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
|
||||
}
|
||||
@@ -293,7 +267,7 @@ func TestScannerUnreadableSymlinkAbortsWithoutSkipErrors(t *testing.T) {
|
||||
sourceDir, linkPath := writeSymlinkSource(t)
|
||||
fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
|
||||
|
||||
_, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, false, nil)
|
||||
_, err := runSkipErrorScan(t, fs, sourceDir, false, nil)
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expected scan to fail on the removed symlink, got %v", err)
|
||||
}
|
||||
@@ -309,7 +283,7 @@ func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
|
||||
fs := &linkRemovedAfterLstatFs{t: t, target: linkPath}
|
||||
uiw := ui.NewWithColor(io.Discard, false)
|
||||
|
||||
repos, err := runSkipErrorScan(context.Background(), t, fs, sourceDir, true, uiw)
|
||||
repos, err := runSkipErrorScan(t, fs, sourceDir, true, uiw)
|
||||
if err != nil {
|
||||
t.Fatalf("expected scan to complete with --skip-errors, got %v", err)
|
||||
}
|
||||
@@ -328,56 +302,3 @@ func TestScannerUnreadableSymlinkSkippedWithSkipErrors(t *testing.T) {
|
||||
t.Fatalf("expected %s not to be recorded", linkPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScannerCancelStopsSkipErrorsRun checks that a --skip-errors backup
|
||||
// cancelled partway through processing returns the cancellation error,
|
||||
// opens no further file, and reports no file as failed.
|
||||
func TestScannerCancelStopsSkipErrorsRun(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
targetContent string
|
||||
}{
|
||||
// The cancellation lands while the target is being read.
|
||||
{name: "while reading a file", targetContent: "first file content"},
|
||||
// An empty target has no chunks, so the cancellation goes unnoticed
|
||||
// until the run moves on to the next file.
|
||||
{name: "between files", targetContent: ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const target = "/source/a.txt"
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
fs := &cancelOnOpenFs{
|
||||
Fs: afero.NewMemMapFs(), target: target, cancel: cancel,
|
||||
}
|
||||
writeSkipErrorTestFile(t, fs, target, tt.targetContent)
|
||||
writeSkipErrorTestFile(t, fs, "/source/b.txt", "second file content")
|
||||
writeSkipErrorTestFile(t, fs, "/source/c.txt", "third file content")
|
||||
|
||||
uiw := ui.NewWithColor(io.Discard, false)
|
||||
|
||||
_, err := runSkipErrorScan(ctx, t, fs, "/source", true, uiw)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected the cancellation error, got %v", err)
|
||||
}
|
||||
|
||||
if len(fs.openedAfterCancel) != 0 {
|
||||
t.Fatalf("expected no file opened after the cancellation, got %v",
|
||||
fs.openedAfterCancel)
|
||||
}
|
||||
|
||||
if uiw.ErrorCount() != 0 {
|
||||
t.Fatalf("expected no file reported as failed, got %d error lines",
|
||||
uiw.ErrorCount())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package vaultik
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -48,10 +47,6 @@ const (
|
||||
year = 365 * day
|
||||
)
|
||||
|
||||
// maxDuration is the largest value a time.Duration can hold, about 292
|
||||
// years.
|
||||
const maxDuration = time.Duration(math.MaxInt64)
|
||||
|
||||
// A snapshot ID split on "_" has at least a hostname and a trailing
|
||||
// timestamp.
|
||||
const minSnapshotIDParts = 2
|
||||
@@ -150,10 +145,10 @@ func parseSnapshotName(snapshotID, hostname string) string {
|
||||
// parseDuration parses a duration string with support for human-friendly units:
|
||||
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
||||
// duration units. Following Go, m is minutes and mo is months. A bare number,
|
||||
// an unknown unit, a negative value, and a total longer than maxDuration are
|
||||
// all rejected. Outside the Go units the input must be whole numbers each
|
||||
// followed directly by a unit (2w3d), so 1.0y and 30 days are errors;
|
||||
// decimals work only in Go units (1.5h).
|
||||
// an unknown unit, and a negative value are all rejected. Outside the Go
|
||||
// units the input must be whole numbers each followed directly by a unit
|
||||
// (2w3d), so 1.0y and 30 days are errors; decimals work only in Go units
|
||||
// (1.5h).
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
if strings.HasPrefix(strings.TrimSpace(s), "-") {
|
||||
return 0, errNegativeDuration
|
||||
@@ -186,30 +181,19 @@ func parseDuration(s string) (time.Duration, error) {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
|
||||
}
|
||||
|
||||
var unitLength time.Duration
|
||||
|
||||
unit := strings.ToLower(match[2])
|
||||
switch unit {
|
||||
case "d", "day", "days":
|
||||
unitLength = day
|
||||
total += time.Duration(n) * day
|
||||
case "w", "week", "weeks":
|
||||
unitLength = week
|
||||
total += time.Duration(n) * week
|
||||
case "mo", "month", "months":
|
||||
unitLength = month
|
||||
total += time.Duration(n) * month
|
||||
case "y", "year", "years":
|
||||
unitLength = year
|
||||
total += time.Duration(n) * year
|
||||
default:
|
||||
return 0, fmt.Errorf("%w %q", errUnknownTimeUnit, unit)
|
||||
}
|
||||
|
||||
// Past maxDuration the sum wraps to a negative value, which would
|
||||
// put the purge cutoff in the future.
|
||||
if time.Duration(n) > (maxDuration-total)/unitLength {
|
||||
return 0, fmt.Errorf("%w: %q is longer than about 292 years",
|
||||
errInvalidDuration, s)
|
||||
}
|
||||
|
||||
total += time.Duration(n) * unitLength
|
||||
}
|
||||
|
||||
return total, nil
|
||||
|
||||
@@ -111,17 +111,6 @@ func TestParseDuration(t *testing.T) {
|
||||
{"30 days", 0, true},
|
||||
{"30 days ago", 0, true},
|
||||
{"x7d", 0, true},
|
||||
// The largest time.Duration is about 292 years. A larger value
|
||||
// must be rejected, not wrapped to a negative one, which would
|
||||
// put the cutoff in the future and select every snapshot.
|
||||
{"292y", 292 * 365 * 24 * time.Hour, false},
|
||||
{"293y", 0, true},
|
||||
{"3559mo", 0, true},
|
||||
{"15251w", 0, true},
|
||||
{"106752d", 0, true},
|
||||
{"2562048h", 0, true},
|
||||
{"200y200y", 0, true},
|
||||
{"292y6mo", 0, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -10,44 +10,30 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// TestNukeRemoteLeavesNoFiles checks that remote nuke leaves no file
|
||||
// under a file:// destination, including the `.partial` files uploads
|
||||
// killed part-way leave next to a snapshot's metadata and next to where a
|
||||
// blob would have been.
|
||||
// under a file:// destination, including the `.partial` file an upload
|
||||
// killed part-way leaves next to where its blob would have been.
|
||||
func TestNukeRemoteLeavesNoFiles(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "store")
|
||||
v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
|
||||
v, _, _ := backUpToFileDestination(ctx, t, storeDir)
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, snapshots, 1)
|
||||
|
||||
snapshotKey := snapshot.RemoteSnapshotKey(snapshots[0].ID.String())
|
||||
hash := testBlobHashA
|
||||
leftovers := []string{
|
||||
filepath.Join(storeDir, "metadata", snapshotKey,
|
||||
"db.zst.age-123456.partial"),
|
||||
filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
|
||||
hash+"-123456.partial"),
|
||||
}
|
||||
|
||||
for _, leftover := range leftovers {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(leftover), 0o750))
|
||||
require.NoError(t, os.WriteFile(leftover, []byte("half an upload"), 0o600))
|
||||
}
|
||||
leftover := filepath.Join(storeDir, "blobs", hash[:2], hash[2:4],
|
||||
hash+"-123456.partial")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(leftover), 0o750))
|
||||
require.NoError(t, os.WriteFile(leftover, []byte("half a blob"), 0o600))
|
||||
|
||||
require.NoError(t, v.NukeRemote(true))
|
||||
|
||||
var files []string
|
||||
|
||||
err = filepath.WalkDir(storeDir,
|
||||
err := filepath.WalkDir(storeDir,
|
||||
func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -267,29 +267,6 @@ func TestPurgeOlderThan_WithNameFilter(t *testing.T) {
|
||||
assert.Contains(t, remaining, snapHomeT0)
|
||||
}
|
||||
|
||||
func TestPurgeOlderThan_TooLargeDeletesNothing(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
snapshotIDs := []string{snapHomeT0, snapHomeT1}
|
||||
|
||||
v := setupPurgeTest(t, snapHostname, snapshotIDs)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
OlderThan: "1000y",
|
||||
Force: true,
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
err = v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
OlderThan: "292y",
|
||||
Force: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.ElementsMatch(t, snapshotIDs, listRemainingSnapshots(t, v))
|
||||
}
|
||||
|
||||
func TestPurgeKeepLatest_ThreeNames(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestVerifyPassesWhenBackupReusesStoredBlob backs up a file, then backs it
|
||||
// up again from an empty local index, as after `database delete`, with an
|
||||
// extra age recipient. The second backup builds the same blob, finds it
|
||||
// already stored and skips the upload, but its own copy is larger, because
|
||||
// age writes a header entry for each recipient. The second snapshot must
|
||||
// record the size of the stored object, so shallow and deep verify pass.
|
||||
func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
secondDBPath := filepath.Join(tempDir, "second.sqlite")
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
maxBlobSize := int64(512 * 1024)
|
||||
|
||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs,
|
||||
filepath.Join(dataDir, "data.bin"),
|
||||
bytesPattern("reuse-", int(chunkSize*3)), 0o644))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
runFileStorageBackup(ctx, t, fs, dataDir, storeDir,
|
||||
filepath.Join(tempDir, "first.sqlite"), chunkSize, maxBlobSize)
|
||||
|
||||
env := setupDedupBackupEnv(
|
||||
ctx, t, fs, storeDir, secondDBPath, chunkSize, maxBlobSize)
|
||||
|
||||
defer func() { _ = env.db.Close() }()
|
||||
|
||||
extra, err := age.GenerateX25519Identity()
|
||||
require.NoError(t, err)
|
||||
|
||||
env.cfg.AgeRecipients = append(env.cfg.AgeRecipients,
|
||||
extra.Recipient().String())
|
||||
|
||||
snapshotID, result := runDedupSnapshot(ctx, t, env.sm, env.makeScanner(),
|
||||
env.cfg.Hostname, "second", dataDir, secondDBPath)
|
||||
require.Positive(t, result.BlobsCreated)
|
||||
require.Zero(t, result.BlobsUploaded,
|
||||
"the second backup should find every blob already stored")
|
||||
|
||||
require.NoError(t, env.db.Close())
|
||||
|
||||
verifier := &vaultik.Vaultik{
|
||||
Config: env.cfg,
|
||||
Storage: env.storer,
|
||||
Fs: fs,
|
||||
Stdout: io.Discard,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
verifier.SetContext(ctx)
|
||||
|
||||
require.NoError(t,
|
||||
verifier.VerifySnapshotWithOptions(snapshotID, &vaultik.VerifyOptions{}),
|
||||
"shallow verify should pass")
|
||||
require.NoError(t,
|
||||
verifier.VerifySnapshotWithOptions(snapshotID,
|
||||
&vaultik.VerifyOptions{Deep: true}),
|
||||
"deep verify should pass")
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// --skip-errors skips only a file that cannot be opened or read. A local
|
||||
// index error while a directory is recorded stops the backup, and the
|
||||
// snapshot is not recorded as complete. See
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/284.
|
||||
func TestSkipErrorsBackupStopsOnIndexErrorRecordingDirectory(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// The scan walks the source path with symlinks resolved, so dirPath
|
||||
// must be spelled the same way to match the row the scan inserts.
|
||||
tempDir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
srcDir := filepath.Join(tempDir, "src")
|
||||
dirPath := filepath.Join(srcDir, "dir")
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
require.NoError(t, fs.MkdirAll(dirPath, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs,
|
||||
filepath.Join(dirPath, "file.txt"), []byte("file content"), 0o644))
|
||||
|
||||
cfg := faultTestConfig()
|
||||
cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
|
||||
cfg.Snapshots = map[string]config.SnapshotConfig{
|
||||
"tree": {Paths: []string{srcDir}},
|
||||
}
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, cfg.IndexPath)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
// The local index refuses the directory's files row.
|
||||
_, err = db.Conn().ExecContext(ctx, fmt.Sprintf(`
|
||||
CREATE TRIGGER refuse_directory BEFORE INSERT ON files
|
||||
WHEN NEW.path = '%s'
|
||||
BEGIN SELECT RAISE(ABORT, 'simulated index error'); END`, dirPath))
|
||||
require.NoError(t, err)
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||
|
||||
err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||
SkipErrors: true,
|
||||
Snapshots: []string{"tree"},
|
||||
})
|
||||
require.ErrorContains(t, err, "simulated index error")
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, snapshots, 1)
|
||||
assert.Nil(t, snapshots[0].CompletedAt)
|
||||
}
|
||||
Reference in New Issue
Block a user