Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d2108fbac4 |
+24
-3
@@ -5,7 +5,16 @@
|
||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
# Without the time limit a stalled connection to the module proxy holds
|
||||
# this step, and every build on the host sharing it, until the kernel
|
||||
# drops the connection, which can take an hour. The limit is well above
|
||||
# the three minutes a working download can take on a busy build host.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
# `golangci-lint run` silently ignores an unknown top-level key in
|
||||
# .golangci.yml, such as a misspelt `linters:`; `config verify` fails on it.
|
||||
@@ -18,7 +27,13 @@ RUN golangci-lint run --config .golangci.yml ./...
|
||||
FROM golang:1.26.1@sha256:cd78d88e00afadbedd272f977d375a6247455f3a4b1178f8ae8bbcb201743a8a AS test
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
# Limited and retried as in the lint phase above.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
RUN go test -timeout 90s -race -cover ./... || \
|
||||
{ echo "--- Rerunning with -v for details ---"; \
|
||||
@@ -36,7 +51,13 @@ RUN apk add --no-cache git
|
||||
RUN git config --system --add safe.directory /src
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
# Limited and retried as in the lint phase above.
|
||||
RUN for attempt in 1 2 3; do \
|
||||
timeout 300 go mod download && exit 0; \
|
||||
echo "go mod download attempt $attempt failed or ran over 300s" >&2; \
|
||||
done; \
|
||||
echo "module download failed: giving up after 3 attempts" >&2; \
|
||||
exit 1
|
||||
COPY . .
|
||||
|
||||
# The VERSION build arg when one is given, otherwise
|
||||
|
||||
@@ -294,8 +294,7 @@ needed.
|
||||
of the local index and the backup destination store — with timestamps
|
||||
and three sizes per snapshot (compressed remote size; total
|
||||
uncompressed chunk size; size of the chunks that snapshot stored that
|
||||
were not stored before). Both chunk sizes are recorded when the
|
||||
snapshot's backup runs.
|
||||
were not stored before, recorded when its backup ran).
|
||||
|
||||
Listing the destination store does **not** require the age secret key,
|
||||
so it works in vaultik's intended configuration, where the backed-up
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package main_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDockerfileBoundsEveryModuleDownload fails unless every
|
||||
// `go mod download` in the Dockerfile runs under a time limit, is tried
|
||||
// a fixed number of times, and then fails the build with a message that
|
||||
// names the module download (issue #351). Like the version tests, this
|
||||
// parses the Dockerfile; that a stalled download really is cut off and
|
||||
// retried was checked by hand.
|
||||
func TestDockerfileBoundsEveryModuleDownload(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var downloads []string
|
||||
|
||||
for _, instruction := range instructions(t, productDockerfile) {
|
||||
if strings.Contains(instruction, "go mod download") {
|
||||
downloads = append(downloads, instruction)
|
||||
}
|
||||
}
|
||||
|
||||
require.NotEmpty(t, downloads, "%s must download modules",
|
||||
productDockerfile)
|
||||
|
||||
for _, download := range downloads {
|
||||
assert.Contains(t, download, "timeout 300 go mod download",
|
||||
"%s: a module download must have a time limit: %s",
|
||||
productDockerfile, download)
|
||||
assert.Contains(t, download, "for attempt in 1 2 3",
|
||||
"%s: a module download must be retried: %s",
|
||||
productDockerfile, download)
|
||||
assert.Contains(t, download, "module download failed",
|
||||
"%s: a module download that never succeeds must say so: %s",
|
||||
productDockerfile, download)
|
||||
assert.True(t, strings.HasSuffix(download, "exit 1"),
|
||||
"%s: a module download that never succeeds must fail the"+
|
||||
" build: %s", productDockerfile, download)
|
||||
}
|
||||
}
|
||||
@@ -122,7 +122,6 @@ Tracks backup snapshots.
|
||||
- `new_chunk_size` (INTEGER) - Total size of those chunks before compression; `snapshot list` shows it as NEW CHUNK SIZE
|
||||
- `blob_count` (INTEGER) - Number of blobs this snapshot created
|
||||
- `total_size` (INTEGER) - Total size of all files
|
||||
- `uncompressed_chunk_size` (INTEGER) - Total size before compression of the distinct chunks the snapshot's files reference, recorded when its backup ran; `snapshot list` shows it as UNCOMPRESSED SIZE
|
||||
- `blob_size` (INTEGER) - Total compressed size of all referenced blobs
|
||||
- `blob_uncompressed_size` (INTEGER) - Total uncompressed size of all referenced blobs
|
||||
- `compression_ratio` (REAL) - Compression ratio achieved
|
||||
|
||||
@@ -89,7 +89,6 @@ CREATE TABLE IF NOT EXISTS snapshots (
|
||||
new_chunk_size INTEGER NOT NULL DEFAULT 0,
|
||||
blob_count INTEGER NOT NULL DEFAULT 0,
|
||||
total_size INTEGER NOT NULL DEFAULT 0,
|
||||
uncompressed_chunk_size INTEGER NOT NULL DEFAULT 0,
|
||||
blob_size INTEGER NOT NULL DEFAULT 0,
|
||||
blob_uncompressed_size INTEGER NOT NULL DEFAULT 0,
|
||||
compression_ratio REAL NOT NULL DEFAULT 1.0,
|
||||
|
||||
@@ -120,11 +120,7 @@ func (r *SnapshotRepository) UpdateCounts(
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateExtendedStats updates extended statistics for a snapshot. It also
|
||||
// records the total plaintext size of the distinct chunks the snapshot's
|
||||
// files reference. Only the snapshot's own backup can take that total: a
|
||||
// file's row is shared by every snapshot that lists it, and a later
|
||||
// backup of a changed file rewrites the row with the new chunks.
|
||||
// UpdateExtendedStats updates extended statistics for a snapshot
|
||||
func (r *SnapshotRepository) UpdateExtendedStats(
|
||||
ctx context.Context,
|
||||
tx *sql.Tx,
|
||||
@@ -149,17 +145,7 @@ func (r *SnapshotRepository) UpdateExtendedStats(
|
||||
compression_level = ?,
|
||||
upload_bytes = ?,
|
||||
upload_duration_ms = ?,
|
||||
new_chunk_size = ?,
|
||||
uncompressed_chunk_size = (
|
||||
SELECT COALESCE(SUM(c.size), 0)
|
||||
FROM (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshot_files sf
|
||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
||||
WHERE sf.snapshot_id = snapshots.id
|
||||
) sc
|
||||
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
|
||||
)
|
||||
new_chunk_size = ?
|
||||
WHERE id = ?
|
||||
`
|
||||
|
||||
@@ -569,22 +555,30 @@ func (r *SnapshotRepository) GetSnapshotBlobSizes(
|
||||
return compressed, uncompressed, nil
|
||||
}
|
||||
|
||||
// GetSnapshotUncompressedChunkSize returns the total plaintext size of the
|
||||
// distinct chunks the snapshot's files reference, as the snapshot's backup
|
||||
// recorded it (see UpdateExtendedStats).
|
||||
// GetSnapshotUncompressedChunkSize returns the sum of plaintext sizes of all unique
|
||||
// chunks referenced by a snapshot (via snapshot_files → file_chunks → chunks).
|
||||
func (r *SnapshotRepository) GetSnapshotUncompressedChunkSize(
|
||||
ctx context.Context, snapshotID string,
|
||||
) (int64, error) {
|
||||
query := `SELECT uncompressed_chunk_size FROM snapshots WHERE id = ?`
|
||||
query := `
|
||||
SELECT COALESCE(SUM(c.size), 0)
|
||||
FROM (
|
||||
SELECT DISTINCT fc.chunk_hash
|
||||
FROM snapshot_files sf
|
||||
JOIN file_chunks fc ON sf.file_id = fc.file_id
|
||||
WHERE sf.snapshot_id = ?
|
||||
) sc
|
||||
JOIN chunks c ON sc.chunk_hash = c.chunk_hash
|
||||
`
|
||||
|
||||
var uncompressedChunkSize int64
|
||||
var totalSize int64
|
||||
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&uncompressedChunkSize)
|
||||
err := r.db.conn.QueryRowContext(ctx, query, snapshotID).Scan(&totalSize)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("querying uncompressed chunk size: %w", err)
|
||||
}
|
||||
|
||||
return uncompressedChunkSize, nil
|
||||
return totalSize, nil
|
||||
}
|
||||
|
||||
// GetSnapshotNewChunkSize returns the total plaintext size of the chunks the
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -11,9 +9,7 @@ import (
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
@@ -81,174 +77,6 @@ func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
||||
"deep verify should fail on a corrupted blob")
|
||||
}
|
||||
|
||||
// TestDeepVerifyJSONCountsBlobs checks the verified and missing counts
|
||||
// that deep verify reports under --json: on a healthy snapshot, with a
|
||||
// damaged blob, and with missing blobs. Deep verify stops at the first
|
||||
// damaged blob, so the blobs read before it are the ones verified; when
|
||||
// blobs are missing it reads no blob contents, but counts every missing
|
||||
// blob.
|
||||
func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
|
||||
healthy, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "ok", healthy.Status)
|
||||
require.Len(t, keys, healthy.BlobCount)
|
||||
require.Equal(t, healthy.BlobCount, healthy.Verified)
|
||||
require.Zero(t, healthy.Missing)
|
||||
|
||||
// Damage the last blob read, keeping its length, so every blob before
|
||||
// it is read and passes first.
|
||||
lastPath := filepath.Join(storeDir, keys[len(keys)-1])
|
||||
blob, err := afero.ReadFile(fs, lastPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob[len(blob)/2] ^= 0xff
|
||||
require.NoError(t, afero.WriteFile(fs, lastPath, blob, 0o600))
|
||||
|
||||
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "failed", got.Status)
|
||||
require.Equal(t, healthy.BlobCount-1, got.Verified)
|
||||
require.Zero(t, got.Missing)
|
||||
|
||||
// Delete the first blob and the damaged last one: both must be
|
||||
// counted as missing, not only the first one found.
|
||||
var deletedSize int64
|
||||
|
||||
for _, key := range []string{keys[0], keys[len(keys)-1]} {
|
||||
info, statErr := storer.Stat(ctx, key)
|
||||
require.NoError(t, statErr)
|
||||
|
||||
deletedSize += info.Size
|
||||
|
||||
require.NoError(t, storer.Delete(ctx, key))
|
||||
}
|
||||
|
||||
got, err = deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "failed", got.Status)
|
||||
require.Equal(t, 2, got.Missing)
|
||||
require.Equal(t, deletedSize, got.MissingSize)
|
||||
require.Zero(t, got.Verified)
|
||||
}
|
||||
|
||||
// TestDeepVerifyJSONCountsWrongSizeBlob checks that under --deep --json a
|
||||
// blob of the wrong size is counted in mismatched and fails the run, and
|
||||
// that the check goes on past it to count a missing blob after it.
|
||||
func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
|
||||
// Make the first blob checked longer, and delete the last one.
|
||||
firstPath := filepath.Join(storeDir, keys[0])
|
||||
blob, err := afero.ReadFile(fs, firstPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
blob = append(blob, []byte("extra")...)
|
||||
require.NoError(t, afero.WriteFile(fs, firstPath, blob, 0o600))
|
||||
require.NoError(t, storer.Delete(ctx, keys[len(keys)-1]))
|
||||
|
||||
got, err := deepVerifyJSON(ctx, t, cfg, storer, snapshotID)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "failed", got.Status)
|
||||
require.Equal(t, 1, got.Mismatched)
|
||||
require.Equal(t, 1, got.Missing)
|
||||
require.Zero(t, got.Verified)
|
||||
}
|
||||
|
||||
// TestDeepVerifyStopsOnBlobThatCannotBeChecked checks that a blob that is
|
||||
// present in storage but cannot be checked is not counted as missing:
|
||||
// deep verify stops and fails with the storage error.
|
||||
func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
|
||||
|
||||
got, err := deepVerifyJSON(ctx, t, cfg, failing, snapshotID)
|
||||
require.Error(t, err)
|
||||
require.Equal(t, "failed", got.Status)
|
||||
require.Zero(t, got.Missing)
|
||||
require.Contains(t, got.ErrorMessage, errStatDenied.Error())
|
||||
}
|
||||
|
||||
// backUpSeveralBlobs backs up random data, which does not compress or
|
||||
// deduplicate, so it fills at least three blobs in a store on disk at
|
||||
// storeDir. It returns the blobs' storage keys sorted, which is hash
|
||||
// order, the order deep verify reads them in.
|
||||
func backUpSeveralBlobs(
|
||||
ctx context.Context, t *testing.T, storeDir string,
|
||||
) (*config.Config, *storage.FileStorer, string, []string) {
|
||||
t.Helper()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
maxBlobSize := int64(128 * 1024)
|
||||
|
||||
data := make([]byte, maxBlobSize*4)
|
||||
_, err := rand.Read(data)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, fs.MkdirAll(dataDir, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs,
|
||||
filepath.Join(dataDir, "data.bin"), data, 0o644))
|
||||
|
||||
cfg, storer, snapshotID := runFileStorageBackup(
|
||||
ctx, t, fs, dataDir, storeDir, dbPath, chunkSize, maxBlobSize)
|
||||
|
||||
keys := storedBlobKeys(t, storeDir)
|
||||
require.GreaterOrEqual(t, len(keys), 3)
|
||||
|
||||
return cfg, storer, snapshotID, keys
|
||||
}
|
||||
|
||||
// deepVerifyJSON runs deep verify with --json on snapshotID and returns
|
||||
// the decoded result and the error RunDeepVerify returned.
|
||||
func deepVerifyJSON(
|
||||
ctx context.Context, t *testing.T,
|
||||
cfg *config.Config, storer storage.Storer, snapshotID string,
|
||||
) (vaultik.VerifyResult, error) {
|
||||
t.Helper()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: cfg,
|
||||
Storage: storer,
|
||||
Fs: afero.NewOsFs(),
|
||||
Stdout: &out,
|
||||
Stderr: io.Discard,
|
||||
UI: ui.NewWithColor(io.Discard, false),
|
||||
}
|
||||
v.SetContext(ctx)
|
||||
|
||||
err := v.RunDeepVerify(snapshotID,
|
||||
&vaultik.VerifyOptions{Deep: true, JSON: true})
|
||||
|
||||
return decodeVerifyResult(t, out.Bytes()), err
|
||||
}
|
||||
|
||||
// corruptOneBlob flips a middle byte of the first blob file found under
|
||||
// blobsDir, leaving the file length unchanged.
|
||||
func corruptOneBlob(t *testing.T, fs afero.Fs, blobsDir string) {
|
||||
|
||||
@@ -483,14 +483,13 @@ func TestListSnapshots_UnreadableManifestDoesNotHideOthers(t *testing.T) {
|
||||
//
|
||||
//nolint:tagliatelle // snake_case is the established output format
|
||||
type listJSONRow struct {
|
||||
ID string `json:"id"`
|
||||
RemoteKey string `json:"remote_key"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
UncompressedSize int64 `json:"uncompressed_size"`
|
||||
NewChunkSize int64 `json:"new_chunk_size"`
|
||||
LocallyTracked bool `json:"locally_tracked"`
|
||||
RemotePresent *bool `json:"remote_present"`
|
||||
ID string `json:"id"`
|
||||
RemoteKey string `json:"remote_key"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
CompressedSize int64 `json:"compressed_size"`
|
||||
NewChunkSize int64 `json:"new_chunk_size"`
|
||||
LocallyTracked bool `json:"locally_tracked"`
|
||||
RemotePresent *bool `json:"remote_present"`
|
||||
}
|
||||
|
||||
// decodeListJSON parses the command's stdout, which must contain
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestUncompressedSizeOfChangedFileStaysFixed covers
|
||||
// https://git.eeqj.de/sneak/vaultik/issues/360. A backup that changes a
|
||||
// file leaves the earlier snapshot's UNCOMPRESSED SIZE as it was when that
|
||||
// snapshot's backup completed. The file's row in the local index is shared
|
||||
// by both snapshots and rewritten by the second backup.
|
||||
func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
changedPath := filepath.Join(dataDir, "changed.bin")
|
||||
|
||||
ctx := context.Background()
|
||||
files := writeFaultSourceTree(t, fs, dataDir)
|
||||
cfg := changedFileConfig(dataDir, dbPath)
|
||||
|
||||
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
|
||||
// With nothing to compare against, sizeOfNewChunks totals every
|
||||
// distinct chunk of the files.
|
||||
firstID := localSnapshotID(ctx, t, repos, "first")
|
||||
firstSize := sizeOfNewChunks(t, files, nil)
|
||||
require.Equal(t, firstSize, listedUncompressedSizes(t, v)[firstID],
|
||||
"the first snapshot must count the chunks of every file it holds")
|
||||
|
||||
appendRandomBytes(t, fs, files, changedPath, faultChunkSize)
|
||||
require.NoError(t, backUp(v, "second"))
|
||||
|
||||
secondID := localSnapshotID(ctx, t, repos, "second")
|
||||
sizes := listedUncompressedSizes(t, v)
|
||||
assert.Equal(t, firstSize, sizes[firstID],
|
||||
"a later backup must not change the first snapshot's figure")
|
||||
assert.Equal(t, sizeOfNewChunks(t, files, nil), sizes[secondID],
|
||||
"the second snapshot must count the changed file's chunks")
|
||||
}
|
||||
|
||||
// listedUncompressedSizes runs `snapshot list --json` and returns each
|
||||
// listed snapshot's uncompressed_size by snapshot ID.
|
||||
func listedUncompressedSizes(t *testing.T, v *vaultik.Vaultik) map[string]int64 {
|
||||
t.Helper()
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v.Stdout = &out
|
||||
require.NoError(t, v.ListSnapshots(true))
|
||||
|
||||
sizes := make(map[string]int64)
|
||||
for _, row := range decodeListJSON(t, out.String()) {
|
||||
sizes[row.ID] = row.UncompressedSize
|
||||
}
|
||||
|
||||
return sizes
|
||||
}
|
||||
+17
-39
@@ -17,7 +17,6 @@ import (
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// Sentinel errors for snapshot verification failures.
|
||||
@@ -33,9 +32,7 @@ var (
|
||||
errManifestExtraBlob = errors.New("manifest contains blob not in database")
|
||||
errManifestMissingBlob = errors.New(
|
||||
"manifest omits blob present in database")
|
||||
errBlobSizeMismatch = errors.New("blob size mismatch")
|
||||
errBlobsNotInStorage = errors.New(
|
||||
"blobs missing from storage or stored with the wrong size")
|
||||
errBlobSizeMismatch = errors.New("blob size mismatch")
|
||||
)
|
||||
|
||||
// verifyStatusFailed is the JSON status value for a failed verification.
|
||||
@@ -152,6 +149,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
}
|
||||
|
||||
result.Status = "ok"
|
||||
result.Verified = len(dbBlobs)
|
||||
|
||||
if opts.JSON {
|
||||
return v.outputVerifyJSON(result)
|
||||
@@ -308,7 +306,7 @@ func (v *Vaultik) runVerificationSteps(
|
||||
v.stdoutf("Checking blob existence in remote storage...\n")
|
||||
}
|
||||
|
||||
err = v.verifyBlobExistenceFromDB(dbBlobs, result)
|
||||
err = v.verifyBlobExistenceFromDB(dbBlobs)
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
@@ -319,8 +317,7 @@ func (v *Vaultik) runVerificationSteps(
|
||||
len(dbBlobs), ubytes(totalSize))
|
||||
}
|
||||
|
||||
err = v.performDeepVerificationFromDB(
|
||||
dbBlobs, tdb.db.Conn(), opts, result, identities)
|
||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts, identities)
|
||||
if err != nil {
|
||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||
}
|
||||
@@ -672,14 +669,8 @@ func (v *Vaultik) verifyManifestAgainstDatabase(
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyBlobExistenceFromDB checks that every blob from the database is
|
||||
// in storage with the size the database records. It checks them all,
|
||||
// counting the missing and wrong-size blobs in result, and fails if it
|
||||
// found any. Any storage error other than not found stops the check and
|
||||
// fails with that error.
|
||||
func (v *Vaultik) verifyBlobExistenceFromDB(
|
||||
blobs []snapshot.BlobInfo, result *VerifyResult,
|
||||
) error {
|
||||
// verifyBlobExistenceFromDB checks that all blobs from database exist in S3
|
||||
func (v *Vaultik) verifyBlobExistenceFromDB(blobs []snapshot.BlobInfo) error {
|
||||
log.Info("Verifying blob existence in S3", "blob_count", len(blobs))
|
||||
|
||||
for i, blob := range blobs {
|
||||
@@ -692,23 +683,17 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
|
||||
// Construct blob path
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
||||
|
||||
// Check blob exists
|
||||
stat, err := v.Storage.Stat(v.ctx, blobPath)
|
||||
switch {
|
||||
case errors.Is(err, storage.ErrNotFound):
|
||||
log.Warn("Blob missing from storage", "hash", blob.Hash)
|
||||
if err != nil {
|
||||
return fmt.Errorf("blob %s missing from storage: %w", blob.Hash, err)
|
||||
}
|
||||
|
||||
result.Missing++
|
||||
result.MissingSize += blob.CompressedSize
|
||||
case err != nil:
|
||||
return fmt.Errorf("checking blob %s in storage: %w",
|
||||
blob.Hash, err)
|
||||
case stat.Size != blob.CompressedSize:
|
||||
log.Warn("Blob has the wrong size in storage",
|
||||
"hash", blob.Hash,
|
||||
"storage_size", stat.Size,
|
||||
"database_size", blob.CompressedSize)
|
||||
|
||||
result.Mismatched++
|
||||
// Verify size matches
|
||||
if stat.Size != blob.CompressedSize {
|
||||
return fmt.Errorf(
|
||||
"%w: blob %s: S3 has %d bytes, database has %d bytes",
|
||||
errBlobSizeMismatch, blob.Hash, stat.Size, blob.CompressedSize)
|
||||
}
|
||||
|
||||
// Progress update every 100 blobs
|
||||
@@ -722,22 +707,16 @@ func (v *Vaultik) verifyBlobExistenceFromDB(
|
||||
}
|
||||
}
|
||||
|
||||
if result.Missing > 0 || result.Mismatched > 0 {
|
||||
return fmt.Errorf("%w: %d missing, %d with the wrong size",
|
||||
errBlobsNotInStorage, result.Missing, result.Mismatched)
|
||||
}
|
||||
|
||||
log.Info("✓ All blobs exist in storage")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// performDeepVerificationFromDB downloads and verifies the content of
|
||||
// each blob using the database as source, stopping at the first blob that
|
||||
// fails. Each blob that passes is counted in result.Verified.
|
||||
// each blob using the database as source.
|
||||
func (v *Vaultik) performDeepVerificationFromDB(
|
||||
blobs []snapshot.BlobInfo, db *sql.DB, opts *VerifyOptions,
|
||||
result *VerifyResult, identities []age.Identity,
|
||||
identities []age.Identity,
|
||||
) error {
|
||||
// Calculate total bytes for ETA
|
||||
var totalBytesExpected int64
|
||||
@@ -760,7 +739,6 @@ func (v *Vaultik) performDeepVerificationFromDB(
|
||||
return fmt.Errorf("blob %s verification failed: %w", blobInfo.Hash, err)
|
||||
}
|
||||
|
||||
result.Verified++
|
||||
bytesProcessed += blobInfo.CompressedSize
|
||||
elapsed := time.Since(startTime)
|
||||
remaining := len(blobs) - (i + 1)
|
||||
|
||||
Reference in New Issue
Block a user