Update golangci-lint to v2.12.2 with canonical config (#62)
check / check (push) Successful in 5s

Updates golangci-lint to v2.12.2 everywhere it is pinned and installs the canonical `.golangci.yml`, then remediates every finding the new linter/config surfaces so `make check` is green.

## Version bump

- `Dockerfile` lint stage: `golangci/golangci-lint:v2.11.3-alpine` -> `v2.12.2-alpine` (digest-pinned, date comment updated)
- `Makefile` `deps` target: `go install` moved from the old v1 module path at `@latest` to the pinned `github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2`
- `.golangci.yml` replaced with the canonical config (v2 schema; settings under `linters.settings` so the thresholds actually apply; `default: all` with the standard six disables)
- `script/bootstrap` installs golangci-lint via the system package manager and carries no version pin, so it is unchanged
- CI (`.gitea/workflows/check.yml`) only runs `script/cibuild`, so it needed no change

## Lint remediation

The canonical config surfaced ~3,300 findings across 56k lines. All are fixed, behavior-preserving; incorporates and supersedes the per-package mechanical passes already merged to `main` (refs #61). Highlights:

- `err113`: dynamic errors replaced with package sentinels + `%w` wrapping; comparisons via `errors.Is`
- `goprintffuncname`: printf-style helpers renamed with an `f` suffix (`ui.Writer` message methods, `cli.ReportErrorf`, `database.Fatalf`) and all call sites updated
- `revive` stutter renames: `blob.Handler`, `blob.WithReader`, `blob.ChunkPosition`, `storage.URL`, `storage.Info`; missing doc comments added
- `contextcheck`/`noctx`: `context.Context` threaded through `blob.Packer` and the scanner call sites; context-aware `exec`/`sql` variants
- `funlen`/`cyclop`/`gocognit`/`dupl`: oversized and duplicated functions split into focused helpers (production and test code)
- tests: `t.Parallel()` added where safe (global logger init kept in the serial phase for `-race`), `t.TempDir()`/`t.Helper()` adopted, several suites converted to external test packages
- `gosec`: bounded integer conversions, `ReadHeaderTimeout` on the test HTTP server; remaining warnings suppressed per-site with justifications
- remaining `nolint` directives are rare, targeted, and each carries a reason (e.g. `nilnil` not-found contract in the repository layer, fx module globals, on-disk snake_case struct tags)
- removed the deprecated `log.LogOptions` alias (callers migrated to `log.Options`)

`make check` (tests with `-race`, lint, fmt-check) passes.

Co-authored-by: sneak <sneak@sneak.berlin>
Reviewed-on: #62
Co-authored-by: clawbot <clawbot@noreply.example.org>
Co-committed-by: clawbot <clawbot@noreply.example.org>
This commit was merged in pull request #62.
This commit is contained in:
2026-08-07 23:22:48 +02:00
committed by sneak
co-authored by sneak
parent b87b72d4b9
commit cc58583130
126 changed files with 8145 additions and 5431 deletions
+381 -243
View File
@@ -14,7 +14,6 @@ import (
"time"
"filippo.io/age"
"github.com/dustin/go-humanize"
"github.com/spf13/afero"
"sneak.berlin/go/vaultik/internal/blobgen"
"sneak.berlin/go/vaultik/internal/database"
@@ -23,6 +22,34 @@ import (
"sneak.berlin/go/vaultik/internal/types"
)
// Sentinel errors for restore failures.
var (
errFilesFailedRestore = errors.New("file(s) failed to restore")
errFilesFailedVerify = errors.New("files failed verification")
errDecryptionKeyRequired = errors.New(
"decryption key required for restore\n\n" +
"Set the VAULTIK_AGE_SECRET_KEY environment variable to your " +
"age private key:\n" +
" export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'")
errBlobMissingFromIndex = errors.New("blob hash missing from blob index")
errChunkNotInAnyBlob = errors.New("chunk not found in any blob")
errBlobIDNotInHashIndex = errors.New("blob id missing from hash index")
errShortChunkRead = errors.New("short read")
)
// restoreDirMode is the permission mode for directories created while
// restoring (parent directories and the target root; restored
// directories themselves get their stored mode).
const restoreDirMode = 0o755
// sweepIntervalDivisor sets the sweeper threshold to one N-th of the
// configured blob size limit.
const sweepIntervalDivisor = 100
// restoreStatusInterval is how often periodic progress lines are
// printed during restore and verify.
const restoreStatusInterval = 15 * time.Second
// RestoreOptions contains options for the restore operation
type RestoreOptions struct {
SnapshotID string
@@ -91,16 +118,17 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
if len(files) == 0 {
log.Warn("No files found to restore")
v.UI.Warning("No files found to restore.")
v.UI.Warningf("No files found to restore.")
return nil
}
log.Info("Found files to restore", "count", len(files))
v.UI.Info("Found %s files to restore.", v.UI.Count(len(files)))
v.UI.Infof("Found %s files to restore.", v.UI.Count(len(files)))
// Step 3: Create target directory
if err := v.Fs.MkdirAll(opts.TargetDir, 0755); err != nil {
err = v.Fs.MkdirAll(opts.TargetDir, restoreDirMode)
if err != nil {
return fmt.Errorf("creating target directory: %w", err)
}
@@ -120,27 +148,40 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
log.Info("Restore complete",
"files_restored", result.FilesRestored,
"bytes_restored", humanize.Bytes(uint64(result.BytesRestored)),
"bytes_restored", ubytes(result.BytesRestored),
"blobs_downloaded", result.BlobsDownloaded,
"bytes_downloaded", humanize.Bytes(uint64(result.BytesDownloaded)),
"bytes_downloaded", ubytes(result.BytesDownloaded),
"duration", result.Duration,
)
v.UI.Complete("Restored %s files (%s) in %s.",
v.UI.Completef("Restored %s files (%s) in %s.",
v.UI.Count(result.FilesRestored),
v.UI.Size(result.BytesRestored),
v.UI.Duration(result.Duration),
)
return v.finishRestore(repos, files, opts, result)
}
// finishRestore emits the post-restore warnings, runs optional
// verification, and converts any failed-file count into an error.
func (v *Vaultik) finishRestore(
repos *database.Repositories,
files []*database.File,
opts *RestoreOptions,
result *RestoreResult,
) error {
if os.Geteuid() != 0 {
v.UI.Warning("Restore did not preserve file ownership: chown(2) requires root. Re-run as root (e.g. with sudo) if you need original UID/GID preserved.")
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
"requires root. Re-run as root (e.g. with sudo) if you need " +
"original UID/GID preserved.")
}
if result.FilesFailed > 0 {
v.UI.Warning("%d file(s) failed to restore:", result.FilesFailed)
v.UI.Warningf("%d file(s) failed to restore:", result.FilesFailed)
for _, path := range result.FailedFiles {
v.UI.Detail("%s", v.UI.Path(path))
v.UI.Detailf("%s", v.UI.Path(path))
}
}
@@ -153,16 +194,19 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
}
if result.FilesFailed > 0 {
return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed)
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedRestore)
}
return nil
}
// prepareRestoreIdentity validates that an age secret key is configured and parses it
// prepareRestoreIdentity validates that an age secret key is configured
// and parses it.
//
//nolint:ireturn // age.Identity is the decryption abstraction by design
func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) {
if v.Config.AgeSecretKey == "" {
return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'")
return nil, errDecryptionKeyRequired
}
identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey)
@@ -212,22 +256,12 @@ func (v *Vaultik) restoreAllFiles(
// Per-restore sweep state: every blob_size_limit/100 bytes written,
// scan the cache and delete any blob whose remaining file references
// are all already restored.
sweeper := newRestoreSweeper(v.ctx, repos, blobCache, v.Config.BlobSizeLimit.Int64()/100)
sweeper := newRestoreSweeper(v.ctx, repos, blobCache,
v.Config.BlobSizeLimit.Int64()/sweepIntervalDivisor)
// Pre-fetch every blob row once so chunk extraction can map a
// blob_id to its hash without a DB round-trip per chunk.
blobsByID, err := repos.Blobs.GetAll(v.ctx)
blobByHash, blobIDToHash, err := v.buildBlobIndexes(repos)
if err != nil {
return nil, fmt.Errorf("fetching blob index: %w", err)
}
blobIDToHash := make(map[string]string, len(blobsByID))
blobByHash := make(map[string]*database.Blob, len(blobsByID))
for id, blob := range blobsByID {
hash := blob.Hash.String()
blobIDToHash[id] = hash
blobByHash[hash] = blob
return nil, err
}
plan, err := newRestorePlan(v.ctx, repos, files, chunkToBlobMap, blobIDToHash)
@@ -235,20 +269,9 @@ func (v *Vaultik) restoreAllFiles(
return nil, fmt.Errorf("building restore plan: %w", err)
}
// Index files by ID so the loop can look them up by the IDs the
// plan hands back.
filesByID := make(map[types.FileID]*database.File, len(files))
for _, f := range files {
filesByID[f.ID] = f
}
filesByID, totalBytesExpected := indexRestoreFiles(files)
// Calculate total bytes expected for percentage / ETA arithmetic.
var totalBytesExpected int64
for _, file := range files {
totalBytesExpected += file.Size
}
v.UI.Begin("Restoring %s files (%s) to %s.",
v.UI.Beginf("Restoring %s files (%s) to %s.",
v.UI.Count(len(files)),
v.UI.Size(totalBytesExpected),
v.UI.Path(opts.TargetDir))
@@ -268,52 +291,41 @@ func (v *Vaultik) restoreAllFiles(
runningAsRoot: os.Geteuid() == 0,
}
err = v.runRestoreLoop(session, plan, filesByID, totalBytesExpected)
if err != nil {
return nil, err
}
return result, nil
}
// runRestoreLoop drains the restore plan: restore files as their blobs
// become available, download the next blob set when nothing is ready,
// and emit periodic progress.
func (v *Vaultik) runRestoreLoop(
session *restoreSession, plan *restorePlan,
filesByID map[types.FileID]*database.File, totalBytesExpected int64,
) error {
// Periodic progress output, matching the snapshot create cadence.
startTime := time.Now()
lastStatusTime := startTime
const statusInterval = 15 * time.Second
processed := 0
totalFiles := len(filesByID)
for plan.hasPending() {
if v.ctx.Err() != nil {
return nil, v.ctx.Err()
return v.ctx.Err()
}
fileID, ready := plan.popReady()
if !ready {
// No file is fully cache-served. First free any blobs
// whose file sets are exhausted — without this, the
// blob whose last file we just finished would still be
// cached when we Put the next one, briefly pushing
// peak occupancy from 1 to 2.
sweeper.sweep()
// Pick the pending file with the smallest uncached
// blob set and download its blobs. After each blob
// lands, the plan moves any pending file whose set
// just emptied onto the ready queue.
next := plan.pickNextDownload()
if next.IsZero() {
break
downloaded, err := session.downloadNextBlobSet(plan)
if err != nil {
return err
}
for _, hash := range plan.blobsNeeded(next) {
blob, ok := blobByHash[hash]
if !ok {
return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16])
}
err := session.downloadBlobToCache(hash, blob.CompressedSize)
if err != nil {
return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err)
}
result.BlobsDownloaded++
result.BytesDownloaded += blob.CompressedSize
plan.markBlobCached(hash)
if !downloaded {
break
}
continue
@@ -323,54 +335,173 @@ func (v *Vaultik) restoreAllFiles(
err := session.restoreFile(file)
if err != nil {
log.Error("Failed to restore file", "path", file.Path, "error", err)
if !opts.SkipErrors {
return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err)
err = v.handleRestoreFileError(
plan, session.opts, session.result, file, fileID, err)
if err != nil {
return err
}
v.UI.Error("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err)
result.FilesFailed++
result.FailedFiles = append(result.FailedFiles, file.Path.String())
plan.finishFile(fileID)
continue
}
// Record the file as restored so the sweeper can free blobs
// once all referencing files are done, and drop it from the
// plan's indexes so future picks ignore it.
sweeper.fileRestored(fileID.String())
session.sweeper.fileRestored(fileID.String())
plan.finishFile(fileID)
processed++
if time.Since(lastStatusTime) >= statusInterval {
v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime)
lastStatusTime = time.Now()
}
// Structured progress log for --verbose / JSON consumers.
if processed%100 == 0 || processed == len(files) {
log.Info("Restore progress",
"files", fmt.Sprintf("%d/%d", processed, len(files)),
"bytes", humanize.Bytes(uint64(result.BytesRestored)),
)
}
v.restoreProgressTick(processed, totalFiles,
session.result.BytesRestored,
totalBytesExpected, startTime, &lastStatusTime)
}
return result, nil
return nil
}
// downloadNextBlobSet is invoked when no file is fully cache-served.
// It first frees any blobs whose file sets are exhausted — without
// this, the blob whose last file we just finished would still be
// cached when we Put the next one, briefly pushing peak occupancy from
// 1 to 2. It then picks the pending file with the smallest uncached
// blob set and downloads its blobs; after each blob lands, the plan
// moves any pending file whose set just emptied onto the ready queue.
// Returns false when nothing is pending download (the caller stops).
func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
s.sweeper.sweep()
next := plan.pickNextDownload()
if next.IsZero() {
return false, nil
}
for _, hash := range plan.blobsNeeded(next) {
blob, ok := s.blobByHash[hash]
if !ok {
return false, fmt.Errorf("%w: %s", errBlobMissingFromIndex, hash[:16])
}
err := s.downloadBlobToCache(hash, blob.CompressedSize)
if err != nil {
return false, fmt.Errorf("downloading blob %s: %w", hash[:16], err)
}
s.result.BlobsDownloaded++
s.result.BytesDownloaded += blob.CompressedSize
plan.markBlobCached(hash)
}
return true, nil
}
// indexRestoreFiles indexes files by ID for plan lookups and sums the
// expected byte total for percentage / ETA arithmetic.
func indexRestoreFiles(
files []*database.File,
) (map[types.FileID]*database.File, int64) {
filesByID := make(map[types.FileID]*database.File, len(files))
var totalBytesExpected int64
for _, f := range files {
filesByID[f.ID] = f
totalBytesExpected += f.Size
}
return filesByID, totalBytesExpected
}
// buildBlobIndexes pre-fetches every blob row once so chunk extraction
// can map a blob_id to its hash without a DB round-trip per chunk.
func (v *Vaultik) buildBlobIndexes(
repos *database.Repositories,
) (map[string]*database.Blob, map[string]string, error) {
blobsByID, err := repos.Blobs.GetAll(v.ctx)
if err != nil {
return nil, nil, fmt.Errorf("fetching blob index: %w", err)
}
blobIDToHash := make(map[string]string, len(blobsByID))
blobByHash := make(map[string]*database.Blob, len(blobsByID))
for id, blob := range blobsByID {
hash := blob.Hash.String()
blobIDToHash[id] = hash
blobByHash[hash] = blob
}
return blobByHash, blobIDToHash, nil
}
// restoreProgressTick emits the periodic UI status line and structured
// progress log during the restore loop.
func (v *Vaultik) restoreProgressTick(
processed, totalFiles int, bytesRestored, totalBytesExpected int64,
startTime time.Time, lastStatusTime *time.Time,
) {
if time.Since(*lastStatusTime) >= restoreStatusInterval {
v.printRestoreProgress(
processed, totalFiles, bytesRestored,
totalBytesExpected, startTime)
*lastStatusTime = time.Now()
}
// Structured progress log for --verbose / JSON consumers.
if processed%progressLogEvery == 0 || processed == totalFiles {
log.Info("Restore progress",
"files", fmt.Sprintf("%d/%d", processed, totalFiles),
"bytes", ubytes(bytesRestored),
)
}
}
// handleRestoreFileError records a per-file restore failure: fatal unless
// --skip-errors is set, in which case the file is counted as failed and
// dropped from the plan.
func (v *Vaultik) handleRestoreFileError(
plan *restorePlan, opts *RestoreOptions, result *RestoreResult,
file *database.File, fileID types.FileID, err error,
) error {
log.Error("Failed to restore file", "path", file.Path, "error", err)
if !opts.SkipErrors {
return fmt.Errorf(
"restoring %s: %w (pass --skip-errors to continue past "+
"restore failures)", file.Path, err)
}
v.UI.Errorf("Failed to restore %s: %v. Skipping (--skip-errors).",
v.UI.Path(file.Path.String()), err)
result.FilesFailed++
result.FailedFiles = append(result.FailedFiles, file.Path.String())
plan.finishFile(fileID)
return nil
}
// printRestoreProgress emits a periodic restore-phase status line via
// the UI writer, mirroring scanner.printProcessingProgress so the two
// long-running commands have the same on-screen rhythm.
func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) {
func (v *Vaultik) printRestoreProgress(
filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time,
) {
v.printPhaseProgress("Restore", "restore",
filesDone, totalFiles, bytesDone, totalBytes, startTime)
}
// printPhaseProgress emits a periodic status line for a long-running
// phase (restore or verify) so user-facing pacing is uniform.
func (v *Vaultik) printPhaseProgress(
title, phase string,
filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time,
) {
elapsed := time.Since(startTime)
pct := float64(bytesDone) / float64(totalBytes) * 100
pct := float64(bytesDone) / float64(totalBytes) * percentScale
byteRate := float64(bytesDone) / elapsed.Seconds()
fileRate := float64(filesDone) / elapsed.Seconds()
@@ -382,7 +513,9 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot
}
if eta > 0 {
v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s, restore ETA: %s (est remain %s).",
v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+
"%s elapsed: %s, %s ETA: %s (est remain %s).",
title,
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
v.UI.Percent(pct),
@@ -390,14 +523,18 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot
v.UI.Size(totalBytes),
v.UI.Speed(byteRate),
fileRate,
phase,
v.UI.Duration(elapsed),
phase,
v.UI.Time(time.Now().Add(eta)),
v.UI.Duration(eta))
return
}
v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.",
v.UI.Progressf("%s: %s/%s files (%s), %s/%s, %s, %.0f files/sec, "+
"%s elapsed: %s.",
title,
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
v.UI.Percent(pct),
@@ -405,6 +542,7 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot
v.UI.Size(totalBytes),
v.UI.Speed(byteRate),
fileRate,
phase,
v.UI.Duration(elapsed))
}
@@ -421,17 +559,17 @@ func (v *Vaultik) handleRestoreVerification(
}
if result.FilesFailed > 0 {
v.UI.Error("Verification failed: %s files did not match expected checksums.",
v.UI.Errorf("Verification failed: %s files did not match expected checksums.",
v.UI.Count(result.FilesFailed))
for _, path := range result.FailedFiles {
v.UI.Detail("%s", v.UI.Path(path))
v.UI.Detailf("%s", v.UI.Path(path))
}
return fmt.Errorf("%d files failed verification", result.FilesFailed)
return fmt.Errorf("%d %w", result.FilesFailed, errFilesFailedVerify)
}
v.UI.Complete("Verified %s files (%s).",
v.UI.Completef("Verified %s files (%s).",
v.UI.Count(result.FilesVerified),
v.UI.Size(result.BytesVerified))
@@ -441,9 +579,12 @@ func (v *Vaultik) handleRestoreVerification(
// downloadSnapshotDB downloads and decrypts the snapshot metadata
// database. The snapshotID is the human ID; we hash it to the remote
// key for the storage path.
func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (*database.DB, error) {
func (v *Vaultik) downloadSnapshotDB(
snapshotID string, identity age.Identity,
) (*database.DB, error) {
// Download encrypted database from storage
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshot.RemoteSnapshotKey(snapshotID))
dbKey := fmt.Sprintf("metadata/%s/db.zst.age",
snapshot.RemoteSnapshotKey(snapshotID))
reader, err := v.Storage.Get(v.ctx, dbKey)
if err != nil {
@@ -457,7 +598,8 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
return nil, fmt.Errorf("reading encrypted data: %w", err)
}
log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData))))
log.Debug("Downloaded encrypted database",
"size", ubytes(int64(len(encryptedData))))
// Decrypt and decompress using blobgen.Reader
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
@@ -472,7 +614,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
return nil, fmt.Errorf("decrypting and decompressing: %w", err)
}
log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData))))
log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
// Create a temporary database file and write the binary SQLite data directly
tempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.db")
@@ -510,7 +652,9 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
}
// getFilesToRestore returns the list of files to restore based on path filters
func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Repositories, pathFilters []string) ([]*database.File, error) {
func (v *Vaultik) getFilesToRestore(
ctx context.Context, repos *database.Repositories, pathFilters []string,
) ([]*database.File, error) {
// If no filters, get all files
if len(pathFilters) == 0 {
return repos.Files.ListAll(ctx)
@@ -543,7 +687,9 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit
}
// buildChunkToBlobMap creates a mapping from chunk hash to blob information
func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) {
func (v *Vaultik) buildChunkToBlobMap(
ctx context.Context, repos *database.Repositories,
) (map[string]*database.BlobChunk, error) {
// Query all blob_chunks
query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks`
@@ -588,7 +734,7 @@ func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repos
// readable: restoreFile(file) instead of a ten-argument helper.
type restoreSession struct {
v *Vaultik
ctx context.Context
ctx context.Context //nolint:containedctx // per-restore state by design
repos *database.Repositories
opts *RestoreOptions
identity age.Identity
@@ -613,7 +759,7 @@ func (s *restoreSession) restoreFile(file *database.File) error {
parentDir := filepath.Dir(targetPath)
err := s.v.Fs.MkdirAll(parentDir, 0755)
err := s.v.Fs.MkdirAll(parentDir, restoreDirMode)
if err != nil {
return fmt.Errorf("creating parent directory: %w", err)
}
@@ -640,7 +786,8 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string)
return fmt.Errorf("creating symlink: %w", err)
}
} else {
log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget)
log.Debug("Symlink creation not supported on this filesystem",
"path", file.Path, "target", file.LinkTarget)
}
s.result.FilesRestored++
@@ -652,34 +799,51 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string)
// restoreDirectory restores a directory with its permissions, mtime,
// and (on real filesystems, with sufficient privileges) ownership.
func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error {
func (s *restoreSession) restoreDirectory(
file *database.File, targetPath string,
) error {
err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode))
if err != nil {
return fmt.Errorf("creating directory: %w", err)
}
err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode))
s.applyFileMetadata(file, targetPath)
s.result.FilesRestored++
return nil
}
// applyFileMetadata applies stored permissions, ownership (when running
// as root on a real filesystem), and mtime to a restored path. Failures
// are logged at debug level and do not abort the restore.
func (s *restoreSession) applyFileMetadata(file *database.File, targetPath string) {
err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode))
if err != nil {
log.Debug("Failed to set directory permissions", "path", targetPath, "error", err)
log.Debug("Failed to set permissions", "path", targetPath, "error", err)
}
if s.runningAsRoot {
if _, ok := s.v.Fs.(*afero.OsFs); ok {
err := os.Chown(targetPath, int(file.UID), int(file.GID))
err = os.Chown(targetPath, int(file.UID), int(file.GID))
if err != nil {
log.Debug("Failed to set directory ownership", "path", targetPath, "error", err)
log.Debug("Failed to set ownership", "path", targetPath, "error", err)
}
}
}
err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime)
if err != nil {
log.Debug("Failed to set directory mtime", "path", targetPath, "error", err)
log.Debug("Failed to set mtime", "path", targetPath, "error", err)
}
}
s.result.FilesRestored++
return nil
// chunkWriteTimings accumulates per-phase durations while writing a
// file's chunks out of the blob cache. Debug instrumentation only.
type chunkWriteTimings struct {
readAt time.Duration
write time.Duration
sweeper time.Duration
}
// restoreRegularFile reconstructs a regular file by reading chunks
@@ -687,7 +851,9 @@ func (s *restoreSession) restoreDirectory(file *database.File, targetPath string
// method runs is that every blob this file needs is already in the
// disk cache — the planner guarantees that by only marking files
// "ready" once their full blob set is on disk.
func (s *restoreSession) restoreRegularFile(file *database.File, targetPath string) error {
func (s *restoreSession) restoreRegularFile(
file *database.File, targetPath string,
) error {
fileStart := time.Now()
t0 := time.Now()
@@ -709,49 +875,9 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
defer func() { _ = outFile.Close() }()
var (
readAtDur time.Duration
writeDur time.Duration
sweeperDur time.Duration
bytesWritten int64
)
for _, fc := range fileChunks {
chunkHashStr := fc.ChunkHash.String()
blobChunk, ok := s.chunkToBlobMap[chunkHashStr]
if !ok {
return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16])
}
blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()]
if !ok {
return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID)
}
t0 = time.Now()
chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length)
readAtDur += time.Since(t0)
if err != nil {
return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err)
}
t0 = time.Now()
n, err := outFile.Write(chunkData)
writeDur += time.Since(t0)
if err != nil {
return fmt.Errorf("writing chunk: %w", err)
}
bytesWritten += int64(n)
t0 = time.Now()
s.sweeper.chunkRestored(int64(n))
sweeperDur += time.Since(t0)
bytesWritten, timings, err := s.writeFileChunks(outFile, fileChunks)
if err != nil {
return err
}
log.Debug("Restored regular file (timings)",
@@ -761,9 +887,9 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
"ms_total", time.Since(fileStart).Milliseconds(),
"ms_file_chunks_query", fileChunksQueryDur.Milliseconds(),
"ms_create", createDur.Milliseconds(),
"ms_readat", readAtDur.Milliseconds(),
"ms_writes", writeDur.Milliseconds(),
"ms_sweeper", sweeperDur.Milliseconds(),
"ms_readat", timings.readAt.Milliseconds(),
"ms_writes", timings.write.Milliseconds(),
"ms_sweeper", timings.sweeper.Milliseconds(),
)
err = outFile.Close()
@@ -771,40 +897,82 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
return fmt.Errorf("closing output file: %w", err)
}
err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode))
if err != nil {
log.Debug("Failed to set file permissions", "path", targetPath, "error", err)
}
if s.runningAsRoot {
if _, ok := s.v.Fs.(*afero.OsFs); ok {
err := os.Chown(targetPath, int(file.UID), int(file.GID))
if err != nil {
log.Debug("Failed to set file ownership", "path", targetPath, "error", err)
}
}
}
err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime)
if err != nil {
log.Debug("Failed to set file mtime", "path", targetPath, "error", err)
}
s.applyFileMetadata(file, targetPath)
s.result.FilesRestored++
s.result.BytesRestored += bytesWritten
log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten)))
log.Debug("Restored file", "path", file.Path, "size", ubytes(bytesWritten))
return nil
}
// writeFileChunks streams each of the file's chunks from the blob disk
// cache into outFile, crediting restored bytes to the sweeper as it
// goes. Returns the bytes written plus per-phase timing accumulators.
func (s *restoreSession) writeFileChunks(
outFile afero.File, fileChunks []*database.FileChunk,
) (int64, chunkWriteTimings, error) {
var (
timings chunkWriteTimings
bytesWritten int64
)
for _, fc := range fileChunks {
chunkHashStr := fc.ChunkHash.String()
blobChunk, ok := s.chunkToBlobMap[chunkHashStr]
if !ok {
return bytesWritten, timings, fmt.Errorf(
"%w: %s", errChunkNotInAnyBlob, chunkHashStr[:16])
}
blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()]
if !ok {
return bytesWritten, timings, fmt.Errorf(
"%w: %s", errBlobIDNotInHashIndex, blobChunk.BlobID)
}
t0 := time.Now()
chunkData, err := s.blobCache.ReadAt(
blobHash, blobChunk.Offset, blobChunk.Length)
timings.readAt += time.Since(t0)
if err != nil {
return bytesWritten, timings, fmt.Errorf(
"reading chunk %s from cached blob %s: %w",
fc.ChunkHash[:16], blobHash[:16], err)
}
t0 = time.Now()
n, err := outFile.Write(chunkData)
timings.write += time.Since(t0)
if err != nil {
return bytesWritten, timings, fmt.Errorf("writing chunk: %w", err)
}
bytesWritten += int64(n)
t0 = time.Now()
s.sweeper.chunkRestored(int64(n))
timings.sweeper += time.Since(t0)
}
return bytesWritten, timings, nil
}
// downloadBlobToCache streams a blob from remote storage straight into
// the disk cache, decrypting and decompressing on the fly. The
// plaintext never lives fully in memory — io.Copy through
// blobDiskCache.PutFromReader uses a 32 KiB buffer regardless of blob
// size, which is what makes multi-GB blobs tractable on machines with
// less RAM than the blob.
func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64) error {
func (s *restoreSession) downloadBlobToCache(
blobHash string, expectedSize int64,
) error {
start := time.Now()
t0 := time.Now()
@@ -840,7 +1008,8 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64
return nil
}
// verifyRestoredFiles verifies that all restored files match their expected chunk hashes
// verifyRestoredFiles verifies that all restored files match their
// expected chunk hashes.
func (v *Vaultik) verifyRestoredFiles(
ctx context.Context,
repos *database.Repositories,
@@ -870,17 +1039,15 @@ func (v *Vaultik) verifyRestoredFiles(
log.Info("Verifying restored files",
"files", len(regularFiles),
"bytes", humanize.Bytes(uint64(totalBytes)),
"bytes", ubytes(totalBytes),
)
v.UI.Begin("Verifying %s files (%s).",
v.UI.Beginf("Verifying %s files (%s).",
v.UI.Count(len(regularFiles)),
v.UI.Size(totalBytes))
startTime := time.Now()
lastStatusTime := startTime
const statusInterval = 15 * time.Second
var bytesProcessed int64
for i, file := range regularFiles {
@@ -903,8 +1070,9 @@ func (v *Vaultik) verifyRestoredFiles(
bytesProcessed += file.Size
if time.Since(lastStatusTime) >= statusInterval {
v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime)
if time.Since(lastStatusTime) >= restoreStatusInterval {
v.printVerifyProgress(
i+1, len(regularFiles), bytesProcessed, totalBytes, startTime)
lastStatusTime = time.Now()
}
@@ -912,7 +1080,7 @@ func (v *Vaultik) verifyRestoredFiles(
log.Info("Verification complete",
"files_verified", result.FilesVerified,
"bytes_verified", humanize.Bytes(uint64(result.BytesVerified)),
"bytes_verified", ubytes(result.BytesVerified),
"files_failed", result.FilesFailed,
)
@@ -922,44 +1090,11 @@ func (v *Vaultik) verifyRestoredFiles(
// printVerifyProgress emits a periodic verify-phase status line. Same
// shape as the restore progress line so user-facing pacing is uniform
// across the two phases.
func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time) {
elapsed := time.Since(startTime)
pct := float64(bytesDone) / float64(totalBytes) * 100
byteRate := float64(bytesDone) / elapsed.Seconds()
fileRate := float64(filesDone) / elapsed.Seconds()
remainingBytes := totalBytes - bytesDone
var eta time.Duration
if byteRate > 0 && remainingBytes > 0 {
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
}
if eta > 0 {
v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s, verify ETA: %s (est remain %s).",
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
v.UI.Percent(pct),
v.UI.Size(bytesDone),
v.UI.Size(totalBytes),
v.UI.Speed(byteRate),
fileRate,
v.UI.Duration(elapsed),
v.UI.Time(time.Now().Add(eta)),
v.UI.Duration(eta))
return
}
v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.",
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
v.UI.Percent(pct),
v.UI.Size(bytesDone),
v.UI.Size(totalBytes),
v.UI.Speed(byteRate),
fileRate,
v.UI.Duration(elapsed))
func (v *Vaultik) printVerifyProgress(
filesDone, totalFiles int, bytesDone, totalBytes int64, startTime time.Time,
) {
v.printPhaseProgress("Verify", "verify",
filesDone, totalFiles, bytesDone, totalBytes, startTime)
}
// verifyFile verifies a single restored file by checking its chunk hashes
@@ -989,7 +1124,8 @@ func (v *Vaultik) verifyFile(
// Get chunk size from database
chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String())
if err != nil {
return bytesVerified, fmt.Errorf("getting chunk %s: %w", fc.ChunkHash.String()[:16], err)
return bytesVerified, fmt.Errorf("getting chunk %s: %w",
fc.ChunkHash.String()[:16], err)
}
// Read chunk data from file
@@ -1001,7 +1137,8 @@ func (v *Vaultik) verifyFile(
}
if int64(n) != chunk.Size {
return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n)
return bytesVerified, fmt.Errorf("%w: expected %d bytes, got %d",
errShortChunkRead, chunk.Size, n)
}
// Calculate hash and compare
@@ -1010,14 +1147,15 @@ func (v *Vaultik) verifyFile(
expectedHash := fc.ChunkHash.String()
if actualHash != expectedHash {
return bytesVerified, fmt.Errorf("chunk %d hash mismatch: expected %s, got %s",
fc.Idx, expectedHash[:16], actualHash[:16])
return bytesVerified, fmt.Errorf("%w: chunk %d: expected %s, got %s",
errChunkHashMismatch, fc.Idx, expectedHash[:16], actualHash[:16])
}
bytesVerified += int64(n)
}
log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks))
log.Debug("File verified",
"path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks))
return bytesVerified, nil
}