sneak/vaultik
1
0
Fork 0
Code Issues 14 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Apply linter autofixes: internal/vaultik (refs #61)

Browse Source
This commit is contained in:
sneak
2026-08-07 16:53:23 +00:00
parent 0296e26210
commit 82eb352eb5
21 changed files with 730 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/vaultik/restore.go
+124 -16
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"math"
@@ -62,16 +63,20 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
// Step 1: Download and decrypt the snapshot metadata database
log.Info("Downloading snapshot metadata...")
tempDB, err := v.downloadSnapshotDB(opts.SnapshotID, identity)
if err != nil {
return fmt.Errorf("downloading snapshot database: %w", err)
}
defer func() {
if err := tempDB.Close(); err != nil {
err := tempDB.Close()
if err != nil {
log.Debug("Failed to close temp database", "error", err)
}
// Clean up temp file
if err := v.Fs.Remove(tempDB.Path()); err != nil {
err = v.Fs.Remove(tempDB.Path())
if err != nil {
log.Debug("Failed to remove temp database", "error", err)
}
}()
@@ -87,6 +92,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
if len(files) == 0 {
log.Warn("No files found to restore")
v.UI.Warning("No files found to restore.")
return nil
}
@@ -132,6 +138,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
if result.FilesFailed > 0 {
v.UI.Warning("%d file(s) failed to restore:", result.FilesFailed)
for _, path := range result.FailedFiles {
v.UI.Detail("%s", v.UI.Path(path))
}
@@ -139,7 +146,8 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
// Run verification if requested
if opts.Verify {
if err := v.handleRestoreVerification(repos, files, opts, result); err != nil {
err := v.handleRestoreVerification(repos, files, opts, result)
if err != nil {
return err
}
}
@@ -154,13 +162,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
// prepareRestoreIdentity validates that an age secret key is configured and parses it
func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) {
if v.Config.AgeSecretKey == "" {
return nil, fmt.Errorf("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'")
return nil, errors.New("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'")
}
identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey)
if err != nil {
return nil, fmt.Errorf("parsing age secret key: %w", err)
}
return identity, nil
}
@@ -187,13 +196,16 @@ func (v *Vaultik) restoreAllFiles(
if err != nil {
return nil, fmt.Errorf("creating blob cache: %w", err)
}
if v.restoreCacheObserver != nil {
v.restoreCacheObserver(blobCache)
}
defer func() {
if v.restoreCacheObserver != nil {
v.restoreCacheObserver(blobCache)
}
_ = blobCache.Close()
}()
@@ -208,7 +220,9 @@ func (v *Vaultik) restoreAllFiles(
if err != nil {
return nil, fmt.Errorf("fetching blob index: %w", err)
}
blobIDToHash := make(map[string]string, len(blobsByID))
blobByHash := make(map[string]*database.Blob, len(blobsByID))
for id, blob := range blobsByID {
hash := blob.Hash.String()
@@ -257,9 +271,11 @@ func (v *Vaultik) restoreAllFiles(
// Periodic progress output, matching the snapshot create cadence.
startTime := time.Now()
lastStatusTime := startTime
const statusInterval = 15 * time.Second
processed := 0
for plan.hasPending() {
if v.ctx.Err() != nil {
return nil, v.ctx.Err()
@@ -282,31 +298,44 @@ func (v *Vaultik) restoreAllFiles(
if next.IsZero() {
break
}
for _, hash := range plan.blobsNeeded(next) {
blob, ok := blobByHash[hash]
if !ok {
return nil, fmt.Errorf("blob hash %s missing from blob index", hash[:16])
}
if err := session.downloadBlobToCache(hash, blob.CompressedSize); err != nil {
err := session.downloadBlobToCache(hash, blob.CompressedSize)
if err != nil {
return nil, fmt.Errorf("downloading blob %s: %w", hash[:16], err)
}
result.BlobsDownloaded++
result.BytesDownloaded += blob.CompressedSize
plan.markBlobCached(hash)
}
continue
}
file := filesByID[fileID]
if err := session.restoreFile(file); err != nil {
err := session.restoreFile(file)
if err != nil {
log.Error("Failed to restore file", "path", file.Path, "error", err)
if !opts.SkipErrors {
return nil, fmt.Errorf("restoring %s: %w (pass --skip-errors to continue past restore failures)", file.Path, err)
}
v.UI.Error("Failed to restore %s: %v. Skipping (--skip-errors).", v.UI.Path(file.Path.String()), err)
result.FilesFailed++
result.FailedFiles = append(result.FailedFiles, file.Path.String())
plan.finishFile(fileID)
continue
}
@@ -315,10 +344,12 @@ func (v *Vaultik) restoreAllFiles(
// plan's indexes so future picks ignore it.
sweeper.fileRestored(fileID.String())
plan.finishFile(fileID)
processed++
if time.Since(lastStatusTime) >= statusInterval {
v.printRestoreProgress(processed, len(files), result.BytesRestored, totalBytesExpected, startTime)
lastStatusTime = time.Now()
}
@@ -344,6 +375,7 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot
fileRate := float64(filesDone) / elapsed.Seconds()
remainingBytes := totalBytes - bytesDone
var eta time.Duration
if byteRate > 0 && remainingBytes > 0 {
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
@@ -361,8 +393,10 @@ func (v *Vaultik) printRestoreProgress(filesDone, totalFiles int, bytesDone, tot
v.UI.Duration(elapsed),
v.UI.Time(time.Now().Add(eta)),
v.UI.Duration(eta))
return
}
v.UI.Progress("Restore: %s/%s files (%s), %s/%s, %s, %.0f files/sec, restore elapsed: %s.",
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
@@ -381,22 +415,26 @@ func (v *Vaultik) handleRestoreVerification(
opts *RestoreOptions,
result *RestoreResult,
) error {
if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil {
err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result)
if err != nil {
return fmt.Errorf("verification failed: %w", err)
}
if result.FilesFailed > 0 {
v.UI.Error("Verification failed: %s files did not match expected checksums.",
v.UI.Count(result.FilesFailed))
for _, path := range result.FailedFiles {
v.UI.Detail("%s", v.UI.Path(path))
}
return fmt.Errorf("%d files failed verification", result.FilesFailed)
}
v.UI.Complete("Verified %s files (%s).",
v.UI.Count(result.FilesVerified),
v.UI.Size(result.BytesVerified))
return nil
}
@@ -418,6 +456,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
if err != nil {
return nil, fmt.Errorf("reading encrypted data: %w", err)
}
log.Debug("Downloaded encrypted database", "size", humanize.Bytes(uint64(len(encryptedData))))
// Decrypt and decompress using blobgen.Reader
@@ -432,6 +471,7 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
if err != nil {
return nil, fmt.Errorf("decrypting and decompressing: %w", err)
}
log.Debug("Decrypted database", "size", humanize.Bytes(uint64(len(dbData))))
// Create a temporary database file and write the binary SQLite data directly
@@ -439,18 +479,23 @@ func (v *Vaultik) downloadSnapshotDB(snapshotID string, identity age.Identity) (
if err != nil {
return nil, fmt.Errorf("creating temp file: %w", err)
}
tempPath := tempFile.Name()
// Write the binary SQLite database directly
if _, err := tempFile.Write(dbData); err != nil {
_ = tempFile.Close()
_ = v.Fs.Remove(tempPath)
return nil, fmt.Errorf("writing database file: %w", err)
}
if err := tempFile.Close(); err != nil {
_ = v.Fs.Remove(tempPath)
return nil, fmt.Errorf("closing temp file: %w", err)
}
log.Debug("Created restore database", "path", tempPath)
// Open the database
@@ -471,6 +516,7 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit
// Get files matching the path filters
var result []*database.File
seen := make(map[string]bool)
for _, filter := range pathFilters {
@@ -498,23 +544,30 @@ func (v *Vaultik) getFilesToRestore(ctx context.Context, repos *database.Reposit
func (v *Vaultik) buildChunkToBlobMap(ctx context.Context, repos *database.Repositories) (map[string]*database.BlobChunk, error) {
// Query all blob_chunks
query := `SELECT blob_id, chunk_hash, offset, length FROM blob_chunks`
rows, err := repos.DB().Conn().QueryContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("querying blob_chunks: %w", err)
}
defer func() { _ = rows.Close() }()
result := make(map[string]*database.BlobChunk)
for rows.Next() {
var bc database.BlobChunk
var blobIDStr, chunkHashStr string
var (
bc database.BlobChunk
blobIDStr, chunkHashStr string
)
if err := rows.Scan(&blobIDStr, &chunkHashStr, &bc.Offset, &bc.Length); err != nil {
return nil, fmt.Errorf("scanning blob_chunk: %w", err)
}
blobID, err := types.ParseBlobID(blobIDStr)
if err != nil {
return nil, fmt.Errorf("parsing blob ID: %w", err)
}
bc.BlobID = blobID
bc.ChunkHash = types.ChunkHash(chunkHashStr)
result[chunkHashStr] = &bc
@@ -553,16 +606,22 @@ type restoreSession struct {
// restoreFile dispatches to the right per-kind restorer.
func (s *restoreSession) restoreFile(file *database.File) error {
targetPath := filepath.Join(s.opts.TargetDir, file.Path.String())
parentDir := filepath.Dir(targetPath)
if err := s.v.Fs.MkdirAll(parentDir, 0755); err != nil {
err := s.v.Fs.MkdirAll(parentDir, 0755)
if err != nil {
return fmt.Errorf("creating parent directory: %w", err)
}
if file.IsSymlink() {
return s.restoreSymlink(file, targetPath)
}
if file.Mode&uint32(os.ModeDir) != 0 {
return s.restoreDirectory(file, targetPath)
}
return s.restoreRegularFile(file, targetPath)
}
@@ -572,37 +631,50 @@ func (s *restoreSession) restoreSymlink(file *database.File, targetPath string)
// afero.MemMapFs doesn't support symlinks, so route real-FS
// symlinks through os.
if _, ok := s.v.Fs.(*afero.OsFs); ok {
if err := os.Symlink(file.LinkTarget.String(), targetPath); err != nil {
err := os.Symlink(file.LinkTarget.String(), targetPath)
if err != nil {
return fmt.Errorf("creating symlink: %w", err)
}
} else {
log.Debug("Symlink creation not supported on this filesystem", "path", file.Path, "target", file.LinkTarget)
}
s.result.FilesRestored++
log.Debug("Restored symlink", "path", file.Path, "target", file.LinkTarget)
return nil
}
// restoreDirectory restores a directory with its permissions, mtime,
// and (on real filesystems, with sufficient privileges) ownership.
func (s *restoreSession) restoreDirectory(file *database.File, targetPath string) error {
if err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode)); err != nil {
err := s.v.Fs.MkdirAll(targetPath, os.FileMode(file.Mode))
if err != nil {
return fmt.Errorf("creating directory: %w", err)
}
if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil {
err = s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode))
if err != nil {
log.Debug("Failed to set directory permissions", "path", targetPath, "error", err)
}
if s.runningAsRoot {
if _, ok := s.v.Fs.(*afero.OsFs); ok {
if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil {
err := os.Chown(targetPath, int(file.UID), int(file.GID))
if err != nil {
log.Debug("Failed to set directory ownership", "path", targetPath, "error", err)
}
}
}
if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil {
err = s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime)
if err != nil {
log.Debug("Failed to set directory mtime", "path", targetPath, "error", err)
}
s.result.FilesRestored++
return nil
}
@@ -617,16 +689,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
t0 := time.Now()
fileChunks, err := s.repos.FileChunks.GetByFileID(s.ctx, file.ID)
fileChunksQueryDur := time.Since(t0)
if err != nil {
return fmt.Errorf("getting file chunks: %w", err)
}
t0 = time.Now()
outFile, err := s.v.Fs.Create(targetPath)
createDur := time.Since(t0)
if err != nil {
return fmt.Errorf("creating output file: %w", err)
}
defer func() { _ = outFile.Close() }()
var (
@@ -638,10 +714,12 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
for _, fc := range fileChunks {
chunkHashStr := fc.ChunkHash.String()
blobChunk, ok := s.chunkToBlobMap[chunkHashStr]
if !ok {
return fmt.Errorf("chunk %s not found in any blob", chunkHashStr[:16])
}
blobHash, ok := s.blobIDToHash[blobChunk.BlobID.String()]
if !ok {
return fmt.Errorf("blob id %s missing from hash index", blobChunk.BlobID)
@@ -650,6 +728,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
t0 = time.Now()
chunkData, err := s.blobCache.ReadAt(blobHash, blobChunk.Offset, blobChunk.Length)
readAtDur += time.Since(t0)
if err != nil {
return fmt.Errorf("reading chunk %s from cached blob %s: %w", fc.ChunkHash[:16], blobHash[:16], err)
}
@@ -657,13 +736,17 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
t0 = time.Now()
n, err := outFile.Write(chunkData)
writeDur += time.Since(t0)
if err != nil {
return fmt.Errorf("writing chunk: %w", err)
}
bytesWritten += int64(n)
t0 = time.Now()
s.sweeper.chunkRestored(int64(n))
sweeperDur += time.Since(t0)
}
@@ -682,16 +765,20 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
if err := outFile.Close(); err != nil {
return fmt.Errorf("closing output file: %w", err)
}
if err := s.v.Fs.Chmod(targetPath, os.FileMode(file.Mode)); err != nil {
log.Debug("Failed to set file permissions", "path", targetPath, "error", err)
}
if s.runningAsRoot {
if _, ok := s.v.Fs.(*afero.OsFs); ok {
if err := os.Chown(targetPath, int(file.UID), int(file.GID)); err != nil {
err := os.Chown(targetPath, int(file.UID), int(file.GID))
if err != nil {
log.Debug("Failed to set file ownership", "path", targetPath, "error", err)
}
}
}
if err := s.v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil {
log.Debug("Failed to set file mtime", "path", targetPath, "error", err)
}
@@ -700,6 +787,7 @@ func (s *restoreSession) restoreRegularFile(file *database.File, targetPath stri
s.result.BytesRestored += bytesWritten
log.Debug("Restored file", "path", file.Path, "size", humanize.Bytes(uint64(bytesWritten)))
return nil
}
@@ -715,6 +803,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64
t0 := time.Now()
rc, err := s.v.FetchAndDecryptBlob(s.ctx, blobHash, expectedSize, s.identity)
fetchSetupDur := time.Since(t0)
if err != nil {
return err
}
@@ -723,9 +812,11 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64
written, copyErr := s.blobCache.PutFromReader(blobHash, rc)
streamDur := time.Since(t0)
closeErr := rc.Close()
if copyErr != nil {
return copyErr
}
if closeErr != nil {
return closeErr
}
@@ -738,6 +829,7 @@ func (s *restoreSession) downloadBlobToCache(blobHash string, expectedSize int64
"ms_fetch_setup", fetchSetupDur.Milliseconds(),
"ms_stream_decrypt_decompress", streamDur.Milliseconds(),
)
return nil
}
@@ -751,18 +843,21 @@ func (v *Vaultik) verifyRestoredFiles(
) error {
// Calculate total bytes to verify for progress bar
var totalBytes int64
regularFiles := make([]*database.File, 0, len(files))
for _, file := range files {
// Skip symlinks and directories - only verify regular files
if file.IsSymlink() || file.Mode&uint32(os.ModeDir) != 0 {
continue
}
regularFiles = append(regularFiles, file)
totalBytes += file.Size
}
if len(regularFiles) == 0 {
log.Info("No regular files to verify")
return nil
}
@@ -776,28 +871,34 @@ func (v *Vaultik) verifyRestoredFiles(
startTime := time.Now()
lastStatusTime := startTime
const statusInterval = 15 * time.Second
var bytesProcessed int64
for i, file := range regularFiles {
if ctx.Err() != nil {
return ctx.Err()
}
targetPath := filepath.Join(targetDir, file.Path.String())
bytesVerified, err := v.verifyFile(ctx, repos, file, targetPath)
if err != nil {
log.Error("File verification failed", "path", file.Path, "error", err)
result.FilesFailed++
result.FailedFiles = append(result.FailedFiles, file.Path.String())
} else {
result.FilesVerified++
result.BytesVerified += bytesVerified
}
bytesProcessed += file.Size
if time.Since(lastStatusTime) >= statusInterval {
v.printVerifyProgress(i+1, len(regularFiles), bytesProcessed, totalBytes, startTime)
lastStatusTime = time.Now()
}
}
@@ -821,6 +922,7 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota
fileRate := float64(filesDone) / elapsed.Seconds()
remainingBytes := totalBytes - bytesDone
var eta time.Duration
if byteRate > 0 && remainingBytes > 0 {
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
@@ -838,8 +940,10 @@ func (v *Vaultik) printVerifyProgress(filesDone, totalFiles int, bytesDone, tota
v.UI.Duration(elapsed),
v.UI.Time(time.Now().Add(eta)),
v.UI.Duration(eta))
return
}
v.UI.Progress("Verify: %s/%s files (%s), %s/%s, %s, %.0f files/sec, verify elapsed: %s.",
v.UI.Count(filesDone),
v.UI.Count(totalFiles),
@@ -873,6 +977,7 @@ func (v *Vaultik) verifyFile(
// Verify each chunk
var bytesVerified int64
for _, fc := range fileChunks {
// Get chunk size from database
chunk, err := repos.Chunks.GetByHash(ctx, fc.ChunkHash.String())
@@ -882,10 +987,12 @@ func (v *Vaultik) verifyFile(
// Read chunk data from file
chunkData := make([]byte, chunk.Size)
n, err := io.ReadFull(f, chunkData)
if err != nil {
return bytesVerified, fmt.Errorf("reading chunk data: %w", err)
}
if int64(n) != chunk.Size {
return bytesVerified, fmt.Errorf("short read: expected %d bytes, got %d", chunk.Size, n)
}
@@ -904,5 +1011,6 @@ func (v *Vaultik) verifyFile(
}
log.Debug("File verified", "path", file.Path, "bytes", bytesVerified, "chunks", len(fileChunks))
return bytesVerified, nil
}