Compare commits
9 Commits
feature/re
...
fix/ctime-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
25860c03a9 | ||
| c24e7e6360 | |||
| 7a5943958d | |||
| d8a51804d2 | |||
| 76f4421eb3 | |||
| 53ac868c5d | |||
| 8c4ea2b870 | |||
| 76e047bbb2 | |||
| cafb3d45b8 |
8
.dockerignore
Normal file
8
.dockerignore
Normal file
@@ -0,0 +1,8 @@
|
||||
.git
|
||||
.gitea
|
||||
*.md
|
||||
LICENSE
|
||||
vaultik
|
||||
coverage.out
|
||||
coverage.html
|
||||
.DS_Store
|
||||
14
.gitea/workflows/check.yml
Normal file
14
.gitea/workflows/check.yml
Normal file
@@ -0,0 +1,14 @@
|
||||
name: check
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# actions/checkout v4, 2024-09-16
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
|
||||
- name: Build and check
|
||||
run: docker build .
|
||||
61
Dockerfile
Normal file
61
Dockerfile
Normal file
@@ -0,0 +1,61 @@
|
||||
# Lint stage
|
||||
# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17
|
||||
FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint
|
||||
|
||||
RUN apk add --no-cache make build-base
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run formatting check and linter
|
||||
RUN make fmt-check
|
||||
RUN make lint
|
||||
|
||||
# Build stage
|
||||
# golang:1.26.1-alpine, 2026-03-17
|
||||
FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder
|
||||
|
||||
# Depend on lint stage passing
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
# Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests)
|
||||
RUN apk add --no-cache make build-base sqlite
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
# Copy go mod files first for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Run tests
|
||||
RUN make test
|
||||
|
||||
# Build with CGO enabled (required for mattn/go-sqlite3)
|
||||
RUN CGO_ENABLED=1 go build -ldflags "-X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=${VERSION}' -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-02-25
|
||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||
|
||||
RUN apk add --no-cache ca-certificates sqlite
|
||||
|
||||
# Copy binary from builder
|
||||
COPY --from=builder /vaultik /usr/local/bin/vaultik
|
||||
|
||||
# Create non-root user
|
||||
RUN adduser -D -H -s /sbin/nologin vaultik
|
||||
|
||||
USER vaultik
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/vaultik"]
|
||||
40
Makefile
40
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: test fmt lint build clean all
|
||||
.PHONY: test fmt lint fmt-check check build clean all docker hooks
|
||||
|
||||
# Version number
|
||||
VERSION := 0.0.1
|
||||
@@ -14,21 +14,12 @@ LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
all: vaultik
|
||||
|
||||
# Run tests
|
||||
test: lint fmt-check
|
||||
@echo "Running tests..."
|
||||
@if ! go test -v -timeout 10s ./... 2>&1; then \
|
||||
echo ""; \
|
||||
echo "TEST FAILURES DETECTED"; \
|
||||
echo "Run 'go test -v ./internal/database' to see database test details"; \
|
||||
exit 1; \
|
||||
fi
|
||||
test:
|
||||
go test -race -timeout 30s ./...
|
||||
|
||||
# Check if code is formatted
|
||||
# Check if code is formatted (read-only)
|
||||
fmt-check:
|
||||
@if [ -n "$$(go fmt ./...)" ]; then \
|
||||
echo "Error: Code is not formatted. Run 'make fmt' to fix."; \
|
||||
exit 1; \
|
||||
fi
|
||||
@test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1)
|
||||
|
||||
# Format code
|
||||
fmt:
|
||||
@@ -36,7 +27,7 @@ fmt:
|
||||
|
||||
# Run linter
|
||||
lint:
|
||||
golangci-lint run
|
||||
golangci-lint run ./...
|
||||
|
||||
# Build binary
|
||||
vaultik: internal/*/*.go cmd/vaultik/*.go
|
||||
@@ -47,11 +38,6 @@ clean:
|
||||
rm -f vaultik
|
||||
go clean
|
||||
|
||||
# Install dependencies
|
||||
deps:
|
||||
go mod download
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage:
|
||||
go test -v -coverprofile=coverage.out ./...
|
||||
@@ -67,3 +53,17 @@ local:
|
||||
|
||||
install: vaultik
|
||||
cp ./vaultik $(HOME)/bin/
|
||||
|
||||
# Run all checks (formatting, linting, tests) without modifying files
|
||||
check: fmt-check lint test
|
||||
|
||||
# Build Docker image
|
||||
docker:
|
||||
docker build -t vaultik .
|
||||
|
||||
# Install pre-commit hook
|
||||
hooks:
|
||||
@printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit
|
||||
@printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit
|
||||
@printf 'make check\n' >> .git/hooks/pre-commit
|
||||
@chmod +x .git/hooks/pre-commit
|
||||
|
||||
28
README.md
28
README.md
@@ -194,8 +194,9 @@ vaultik [--config <path>] store info
|
||||
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable with age private key
|
||||
* Optional path arguments to restore specific files/directories (default: all)
|
||||
* Downloads and decrypts metadata, fetches required blobs, reconstructs files
|
||||
* Preserves file permissions, timestamps, and ownership (ownership requires root)
|
||||
* Preserves file permissions, mtime, and ownership (ownership requires root)
|
||||
* Handles symlinks and directories
|
||||
* Note: ctime cannot be restored (see [platform notes](#platform-specific-ctime-semantics))
|
||||
|
||||
**prune**: Remove unreferenced blobs from remote storage
|
||||
* Scans all snapshots for referenced blobs
|
||||
@@ -247,11 +248,14 @@ Snapshot IDs follow the format `<hostname>_<snapshot-name>_<timestamp>` (e.g., `
|
||||
CREATE TABLE files (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
source_path TEXT NOT NULL DEFAULT '',
|
||||
mtime INTEGER NOT NULL,
|
||||
ctime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
mode INTEGER NOT NULL,
|
||||
uid INTEGER NOT NULL,
|
||||
gid INTEGER NOT NULL
|
||||
gid INTEGER NOT NULL,
|
||||
link_target TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE file_chunks (
|
||||
@@ -339,7 +343,25 @@ CREATE TABLE snapshot_blobs (
|
||||
1. For each file, get ordered chunk list from file_chunks
|
||||
1. Download required blobs, decrypt, decompress
|
||||
1. Extract chunks and reconstruct files
|
||||
1. Restore permissions, mtime, uid/gid
|
||||
1. Restore permissions, mtime, uid/gid (ctime cannot be restored — see platform notes above)
|
||||
|
||||
### platform-specific ctime semantics
|
||||
|
||||
The `ctime` field in the files table stores a platform-dependent timestamp:
|
||||
|
||||
* **macOS (Darwin)**: `ctime` is the file's **birth time** — when the file was
|
||||
first created on disk. This value never changes after file creation, even if
|
||||
the file's content or metadata is modified.
|
||||
|
||||
* **Linux**: `ctime` is the **inode change time** — the last time the file's
|
||||
metadata (permissions, ownership, link count, etc.) was modified. This is NOT
|
||||
the file creation time. Linux did not expose birth time (via `statx(2)`) until
|
||||
kernel 4.11, and Go's `syscall` package does not yet surface it.
|
||||
|
||||
**Restore limitation**: `ctime` cannot be restored on either platform. On Linux,
|
||||
the kernel manages the inode change time and userspace cannot set it. On macOS,
|
||||
there is no standard POSIX API to set birth time. The `ctime` value is preserved
|
||||
in the snapshot database for informational/forensic purposes only.
|
||||
|
||||
#### prune
|
||||
|
||||
|
||||
2
go.mod
2
go.mod
@@ -1,6 +1,6 @@
|
||||
module git.eeqj.de/sneak/vaultik
|
||||
|
||||
go 1.24.4
|
||||
go 1.26.1
|
||||
|
||||
require (
|
||||
filippo.io/age v1.2.1
|
||||
|
||||
64
internal/blobgen/compress_test.go
Normal file
64
internal/blobgen/compress_test.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package blobgen
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// testRecipient is a static age recipient for tests.
|
||||
const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
|
||||
|
||||
// TestCompressStreamNoDoubleClose is a regression test for issue #28.
|
||||
// It verifies that CompressStream does not panic or return an error due to
|
||||
// double-closing the underlying blobgen.Writer. Before the fix in PR #33,
|
||||
// the explicit Close() on the happy path combined with defer Close() would
|
||||
// cause a double close.
|
||||
func TestCompressStreamNoDoubleClose(t *testing.T) {
|
||||
input := []byte("regression test data for issue #28 double-close fix")
|
||||
var buf bytes.Buffer
|
||||
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient})
|
||||
require.NoError(t, err, "CompressStream should not return an error")
|
||||
assert.True(t, written > 0, "expected bytes written > 0")
|
||||
assert.NotEmpty(t, hash, "expected non-empty hash")
|
||||
assert.True(t, buf.Len() > 0, "expected non-empty output")
|
||||
}
|
||||
|
||||
// TestCompressStreamLargeInput exercises CompressStream with a larger payload
|
||||
// to ensure no double-close issues surface under heavier I/O.
|
||||
func TestCompressStreamLargeInput(t *testing.T) {
|
||||
data := make([]byte, 512*1024) // 512 KB
|
||||
_, err := rand.Read(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
var buf bytes.Buffer
|
||||
written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, written > 0)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressStreamEmptyInput verifies CompressStream handles empty input
|
||||
// without double-close issues.
|
||||
func TestCompressStreamEmptyInput(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
_, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, hash)
|
||||
}
|
||||
|
||||
// TestCompressDataNoDoubleClose mirrors the stream test for CompressData,
|
||||
// ensuring the explicit Close + error-path Close pattern is also safe.
|
||||
func TestCompressDataNoDoubleClose(t *testing.T) {
|
||||
input := []byte("CompressData regression test for double-close")
|
||||
result, err := CompressData(input, 3, []string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.CompressedSize > 0)
|
||||
assert.True(t, result.UncompressedSize == int64(len(input)))
|
||||
assert.NotEmpty(t, result.SHA256)
|
||||
}
|
||||
@@ -16,8 +16,8 @@ type File struct {
|
||||
ID types.FileID // UUID primary key
|
||||
Path types.FilePath // Absolute path of the file
|
||||
SourcePath types.SourcePath // The source directory this file came from (for restore path stripping)
|
||||
MTime time.Time
|
||||
CTime time.Time
|
||||
MTime time.Time // Last modification time
|
||||
CTime time.Time // Creation/change time (platform-specific: birth time on macOS, inode change time on Linux)
|
||||
Size int64
|
||||
Mode uint32
|
||||
UID uint32
|
||||
|
||||
@@ -345,7 +345,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
||||
Size: info.Size(),
|
||||
Mode: uint32(info.Mode()),
|
||||
MTime: info.ModTime(),
|
||||
CTime: info.ModTime(), // Use mtime as ctime for test
|
||||
CTime: fileCTime(info), // platform-specific: birth time on macOS, inode change time on Linux
|
||||
UID: 1000, // Default UID for test
|
||||
GID: 1000, // Default GID for test
|
||||
}
|
||||
|
||||
26
internal/snapshot/ctime_darwin.go
Normal file
26
internal/snapshot/ctime_darwin.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fileCTime returns the file creation time (birth time) on macOS.
|
||||
//
|
||||
// On macOS/Darwin, "ctime" refers to the file's birth time (when the file
|
||||
// was first created on disk). This is stored in the Birthtimespec field of
|
||||
// the syscall.Stat_t structure.
|
||||
//
|
||||
// This differs from Linux where "ctime" means inode change time (the last
|
||||
// time file metadata was modified). See ctime_linux.go for details.
|
||||
//
|
||||
// If the underlying stat information is unavailable (e.g. when using a
|
||||
// virtual filesystem like afero.MemMapFs), this falls back to mtime.
|
||||
func fileCTime(info os.FileInfo) time.Time {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return info.ModTime()
|
||||
}
|
||||
return time.Unix(stat.Birthtimespec.Sec, stat.Birthtimespec.Nsec).UTC()
|
||||
}
|
||||
28
internal/snapshot/ctime_linux.go
Normal file
28
internal/snapshot/ctime_linux.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fileCTime returns the inode change time on Linux.
|
||||
//
|
||||
// On Linux, "ctime" refers to the inode change time — the last time the
|
||||
// file's metadata (permissions, ownership, link count, etc.) was modified.
|
||||
// This is NOT the file creation time; Linux did not expose birth time until
|
||||
// the statx(2) syscall was added in kernel 4.11, and Go's syscall package
|
||||
// does not yet surface it.
|
||||
//
|
||||
// This differs from macOS/Darwin where "ctime" means birth time (file
|
||||
// creation time). See ctime_darwin.go for details.
|
||||
//
|
||||
// If the underlying stat information is unavailable (e.g. when using a
|
||||
// virtual filesystem like afero.MemMapFs), this falls back to mtime.
|
||||
func fileCTime(info os.FileInfo) time.Time {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return info.ModTime()
|
||||
}
|
||||
return time.Unix(stat.Ctim.Sec, stat.Ctim.Nsec).UTC()
|
||||
}
|
||||
133
internal/snapshot/ctime_test.go
Normal file
133
internal/snapshot/ctime_test.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFileCTime_RealFile(t *testing.T) {
|
||||
// Create a temporary file
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "testfile.txt")
|
||||
|
||||
if err := os.WriteFile(path, []byte("hello"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctime := fileCTime(info)
|
||||
|
||||
// ctime should be a valid time (not zero)
|
||||
if ctime.IsZero() {
|
||||
t.Fatal("fileCTime returned zero time")
|
||||
}
|
||||
|
||||
// ctime should be close to now (within a few seconds)
|
||||
diff := time.Since(ctime)
|
||||
if diff < 0 || diff > 5*time.Second {
|
||||
t.Fatalf("fileCTime returned unexpected time: %v (diff from now: %v)", ctime, diff)
|
||||
}
|
||||
|
||||
// ctime should not equal mtime exactly in all cases, but for a freshly
|
||||
// created file they should be very close
|
||||
mtime := info.ModTime()
|
||||
ctimeMtimeDiff := ctime.Sub(mtime)
|
||||
if ctimeMtimeDiff < 0 {
|
||||
ctimeMtimeDiff = -ctimeMtimeDiff
|
||||
}
|
||||
// For a freshly created file, ctime and mtime should be within 1 second
|
||||
if ctimeMtimeDiff > time.Second {
|
||||
t.Fatalf("ctime and mtime differ by too much for a new file: ctime=%v, mtime=%v, diff=%v",
|
||||
ctime, mtime, ctimeMtimeDiff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileCTime_AfterMtimeChange(t *testing.T) {
|
||||
// Create a temporary file
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "testfile.txt")
|
||||
|
||||
if err := os.WriteFile(path, []byte("hello"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Get initial ctime
|
||||
info1, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctime1 := fileCTime(info1)
|
||||
|
||||
// Change mtime to a time in the past
|
||||
pastTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
if err := os.Chtimes(path, pastTime, pastTime); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Get new stats
|
||||
info2, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctime2 := fileCTime(info2)
|
||||
mtime2 := info2.ModTime()
|
||||
|
||||
// mtime should now be in the past
|
||||
if mtime2.Year() != 2020 {
|
||||
t.Fatalf("mtime not set correctly: %v", mtime2)
|
||||
}
|
||||
|
||||
// On macOS: ctime (birth time) should remain unchanged since birth time
|
||||
// doesn't change when mtime is updated.
|
||||
// On Linux: ctime (inode change time) will be updated to ~now because
|
||||
// changing mtime is a metadata change.
|
||||
// Either way, ctime should NOT equal the past mtime we just set.
|
||||
if ctime2.Equal(pastTime) {
|
||||
t.Fatal("ctime should not equal the artificially set past mtime")
|
||||
}
|
||||
|
||||
// ctime should still be a recent time (the original creation time or
|
||||
// the metadata change time, depending on platform)
|
||||
_ = ctime1 // used for reference; both platforms will have a recent ctime2
|
||||
if time.Since(ctime2) > 10*time.Second {
|
||||
t.Fatalf("ctime is unexpectedly old: %v", ctime2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileCTime_NonSyscallFileInfo verifies the fallback to mtime when
|
||||
// the FileInfo doesn't have a *syscall.Stat_t (e.g. afero.MemMapFs).
|
||||
type mockFileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
isDir bool
|
||||
}
|
||||
|
||||
func (m *mockFileInfo) Name() string { return m.name }
|
||||
func (m *mockFileInfo) Size() int64 { return m.size }
|
||||
func (m *mockFileInfo) Mode() os.FileMode { return m.mode }
|
||||
func (m *mockFileInfo) ModTime() time.Time { return m.modTime }
|
||||
func (m *mockFileInfo) IsDir() bool { return m.isDir }
|
||||
func (m *mockFileInfo) Sys() interface{} { return nil } // No syscall.Stat_t
|
||||
|
||||
func TestFileCTime_FallbackToMtime(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
info := &mockFileInfo{
|
||||
name: "test.txt",
|
||||
size: 100,
|
||||
mode: 0644,
|
||||
modTime: now,
|
||||
}
|
||||
|
||||
ctime := fileCTime(info)
|
||||
if !ctime.Equal(now) {
|
||||
t.Fatalf("expected fallback to mtime %v, got %v", now, ctime)
|
||||
}
|
||||
}
|
||||
@@ -728,7 +728,7 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath), // Store source directory for restore path stripping
|
||||
MTime: info.ModTime(),
|
||||
CTime: info.ModTime(), // afero doesn't provide ctime
|
||||
CTime: fileCTime(info), // platform-specific: birth time on macOS, inode change time on Linux
|
||||
Size: info.Size(),
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
|
||||
55
internal/vaultik/blob_fetch_stub.go
Normal file
55
internal/vaultik/blob_fetch_stub.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// FetchAndDecryptBlobResult holds the result of fetching and decrypting a blob.
|
||||
type FetchAndDecryptBlobResult struct {
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// FetchAndDecryptBlob downloads a blob, decrypts it, and returns the plaintext data.
|
||||
func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*FetchAndDecryptBlobResult, error) {
|
||||
rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rc.Close() }()
|
||||
|
||||
reader, err := blobgen.NewReader(rc, identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating blob reader: %w", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading blob data: %w", err)
|
||||
}
|
||||
|
||||
return &FetchAndDecryptBlobResult{Data: data}, nil
|
||||
}
|
||||
|
||||
// FetchBlob downloads a blob and returns a reader for the encrypted data.
|
||||
func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||
|
||||
rc, err := v.Storage.Get(ctx, blobPath)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
|
||||
}
|
||||
|
||||
info, err := v.Storage.Stat(ctx, blobPath)
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
|
||||
}
|
||||
|
||||
return rc, info.Size, nil
|
||||
}
|
||||
@@ -7,9 +7,6 @@ import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// defaultMaxBlobCacheBytes is the default maximum size of the disk blob cache (10 GB).
|
||||
const defaultMaxBlobCacheBytes = 10 << 30 // 10 GiB
|
||||
|
||||
// blobDiskCacheEntry tracks a cached blob on disk.
|
||||
type blobDiskCacheEntry struct {
|
||||
key string
|
||||
|
||||
@@ -22,6 +22,13 @@ import (
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
const (
|
||||
// progressBarWidth is the character width of the progress bar display.
|
||||
progressBarWidth = 40
|
||||
// progressBarThrottle is the minimum interval between progress bar redraws.
|
||||
progressBarThrottle = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// RestoreOptions contains options for the restore operation
|
||||
type RestoreOptions struct {
|
||||
SnapshotID string
|
||||
@@ -109,75 +116,47 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
|
||||
// Step 5: Restore files
|
||||
result := &RestoreResult{}
|
||||
blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes)
|
||||
blobCache, err := newBlobDiskCache(4 * v.Config.BlobSizeLimit.Int64())
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob cache: %w", err)
|
||||
}
|
||||
defer func() { _ = blobCache.Close() }()
|
||||
|
||||
// Calculate total bytes for progress bar
|
||||
var totalBytes int64
|
||||
var totalBytesExpected int64
|
||||
for _, file := range files {
|
||||
totalBytes += file.Size
|
||||
totalBytesExpected += file.Size
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Restoring %d files (%s)...\n",
|
||||
len(files),
|
||||
humanize.Bytes(uint64(totalBytes)),
|
||||
)
|
||||
// Create progress bar if output is a terminal
|
||||
bar := v.newProgressBar("Restoring", totalBytesExpected)
|
||||
|
||||
// Create progress bar if stderr is a terminal
|
||||
isTTY := isTerminal(v.Stderr)
|
||||
var bar *progressbar.ProgressBar
|
||||
if isTTY {
|
||||
bar = progressbar.NewOptions64(
|
||||
totalBytes,
|
||||
progressbar.OptionSetDescription("Restoring"),
|
||||
progressbar.OptionSetWriter(v.Stderr),
|
||||
progressbar.OptionShowBytes(true),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetWidth(40),
|
||||
progressbar.OptionThrottle(100*time.Millisecond),
|
||||
progressbar.OptionOnCompletion(func() {
|
||||
v.printlnStderr()
|
||||
}),
|
||||
progressbar.OptionSetRenderBlankState(true),
|
||||
)
|
||||
}
|
||||
|
||||
filesProcessed := 0
|
||||
for _, file := range files {
|
||||
for i, file := range files {
|
||||
if v.ctx.Err() != nil {
|
||||
return v.ctx.Err()
|
||||
}
|
||||
|
||||
if err := v.restoreFile(v.ctx, repos, file, opts.TargetDir, identity, chunkToBlobMap, blobCache, result); err != nil {
|
||||
log.Error("Failed to restore file", "path", file.Path, "error", err)
|
||||
filesProcessed++
|
||||
result.FilesFailed++
|
||||
result.FailedFiles = append(result.FailedFiles, file.Path.String())
|
||||
// Update progress bar even on failure
|
||||
if bar != nil {
|
||||
_ = bar.Add64(file.Size)
|
||||
}
|
||||
// Periodic structured log for non-terminal contexts (headless/CI)
|
||||
if !isTTY && filesProcessed%100 == 0 {
|
||||
log.Info("Restore progress",
|
||||
"files", fmt.Sprintf("%d/%d", filesProcessed, len(files)),
|
||||
"bytes_restored", humanize.Bytes(uint64(result.BytesRestored)),
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
filesProcessed++
|
||||
// Update progress bar
|
||||
if bar != nil {
|
||||
_ = bar.Add64(file.Size)
|
||||
}
|
||||
// Periodic structured log for non-terminal contexts (headless/CI)
|
||||
if !isTTY && (filesProcessed%100 == 0 || filesProcessed == len(files)) {
|
||||
|
||||
// Progress logging (for non-terminal or structured logs)
|
||||
if (i+1)%100 == 0 || i+1 == len(files) {
|
||||
log.Info("Restore progress",
|
||||
"files", fmt.Sprintf("%d/%d", filesProcessed, len(files)),
|
||||
"bytes_restored", humanize.Bytes(uint64(result.BytesRestored)),
|
||||
"files", fmt.Sprintf("%d/%d", i+1, len(files)),
|
||||
"bytes", humanize.Bytes(uint64(result.BytesRestored)),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -202,6 +181,13 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
result.Duration.Round(time.Second),
|
||||
)
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nWARNING: %d file(s) failed to restore:\n", result.FilesFailed)
|
||||
for _, path := range result.FailedFiles {
|
||||
_, _ = fmt.Fprintf(v.Stdout, " - %s\n", path)
|
||||
}
|
||||
}
|
||||
|
||||
// Run verification if requested
|
||||
if opts.Verify {
|
||||
if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil {
|
||||
@@ -222,6 +208,10 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
)
|
||||
}
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -421,7 +411,11 @@ func (v *Vaultik) restoreDirectory(file *database.File, targetPath string, resul
|
||||
}
|
||||
}
|
||||
|
||||
// Set mtime
|
||||
// Set mtime (atime is set to mtime as a reasonable default).
|
||||
// Note: ctime cannot be restored. On Linux, ctime (inode change time) is
|
||||
// managed by the kernel and cannot be set by userspace. On macOS, birth
|
||||
// time cannot be set via standard POSIX APIs. The ctime value is preserved
|
||||
// in the snapshot database for informational purposes.
|
||||
if err := v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil {
|
||||
log.Debug("Failed to set directory mtime", "path", targetPath, "error", err)
|
||||
}
|
||||
@@ -518,7 +512,11 @@ func (v *Vaultik) restoreRegularFile(
|
||||
}
|
||||
}
|
||||
|
||||
// Set mtime
|
||||
// Set mtime (atime is set to mtime as a reasonable default).
|
||||
// Note: ctime cannot be restored. On Linux, ctime (inode change time) is
|
||||
// managed by the kernel and cannot be set by userspace. On macOS, birth
|
||||
// time cannot be set via standard POSIX APIs. The ctime value is preserved
|
||||
// in the snapshot database for informational purposes.
|
||||
if err := v.Fs.Chtimes(targetPath, file.MTime, file.MTime); err != nil {
|
||||
log.Debug("Failed to set file mtime", "path", targetPath, "error", err)
|
||||
}
|
||||
@@ -530,53 +528,6 @@ func (v *Vaultik) restoreRegularFile(
|
||||
return nil
|
||||
}
|
||||
|
||||
// BlobFetchResult holds the result of fetching and decrypting a blob.
|
||||
type BlobFetchResult struct {
|
||||
Data []byte
|
||||
CompressedSize int64
|
||||
}
|
||||
|
||||
// FetchAndDecryptBlob downloads a blob from storage, decrypts and decompresses it.
|
||||
func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*BlobFetchResult, error) {
|
||||
// Construct blob path with sharding
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||
|
||||
reader, err := v.Storage.Get(ctx, blobPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("downloading blob: %w", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Read encrypted data
|
||||
encryptedData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading blob data: %w", err)
|
||||
}
|
||||
|
||||
// Decrypt and decompress
|
||||
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
||||
}
|
||||
defer func() { _ = blobReader.Close() }()
|
||||
|
||||
data, err := io.ReadAll(blobReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypting blob: %w", err)
|
||||
}
|
||||
|
||||
log.Debug("Downloaded and decrypted blob",
|
||||
"hash", blobHash[:16],
|
||||
"encrypted_size", humanize.Bytes(uint64(len(encryptedData))),
|
||||
"decrypted_size", humanize.Bytes(uint64(len(data))),
|
||||
)
|
||||
|
||||
return &BlobFetchResult{
|
||||
Data: data,
|
||||
CompressedSize: int64(len(encryptedData)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// downloadBlob downloads and decrypts a blob
|
||||
func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) ([]byte, error) {
|
||||
result, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity)
|
||||
@@ -621,22 +572,7 @@ func (v *Vaultik) verifyRestoredFiles(
|
||||
)
|
||||
|
||||
// Create progress bar if output is a terminal
|
||||
var bar *progressbar.ProgressBar
|
||||
if isTerminal(v.Stderr) {
|
||||
bar = progressbar.NewOptions64(
|
||||
totalBytes,
|
||||
progressbar.OptionSetDescription("Verifying"),
|
||||
progressbar.OptionSetWriter(v.Stderr),
|
||||
progressbar.OptionShowBytes(true),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetWidth(40),
|
||||
progressbar.OptionThrottle(100*time.Millisecond),
|
||||
progressbar.OptionOnCompletion(func() {
|
||||
v.printfStderr("\n")
|
||||
}),
|
||||
progressbar.OptionSetRenderBlankState(true),
|
||||
)
|
||||
}
|
||||
bar := v.newProgressBar("Verifying", totalBytes)
|
||||
|
||||
// Verify each file
|
||||
for _, file := range regularFiles {
|
||||
@@ -730,11 +666,37 @@ func (v *Vaultik) verifyFile(
|
||||
return bytesVerified, nil
|
||||
}
|
||||
|
||||
// isTerminal returns true if the given writer is connected to a terminal.
|
||||
// Returns false if the writer does not expose a file descriptor (e.g. in tests).
|
||||
func isTerminal(w io.Writer) bool {
|
||||
if f, ok := w.(*os.File); ok {
|
||||
return term.IsTerminal(int(f.Fd()))
|
||||
// newProgressBar creates a terminal-aware progress bar with standard options.
|
||||
// It returns nil if stdout is not a terminal.
|
||||
func (v *Vaultik) newProgressBar(description string, total int64) *progressbar.ProgressBar {
|
||||
if !v.isTerminal() {
|
||||
return nil
|
||||
}
|
||||
return false
|
||||
return progressbar.NewOptions64(
|
||||
total,
|
||||
progressbar.OptionSetDescription(description),
|
||||
progressbar.OptionSetWriter(v.Stderr),
|
||||
progressbar.OptionShowBytes(true),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetWidth(progressBarWidth),
|
||||
progressbar.OptionThrottle(progressBarThrottle),
|
||||
progressbar.OptionOnCompletion(func() {
|
||||
v.printfStderr("\n")
|
||||
}),
|
||||
progressbar.OptionSetRenderBlankState(true),
|
||||
)
|
||||
}
|
||||
|
||||
// isTerminal returns true if stdout is a terminal.
|
||||
// It checks whether v.Stdout implements Fd() (i.e. is an *os.File),
|
||||
// and falls back to false for non-file writers (e.g. in tests).
|
||||
func (v *Vaultik) isTerminal() bool {
|
||||
type fder interface {
|
||||
Fd() uintptr
|
||||
}
|
||||
f, ok := v.Stdout.(fder)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return term.IsTerminal(int(f.Fd()))
|
||||
}
|
||||
|
||||
@@ -90,6 +90,24 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
v.printfStdout("\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second))
|
||||
}
|
||||
|
||||
// Prune old snapshots and unreferenced blobs if --prune was specified
|
||||
if opts.Prune {
|
||||
log.Info("Pruning enabled - deleting old snapshots and unreferenced blobs")
|
||||
v.printlnStdout("\nPruning old snapshots (keeping latest)...")
|
||||
|
||||
if err := v.PurgeSnapshots(true, "", true); err != nil {
|
||||
return fmt.Errorf("prune: purging old snapshots: %w", err)
|
||||
}
|
||||
|
||||
v.printlnStdout("Pruning unreferenced blobs...")
|
||||
|
||||
if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil {
|
||||
return fmt.Errorf("prune: removing unreferenced blobs: %w", err)
|
||||
}
|
||||
|
||||
log.Info("Pruning complete")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -306,11 +324,6 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
}
|
||||
v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration))
|
||||
|
||||
if opts.Prune {
|
||||
log.Info("Pruning enabled - will delete old snapshots after snapshot")
|
||||
// TODO: Implement pruning
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1004,16 +1017,16 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error {
|
||||
|
||||
// Delete related records first to avoid foreign key constraints
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot files", "snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil {
|
||||
log.Error("Failed to delete snapshot record", "snapshot_id", snapshotID, "error", err)
|
||||
return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
23
internal/vaultik/snapshot_prune_test.go
Normal file
23
internal/vaultik/snapshot_prune_test.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestSnapshotCreateOptions_PruneFlag verifies the Prune field exists on
|
||||
// SnapshotCreateOptions and can be set.
|
||||
func TestSnapshotCreateOptions_PruneFlag(t *testing.T) {
|
||||
opts := &SnapshotCreateOptions{
|
||||
Prune: true,
|
||||
}
|
||||
if !opts.Prune {
|
||||
t.Error("Expected Prune to be true")
|
||||
}
|
||||
|
||||
opts2 := &SnapshotCreateOptions{
|
||||
Prune: false,
|
||||
}
|
||||
if opts2.Prune {
|
||||
t.Error("Expected Prune to be false")
|
||||
}
|
||||
}
|
||||
@@ -129,7 +129,7 @@ func (v *Vaultik) GetFilesystem() afero.Fs {
|
||||
return v.Fs
|
||||
}
|
||||
|
||||
// printfStdout writes formatted output to stdout for user-facing messages.
|
||||
// printfStdout writes formatted output to stdout.
|
||||
func (v *Vaultik) printfStdout(format string, args ...any) {
|
||||
_, _ = fmt.Fprintf(v.Stdout, format, args...)
|
||||
}
|
||||
@@ -139,28 +139,11 @@ func (v *Vaultik) printlnStdout(args ...any) {
|
||||
_, _ = fmt.Fprintln(v.Stdout, args...)
|
||||
}
|
||||
|
||||
// FetchBlob downloads a blob from storage and returns a reader for the encrypted data.
|
||||
func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
|
||||
|
||||
reader, err := v.Storage.Get(ctx, blobPath)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("downloading blob: %w", err)
|
||||
}
|
||||
|
||||
return reader, expectedSize, nil
|
||||
}
|
||||
|
||||
// printfStderr writes formatted output to stderr.
|
||||
func (v *Vaultik) printfStderr(format string, args ...any) {
|
||||
_, _ = fmt.Fprintf(v.Stderr, format, args...)
|
||||
}
|
||||
|
||||
// printlnStderr writes a line to stderr.
|
||||
func (v *Vaultik) printlnStderr(args ...any) {
|
||||
_, _ = fmt.Fprintln(v.Stderr, args...)
|
||||
}
|
||||
|
||||
// scanStdin reads a line of input from stdin.
|
||||
func (v *Vaultik) scanStdin(a ...any) (int, error) {
|
||||
return fmt.Fscanln(v.Stdin, a...)
|
||||
|
||||
Reference in New Issue
Block a user