Compare commits
28 Commits
20d3a9ac8c
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e534746cf3 | |||
| 5397b37c13 | |||
| 2df2792a75 | |||
| 4fe568f803 | |||
| 27e85f01f2 | |||
| d479bfcd52 | |||
| cb16d6869f | |||
| ff85f1e4f8 | |||
| b2e160944f | |||
| 307867f59e | |||
| 9d12d500fa | |||
| 2e2bf01130 | |||
| e9687c68b7 | |||
| a8970a87fc | |||
| e6ee488d9d | |||
| 2e2b02a056 | |||
| 0b95cb4308 | |||
| 4a3e61f8e1 | |||
| 6fbcac0cd8 | |||
| 34f73f72d8 | |||
| ee240faa32 | |||
| f719ab3adc | |||
| 1a8baf7491 | |||
| 7d5d3fa598 | |||
| ac5d2f4a0d | |||
| b250ddfa94 | |||
| fe3ad13a91 | |||
| ebd6619638 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,5 +1,5 @@
|
||||
# Binary
|
||||
vaultik
|
||||
/vaultik
|
||||
|
||||
# Test artifacts
|
||||
*.out
|
||||
|
||||
@@ -20,8 +20,8 @@ builds:
|
||||
- arm64
|
||||
ldflags:
|
||||
- -s -w
|
||||
- -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version={{ .Version }}'
|
||||
- -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit={{ .Commit }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Version={{ .Version }}'
|
||||
- -X 'sneak.berlin/go/vaultik/internal/globals.Commit={{ .Commit }}'
|
||||
|
||||
archives:
|
||||
- id: default
|
||||
|
||||
@@ -53,8 +53,8 @@ The database tracks five primary entities and their relationships:
|
||||
### Entity Descriptions
|
||||
|
||||
#### File (`database.File`)
|
||||
Represents a file or directory in the backup system. Stores metadata needed for restoration:
|
||||
- Path, mtime
|
||||
Represents a file, directory, or symlink in the backup system. Stores metadata needed for restoration:
|
||||
- Path, source_path (for restore path stripping), mtime
|
||||
- Size, mode, ownership (uid, gid)
|
||||
- Symlink target (if applicable)
|
||||
|
||||
@@ -95,7 +95,7 @@ Maps chunks to their position within blobs:
|
||||
|
||||
#### Snapshot (`database.Snapshot`)
|
||||
Represents a point-in-time backup:
|
||||
- `ID`: Format is `{hostname}-{YYYYMMDD}-{HHMMSS}Z`
|
||||
- `ID`: Format is `{hostname}_{snapshot-name}_{RFC3339}` (e.g. `server1_home_2025-06-01T12:00:00Z`)
|
||||
- Tracks file count, chunk count, blob count, sizes, compression ratio
|
||||
- `CompletedAt`: Null until snapshot finishes successfully
|
||||
|
||||
@@ -127,7 +127,7 @@ fx.New(
|
||||
config.Module, // 5. Config
|
||||
database.Module, // 6. Database + Repositories
|
||||
log.Module, // 7. Logger initialization
|
||||
s3.Module, // 8. S3 client
|
||||
storage.Module, // 8. Storage backend (S3/file/rclone)
|
||||
snapshot.Module, // 9. SnapshotManager + ScannerFactory
|
||||
fx.Provide(vaultik.New), // 10. Vaultik orchestrator
|
||||
)
|
||||
@@ -161,7 +161,7 @@ type Vaultik struct {
|
||||
Config *config.Config
|
||||
DB *database.DB
|
||||
Repositories *database.Repositories
|
||||
S3Client *s3.Client
|
||||
Storage storage.Storer
|
||||
ScannerFactory snapshot.ScannerFactory
|
||||
SnapshotManager *snapshot.SnapshotManager
|
||||
Shutdowner fx.Shutdowner
|
||||
@@ -341,12 +341,11 @@ CreateSnapshot(opts)
|
||||
└─► SnapshotManager.ExportSnapshotMetadata()
|
||||
│
|
||||
├─► Copy database to temp file
|
||||
├─► Clean to only current snapshot data
|
||||
├─► Dump to SQL
|
||||
├─► Compress with zstd
|
||||
├─► Clean to only current snapshot data (VACUUM)
|
||||
├─► Compress binary SQLite with zstd
|
||||
├─► Encrypt with age
|
||||
├─► Upload db.zst.age to S3
|
||||
└─► Upload manifest.json.zst to S3
|
||||
├─► Upload db.zst.age to storage
|
||||
└─► Upload manifest.json.zst to storage
|
||||
```
|
||||
|
||||
## Deduplication Strategy
|
||||
@@ -368,8 +367,8 @@ bucket/
|
||||
│
|
||||
└── metadata/
|
||||
└── {snapshot-id}/
|
||||
├── db.zst.age # Encrypted database dump
|
||||
└── manifest.json.zst # Blob list (for verification)
|
||||
├── db.zst.age # Encrypted binary SQLite database
|
||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
||||
```
|
||||
|
||||
## Thread Safety
|
||||
|
||||
@@ -42,7 +42,7 @@ COPY . .
|
||||
RUN make test
|
||||
|
||||
# Build with CGO enabled (required for mattn/go-sqlite3)
|
||||
RUN CGO_ENABLED=1 go build -ldflags "-X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=${VERSION}' -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||
RUN CGO_ENABLED=0 go build -ldflags "-X 'sneak.berlin/go/vaultik/internal/globals.Version=${VERSION}' -X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-02-25
|
||||
|
||||
4
Makefile
4
Makefile
@@ -7,8 +7,8 @@ VERSION := 1.0.0-rc.1
|
||||
GIT_REVISION := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Linker flags
|
||||
LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(GIT_REVISION)'
|
||||
LDFLAGS := -X 'sneak.berlin/go/vaultik/internal/globals.Version=$(VERSION)' \
|
||||
-X 'sneak.berlin/go/vaultik/internal/globals.Commit=$(GIT_REVISION)'
|
||||
|
||||
# Default target
|
||||
all: vaultik
|
||||
|
||||
573
README.md
573
README.md
@@ -1,43 +1,61 @@
|
||||
# vaultik (ваултик)
|
||||
|
||||
WIP: pre-1.0, some functions may not be fully implemented yet
|
||||
|
||||
`vaultik` is an incremental backup tool written in Go. It encrypts data
|
||||
using an `age` public key and uploads each encrypted blob directly to a
|
||||
remote S3-compatible object store. It requires no private keys, secrets, or
|
||||
credentials (other than those required to PUT to encrypted object storage,
|
||||
such as S3 API keys) stored on the backed-up system.
|
||||
|
||||
It includes table-stakes features such as:
|
||||
## quickstart
|
||||
|
||||
* modern encryption (the excellent `age`)
|
||||
* deduplication
|
||||
* incremental backups
|
||||
* modern multithreaded zstd compression with configurable levels
|
||||
```sh
|
||||
# install
|
||||
go install sneak.berlin/go/vaultik/cmd/vaultik@latest
|
||||
|
||||
# create a default config file (prints the path it wrote to)
|
||||
vaultik config init
|
||||
|
||||
# generate an age keypair; keep the private key file somewhere safe and
|
||||
# offline — you need it to restore, and the backed-up machine does not need it
|
||||
age-keygen -o vaultik_backup_private_key.txt
|
||||
grep 'public key' vaultik_backup_private_key.txt
|
||||
|
||||
# configure the encryption key and backup destination
|
||||
vaultik config set age_recipients.0 age1YOUR_PUBLIC_KEY_HERE
|
||||
vaultik config set storage_url "file:///Volumes/usbstick/mybackup"
|
||||
|
||||
# back up your home directory (the default config includes a "home"
|
||||
# snapshot of ~ with sensible excludes)
|
||||
vaultik snapshot create
|
||||
|
||||
# see what you have
|
||||
vaultik snapshot list
|
||||
```
|
||||
|
||||
Features:
|
||||
|
||||
* modern encryption ([age](https://age-encryption.org/), X25519 + XChaCha20-Poly1305)
|
||||
* content-defined chunking with deduplication (FastCDC)
|
||||
* incremental backups (only changed files are re-chunked)
|
||||
* multithreaded zstd compression at configurable levels
|
||||
* content-addressed immutable storage
|
||||
* local state tracking in standard SQLite database, enables write-only
|
||||
incremental backups to destination
|
||||
* local state tracking in SQLite (enables write-only incremental backups)
|
||||
* no mutable remote metadata
|
||||
* no plaintext file paths or metadata stored in remote
|
||||
* does not create huge numbers of small files (to keep S3 operation counts
|
||||
down) even if the source system has many small files
|
||||
* no plaintext file paths or metadata in remote storage
|
||||
* packs small files into large blobs (keeps S3 operation counts down)
|
||||
* backs up regular files, symlinks, empty directories, and file permissions
|
||||
* pluggable storage backends: S3, local filesystem, rclone (70+ providers)
|
||||
* pure Go (no CGO), cross-compiles to linux/darwin × amd64/arm64
|
||||
|
||||
## why
|
||||
|
||||
Existing backup software fails under one or more of these conditions:
|
||||
|
||||
* Requires secrets (passwords, private keys) on the source system, which
|
||||
compromises encrypted backups in the case of host system compromise
|
||||
* Depends on symmetric encryption unsuitable for zero-trust environments
|
||||
* Creates one-blob-per-file, which results in excessive S3 operation counts
|
||||
* is slow
|
||||
|
||||
Other backup tools like `restic`, `borg`, and `duplicity` are designed for
|
||||
environments where the source host can store secrets and has access to
|
||||
decryption keys. I don't want to store backup decryption keys on my hosts,
|
||||
only public keys for encryption.
|
||||
decryption keys. `vaultik` is for environments where you don't want to
|
||||
store backup decryption keys on your hosts — only public keys for
|
||||
encryption.
|
||||
|
||||
My requirements are:
|
||||
Requirements that no existing tool meets:
|
||||
|
||||
* open source
|
||||
* no passphrases or private keys on the source host
|
||||
@@ -46,96 +64,21 @@ My requirements are:
|
||||
* encrypted
|
||||
* s3 compatible without an intermediate step or tool
|
||||
|
||||
Surprisingly, no existing tool meets these requirements, so I wrote `vaultik`.
|
||||
## daily use
|
||||
|
||||
## design goals
|
||||
```sh
|
||||
# verify a snapshot (shallow: checks all blobs exist)
|
||||
vaultik snapshot verify <snapshot-id>
|
||||
|
||||
1. Backups must require only a public key on the source host.
|
||||
1. No secrets or private keys may exist on the source system.
|
||||
1. Restore must be possible using **only** the backup bucket and a private key.
|
||||
1. Prune must be possible (requires private key, done on different hosts).
|
||||
1. All encryption uses [`age`](https://age-encryption.org/) (X25519, XChaCha20-Poly1305).
|
||||
1. Compression uses `zstd` at a configurable level.
|
||||
1. Files are chunked, and multiple chunks are packed into encrypted blobs
|
||||
to reduce object count for filesystems with many small files.
|
||||
1. All metadata (snapshots) is stored remotely as encrypted SQLite DBs.
|
||||
# deep verify (downloads and cryptographically verifies every blob)
|
||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik snapshot verify --deep <snapshot-id>
|
||||
|
||||
## what
|
||||
# restore (requires the private key)
|
||||
VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...' vaultik restore <snapshot-id> /tmp/restored
|
||||
|
||||
`vaultik` walks a set of configured directories and builds a
|
||||
content-addressable chunk map of changed files using deterministic chunking.
|
||||
Each chunk is streamed into a blob packer. Blobs are compressed with `zstd`,
|
||||
encrypted with `age`, and uploaded directly to remote storage under a
|
||||
content-addressed S3 path. At the end, a pruned snapshot-specific sqlite
|
||||
database of metadata is created, encrypted, and uploaded alongside the
|
||||
blobs.
|
||||
|
||||
No plaintext file contents ever hit disk. No private key or secret
|
||||
passphrase is needed or stored locally.
|
||||
|
||||
## how
|
||||
|
||||
1. **install**
|
||||
|
||||
```sh
|
||||
go install git.eeqj.de/sneak/vaultik@latest
|
||||
```
|
||||
|
||||
1. **generate keypair**
|
||||
|
||||
```sh
|
||||
age-keygen -o agekey.txt
|
||||
grep 'public key:' agekey.txt
|
||||
```
|
||||
|
||||
1. **write config**
|
||||
|
||||
```yaml
|
||||
# Named snapshots - each snapshot can contain multiple paths
|
||||
snapshots:
|
||||
system:
|
||||
paths:
|
||||
- /etc
|
||||
- /var/lib
|
||||
exclude:
|
||||
- '*.cache' # Snapshot-specific exclusions
|
||||
home:
|
||||
paths:
|
||||
- /home/user/documents
|
||||
- /home/user/photos
|
||||
|
||||
# Global exclusions (apply to all snapshots)
|
||||
exclude:
|
||||
- '*.log'
|
||||
- '*.tmp'
|
||||
- '.git'
|
||||
- 'node_modules'
|
||||
|
||||
age_recipients:
|
||||
- age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj
|
||||
s3:
|
||||
endpoint: https://s3.example.com
|
||||
bucket: vaultik-data
|
||||
prefix: host1/
|
||||
access_key_id: ...
|
||||
secret_access_key: ...
|
||||
region: us-east-1
|
||||
chunk_size: 10MB
|
||||
blob_size_limit: 1GB
|
||||
```
|
||||
|
||||
1. **run**
|
||||
|
||||
```sh
|
||||
# Create all configured snapshots
|
||||
vaultik --config /etc/vaultik.yaml snapshot create
|
||||
|
||||
# Create specific snapshots by name
|
||||
vaultik --config /etc/vaultik.yaml snapshot create home system
|
||||
|
||||
# Silent mode for cron
|
||||
vaultik --config /etc/vaultik.yaml snapshot create --cron
|
||||
```
|
||||
# daily cron job: back up, keep a 4-week rolling window of snapshots
|
||||
# 0 3 * * * vaultik snapshot create --cron --prune --keep-newer-than 4w
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
@@ -144,260 +87,324 @@ passphrase is needed or stored locally.
|
||||
### commands
|
||||
|
||||
```sh
|
||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--skip-errors]
|
||||
vaultik [--config <path>] config init
|
||||
vaultik [--config <path>] config edit
|
||||
vaultik [--config <path>] config get <key>
|
||||
vaultik [--config <path>] config set <key> <value>
|
||||
vaultik [--config <path>] snapshot create [snapshot-names...] [--cron] [--prune] [--keep-newer-than <duration>] [--skip-errors]
|
||||
vaultik [--config <path>] snapshot list [--json]
|
||||
vaultik [--config <path>] snapshot verify <snapshot-id> [--deep] [--json]
|
||||
vaultik [--config <path>] snapshot purge [--keep-latest | --older-than <duration>] [--snapshot <name>...] [--force]
|
||||
vaultik [--config <path>] snapshot remove <snapshot-id|--all> [--dry-run] [--force] [--remote] [--json]
|
||||
vaultik [--config <path>] snapshot prune
|
||||
vaultik [--config <path>] snapshot cleanup
|
||||
vaultik [--config <path>] restore <snapshot-id> <target-dir> [paths...] [--verify]
|
||||
vaultik [--config <path>] prune [--force] [--json]
|
||||
vaultik [--config <path>] info
|
||||
vaultik [--config <path>] remote info [--json]
|
||||
vaultik [--config <path>] store info
|
||||
vaultik [--config <path>] database purge [--force]
|
||||
vaultik completion <bash|zsh|fish|powershell>
|
||||
vaultik version
|
||||
```
|
||||
|
||||
### environment
|
||||
### global flags
|
||||
|
||||
* `VAULTIK_AGE_SECRET_KEY`: Required for `restore` and deep `verify`. Contains the age private key for decryption.
|
||||
* `VAULTIK_CONFIG`: Optional path to config file.
|
||||
* `--config <path>`: Path to config file (default: `$VAULTIK_CONFIG`, then platform config dir, then `/etc/vaultik/config.yml`)
|
||||
* `--verbose`, `-v`: Enable verbose output
|
||||
* `--debug`: Enable debug output
|
||||
* `--quiet`, `-q`: Suppress non-error output
|
||||
|
||||
### environment variables
|
||||
|
||||
* `VAULTIK_AGE_SECRET_KEY`: Age private key for decryption (required for `restore` and `verify --deep`)
|
||||
* `VAULTIK_CONFIG`: Path to config file (overridden by `--config`)
|
||||
* `VAULTIK_INDEX_PATH`: Override local SQLite index path
|
||||
|
||||
### shell completion
|
||||
|
||||
```sh
|
||||
# zsh: load for the current session
|
||||
source <(vaultik completion zsh)
|
||||
|
||||
# zsh: install permanently
|
||||
vaultik completion zsh > "${fpath[1]}/_vaultik"
|
||||
|
||||
# bash: load for the current session
|
||||
source <(vaultik completion bash)
|
||||
|
||||
# bash: install permanently (Linux)
|
||||
vaultik completion bash > /etc/bash_completion.d/vaultik
|
||||
|
||||
# fish
|
||||
vaultik completion fish > ~/.config/fish/completions/vaultik.fish
|
||||
```
|
||||
|
||||
### command details
|
||||
|
||||
**snapshot create**: Perform incremental backup of configured snapshots
|
||||
* Config is located at `/etc/vaultik/config.yml` by default
|
||||
**`config init`**: Write a default config file with commented explanations for
|
||||
every setting. Writes to the path from `--config`, `$VAULTIK_CONFIG`, or the
|
||||
platform config directory (`~/Library/Application Support/vaultik/` on macOS,
|
||||
`~/.config/vaultik/` on Linux, `/etc/vaultik/` as root). Refuses to overwrite an
|
||||
existing file. Created with mode `0600` since it will contain credentials.
|
||||
|
||||
**`config edit`**: Open the config file in `$EDITOR` (falls back to `vi`).
|
||||
|
||||
**`config get`**: Print a config value addressed by dotted YAML path
|
||||
(e.g. `vaultik config get s3.bucket`). Non-scalar values print as YAML.
|
||||
|
||||
**`config set`**: Set a scalar config value by dotted YAML path
|
||||
(e.g. `vaultik config set compression_level 9`). Comments and formatting
|
||||
in the file are preserved; intermediate maps are created as needed.
|
||||
|
||||
**`snapshot create`**: Perform incremental backup of configured snapshots.
|
||||
* Optional snapshot names argument to create specific snapshots (default: all)
|
||||
* `--cron`: Silent unless error (for crontab)
|
||||
* `--prune`: After backup, drop older snapshots of each backed-up name (keeping
|
||||
only the latest) and remove orphaned blobs from remote storage
|
||||
* `--prune`: After backup, drop older snapshots of each backed-up name and
|
||||
remove orphaned blobs from remote storage. By default keeps only the latest
|
||||
snapshot per name; use `--keep-newer-than` for a rolling window.
|
||||
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`)
|
||||
* `--skip-errors`: Skip file read errors (log them loudly but continue)
|
||||
|
||||
**snapshot list**: List all snapshots with their timestamps and sizes
|
||||
**`snapshot list`**: List all snapshots with their timestamps and sizes.
|
||||
* `--json`: Output in JSON format
|
||||
|
||||
**snapshot verify**: Verify snapshot integrity
|
||||
* `--deep`: Download and verify blob contents (not just existence)
|
||||
**`snapshot verify`**: Verify snapshot integrity.
|
||||
* Default (shallow): checks that all blobs referenced in the manifest exist in storage
|
||||
* `--deep`: Downloads and decrypts each blob, verifies chunk hashes against the
|
||||
encrypted metadata database
|
||||
* `--json`: Output results as JSON
|
||||
|
||||
**snapshot purge**: Remove old snapshots based on criteria. Retention is
|
||||
applied per-snapshot-name (e.g. `--keep-latest` keeps the latest of each
|
||||
configured name, not the latest globally).
|
||||
**`snapshot purge`**: Remove old snapshots based on criteria. Retention is
|
||||
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
|
||||
latest globally).
|
||||
* `--keep-latest`: Keep only the most recent snapshot of each name
|
||||
* `--older-than`: Remove snapshots older than duration (e.g., 30d, 6mo, 1y)
|
||||
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`, `6m`, `1y`)
|
||||
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
|
||||
* `--force`: Skip confirmation prompt
|
||||
|
||||
**snapshot remove**: Remove a specific snapshot
|
||||
**`snapshot remove`**: Remove a specific snapshot from the local database.
|
||||
* `--remote`: Also remove snapshot metadata from remote storage
|
||||
* `--all`: Remove all snapshots (requires `--force`)
|
||||
* `--dry-run`: Show what would be deleted without deleting
|
||||
* `--force`: Skip confirmation prompt
|
||||
* `--json`: Output result as JSON
|
||||
|
||||
**snapshot prune**: Clean orphaned data from local database
|
||||
**`snapshot prune`**: Clean orphaned data from the local database (files,
|
||||
chunks, blobs not referenced by any snapshot).
|
||||
|
||||
**restore**: Restore snapshot to target directory
|
||||
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable with age private key
|
||||
**`snapshot cleanup`**: Remove stale local snapshot records that have no
|
||||
corresponding metadata in remote storage. These are typically left behind
|
||||
by incomplete or interrupted backups. Does not touch remote storage.
|
||||
|
||||
**`restore`**: Restore files from a backup snapshot.
|
||||
* Requires `VAULTIK_AGE_SECRET_KEY` environment variable
|
||||
* Optional path arguments to restore specific files/directories (default: all)
|
||||
* Downloads and decrypts metadata, fetches required blobs, reconstructs files
|
||||
* Preserves file permissions, timestamps, and ownership (ownership requires root)
|
||||
* Handles symlinks and directories
|
||||
* Preserves file permissions, timestamps, ownership (ownership requires root),
|
||||
symlinks, and empty directories
|
||||
* `--verify`: After restoring, verify every file's chunk hashes match
|
||||
|
||||
**prune**: Remove unreferenced blobs from remote storage
|
||||
* Scans all snapshots for referenced blobs
|
||||
* Deletes orphaned blobs
|
||||
**`prune`**: Remove unreferenced blobs from remote storage.
|
||||
* Scans all snapshot manifests for referenced blobs, deletes any blob not referenced
|
||||
* `--force`: Skip confirmation prompt
|
||||
* `--json`: Output stats as JSON
|
||||
|
||||
**info**: Display system and configuration information
|
||||
**`info`**: Display system configuration, storage settings, encryption
|
||||
recipients, and local database statistics.
|
||||
|
||||
**store info**: Display S3 bucket configuration and storage statistics
|
||||
**`remote info`**: Show detailed remote storage information including per-snapshot
|
||||
metadata sizes, blob counts, and orphaned blob detection.
|
||||
* `--json`: Output as JSON
|
||||
|
||||
**`store info`**: Display storage backend type and statistics.
|
||||
|
||||
**`database purge`**: Delete the local SQLite state database entirely. Remote
|
||||
storage is unaffected; the next backup will do a full scan and re-deduplicate
|
||||
against existing remote blobs.
|
||||
* `--force`: Skip confirmation prompt
|
||||
|
||||
---
|
||||
|
||||
## storage backends
|
||||
|
||||
vaultik supports three storage backends, selected via the `storage_url` config field:
|
||||
|
||||
**S3** (`s3://bucket/prefix?endpoint=host®ion=us-east-1`): Any S3-compatible
|
||||
object store. Credentials are read from `s3.access_key_id` and
|
||||
`s3.secret_access_key` in the config file.
|
||||
|
||||
**Local filesystem** (`file:///path/to/backup`): Stores blobs and metadata on
|
||||
a local or mounted filesystem. Useful for testing or backing up to a NAS.
|
||||
|
||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||
providers. Requires rclone to be configured separately (`rclone config`).
|
||||
|
||||
Legacy S3 configuration via `s3.*` fields (endpoint, bucket, prefix, etc.) is
|
||||
still supported for backward compatibility. `storage_url` takes precedence if
|
||||
both are set.
|
||||
|
||||
---
|
||||
|
||||
## architecture
|
||||
|
||||
### s3 bucket layout
|
||||
### remote storage layout
|
||||
|
||||
```
|
||||
s3://<bucket>/<prefix>/
|
||||
<bucket>/<prefix>/
|
||||
├── blobs/
|
||||
│ └── <aa>/<bb>/<full_blob_hash>
|
||||
└── metadata/
|
||||
├── <snapshot_id>/
|
||||
│ ├── db.zst.age
|
||||
│ └── manifest.json.zst
|
||||
└── <snapshot_id>/
|
||||
├── db.zst.age # Encrypted binary SQLite database
|
||||
└── manifest.json.zst # Unencrypted blob list (for pruning)
|
||||
```
|
||||
|
||||
* `blobs/<aa>/<bb>/...`: Two-level directory sharding using first 4 hex chars of blob hash
|
||||
* `metadata/<snapshot_id>/db.zst.age`: Encrypted, compressed SQLite database
|
||||
* `metadata/<snapshot_id>/manifest.json.zst`: Unencrypted blob list for pruning
|
||||
* Blobs are two-level directory sharded using the first 4 hex chars of the blob hash
|
||||
* `db.zst.age` is a binary SQLite database (zstd compressed, age encrypted)
|
||||
containing all file metadata, chunk mappings, and relationships for the snapshot
|
||||
* `manifest.json.zst` is an unencrypted compressed JSON blob list, enabling
|
||||
pruning without the private key
|
||||
|
||||
### blob manifest format
|
||||
|
||||
The `manifest.json.zst` file is unencrypted (compressed JSON) to enable pruning without decryption:
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot_id": "hostname_snapshotname_2025-01-01T12:00:00Z",
|
||||
"blob_hashes": [
|
||||
"aa1234567890abcdef...",
|
||||
"bb2345678901bcdef0..."
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Snapshot IDs follow the format `<hostname>_<snapshot-name>_<timestamp>` (e.g., `server1_home_2025-01-01T12:00:00Z`).
|
||||
|
||||
### local sqlite schema
|
||||
|
||||
```sql
|
||||
CREATE TABLE files (
|
||||
id TEXT PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
mtime INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
mode INTEGER NOT NULL,
|
||||
uid INTEGER NOT NULL,
|
||||
gid INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE file_chunks (
|
||||
file_id TEXT NOT NULL,
|
||||
idx INTEGER NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
PRIMARY KEY (file_id, idx),
|
||||
FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE chunks (
|
||||
chunk_hash TEXT PRIMARY KEY,
|
||||
size INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE blobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
blob_hash TEXT NOT NULL UNIQUE,
|
||||
uncompressed INTEGER NOT NULL,
|
||||
compressed INTEGER NOT NULL,
|
||||
uploaded_at INTEGER
|
||||
);
|
||||
|
||||
CREATE TABLE blob_chunks (
|
||||
blob_hash TEXT NOT NULL,
|
||||
chunk_hash TEXT NOT NULL,
|
||||
offset INTEGER NOT NULL,
|
||||
length INTEGER NOT NULL,
|
||||
PRIMARY KEY (blob_hash, chunk_hash)
|
||||
);
|
||||
|
||||
CREATE TABLE chunk_files (
|
||||
chunk_hash TEXT NOT NULL,
|
||||
file_id TEXT NOT NULL,
|
||||
file_offset INTEGER NOT NULL,
|
||||
length INTEGER NOT NULL,
|
||||
PRIMARY KEY (chunk_hash, file_id)
|
||||
);
|
||||
|
||||
CREATE TABLE snapshots (
|
||||
id TEXT PRIMARY KEY,
|
||||
hostname TEXT NOT NULL,
|
||||
vaultik_version TEXT NOT NULL,
|
||||
started_at INTEGER NOT NULL,
|
||||
completed_at INTEGER,
|
||||
file_count INTEGER NOT NULL,
|
||||
chunk_count INTEGER NOT NULL,
|
||||
blob_count INTEGER NOT NULL,
|
||||
total_size INTEGER NOT NULL,
|
||||
blob_size INTEGER NOT NULL,
|
||||
compression_ratio REAL NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE snapshot_files (
|
||||
snapshot_id TEXT NOT NULL,
|
||||
file_id TEXT NOT NULL,
|
||||
PRIMARY KEY (snapshot_id, file_id)
|
||||
);
|
||||
|
||||
CREATE TABLE snapshot_blobs (
|
||||
snapshot_id TEXT NOT NULL,
|
||||
blob_id TEXT NOT NULL,
|
||||
blob_hash TEXT NOT NULL,
|
||||
PRIMARY KEY (snapshot_id, blob_id)
|
||||
);
|
||||
```
|
||||
Snapshot IDs follow the format `<hostname>_<snapshot-name>_<RFC3339-timestamp>`
|
||||
(e.g. `server1_home_2025-06-01T12:00:00Z`).
|
||||
|
||||
### data flow
|
||||
|
||||
#### backup
|
||||
**backup:**
|
||||
|
||||
1. Load config, open local SQLite index
|
||||
1. Walk source directories, check mtime/size against index
|
||||
1. For changed/new files: chunk using content-defined chunking
|
||||
1. For each chunk: hash, check if already uploaded, add to blob packer
|
||||
1. When blob reaches threshold: compress, encrypt, upload to S3
|
||||
1. Build snapshot metadata, compress, encrypt, upload
|
||||
1. Create blob manifest (unencrypted) for pruning support
|
||||
1. Open local SQLite index, load known files and chunks into memory
|
||||
2. Walk source directories, compare mtime/size/mode against index
|
||||
3. For changed/new files: chunk using content-defined chunking (FastCDC)
|
||||
4. For symlinks and directories: record metadata (no chunking)
|
||||
5. For each chunk: hash, check dedup, add to blob packer
|
||||
6. When blob reaches size threshold: compress (zstd), encrypt (age), upload
|
||||
7. Build snapshot metadata database, compress, encrypt, upload
|
||||
8. Create unencrypted blob manifest for pruning support
|
||||
|
||||
#### restore
|
||||
**restore:**
|
||||
|
||||
1. Download `metadata/<snapshot_id>/db.zst.age`
|
||||
1. Decrypt and decompress SQLite database
|
||||
1. Query files table (optionally filtered by paths)
|
||||
1. For each file, get ordered chunk list from file_chunks
|
||||
1. Download required blobs, decrypt, decompress
|
||||
1. Extract chunks and reconstruct files
|
||||
1. Restore permissions, mtime, uid/gid
|
||||
1. Download and decrypt `metadata/<snapshot_id>/db.zst.age`
|
||||
2. Open the binary SQLite database
|
||||
3. Query files (optionally filtered by paths)
|
||||
4. Download and decrypt required blobs
|
||||
5. Extract chunks, reconstruct files
|
||||
6. Restore permissions, timestamps, ownership, symlinks
|
||||
|
||||
#### prune
|
||||
**prune:**
|
||||
|
||||
1. List all snapshot manifests
|
||||
1. Build set of all referenced blob hashes
|
||||
1. List all blobs in storage
|
||||
1. Delete any blob not in referenced set
|
||||
2. Build set of all referenced blob hashes
|
||||
3. List all blobs in storage
|
||||
4. Delete any blob not in the referenced set
|
||||
|
||||
### chunking
|
||||
### chunking and deduplication
|
||||
|
||||
* Content-defined chunking using FastCDC algorithm
|
||||
* Content-defined chunking using the FastCDC algorithm
|
||||
* Average chunk size: configurable (default 10MB)
|
||||
* Deduplication at chunk level
|
||||
* Multiple chunks packed into blobs for efficiency
|
||||
* Deduplication at file level (unchanged files skipped) and chunk level
|
||||
(identical chunks across files stored once)
|
||||
* Multiple chunks packed into blobs to reduce object count
|
||||
|
||||
### encryption
|
||||
|
||||
* Asymmetric encryption using age (X25519 + XChaCha20-Poly1305)
|
||||
* Only public key needed on source host
|
||||
* Each blob encrypted independently
|
||||
* Metadata databases also encrypted
|
||||
* Only the public key is needed on the source host
|
||||
* Each blob and each metadata database is encrypted independently
|
||||
* Multiple recipients supported (encrypt to multiple keys)
|
||||
|
||||
### compression
|
||||
|
||||
* zstd compression at configurable level
|
||||
* Applied before encryption
|
||||
* Blob-level compression for efficiency
|
||||
* zstd compression at configurable level (1-19, default 3)
|
||||
* Applied before encryption at the blob level
|
||||
|
||||
---
|
||||
|
||||
## does not
|
||||
## configuration reference
|
||||
|
||||
* Store any secrets on the backed-up machine
|
||||
* Require mutable remote metadata
|
||||
* Use tarballs, restic, rsync, or ssh
|
||||
* Require a symmetric passphrase or password
|
||||
* Trust the source system with anything
|
||||
Run `vaultik config init` to generate a fully commented config file.
|
||||
Key fields:
|
||||
|
||||
## does
|
||||
| Field | Default | Description |
|
||||
|-------|---------|-------------|
|
||||
| `age_recipients` | (required) | Age public keys for encryption |
|
||||
| `snapshots` | (required) | Named snapshot definitions with paths and excludes |
|
||||
| `storage_url` | | Storage backend URL (`s3://`, `file://`, `rclone://`) |
|
||||
| `s3.*` | | Legacy S3 configuration (endpoint, bucket, credentials) |
|
||||
| `exclude` | | Global exclude patterns (applied to all snapshots) |
|
||||
| `chunk_size` | `10MB` | Average chunk size for content-defined chunking |
|
||||
| `blob_size_limit` | `10GB` | Maximum blob size before splitting |
|
||||
| `compression_level` | `3` | zstd compression level (1-19) |
|
||||
| `hostname` | system hostname | Hostname used in snapshot IDs |
|
||||
| `index_path` | platform data dir | Local SQLite index path |
|
||||
|
||||
* Incremental deduplicated backup
|
||||
* Blob-packed chunk encryption
|
||||
* Content-addressed immutable blobs
|
||||
* Public-key encryption only
|
||||
* SQLite-based local and snapshot metadata
|
||||
* Fully stream-processed storage
|
||||
---
|
||||
|
||||
## limitations
|
||||
|
||||
* **No extended attributes (xattrs).** ACLs, macOS Finder metadata,
|
||||
quarantine flags, SELinux labels, and other extended attributes are not
|
||||
backed up or restored.
|
||||
* **No hard link detection.** Two hard links to the same inode are backed
|
||||
up as independent files. Content deduplication means the data is stored
|
||||
once, but the hard link relationship is lost on restore.
|
||||
* **No sparse file support.** Sparse files are fully materialized during
|
||||
backup. A 100 GB sparse VM disk that is mostly zeros will consume the
|
||||
full (compressed) size in storage.
|
||||
* **No bandwidth limiting.** Uploads and downloads use whatever bandwidth
|
||||
is available. There is no `--bwlimit` flag yet.
|
||||
* **No parallel blob downloads during restore.** Blobs are fetched
|
||||
sequentially. Restore speed is bound by single-stream throughput.
|
||||
* **Device nodes, named pipes, and sockets are silently skipped.** Only
|
||||
regular files, directories, and symlinks are backed up.
|
||||
* **No database migrations.** If the local SQLite schema changes between
|
||||
versions, delete the local database (`vaultik database purge`) and run
|
||||
a full backup. Remote storage is unaffected.
|
||||
* **Files that change during backup may be inconsistent.** There is no
|
||||
filesystem snapshot or freeze. If a file is modified between the scan
|
||||
and chunk phases, the backed-up copy may reflect a partial write.
|
||||
* **Ownership restoration requires root.** File uid/gid are recorded
|
||||
and restored, but `chown` requires elevated privileges. Without root,
|
||||
files are restored with the current user's ownership.
|
||||
|
||||
---
|
||||
|
||||
## roadmap
|
||||
|
||||
Items for future releases:
|
||||
|
||||
* Error-condition tests (network failures, disk full, corrupted/missing blobs)
|
||||
* Parallel blob downloads during restore
|
||||
* Bandwidth limiting (`--bwlimit`)
|
||||
* Security audit of encryption implementation
|
||||
* Man pages and richer `--help` examples
|
||||
|
||||
---
|
||||
|
||||
## requirements
|
||||
|
||||
* Go 1.24 or later
|
||||
* S3-compatible object storage
|
||||
* Sufficient disk space for local index (typically <1GB)
|
||||
* Go 1.26 or later
|
||||
* S3-compatible object storage (or local filesystem, or rclone remote)
|
||||
|
||||
## development workflow
|
||||
|
||||
All changes follow this workflow. No exceptions.
|
||||
|
||||
1. Create a feature branch off `main`.
|
||||
2. Write tests.
|
||||
3. Write the implementation.
|
||||
4. Fix implementation errors until it compiles and tests pass.
|
||||
5. Fix linting errors (`make lint`).
|
||||
6. Update documentation and README as required by the change.
|
||||
7. Format code (`make fmt`).
|
||||
8. Run `make check` (lint + fmt-check + test). Fix any issues. Repeat until clean.
|
||||
9. Commit on the branch.
|
||||
10. Merge to `main`.
|
||||
11. Push.
|
||||
|
||||
Do not commit directly to `main`. Do not skip steps.
|
||||
|
||||
Repository policies for AI agents are in [`AGENTS.md`](AGENTS.md).
|
||||
|
||||
## license
|
||||
|
||||
|
||||
105
TODO.md
105
TODO.md
@@ -1,105 +0,0 @@
|
||||
# Vaultik 1.0 TODO
|
||||
|
||||
Linear list of tasks to complete before 1.0 release.
|
||||
|
||||
## Rclone Storage Backend (Complete)
|
||||
|
||||
Add rclone as a storage backend via Go library import, allowing vaultik to use any of rclone's 70+ supported cloud storage providers.
|
||||
|
||||
**Configuration:**
|
||||
```yaml
|
||||
storage_url: "rclone://myremote/path/to/backups"
|
||||
```
|
||||
User must have rclone configured separately (via `rclone config`).
|
||||
|
||||
**Implementation Steps:**
|
||||
1. [x] Add rclone dependency to go.mod
|
||||
2. [x] Create `internal/storage/rclone.go` implementing `Storer` interface
|
||||
- `NewRcloneStorer(remote, path)` - init with `configfile.Install()` and `fs.NewFs()`
|
||||
- `Put` / `PutWithProgress` - use `operations.Rcat()`
|
||||
- `Get` - use `fs.NewObject()` then `obj.Open()`
|
||||
- `Stat` - use `fs.NewObject()` for size/metadata
|
||||
- `Delete` - use `obj.Remove()`
|
||||
- `List` / `ListStream` - use `operations.ListFn()`
|
||||
- `Info` - return remote name
|
||||
3. [x] Update `internal/storage/url.go` - parse `rclone://remote/path` URLs
|
||||
4. [x] Update `internal/storage/module.go` - add rclone case to `storerFromURL()`
|
||||
5. [x] Test with real rclone remote
|
||||
|
||||
**Error Mapping:**
|
||||
- `fs.ErrorObjectNotFound` → `ErrNotFound`
|
||||
- `fs.ErrorDirNotFound` → `ErrNotFound`
|
||||
- `fs.ErrorNotFoundInConfigFile` → `ErrRemoteNotFound` (new)
|
||||
|
||||
---
|
||||
|
||||
## CLI Polish (Priority)
|
||||
|
||||
1. Improve error messages throughout
|
||||
- Ensure all errors include actionable context
|
||||
- Add suggestions for common issues (e.g., "did you set VAULTIK_AGE_SECRET_KEY?")
|
||||
|
||||
## Security (Priority)
|
||||
|
||||
1. Audit encryption implementation
|
||||
- Verify age encryption is used correctly
|
||||
- Ensure no plaintext leaks in logs or errors
|
||||
- Verify blob hashes are computed correctly
|
||||
|
||||
1. Secure memory handling for secrets
|
||||
- Clear S3 credentials from memory after client init
|
||||
- Document that age_secret_key is env-var only (already implemented)
|
||||
|
||||
## Testing
|
||||
|
||||
1. Write integration tests for restore command
|
||||
|
||||
1. Write end-to-end integration test
|
||||
- Create backup
|
||||
- Verify backup
|
||||
- Restore backup
|
||||
- Compare restored files to originals
|
||||
|
||||
1. Add tests for edge cases
|
||||
- Empty directories
|
||||
- Symlinks
|
||||
- Special characters in filenames
|
||||
- Very large files (multi-GB)
|
||||
- Many small files (100k+)
|
||||
|
||||
1. Add tests for error conditions
|
||||
- Network failures during upload
|
||||
- Disk full during restore
|
||||
- Corrupted blobs
|
||||
- Missing blobs
|
||||
|
||||
## Performance
|
||||
|
||||
1. Profile and optimize restore performance
|
||||
- Parallel blob downloads
|
||||
- Streaming decompression/decryption
|
||||
- Efficient chunk reassembly
|
||||
|
||||
1. Add bandwidth limiting option
|
||||
- `--bwlimit` flag for upload/download speed limiting
|
||||
|
||||
## Documentation
|
||||
|
||||
1. Add man page or --help improvements
|
||||
- Detailed help for each command
|
||||
- Examples in help output
|
||||
|
||||
## Final Polish
|
||||
|
||||
1. Ensure version is set correctly in releases
|
||||
|
||||
1. Create release process
|
||||
- Binary releases for supported platforms
|
||||
- Checksums for binaries
|
||||
- Release notes template
|
||||
|
||||
1. Final code review
|
||||
- Remove debug statements
|
||||
- Ensure consistent code style
|
||||
|
||||
1. Tag and release v1.0.0
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/cli"
|
||||
"sneak.berlin/go/vaultik/internal/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
2
go.mod
2
go.mod
@@ -1,4 +1,4 @@
|
||||
module git.eeqj.de/sneak/vaultik
|
||||
module sneak.berlin/go/vaultik
|
||||
|
||||
go 1.26.1
|
||||
|
||||
|
||||
@@ -23,12 +23,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// BlobHandler is a callback function invoked when a blob is finalized and ready for upload.
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -10,16 +10,16 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/pidlock"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/adrg/xdg"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/pidlock"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// AppOptions contains common options for creating the fx application.
|
||||
@@ -125,7 +125,7 @@ func RunApp(ctx context.Context, app *fx.App) error {
|
||||
// It acquires a PID lock before starting to prevent concurrent instances.
|
||||
func RunWithApp(ctx context.Context, opts AppOptions) error {
|
||||
// Acquire PID lock to prevent concurrent instances
|
||||
lockDir := filepath.Join(xdg.DataHome, "berlin.sneak.app.vaultik")
|
||||
lockDir := filepath.Join(xdg.DataHome, "vaultik")
|
||||
lock, err := pidlock.Acquire(lockDir)
|
||||
if err != nil {
|
||||
if errors.Is(err, pidlock.ErrAlreadyRunning) {
|
||||
|
||||
522
internal/cli/config.go
Normal file
522
internal/cli/config.go
Normal file
@@ -0,0 +1,522 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const defaultConfigTemplate = `# vaultik configuration
|
||||
# Documentation: https://sneak.berlin/go/vaultik
|
||||
|
||||
# ─── REQUIRED ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Age recipient public keys for encryption.
|
||||
# Backups are encrypted to ALL listed recipients. Any one of the corresponding
|
||||
# private keys can decrypt. Generate a keypair with:
|
||||
# age-keygen -o vaultik_backup_private_key.txt
|
||||
# grep 'public key' vaultik_backup_private_key.txt
|
||||
age_recipients:
|
||||
- age1REPLACE_WITH_YOUR_PUBLIC_KEY
|
||||
|
||||
# Named snapshots. Each snapshot backs up one or more paths and can have its
|
||||
# own exclude patterns in addition to the global excludes below.
|
||||
#
|
||||
# Exclude pattern semantics:
|
||||
# - Patterns starting with / are anchored to the snapshot path root
|
||||
# (e.g. "/Library/Caches" matches only ~/Library/Caches in a ~ snapshot)
|
||||
# - Patterns without a leading / match anywhere in the tree
|
||||
# (e.g. ".cache" matches any directory named .cache at any depth)
|
||||
# - Globs are supported: *, **, ?
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
exclude:
|
||||
# Trash, temp, and filesystem metadata
|
||||
- "/.Trash"
|
||||
- "/.Trashes"
|
||||
- "/.fseventsd"
|
||||
- "/.Spotlight-V100"
|
||||
- "/.TemporaryItems"
|
||||
- "/tmp"
|
||||
- "/.rnd"
|
||||
- ".DS_Store"
|
||||
# Caches and package manager state (rebuildable)
|
||||
- ".cache"
|
||||
- ".bundle"
|
||||
- "/.cpan/build"
|
||||
- "/.cpan/sources"
|
||||
- "/.gradle/caches"
|
||||
- "/.dropbox"
|
||||
- "/.minikube/cache"
|
||||
- "/.local/share/containers/podman/machine"
|
||||
- "/.persepolis"
|
||||
- "/Library/Caches"
|
||||
- "/Library/Logs"
|
||||
- "/Library/Cookies"
|
||||
- "/Library/Metadata"
|
||||
- "/Library/Suggestions"
|
||||
- "/Library/PubSub"
|
||||
- "/Library/Homebrew"
|
||||
- "/Library/Developer"
|
||||
- "/Library/Google/GoogleSoftwareUpdate"
|
||||
- "/Library/Preferences/Macromedia/Flash Player"
|
||||
- "/Library/Preferences/SDMHelpData"
|
||||
- "/Library/VoiceTrigger/SAT"
|
||||
# Language/toolchain package caches (rebuildable from registries)
|
||||
- "/.npm"
|
||||
- "/.cargo/registry"
|
||||
- "/.cargo/git"
|
||||
- "/.rustup/toolchains"
|
||||
- "/go/pkg/mod"
|
||||
- "/.m2/repository"
|
||||
- "/.vagrant.d/boxes"
|
||||
- "node_modules"
|
||||
- "__pycache__"
|
||||
- ".venv"
|
||||
# Virtual machine disk images (huge; remove these lines to back them up)
|
||||
- "/Parallels"
|
||||
- "/Virtual Machines.localized"
|
||||
- "/VirtualBox VMs"
|
||||
- "/.orbstack"
|
||||
- "/Library/Containers/com.utmapp.UTM"
|
||||
# Downloaded LLM models (huge, re-downloadable)
|
||||
- "/.ollama/models"
|
||||
- "/.lmstudio/models"
|
||||
# Cloud-synced storage. These are synced to a provider already, and on
|
||||
# modern macOS may contain dataless placeholder files that the backup
|
||||
# would force-download in full.
|
||||
- "/Library/CloudStorage"
|
||||
- "/Library/Mobile Documents"
|
||||
# Android SDK and emulator images (re-downloadable)
|
||||
- "/Library/Android/sdk"
|
||||
- "/.android/avd"
|
||||
# Cloud-synced or restorable-from-server data
|
||||
- "/Library/Mail"
|
||||
- "/Library/Mail Downloads"
|
||||
- "/Library/Safari"
|
||||
- "/Library/Application Support/Evernote"
|
||||
- "/Library/Application Support/MobileSync"
|
||||
- "/Library/Application Support/SyncServices"
|
||||
- "/Library/Application Support/protonmail/bridge/cache"
|
||||
- "/Library/Application Support/Syncthing/index-*"
|
||||
- "/Library/Syncthing/folders"
|
||||
- "/Documents/Dropbox/.dropbox.cache"
|
||||
# Large rebuildable app data (games, media caches, device backups)
|
||||
- "/Applications/Fortnite"
|
||||
- "/Documents/Steam Content"
|
||||
- "/Library/Application Support/Ableton"
|
||||
- "/Library/Application Support/CrossOver Games"
|
||||
- "/Library/Application Support/SecondLife/cache"
|
||||
- "/Library/Application Support/Steam/SteamApps"
|
||||
- "/Library/Containers/com.docker.docker"
|
||||
- "/Library/Group Containers/group.com.apple.secure-control-center-preferences"
|
||||
- "/Library/iTunes/iPad Software Updates"
|
||||
- "/Library/iTunes/iPhone Software Updates"
|
||||
- "/Movies/CacheClip"
|
||||
- "/Movies/ProxyMedia"
|
||||
- "/Music/iTunes/Album Artwork"
|
||||
- "/Pictures/iPod Photo Cache"
|
||||
|
||||
# Third-party applications. OS-provided apps live in /System/Applications
|
||||
# on modern macOS and are never in /Applications, but Apple-installed
|
||||
# App Store apps (Safari, GarageBand, iWork, iMovie) are excluded since
|
||||
# they are re-downloadable.
|
||||
apps:
|
||||
paths:
|
||||
- /Applications
|
||||
exclude:
|
||||
- ".DS_Store"
|
||||
- "/Safari.app"
|
||||
- "/GarageBand.app"
|
||||
- "/iMovie.app"
|
||||
- "/Keynote.app"
|
||||
- "/Numbers.app"
|
||||
- "/Pages.app"
|
||||
- "/Xcode.app"
|
||||
- "/Spotify.app"
|
||||
- "/Steam.app"
|
||||
- "/VirtualBox.app"
|
||||
- "/Utilities/Adobe Installers"
|
||||
|
||||
# Storage backend (pick ONE of the three forms below).
|
||||
#
|
||||
# S3-compatible:
|
||||
# storage_url: "s3://mybucket/backups?endpoint=s3.example.com®ion=us-east-1"
|
||||
# (also set s3.access_key_id and s3.secret_access_key below)
|
||||
#
|
||||
# Local filesystem:
|
||||
# storage_url: "file:///mnt/backups/vaultik"
|
||||
#
|
||||
# Rclone (requires rclone configured separately):
|
||||
# storage_url: "rclone://myremote/path/to/backups"
|
||||
storage_url: ""
|
||||
|
||||
# ─── S3 CREDENTIALS (required for s3:// storage_url) ────────────────────────
|
||||
|
||||
# s3:
|
||||
# access_key_id: YOUR_ACCESS_KEY
|
||||
# secret_access_key: YOUR_SECRET_KEY
|
||||
# # region: us-east-1 # Default: us-east-1
|
||||
# # use_ssl: true # Default: true
|
||||
# # part_size: 5MB # Multipart upload part size. Default: 5MB
|
||||
|
||||
# ─── OPTIONAL ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Global exclude patterns applied to ALL snapshots.
|
||||
# Snapshot-specific excludes are additive.
|
||||
# exclude:
|
||||
# - "*.log"
|
||||
# - "*.tmp"
|
||||
# - ".git"
|
||||
# - "node_modules"
|
||||
|
||||
# Average chunk size for content-defined chunking (FastCDC).
|
||||
# Smaller = better deduplication but more metadata overhead.
|
||||
# Accepts: 1MB, 10M, 64KB, etc.
|
||||
# Default: 10MB
|
||||
# chunk_size: 10MB
|
||||
|
||||
# Maximum blob size before splitting into a new blob.
|
||||
# Accepts: 1GB, 10G, 500MB, etc.
|
||||
# Default: 10GB
|
||||
# blob_size_limit: 10GB
|
||||
|
||||
# Zstd compression level (1-19). Higher = better ratio but slower.
|
||||
# Default: 3
|
||||
# compression_level: 3
|
||||
|
||||
# Hostname used in snapshot IDs. Default: system hostname.
|
||||
# hostname: myserver
|
||||
|
||||
# Path to the local SQLite index database.
|
||||
# Default: the platform data directory, e.g.
|
||||
# macOS: ~/Library/Application Support/vaultik/index.sqlite
|
||||
# Linux: ~/.local/share/vaultik/index.sqlite
|
||||
# index_path: /path/to/index.sqlite
|
||||
`
|
||||
|
||||
// NewConfigCommand creates the config command group.
|
||||
func NewConfigCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "config",
|
||||
Short: "Manage the configuration file",
|
||||
Long: "Commands for creating, editing, and querying the vaultik config file.",
|
||||
}
|
||||
|
||||
cmd.AddCommand(newConfigInitCommand())
|
||||
cmd.AddCommand(newConfigEditCommand())
|
||||
cmd.AddCommand(newConfigGetCommand())
|
||||
cmd.AddCommand(newConfigSetCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newConfigInitCommand creates the 'config init' subcommand.
|
||||
func newConfigInitCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "init",
|
||||
Short: "Write a default config file",
|
||||
Long: `Creates a default configuration file with commented explanations
|
||||
for every setting. If a config file already exists at the target path,
|
||||
the command refuses to overwrite it.
|
||||
|
||||
The config is written to the path from --config, $VAULTIK_CONFIG, or
|
||||
the platform default config directory (e.g. ~/Library/Application Support/
|
||||
on macOS, ~/.config/ on Linux, /etc/vaultik/ as root).`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path := configPathForInit()
|
||||
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return fmt.Errorf("config file already exists: %s", path)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return fmt.Errorf("creating config directory %s: %w", dir, err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, []byte(defaultConfigTemplate), 0o600); err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Config written to %s\n", path)
|
||||
fmt.Println("Edit it to set your age_recipients, snapshots, and storage_url.")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigEditCommand creates the 'config edit' subcommand.
|
||||
func newConfigEditCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "edit",
|
||||
Short: "Open the config file in $EDITOR",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
editor := os.Getenv("EDITOR")
|
||||
if editor == "" {
|
||||
editor = "vi"
|
||||
}
|
||||
|
||||
ed := exec.Command(editor, path)
|
||||
ed.Stdin = os.Stdin
|
||||
ed.Stdout = os.Stdout
|
||||
ed.Stderr = os.Stderr
|
||||
return ed.Run()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigGetCommand creates the 'config get' subcommand.
|
||||
func newConfigGetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "get <key>",
|
||||
Short: "Print a config value by dotted path (e.g. s3.bucket)",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
node, err := yamlPathGet(root, strings.Split(args[0], "."))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if node.Kind == yaml.ScalarNode {
|
||||
fmt.Println(node.Value)
|
||||
return nil
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling value: %w", err)
|
||||
}
|
||||
fmt.Print(string(out))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newConfigSetCommand creates the 'config set' subcommand.
|
||||
func newConfigSetCommand() *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "set <key> <value>",
|
||||
Short: "Set a config value by dotted path (e.g. compression_level 5)",
|
||||
Long: `Sets a scalar config value addressed by dotted YAML path and writes
|
||||
the file back, preserving comments and formatting. Intermediate maps
|
||||
are created as needed.
|
||||
|
||||
Examples:
|
||||
vaultik config set compression_level 9
|
||||
vaultik config set s3.bucket mybucket
|
||||
vaultik config set storage_url "file:///mnt/backups"`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
root, err := loadYAMLFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := yamlPathSet(root, strings.Split(args[0], "."), args[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshaling config: %w", err)
|
||||
}
|
||||
|
||||
mode := os.FileMode(0o600)
|
||||
if info, err := os.Stat(path); err == nil {
|
||||
mode = info.Mode().Perm()
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, out, mode); err != nil {
|
||||
return fmt.Errorf("writing config file: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%s = %s\n", args[0], args[1])
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// loadYAMLFile parses a YAML file into a yaml.Node document tree,
|
||||
// which preserves comments and ordering for round-tripping.
|
||||
func loadYAMLFile(path string) (*yaml.Node, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading config file: %w", err)
|
||||
}
|
||||
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal(data, &root); err != nil {
|
||||
return nil, fmt.Errorf("parsing config file: %w", err)
|
||||
}
|
||||
|
||||
// An empty file yields a zero node; normalize to an empty mapping document.
|
||||
if root.Kind == 0 {
|
||||
root = yaml.Node{
|
||||
Kind: yaml.DocumentNode,
|
||||
Content: []*yaml.Node{{Kind: yaml.MappingNode}},
|
||||
}
|
||||
}
|
||||
|
||||
return &root, nil
|
||||
}
|
||||
|
||||
// yamlPathGet navigates a dotted key path through mapping and sequence
|
||||
// nodes and returns the value node. Numeric path components index into
|
||||
// sequences (e.g. "age_recipients.0").
|
||||
func yamlPathGet(root *yaml.Node, keys []string) (*yaml.Node, error) {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
return nil, fmt.Errorf("empty config file")
|
||||
}
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i, key := range keys {
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
found := false
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
node = node.Content[j+1]
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("key not found: %s", strings.Join(keys[:i+1], "."))
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
idx, err := strconv.Atoi(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
||||
}
|
||||
if idx < 0 || idx >= len(node.Content) {
|
||||
return nil, fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
||||
}
|
||||
node = node.Content[idx]
|
||||
default:
|
||||
return nil, fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// yamlPathSet navigates a dotted key path, creating intermediate maps as
|
||||
// needed, and sets the final key to the given scalar value. Numeric path
|
||||
// components index into sequences; an index equal to the sequence length
|
||||
// appends a new element (e.g. "age_recipients.1" on a 1-element list).
|
||||
func yamlPathSet(root *yaml.Node, keys []string, value string) error {
|
||||
node := root
|
||||
if node.Kind == yaml.DocumentNode {
|
||||
if len(node.Content) == 0 {
|
||||
node.Content = []*yaml.Node{{Kind: yaml.MappingNode}}
|
||||
}
|
||||
node = node.Content[0]
|
||||
}
|
||||
|
||||
for i, key := range keys {
|
||||
last := i == len(keys)-1
|
||||
|
||||
switch node.Kind {
|
||||
case yaml.MappingNode:
|
||||
var valueNode *yaml.Node
|
||||
for j := 0; j+1 < len(node.Content); j += 2 {
|
||||
if node.Content[j].Value == key {
|
||||
valueNode = node.Content[j+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if valueNode == nil {
|
||||
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
|
||||
valueNode = &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
valueNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
node.Content = append(node.Content, keyNode, valueNode)
|
||||
} else if last {
|
||||
setScalar(valueNode, value)
|
||||
}
|
||||
|
||||
node = valueNode
|
||||
|
||||
case yaml.SequenceNode:
|
||||
idx, err := strconv.Atoi(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("key %q is a list; use a numeric index", strings.Join(keys[:i], "."))
|
||||
}
|
||||
if idx < 0 || idx > len(node.Content) {
|
||||
return fmt.Errorf("index %d out of range for %s (len %d)", idx, strings.Join(keys[:i], "."), len(node.Content))
|
||||
}
|
||||
if idx == len(node.Content) {
|
||||
newNode := &yaml.Node{Kind: yaml.MappingNode}
|
||||
if last {
|
||||
newNode = &yaml.Node{Kind: yaml.ScalarNode, Value: value}
|
||||
}
|
||||
node.Content = append(node.Content, newNode)
|
||||
} else if last {
|
||||
setScalar(node.Content[idx], value)
|
||||
}
|
||||
node = node.Content[idx]
|
||||
|
||||
default:
|
||||
return fmt.Errorf("key %q is not a map or list", strings.Join(keys[:i], "."))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setScalar overwrites a node in place with a plain scalar value.
|
||||
func setScalar(n *yaml.Node, value string) {
|
||||
n.Kind = yaml.ScalarNode
|
||||
n.Tag = ""
|
||||
n.Value = value
|
||||
n.Content = nil
|
||||
n.Style = 0
|
||||
}
|
||||
|
||||
// configPathForInit returns the config path to write, checking --config flag,
|
||||
// VAULTIK_CONFIG env, and the platform default.
|
||||
func configPathForInit() string {
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath
|
||||
}
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath
|
||||
}
|
||||
return DefaultConfigPath()
|
||||
}
|
||||
161
internal/cli/config_test.go
Normal file
161
internal/cli/config_test.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
)
|
||||
|
||||
// TestDefaultConfigTemplateParses ensures the init template is valid YAML
|
||||
// that unmarshals into the Config struct with the expected snapshots.
|
||||
func TestDefaultConfigTemplateParses(t *testing.T) {
|
||||
var cfg config.Config
|
||||
if err := yaml.Unmarshal([]byte(defaultConfigTemplate), &cfg); err != nil {
|
||||
t.Fatalf("default config template is not valid YAML: %v", err)
|
||||
}
|
||||
|
||||
if len(cfg.AgeRecipients) != 1 {
|
||||
t.Errorf("expected 1 placeholder age recipient, got %d", len(cfg.AgeRecipients))
|
||||
}
|
||||
|
||||
home, ok := cfg.Snapshots["home"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'home' snapshot in default config")
|
||||
}
|
||||
if len(home.Paths) == 0 {
|
||||
t.Error("home snapshot should have at least one path")
|
||||
}
|
||||
if len(home.Exclude) == 0 {
|
||||
t.Error("home snapshot should have exclude patterns")
|
||||
}
|
||||
|
||||
apps, ok := cfg.Snapshots["apps"]
|
||||
if !ok {
|
||||
t.Fatal("expected 'apps' snapshot in default config")
|
||||
}
|
||||
if len(apps.Paths) != 1 || apps.Paths[0] != "/Applications" {
|
||||
t.Errorf("apps snapshot should back up /Applications, got %v", apps.Paths)
|
||||
}
|
||||
if len(apps.Exclude) == 0 {
|
||||
t.Error("apps snapshot should have exclude patterns")
|
||||
}
|
||||
}
|
||||
|
||||
const testYAML = `# top comment
|
||||
compression_level: 3
|
||||
age_recipients:
|
||||
- age1aaa
|
||||
s3:
|
||||
bucket: oldbucket # inline comment
|
||||
region: us-east-1
|
||||
snapshots:
|
||||
home:
|
||||
paths:
|
||||
- "~"
|
||||
`
|
||||
|
||||
func parseTestYAML(t *testing.T) *yaml.Node {
|
||||
t.Helper()
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal([]byte(testYAML), &root); err != nil {
|
||||
t.Fatalf("parsing test yaml: %v", err)
|
||||
}
|
||||
return &root
|
||||
}
|
||||
|
||||
func TestYAMLPathGet(t *testing.T) {
|
||||
root := parseTestYAML(t)
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want string
|
||||
err bool
|
||||
}{
|
||||
{"compression_level", "3", false},
|
||||
{"s3.bucket", "oldbucket", false},
|
||||
{"s3.region", "us-east-1", false},
|
||||
{"age_recipients.0", "age1aaa", false},
|
||||
{"age_recipients.5", "", true},
|
||||
{"age_recipients.notanumber", "", true},
|
||||
{"s3.nonexistent", "", true},
|
||||
{"nonexistent", "", true},
|
||||
{"compression_level.sub", "", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
node, err := yamlPathGet(root, splitPath(tt.path))
|
||||
if tt.err {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q", tt.path)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if node.Value != tt.want {
|
||||
t.Errorf("get %q = %q, want %q", tt.path, node.Value, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestYAMLPathSet(t *testing.T) {
|
||||
root := parseTestYAML(t)
|
||||
|
||||
// Overwrite existing nested value
|
||||
if err := yamlPathSet(root, splitPath("s3.bucket"), "newbucket"); err != nil {
|
||||
t.Fatalf("set s3.bucket: %v", err)
|
||||
}
|
||||
|
||||
// Create new nested key with intermediate map
|
||||
if err := yamlPathSet(root, splitPath("s3.endpoint"), "s3.example.com"); err != nil {
|
||||
t.Fatalf("set s3.endpoint: %v", err)
|
||||
}
|
||||
if err := yamlPathSet(root, splitPath("newmap.newkey"), "val"); err != nil {
|
||||
t.Fatalf("set newmap.newkey: %v", err)
|
||||
}
|
||||
|
||||
// Overwrite a sequence element and append a new one
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.0"), "age1bbb"); err != nil {
|
||||
t.Fatalf("set age_recipients.0: %v", err)
|
||||
}
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.1"), "age1ccc"); err != nil {
|
||||
t.Fatalf("append age_recipients.1: %v", err)
|
||||
}
|
||||
if err := yamlPathSet(root, splitPath("age_recipients.5"), "age1ddd"); err == nil {
|
||||
t.Error("expected out-of-range append to fail")
|
||||
}
|
||||
|
||||
// Round-trip and verify values + comment preservation
|
||||
out, err := yaml.Marshal(root)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
text := string(out)
|
||||
|
||||
for _, want := range []string{"newbucket", "s3.example.com", "newkey: val", "# top comment", "# inline comment", "age1bbb", "age1ccc"} {
|
||||
if !contains(text, want) {
|
||||
t.Errorf("round-tripped YAML missing %q:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := yamlPathGet(root, splitPath("s3.bucket"))
|
||||
if err != nil {
|
||||
t.Fatalf("get after set: %v", err)
|
||||
}
|
||||
if got.Value != "newbucket" {
|
||||
t.Errorf("s3.bucket = %q after set, want newbucket", got.Value)
|
||||
}
|
||||
}
|
||||
|
||||
func splitPath(s string) []string {
|
||||
return strings.Split(s, ".")
|
||||
}
|
||||
|
||||
func contains(haystack, needle string) bool {
|
||||
return strings.Contains(haystack, needle)
|
||||
}
|
||||
@@ -4,9 +4,9 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// NewDatabaseCommand creates the database command group
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestCLIEntry(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify all subcommands are registered
|
||||
expectedCommands := []string{"snapshot", "store", "restore", "prune", "info", "version", "remote", "database"}
|
||||
expectedCommands := []string{"config", "snapshot", "store", "restore", "prune", "info", "version", "remote", "database"}
|
||||
for _, expected := range expectedCommands {
|
||||
found := false
|
||||
for _, cmd := range cmd.Commands() {
|
||||
@@ -38,7 +38,7 @@ func TestCLIEntry(t *testing.T) {
|
||||
t.Errorf("Failed to find snapshot command: %v", err)
|
||||
} else {
|
||||
// Check snapshot subcommands
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify"}
|
||||
expectedSubCommands := []string{"create", "list", "purge", "verify", "cleanup"}
|
||||
for _, expected := range expectedSubCommands {
|
||||
found := false
|
||||
for _, subcmd := range snapshotCmd.Commands() {
|
||||
|
||||
@@ -4,10 +4,10 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewInfoCommand creates the info command
|
||||
|
||||
@@ -4,10 +4,10 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewPruneCommand creates the prune command
|
||||
|
||||
@@ -4,10 +4,10 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewRemoteCommand creates the remote command and subcommands
|
||||
|
||||
@@ -4,13 +4,13 @@ import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// RestoreOptions contains options for the restore command
|
||||
|
||||
@@ -3,7 +3,9 @@ package cli
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/adrg/xdg"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -32,13 +34,14 @@ on the source system.`,
|
||||
}
|
||||
|
||||
// Add global flags
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or /etc/vaultik/config.yml)")
|
||||
cmd.PersistentFlags().StringVar(&rootFlags.ConfigPath, "config", "", "Path to config file (default: $VAULTIK_CONFIG or platform config dir)")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Verbose, "verbose", "v", false, "Enable verbose output")
|
||||
cmd.PersistentFlags().BoolVar(&rootFlags.Debug, "debug", false, "Enable debug output")
|
||||
cmd.PersistentFlags().BoolVarP(&rootFlags.Quiet, "quiet", "q", false, "Suppress non-error output")
|
||||
|
||||
// Add subcommands
|
||||
cmd.AddCommand(
|
||||
NewConfigCommand(),
|
||||
NewRestoreCommand(),
|
||||
NewPruneCommand(),
|
||||
NewStoreCommand(),
|
||||
@@ -59,25 +62,41 @@ func GetRootFlags() RootFlags {
|
||||
}
|
||||
|
||||
// ResolveConfigPath resolves the config file path from flags, environment, or default.
|
||||
// It checks in order: 1) --config flag, 2) VAULTIK_CONFIG environment variable,
|
||||
// 3) default location /etc/vaultik/config.yml. Returns an error if no valid
|
||||
// config file can be found through any of these methods.
|
||||
// Search order: --config flag, VAULTIK_CONFIG env, XDG config dir, /etc/vaultik/config.yml.
|
||||
func ResolveConfigPath() (string, error) {
|
||||
// First check global flag
|
||||
if rootFlags.ConfigPath != "" {
|
||||
return rootFlags.ConfigPath, nil
|
||||
}
|
||||
|
||||
// Then check environment variable
|
||||
if envPath := os.Getenv("VAULTIK_CONFIG"); envPath != "" {
|
||||
return envPath, nil
|
||||
}
|
||||
|
||||
// Finally check default location
|
||||
defaultPath := "/etc/vaultik/config.yml"
|
||||
if _, err := os.Stat(defaultPath); err == nil {
|
||||
return defaultPath, nil
|
||||
for _, path := range defaultConfigPaths() {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("no config file specified, VAULTIK_CONFIG not set, and %s not found", defaultPath)
|
||||
return "", fmt.Errorf("no config file found; run 'vaultik config init' to create one, or specify with --config")
|
||||
}
|
||||
|
||||
// defaultConfigPaths returns the ordered list of config paths to search.
|
||||
// On macOS: ~/Library/Application Support/vaultik/config.yml
|
||||
// On Linux: ~/.config/vaultik/config.yml
|
||||
// Fallback: /etc/vaultik/config.yml
|
||||
func defaultConfigPaths() []string {
|
||||
return []string{
|
||||
filepath.Join(xdg.ConfigHome, "vaultik", "config.yml"),
|
||||
"/etc/vaultik/config.yml",
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultConfigPath returns the platform-appropriate default config path.
|
||||
// Used by the init command and in help text.
|
||||
func DefaultConfigPath() string {
|
||||
if os.Getuid() == 0 {
|
||||
return "/etc/vaultik/config.yml"
|
||||
}
|
||||
return filepath.Join(xdg.ConfigHome, "vaultik", "config.yml")
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// NewSnapshotCommand creates the snapshot command and subcommands
|
||||
@@ -26,6 +27,7 @@ func NewSnapshotCommand() *cobra.Command {
|
||||
cmd.AddCommand(newSnapshotVerifyCommand())
|
||||
cmd.AddCommand(newSnapshotRemoveCommand())
|
||||
cmd.AddCommand(newSnapshotPruneCommand())
|
||||
cmd.AddCommand(newSnapshotCleanupCommand())
|
||||
|
||||
return cmd
|
||||
}
|
||||
@@ -71,7 +73,9 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
OnStart: func(ctx context.Context) error {
|
||||
// Start the snapshot creation in a goroutine
|
||||
go func() {
|
||||
// Run the snapshot creation
|
||||
if opts.Cron {
|
||||
v.Stdout = io.Discard
|
||||
}
|
||||
if err := v.CreateSnapshot(opts); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Snapshot creation failed", "error", err)
|
||||
@@ -101,6 +105,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
|
||||
cmd.Flags().BoolVar(&opts.Cron, "cron", false, "Run in cron mode (silent unless error)")
|
||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false, "After backup, drop older snapshots of the same name and remove orphaned blobs")
|
||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "", "With --prune: keep snapshots newer than this duration (e.g. 4w, 30d, 6mo) instead of only the latest")
|
||||
cmd.Flags().BoolVar(&opts.SkipErrors, "skip-errors", false, "Skip file read errors (log them loudly but continue)")
|
||||
|
||||
return cmd
|
||||
@@ -462,3 +467,60 @@ accumulate from incomplete backups or deleted snapshots.`,
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// newSnapshotCleanupCommand creates the 'snapshot cleanup' subcommand
|
||||
func newSnapshotCleanupCommand() *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "cleanup",
|
||||
Short: "Remove stale local snapshot records not found in remote storage",
|
||||
Long: `Removes local database records for snapshots whose metadata no longer
|
||||
exists in remote storage. These are typically left behind by incomplete
|
||||
or interrupted backups.
|
||||
|
||||
This command does not delete anything from remote storage.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
configPath, err := ResolveConfigPath()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rootFlags := GetRootFlags()
|
||||
return RunWithApp(cmd.Context(), AppOptions{
|
||||
ConfigPath: configPath,
|
||||
LogOptions: log.LogOptions{
|
||||
Verbose: rootFlags.Verbose,
|
||||
Debug: rootFlags.Debug,
|
||||
Quiet: rootFlags.Quiet,
|
||||
},
|
||||
Modules: []fx.Option{},
|
||||
Invokes: []fx.Option{
|
||||
fx.Invoke(func(v *vaultik.Vaultik, lc fx.Lifecycle) {
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(ctx context.Context) error {
|
||||
go func() {
|
||||
if err := v.CleanupLocalSnapshots(); err != nil {
|
||||
if err != context.Canceled {
|
||||
log.Error("Cleanup failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := v.Shutdowner.Shutdown(); err != nil {
|
||||
log.Error("Failed to shutdown", "error", err)
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
},
|
||||
OnStop: func(ctx context.Context) error {
|
||||
v.Cancel()
|
||||
return nil
|
||||
},
|
||||
})
|
||||
}),
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// StoreApp contains dependencies for store commands
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"github.com/spf13/cobra"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
)
|
||||
|
||||
// NewVersionCommand creates the version command
|
||||
|
||||
@@ -9,13 +9,13 @@ import (
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/smartconfig"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/adrg/xdg"
|
||||
"go.uber.org/fx"
|
||||
"gopkg.in/yaml.v3"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const appName = "berlin.sneak.app.vaultik"
|
||||
const appName = "vaultik"
|
||||
|
||||
// expandTilde expands ~ at the start of a path to the user's home directory.
|
||||
func expandTilde(path string) string {
|
||||
@@ -236,11 +236,11 @@ func Load(path string) (*Config, error) {
|
||||
// Returns an error describing the first validation failure encountered.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.AgeRecipients) == 0 {
|
||||
return fmt.Errorf("at least one age_recipient is required")
|
||||
return fmt.Errorf("at least one age_recipient is required (generate with: age-keygen)")
|
||||
}
|
||||
|
||||
if len(c.Snapshots) == 0 {
|
||||
return fmt.Errorf("at least one snapshot must be configured")
|
||||
return fmt.Errorf("at least one snapshot must be configured (see config.example.yml)")
|
||||
}
|
||||
|
||||
for name, snap := range c.Snapshots {
|
||||
@@ -299,7 +299,7 @@ func (c *Config) validateStorage() error {
|
||||
|
||||
// Legacy S3 configuration
|
||||
if c.S3.Endpoint == "" {
|
||||
return fmt.Errorf("s3.endpoint is required (or set storage_url)")
|
||||
return fmt.Errorf("storage not configured; set storage_url or provide s3.endpoint + s3.bucket + credentials")
|
||||
}
|
||||
|
||||
if c.S3.Bucket == "" {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobChunkRepository(t *testing.T) {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
type BlobRepository struct {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestBlobRepository(t *testing.T) {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestCascadeDeleteDebug tests cascade delete with debug output
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
type ChunkFileRepository struct {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestChunkFileRepository(t *testing.T) {
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
type ChunkRepository struct {
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestChunkRepository(t *testing.T) {
|
||||
|
||||
@@ -22,8 +22,8 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
//go:embed schema/*.sql
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
type FileChunkRepository struct {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestFileChunkRepository(t *testing.T) {
|
||||
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
type FileRepository struct {
|
||||
|
||||
@@ -5,7 +5,7 @@ package database
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// File represents a file or directory in the backup system.
|
||||
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// Module provides database dependencies
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestRepositoriesTransaction(t *testing.T) {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestFileRepositoryUUIDGeneration tests that files get unique UUIDs
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestFileRepositoryEdgeCases tests edge cases for file repository
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
type SnapshotRepository struct {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// Upload represents a blob upload record
|
||||
|
||||
@@ -2,6 +2,7 @@ package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync/atomic"
|
||||
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go/logging"
|
||||
)
|
||||
|
||||
@@ -203,9 +205,12 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
|
||||
Key: aws.String(fullKey),
|
||||
})
|
||||
if err != nil {
|
||||
// Check if it's a not found error
|
||||
// TODO: Add proper error type checking
|
||||
return false, nil
|
||||
var notFound *s3types.NotFound
|
||||
var noSuchKey *s3types.NoSuchKey
|
||||
if errors.As(err, ¬Found) || errors.As(err, &noSuchKey) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"sneak.berlin/go/vaultik/internal/s3"
|
||||
)
|
||||
|
||||
func TestClient(t *testing.T) {
|
||||
|
||||
@@ -3,8 +3,8 @@ package s3
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
)
|
||||
|
||||
// Module exports S3 functionality as an fx module.
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// MockS3Client is a mock implementation of S3 operations for testing
|
||||
|
||||
@@ -7,12 +7,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func setupExcludeTestFS(t *testing.T) afero.Fs {
|
||||
|
||||
@@ -6,13 +6,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// TestFileContentChange verifies that when a file's content changes,
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// ScannerParams holds parameters for scanner creation
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -5,21 +5,22 @@ import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/blob"
|
||||
"git.eeqj.de/sneak/vaultik/internal/chunker"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/blob"
|
||||
"sneak.berlin/go/vaultik/internal/chunker"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// FileToProcess holds information about a file that needs processing
|
||||
@@ -58,7 +59,8 @@ type Scanner struct {
|
||||
exclude []string // Glob patterns for files/directories to exclude
|
||||
compiledExclude []compiledPattern // Compiled glob patterns
|
||||
progress *ProgressReporter
|
||||
skipErrors bool // Skip file read errors (log loudly but continue)
|
||||
skipErrors bool // Skip file read errors (log loudly but continue)
|
||||
output io.Writer // User-facing output (os.Stdout or io.Discard in cron mode)
|
||||
|
||||
// In-memory cache of known chunk hashes for fast existence checks
|
||||
knownChunks map[string]struct{}
|
||||
@@ -139,6 +141,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
// Compile exclude patterns
|
||||
compiledExclude := compileExcludePatterns(cfg.Exclude)
|
||||
|
||||
output := io.Writer(io.Discard)
|
||||
if cfg.EnableProgress {
|
||||
output = os.Stdout
|
||||
}
|
||||
|
||||
return &Scanner{
|
||||
fs: cfg.FS,
|
||||
chunker: chunker.NewChunker(cfg.ChunkSize),
|
||||
@@ -152,6 +159,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
||||
compiledExclude: compiledExclude,
|
||||
progress: progress,
|
||||
skipErrors: cfg.SkipErrors,
|
||||
output: output,
|
||||
pendingChunkHashes: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
@@ -202,7 +210,7 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
|
||||
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
|
||||
if len(scanResult.UnchangedFileIDs) > 0 {
|
||||
fmt.Printf("Associating %s unchanged files with snapshot...\n", formatNumber(len(scanResult.UnchangedFileIDs)))
|
||||
_, _ = fmt.Fprintf(s.output, "Associating %s unchanged files with snapshot...\n", formatNumber(len(scanResult.UnchangedFileIDs)))
|
||||
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
|
||||
return nil, fmt.Errorf("associating unchanged files: %w", err)
|
||||
}
|
||||
@@ -213,13 +221,13 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
|
||||
// Phase 2: Process files and create chunks
|
||||
if len(filesToProcess) > 0 {
|
||||
fmt.Printf("Processing %s files...\n", formatNumber(len(filesToProcess)))
|
||||
_, _ = fmt.Fprintf(s.output, "Processing %s files...\n", formatNumber(len(filesToProcess)))
|
||||
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
|
||||
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
|
||||
return nil, fmt.Errorf("process phase failed: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("No files need processing. Creating metadata-only snapshot.\n")
|
||||
_, _ = fmt.Fprintf(s.output, "No files need processing. Creating metadata-only snapshot.\n")
|
||||
log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)")
|
||||
}
|
||||
|
||||
@@ -232,18 +240,18 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
||||
// loadDatabaseState loads known files and chunks from the database into memory for fast lookup
|
||||
// This avoids per-file and per-chunk database queries during the scan and process phases
|
||||
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
|
||||
fmt.Println("Loading known files from database...")
|
||||
_, _ = fmt.Fprintln(s.output, "Loading known files from database...")
|
||||
knownFiles, err := s.loadKnownFiles(ctx, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading known files: %w", err)
|
||||
}
|
||||
fmt.Printf("Loaded %s known files from database\n", formatNumber(len(knownFiles)))
|
||||
_, _ = fmt.Fprintf(s.output, "Loaded %s known files from database\n", formatNumber(len(knownFiles)))
|
||||
|
||||
fmt.Println("Loading known chunks from database...")
|
||||
_, _ = fmt.Fprintln(s.output, "Loading known chunks from database...")
|
||||
if err := s.loadKnownChunks(ctx); err != nil {
|
||||
return nil, fmt.Errorf("loading known chunks: %w", err)
|
||||
}
|
||||
fmt.Printf("Loaded %s known chunks from database\n", formatNumber(len(s.knownChunks)))
|
||||
_, _ = fmt.Fprintf(s.output, "Loaded %s known chunks from database\n", formatNumber(len(s.knownChunks)))
|
||||
|
||||
return knownFiles, nil
|
||||
}
|
||||
@@ -267,17 +275,17 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
|
||||
"files_skipped", result.FilesSkipped,
|
||||
"bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped)))
|
||||
|
||||
fmt.Printf("Scan complete: %s examined (%s), %s to process (%s)",
|
||||
_, _ = fmt.Fprintf(s.output, "Scan complete: %s examined (%s), %s to process (%s)",
|
||||
formatNumber(result.FilesScanned),
|
||||
humanize.Bytes(uint64(totalSizeToProcess+result.BytesSkipped)),
|
||||
formatNumber(len(filesToProcess)),
|
||||
humanize.Bytes(uint64(totalSizeToProcess)))
|
||||
if result.FilesDeleted > 0 {
|
||||
fmt.Printf(", %s deleted (%s)",
|
||||
_, _ = fmt.Fprintf(s.output, ", %s deleted (%s)",
|
||||
formatNumber(result.FilesDeleted),
|
||||
humanize.Bytes(uint64(result.BytesDeleted)))
|
||||
}
|
||||
fmt.Println()
|
||||
_, _ = fmt.Fprintln(s.output)
|
||||
}
|
||||
|
||||
// finalizeScanResult populates final blob statistics in the scan result
|
||||
@@ -619,7 +627,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
if err != nil {
|
||||
if s.skipErrors {
|
||||
log.Error("ERROR: Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
|
||||
fmt.Printf("ERROR: Failed to access %s: %v (skipping)\n", filePath, err)
|
||||
_, _ = fmt.Fprintf(s.output, "ERROR: Failed to access %s: %v (skipping)\n", filePath, err)
|
||||
return nil // Continue scanning
|
||||
}
|
||||
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
|
||||
@@ -641,7 +649,40 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip non-regular files for processing (but still count them)
|
||||
// Handle symlinks
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
file := s.buildSymlinkEntry(filePath, info)
|
||||
if file != nil {
|
||||
existingFiles[filePath] = struct{}{}
|
||||
mu.Lock()
|
||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||
Path: filePath,
|
||||
FileInfo: info,
|
||||
File: file,
|
||||
})
|
||||
filesScanned++
|
||||
mu.Unlock()
|
||||
s.updateScanEntryStats(result, true, info)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handle directories (record for permission/ownership preservation and empty-dir support)
|
||||
if info.IsDir() {
|
||||
file := s.buildDirectoryEntry(filePath, info)
|
||||
existingFiles[filePath] = struct{}{}
|
||||
mu.Lock()
|
||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||
Path: filePath,
|
||||
FileInfo: info,
|
||||
File: file,
|
||||
})
|
||||
filesScanned++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip other non-regular files (devices, sockets, etc.)
|
||||
if !info.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
@@ -673,7 +714,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
||||
|
||||
// Output periodic status
|
||||
if time.Since(lastStatusTime) >= statusInterval {
|
||||
printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
|
||||
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
|
||||
@@ -714,7 +755,7 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool,
|
||||
|
||||
// printScanProgressLine prints a periodic progress line during the scan phase,
|
||||
// showing files scanned, percentage complete (if estimate available), and ETA
|
||||
func printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time) {
|
||||
func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time) {
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(filesScanned) / elapsed.Seconds()
|
||||
|
||||
@@ -732,19 +773,19 @@ func printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal
|
||||
if rate > 0 && remaining > 0 {
|
||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||
}
|
||||
fmt.Printf("Scan: %s files (~%.0f%%), %s changed/new, %.0f files/sec, %s elapsed",
|
||||
_, _ = fmt.Fprintf(s.output, "Scan: %s files (~%.0f%%), %s changed/new, %.0f files/sec, %s elapsed",
|
||||
formatNumber(int(filesScanned)),
|
||||
pct,
|
||||
formatNumber(changedCount),
|
||||
rate,
|
||||
elapsed.Round(time.Second))
|
||||
if eta > 0 {
|
||||
fmt.Printf(", ETA %s", eta.Round(time.Second))
|
||||
_, _ = fmt.Fprintf(s.output, ", ETA %s", eta.Round(time.Second))
|
||||
}
|
||||
fmt.Println()
|
||||
_, _ = fmt.Fprintln(s.output)
|
||||
} else {
|
||||
// First backup - no estimate available
|
||||
fmt.Printf("Scan: %s files, %s changed/new, %.0f files/sec, %s elapsed\n",
|
||||
_, _ = fmt.Fprintf(s.output, "Scan: %s files, %s changed/new, %.0f files/sec, %s elapsed\n",
|
||||
formatNumber(int(filesScanned)),
|
||||
formatNumber(changedCount),
|
||||
rate,
|
||||
@@ -752,6 +793,71 @@ func printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal
|
||||
}
|
||||
}
|
||||
|
||||
// buildSymlinkEntry creates a File record for a symlink.
|
||||
// Returns nil if the link target cannot be read.
|
||||
func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.File {
|
||||
target, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
log.Debug("Cannot read symlink target", "path", path, "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
var uid, gid uint32
|
||||
if stat, ok := info.Sys().(interface {
|
||||
Uid() uint32
|
||||
Gid() uint32
|
||||
}); ok {
|
||||
uid = stat.Uid()
|
||||
gid = stat.Gid()
|
||||
}
|
||||
|
||||
return &database.File{
|
||||
ID: types.NewFileID(),
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath),
|
||||
MTime: info.ModTime(),
|
||||
Size: 0,
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
GID: gid,
|
||||
LinkTarget: types.FilePath(target),
|
||||
}
|
||||
}
|
||||
|
||||
// buildDirectoryEntry creates a File record for a directory.
|
||||
func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.File {
|
||||
var uid, gid uint32
|
||||
if stat, ok := info.Sys().(interface {
|
||||
Uid() uint32
|
||||
Gid() uint32
|
||||
}); ok {
|
||||
uid = stat.Uid()
|
||||
gid = stat.Gid()
|
||||
}
|
||||
|
||||
return &database.File{
|
||||
ID: types.NewFileID(),
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath),
|
||||
MTime: info.ModTime(),
|
||||
Size: 0,
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
GID: gid,
|
||||
}
|
||||
}
|
||||
|
||||
// recordNonRegularFile writes a symlink or directory entry to the database
|
||||
// and associates it with the current snapshot. No chunking is performed.
|
||||
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
|
||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||
if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil {
|
||||
return fmt.Errorf("creating non-regular file record: %w", err)
|
||||
}
|
||||
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
|
||||
})
|
||||
}
|
||||
|
||||
// checkFileInMemory checks if a file needs processing using the in-memory map
|
||||
// No database access is performed - this is purely CPU/memory work
|
||||
func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles map[string]*database.File) (*database.File, bool) {
|
||||
@@ -849,7 +955,7 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(end) / elapsed.Seconds()
|
||||
pct := float64(end) / float64(len(fileIDs)) * 100
|
||||
fmt.Printf("Associating files: %s/%s (%.1f%%), %.0f files/sec\n",
|
||||
_, _ = fmt.Fprintf(s.output, "Associating files: %s/%s (%.1f%%), %.0f files/sec\n",
|
||||
formatNumber(end), formatNumber(len(fileIDs)), pct, rate)
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
@@ -857,7 +963,7 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
||||
|
||||
elapsed := time.Since(startTime)
|
||||
rate := float64(len(fileIDs)) / elapsed.Seconds()
|
||||
fmt.Printf("Associated %s unchanged files in %s (%.0f files/sec)\n",
|
||||
_, _ = fmt.Fprintf(s.output, "Associated %s unchanged files in %s (%.0f files/sec)\n",
|
||||
formatNumber(len(fileIDs)), elapsed.Round(time.Second), rate)
|
||||
|
||||
return nil
|
||||
@@ -905,7 +1011,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
|
||||
// Output periodic status
|
||||
if time.Since(lastStatusTime) >= statusInterval {
|
||||
printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
|
||||
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
|
||||
lastStatusTime = time.Now()
|
||||
}
|
||||
}
|
||||
@@ -927,7 +1033,7 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces
|
||||
// Skip file read errors if --skip-errors is enabled
|
||||
if s.skipErrors {
|
||||
log.Error("ERROR: Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
|
||||
fmt.Printf("ERROR: Failed to process %s: %v (skipping)\n", fileToProcess.Path, err)
|
||||
_, _ = fmt.Fprintf(s.output, "ERROR: Failed to process %s: %v (skipping)\n", fileToProcess.Path, err)
|
||||
result.FilesSkipped++
|
||||
return true, nil
|
||||
}
|
||||
@@ -938,7 +1044,7 @@ func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProces
|
||||
|
||||
// printProcessingProgress prints a periodic progress line during the process phase,
|
||||
// showing files processed, bytes transferred, throughput, and ETA
|
||||
func printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, startTime time.Time) {
|
||||
func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, startTime time.Time) {
|
||||
elapsed := time.Since(startTime)
|
||||
pct := float64(bytesProcessed) / float64(totalBytes) * 100
|
||||
byteRate := float64(bytesProcessed) / elapsed.Seconds()
|
||||
@@ -952,7 +1058,7 @@ func printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, tot
|
||||
}
|
||||
|
||||
// Format: Progress [5.7k/610k] 6.7 GB/44 GB (15.4%), 106MB/sec, 500 files/sec, running for 1m30s, ETA: 5m49s
|
||||
fmt.Printf("Progress [%s/%s] %s/%s (%.1f%%), %s/sec, %.0f files/sec, running for %s",
|
||||
_, _ = fmt.Fprintf(s.output, "Progress [%s/%s] %s/%s (%.1f%%), %s/sec, %.0f files/sec, running for %s",
|
||||
formatCompact(filesProcessed),
|
||||
formatCompact(totalFiles),
|
||||
humanize.Bytes(uint64(bytesProcessed)),
|
||||
@@ -962,9 +1068,9 @@ func printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, tot
|
||||
fileRate,
|
||||
elapsed.Round(time.Second))
|
||||
if eta > 0 {
|
||||
fmt.Printf(", ETA: %s", eta.Round(time.Second))
|
||||
_, _ = fmt.Fprintf(s.output, ", ETA: %s", eta.Round(time.Second))
|
||||
}
|
||||
fmt.Println()
|
||||
_, _ = fmt.Fprintln(s.output)
|
||||
}
|
||||
|
||||
// finalizeProcessPhase flushes the packer, writes remaining pending files to the database,
|
||||
@@ -1056,7 +1162,7 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
|
||||
if _, err := s.storage.Stat(ctx, blobPath); err == nil {
|
||||
log.Info("Blob already exists in storage, skipping upload",
|
||||
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
|
||||
fmt.Printf("Blob exists: %s (%s, skipped upload)\n",
|
||||
_, _ = fmt.Fprintf(s.output, "Blob exists: %s (%s, skipped upload)\n",
|
||||
finishedBlob.Hash[:12]+"...", humanize.Bytes(uint64(finishedBlob.Compressed)))
|
||||
return true, nil
|
||||
}
|
||||
@@ -1071,7 +1177,7 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
|
||||
uploadDuration := time.Since(startTime)
|
||||
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
|
||||
|
||||
fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n",
|
||||
_, _ = fmt.Fprintf(s.output, "Blob stored: %s (%s, %s/sec, %s)\n",
|
||||
finishedBlob.Hash[:12]+"...",
|
||||
humanize.Bytes(uint64(finishedBlob.Compressed)),
|
||||
humanize.Bytes(uint64(uploadSpeedBps)),
|
||||
@@ -1176,6 +1282,12 @@ type streamingChunkInfo struct {
|
||||
|
||||
// processFileStreaming processes a file by streaming chunks directly to the packer
|
||||
func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) error {
|
||||
// Symlinks and directories have no data to chunk — just record them in the DB.
|
||||
mode := os.FileMode(fileToProcess.File.Mode)
|
||||
if mode&os.ModeSymlink != 0 || mode.IsDir() {
|
||||
return s.recordNonRegularFile(ctx, fileToProcess)
|
||||
}
|
||||
|
||||
file, err := s.fs.Open(fileToProcess.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening file: %w", err)
|
||||
@@ -1329,7 +1441,7 @@ func (s *Scanner) detectDeletedFilesFromMap(ctx context.Context, knownFiles map[
|
||||
}
|
||||
|
||||
if result.FilesDeleted > 0 {
|
||||
fmt.Printf("Found %s deleted files\n", formatNumber(result.FilesDeleted))
|
||||
_, _ = fmt.Fprintf(s.output, "Found %s deleted files\n", formatNumber(result.FilesDeleted))
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -7,11 +7,11 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
func TestScannerSimpleDirectory(t *testing.T) {
|
||||
@@ -110,15 +110,15 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
||||
t.Errorf("expected at least 97 bytes scanned, got %d", result.BytesScanned)
|
||||
}
|
||||
|
||||
// Verify files in database - only regular files are stored
|
||||
// Verify files in database - includes regular files and directories
|
||||
files, err := repos.Files.ListByPrefix(ctx, "/source")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to list files: %v", err)
|
||||
}
|
||||
|
||||
// We should have 6 files (directories are not stored)
|
||||
if len(files) != 6 {
|
||||
t.Errorf("expected 6 files in database, got %d", len(files))
|
||||
// 6 regular files + 3 directories (/source, /source/subdir, /source/subdir2)
|
||||
if len(files) != 9 {
|
||||
t.Errorf("expected 9 entries in database (6 files + 3 dirs), got %d", len(files))
|
||||
}
|
||||
|
||||
// Verify specific file
|
||||
|
||||
@@ -44,15 +44,15 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// SnapshotManager handles snapshot creation and metadata export
|
||||
|
||||
@@ -7,10 +7,10 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/s3"
|
||||
)
|
||||
|
||||
// Module exports storage functionality as an fx module.
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"sneak.berlin/go/vaultik/internal/s3"
|
||||
)
|
||||
|
||||
// S3Storer wraps the existing s3.Client to implement Storer.
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"io"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
)
|
||||
|
||||
// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks
|
||||
|
||||
@@ -2,11 +2,12 @@ package vaultik
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// SnapshotInfo contains information about a snapshot
|
||||
@@ -95,18 +96,39 @@ func parseSnapshotName(snapshotID string) string {
|
||||
return strings.Join(parts[1:len(parts)-1], "_")
|
||||
}
|
||||
|
||||
// parseDuration parses a duration string with support for days
|
||||
// parseDuration parses a duration string with support for human-friendly units:
|
||||
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
|
||||
// duration units (h, m, s).
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
// Check for days suffix
|
||||
if strings.HasSuffix(s, "d") {
|
||||
daysStr := strings.TrimSuffix(s, "d")
|
||||
days, err := strconv.Atoi(daysStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid days value: %w", err)
|
||||
}
|
||||
return time.Duration(days) * 24 * time.Hour, nil
|
||||
if d, err := time.ParseDuration(s); err == nil {
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// Otherwise use standard Go duration parsing
|
||||
return time.ParseDuration(s)
|
||||
re := regexp.MustCompile(`(\d+)\s*([a-zA-Z]+)`)
|
||||
matches := re.FindAllStringSubmatch(s, -1)
|
||||
if len(matches) == 0 {
|
||||
return 0, fmt.Errorf("invalid duration: %q", s)
|
||||
}
|
||||
|
||||
var total time.Duration
|
||||
for _, match := range matches {
|
||||
n, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid number %q: %w", match[1], err)
|
||||
}
|
||||
unit := strings.ToLower(match[2])
|
||||
switch unit {
|
||||
case "d", "day", "days":
|
||||
total += time.Duration(n) * 24 * time.Hour
|
||||
case "w", "week", "weeks":
|
||||
total += time.Duration(n) * 7 * 24 * time.Hour
|
||||
case "mo", "month", "months":
|
||||
total += time.Duration(n) * 30 * 24 * time.Hour
|
||||
case "y", "year", "years":
|
||||
total += time.Duration(n) * 365 * 24 * time.Hour
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown time unit %q", unit)
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package vaultik
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseSnapshotName(t *testing.T) {
|
||||
@@ -37,6 +38,41 @@ func TestParseSnapshotName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want time.Duration
|
||||
err bool
|
||||
}{
|
||||
{"30d", 30 * 24 * time.Hour, false},
|
||||
{"4w", 4 * 7 * 24 * time.Hour, false},
|
||||
{"6mo", 6 * 30 * 24 * time.Hour, false},
|
||||
{"1y", 365 * 24 * time.Hour, false},
|
||||
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
|
||||
{"1h", time.Hour, false},
|
||||
{"30s", 30 * time.Second, false},
|
||||
{"garbage", 0, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
got, err := parseDuration(tt.input)
|
||||
if tt.err {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q, got %v", tt.input, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for %q: %v", tt.input, err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("parseDuration(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSnapshotTimestamp(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// ShowInfo displays system and configuration information
|
||||
|
||||
@@ -11,16 +11,16 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// MockStorer implements storage.Storer for testing
|
||||
@@ -585,6 +585,19 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
|
||||
}
|
||||
|
||||
// Create a file with non-default permissions.
|
||||
restrictedPath := filepath.Join(dataDir, "restricted.txt")
|
||||
require.NoError(t, afero.WriteFile(fs, restrictedPath, []byte("secret"), 0o600))
|
||||
testFiles[restrictedPath] = []byte("secret")
|
||||
|
||||
// Create an empty directory (should survive round-trip).
|
||||
emptyDir := filepath.Join(dataDir, "emptydir")
|
||||
require.NoError(t, fs.MkdirAll(emptyDir, 0o755))
|
||||
|
||||
// Create a symlink.
|
||||
symlinkPath := filepath.Join(dataDir, "link-to-small")
|
||||
require.NoError(t, os.Symlink("small.txt", symlinkPath))
|
||||
|
||||
// FileStorer is the real-world local-disk backend.
|
||||
storer, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
@@ -669,6 +682,25 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
require.NoError(t, err, "restored file missing: %s", restoredPath)
|
||||
require.Equalf(t, expected, got, "byte-equality failed for %s", origPath)
|
||||
}
|
||||
|
||||
// Verify the restricted file kept its permissions.
|
||||
restoredRestricted := filepath.Join(restoreDir, restrictedPath)
|
||||
rInfo, err := os.Stat(restoredRestricted)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o600), rInfo.Mode().Perm(),
|
||||
"restricted file should preserve 0600 permissions")
|
||||
|
||||
// Verify the empty directory was restored.
|
||||
restoredEmptyDir := filepath.Join(restoreDir, emptyDir)
|
||||
dInfo, err := os.Stat(restoredEmptyDir)
|
||||
require.NoError(t, err, "empty directory should be restored")
|
||||
assert.True(t, dInfo.IsDir(), "emptydir should be a directory")
|
||||
|
||||
// Verify the symlink was restored with the correct target.
|
||||
restoredSymlink := filepath.Join(restoreDir, symlinkPath)
|
||||
target, err := os.Readlink(restoredSymlink)
|
||||
require.NoError(t, err, "symlink should be restored")
|
||||
assert.Equal(t, "small.txt", target, "symlink target should be preserved")
|
||||
}
|
||||
|
||||
// bytesPattern returns a deterministic byte slice of length n with a tag prefix,
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
)
|
||||
|
||||
// PruneOptions contains options for the prune command
|
||||
|
||||
@@ -8,12 +8,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"git.eeqj.de/sneak/vaultik/internal/vaultik"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// setupPurgeTest creates a Vaultik instance with an in-memory database and mock
|
||||
|
||||
351
internal/vaultik/remove_snapshot_test.go
Normal file
351
internal/vaultik/remove_snapshot_test.go
Normal file
@@ -0,0 +1,351 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// testStorer implements storage.Storer for testing
|
||||
type testStorer struct {
|
||||
mu sync.Mutex
|
||||
data map[string][]byte
|
||||
}
|
||||
|
||||
func newTestStorer() *testStorer {
|
||||
return &testStorer{
|
||||
data: make(map[string][]byte),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *testStorer) Put(ctx context.Context, key string, reader io.Reader) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.data[key] = data
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *testStorer) PutWithProgress(ctx context.Context, key string, reader io.Reader, size int64, progress storage.ProgressCallback) error {
|
||||
return s.Put(ctx, key, reader)
|
||||
}
|
||||
|
||||
func (s *testStorer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
data, exists := s.data[key]
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(data)), nil
|
||||
}
|
||||
|
||||
func (s *testStorer) Stat(ctx context.Context, key string) (*storage.ObjectInfo, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
data, exists := s.data[key]
|
||||
if !exists {
|
||||
return nil, storage.ErrNotFound
|
||||
}
|
||||
return &storage.ObjectInfo{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *testStorer) Delete(ctx context.Context, key string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
delete(s.data, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *testStorer) List(ctx context.Context, prefix string) ([]string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
var keys []string
|
||||
for key := range s.data {
|
||||
if prefix == "" || strings.HasPrefix(key, prefix) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (s *testStorer) ListStream(ctx context.Context, prefix string) <-chan storage.ObjectInfo {
|
||||
ch := make(chan storage.ObjectInfo)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
for key, data := range s.data {
|
||||
if prefix == "" || strings.HasPrefix(key, prefix) {
|
||||
ch <- storage.ObjectInfo{
|
||||
Key: key,
|
||||
Size: int64(len(data)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
func (s *testStorer) hasKey(key string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
_, exists := s.data[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
func (s *testStorer) keyCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.data)
|
||||
}
|
||||
|
||||
func (s *testStorer) Info() storage.StorageInfo {
|
||||
return storage.StorageInfo{
|
||||
Type: "test",
|
||||
Location: "memory",
|
||||
}
|
||||
}
|
||||
|
||||
// addManifest creates a compressed manifest in storage
|
||||
func addManifest(t *testing.T, store *testStorer, snapshotID string, blobHashes []string) {
|
||||
t.Helper()
|
||||
|
||||
blobs := make([]snapshot.BlobInfo, len(blobHashes))
|
||||
for i, hash := range blobHashes {
|
||||
blobs[i] = snapshot.BlobInfo{
|
||||
Hash: hash,
|
||||
CompressedSize: 1000,
|
||||
}
|
||||
}
|
||||
|
||||
manifest := &snapshot.Manifest{
|
||||
SnapshotID: snapshotID,
|
||||
BlobCount: len(blobs),
|
||||
Blobs: blobs,
|
||||
}
|
||||
|
||||
data, err := snapshot.EncodeManifest(manifest, 3)
|
||||
require.NoError(t, err)
|
||||
|
||||
key := "metadata/" + snapshotID + "/manifest.json.zst"
|
||||
err = store.Put(context.Background(), key, bytes.NewReader(data))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// addBlob adds a fake blob to storage
|
||||
func addBlob(t *testing.T, store *testStorer, hash string) {
|
||||
t.Helper()
|
||||
|
||||
// Create zstd compressed data
|
||||
var buf bytes.Buffer
|
||||
writer, _ := zstd.NewWriter(&buf)
|
||||
_, _ = writer.Write([]byte("blob data"))
|
||||
_ = writer.Close()
|
||||
|
||||
key := "blobs/" + hash[:2] + "/" + hash[2:4] + "/" + hash
|
||||
err := store.Put(context.Background(), key, bytes.NewReader(buf.Bytes()))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Unit Tests for RemoveSnapshot
|
||||
// ============================================================================
|
||||
|
||||
func TestRemoveSnapshot_LocalOnly(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
|
||||
blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
addManifest(t, store, "snapshot-001", []string{blobA})
|
||||
addBlob(t, store, blobA)
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{Force: true}
|
||||
result, err := tv.RemoveSnapshot("snapshot-001", opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "snapshot-001", result.SnapshotID)
|
||||
assert.False(t, result.RemoteRemoved)
|
||||
|
||||
// Blobs should NOT be deleted (that's what prune is for)
|
||||
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
|
||||
// Remote metadata should NOT be deleted (no --remote flag)
|
||||
assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
|
||||
|
||||
// Verify output
|
||||
assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database")
|
||||
}
|
||||
|
||||
func TestRemoveSnapshot_WithRemote(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
|
||||
blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
addManifest(t, store, "snapshot-001", []string{blobA})
|
||||
addBlob(t, store, blobA)
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{Force: true, Remote: true}
|
||||
result, err := tv.RemoveSnapshot("snapshot-001", opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "snapshot-001", result.SnapshotID)
|
||||
assert.True(t, result.RemoteRemoved)
|
||||
|
||||
// Blobs should NOT be deleted
|
||||
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
|
||||
// Remote metadata SHOULD be deleted
|
||||
assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
|
||||
|
||||
// Verify output mentions prune
|
||||
assert.Contains(t, tv.Stdout.String(), "Removed snapshot 'snapshot-001' from local database")
|
||||
assert.Contains(t, tv.Stdout.String(), "Removed snapshot metadata from remote storage")
|
||||
assert.Contains(t, tv.Stdout.String(), "Run 'vaultik prune' to remove orphaned blobs")
|
||||
}
|
||||
|
||||
func TestRemoveSnapshot_DryRun(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
|
||||
blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
addManifest(t, store, "snapshot-001", []string{blobA})
|
||||
addBlob(t, store, blobA)
|
||||
|
||||
initialCount := store.keyCount()
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{Force: true, DryRun: true, Remote: true}
|
||||
result, err := tv.RemoveSnapshot("snapshot-001", opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.DryRun)
|
||||
|
||||
// Nothing should be deleted
|
||||
assert.Equal(t, initialCount, store.keyCount())
|
||||
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
|
||||
assert.True(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
|
||||
|
||||
// Verify dry run message
|
||||
assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
|
||||
}
|
||||
|
||||
func TestRemoveAllSnapshots_RequiresForce(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", []string{})
|
||||
addManifest(t, store, "snapshot-002", []string{})
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{All: true} // No Force
|
||||
_, err := tv.RemoveAllSnapshots(opts)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "--all requires --force")
|
||||
}
|
||||
|
||||
func TestRemoveAllSnapshots_WithForce(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
|
||||
blobA := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
addManifest(t, store, "snapshot-001", []string{blobA})
|
||||
addManifest(t, store, "snapshot-002", []string{blobA})
|
||||
addBlob(t, store, blobA)
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{All: true, Force: true, Remote: true}
|
||||
result, err := tv.RemoveAllSnapshots(opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result.SnapshotsRemoved, 2)
|
||||
assert.True(t, result.RemoteRemoved)
|
||||
|
||||
// Blobs should NOT be deleted
|
||||
assert.True(t, store.hasKey("blobs/aa/aa/"+blobA))
|
||||
// Remote metadata SHOULD be deleted
|
||||
assert.False(t, store.hasKey("metadata/snapshot-001/manifest.json.zst"))
|
||||
assert.False(t, store.hasKey("metadata/snapshot-002/manifest.json.zst"))
|
||||
|
||||
// Verify output
|
||||
assert.Contains(t, tv.Stdout.String(), "Removed 2 snapshot(s)")
|
||||
assert.Contains(t, tv.Stdout.String(), "Run 'vaultik prune' to remove orphaned blobs")
|
||||
}
|
||||
|
||||
func TestRemoveAllSnapshots_DryRun(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", []string{})
|
||||
addManifest(t, store, "snapshot-002", []string{})
|
||||
|
||||
initialCount := store.keyCount()
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{All: true, Force: true, DryRun: true}
|
||||
result, err := tv.RemoveAllSnapshots(opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.True(t, result.DryRun)
|
||||
assert.Len(t, result.SnapshotsRemoved, 2)
|
||||
|
||||
// Nothing should be deleted
|
||||
assert.Equal(t, initialCount, store.keyCount())
|
||||
|
||||
// Verify dry run message
|
||||
assert.Contains(t, tv.Stdout.String(), "[Dry run - no changes made]")
|
||||
}
|
||||
|
||||
func TestRemoveAllSnapshots_NoSnapshots(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
store := newTestStorer()
|
||||
// No snapshots added
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
|
||||
opts := &vaultik.RemoveOptions{All: true, Force: true}
|
||||
result, err := tv.RemoveAllSnapshots(opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result.SnapshotsRemoved, 0)
|
||||
|
||||
// Verify output
|
||||
assert.Contains(t, tv.Stdout.String(), "No snapshots found")
|
||||
}
|
||||
@@ -12,14 +12,14 @@ import (
|
||||
"time"
|
||||
|
||||
"filippo.io/age"
|
||||
"git.eeqj.de/sneak/vaultik/internal/blobgen"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"github.com/spf13/afero"
|
||||
"golang.org/x/term"
|
||||
"sneak.berlin/go/vaultik/internal/blobgen"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -12,20 +12,21 @@ import (
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/types"
|
||||
"github.com/dustin/go-humanize"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/types"
|
||||
)
|
||||
|
||||
// SnapshotCreateOptions contains options for the snapshot create command
|
||||
type SnapshotCreateOptions struct {
|
||||
Cron bool
|
||||
Prune bool
|
||||
SkipErrors bool // Skip file read errors (log them loudly but continue)
|
||||
Snapshots []string // Optional list of snapshot names to process (empty = all)
|
||||
Cron bool
|
||||
Prune bool
|
||||
KeepNewerThan string // With --prune: keep snapshots newer than this duration (e.g. "4w"); default: keep only latest
|
||||
SkipErrors bool // Skip file read errors (log them loudly but continue)
|
||||
Snapshots []string // Optional list of snapshot names to process (empty = all)
|
||||
}
|
||||
|
||||
// CreateSnapshot executes the snapshot creation operation
|
||||
@@ -86,7 +87,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
}
|
||||
|
||||
if opts.Prune {
|
||||
if err := v.runPostBackupPrune(snapshotNames); err != nil {
|
||||
if err := v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan); err != nil {
|
||||
return fmt.Errorf("post-backup prune: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -94,19 +95,26 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runPostBackupPrune drops older snapshots of the given names (keeping only
|
||||
// the latest of each) and removes orphan blobs from remote storage. Invoked
|
||||
// when `snapshot create --prune` is used.
|
||||
func (v *Vaultik) runPostBackupPrune(snapshotNames []string) error {
|
||||
log.Info("Running post-backup prune", "snapshots", snapshotNames)
|
||||
// runPostBackupPrune drops older snapshots of the given names and removes
|
||||
// orphan blobs from remote storage. If keepNewerThan is set (e.g. "4w"),
|
||||
// snapshots newer than that duration are kept. Otherwise only the latest
|
||||
// snapshot of each name is kept.
|
||||
func (v *Vaultik) runPostBackupPrune(snapshotNames []string, keepNewerThan string) error {
|
||||
log.Info("Running post-backup prune", "snapshots", snapshotNames, "keep_newer_than", keepNewerThan)
|
||||
v.printlnStdout("\n=== Post-backup prune ===")
|
||||
|
||||
purgeOpts := &SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
Force: true,
|
||||
Names: snapshotNames,
|
||||
Quiet: true,
|
||||
Force: true,
|
||||
Names: snapshotNames,
|
||||
Quiet: true,
|
||||
}
|
||||
|
||||
if keepNewerThan != "" {
|
||||
purgeOpts.OlderThan = keepNewerThan
|
||||
} else {
|
||||
purgeOpts.KeepLatest = true
|
||||
}
|
||||
|
||||
if err := v.PurgeSnapshotsWithOptions(purgeOpts); err != nil {
|
||||
return fmt.Errorf("purging old snapshots: %w", err)
|
||||
}
|
||||
@@ -401,7 +409,26 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
return encoder.Encode(snapshots)
|
||||
}
|
||||
|
||||
return v.printSnapshotTable(snapshots)
|
||||
if err := v.printSnapshotTable(snapshots); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Warn about local snapshots that don't exist in remote storage.
|
||||
var stale []string
|
||||
for id := range localSnapshotMap {
|
||||
if !remoteSnapshots[id] {
|
||||
stale = append(stale, id)
|
||||
}
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
v.printfStdout("\nWarning: %d local snapshot(s) not found in remote storage:\n", len(stale))
|
||||
for _, id := range stale {
|
||||
v.printfStdout(" %s\n", id)
|
||||
}
|
||||
v.printlnStdout("Run 'vaultik snapshot cleanup' to remove stale local records.")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// listRemoteSnapshotIDs returns a set of snapshot IDs found in remote storage
|
||||
@@ -865,6 +892,41 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupLocalSnapshots removes local snapshot records that have no
|
||||
// corresponding metadata in remote storage. These are typically left
|
||||
// behind by incomplete or interrupted backups.
|
||||
func (v *Vaultik) CleanupLocalSnapshots() error {
|
||||
remoteSnapshots, err := v.listRemoteSnapshotIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing local snapshots: %w", err)
|
||||
}
|
||||
|
||||
var removed int
|
||||
for _, snap := range localSnapshots {
|
||||
id := snap.ID.String()
|
||||
if !remoteSnapshots[id] {
|
||||
v.printfStdout("Removing stale local record: %s\n", id)
|
||||
if err := v.deleteSnapshotFromLocalDB(id); err != nil {
|
||||
log.Error("Failed to delete local snapshot", "snapshot_id", id, "error", err)
|
||||
continue
|
||||
}
|
||||
removed++
|
||||
}
|
||||
}
|
||||
|
||||
if removed == 0 {
|
||||
v.printlnStdout("No stale local snapshots found.")
|
||||
} else {
|
||||
v.printfStdout("Removed %d stale local snapshot record(s).\n", removed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Helper methods that were previously on SnapshotApp
|
||||
|
||||
func (v *Vaultik) downloadManifest(snapshotID string) (*snapshot.Manifest, error) {
|
||||
|
||||
@@ -7,14 +7,14 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/crypto"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"git.eeqj.de/sneak/vaultik/internal/storage"
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/globals"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// Vaultik contains all dependencies needed for vaultik operations
|
||||
|
||||
@@ -10,11 +10,11 @@ import (
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
_ "modernc.org/sqlite"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||
)
|
||||
|
||||
// VerifyOptions contains options for the verify command
|
||||
@@ -57,9 +57,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
|
||||
}
|
||||
|
||||
if !v.CanDecrypt() {
|
||||
return v.deepVerifyFailure(result, opts,
|
||||
"VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification",
|
||||
fmt.Errorf("VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification"))
|
||||
msg := "VAULTIK_AGE_SECRET_KEY not set; required for deep verification"
|
||||
return v.deepVerifyFailure(result, opts, msg, fmt.Errorf("%s", msg))
|
||||
}
|
||||
|
||||
log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep")
|
||||
|
||||
92
internal/vaultik/verify_test.go
Normal file
92
internal/vaultik/verify_test.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/crypto"
|
||||
)
|
||||
|
||||
// TestTeeReaderWithDecryption tests that TeeReader correctly hashes all encrypted
|
||||
// bytes when streaming through age decryption and zstd decompression.
|
||||
// This validates the verification path: hash encrypted blob -> decrypt -> decompress.
|
||||
func TestTeeReaderWithDecryption(t *testing.T) {
|
||||
// Test data - use random data that doesn't compress well (5MB)
|
||||
testData := make([]byte, 5*1024*1024)
|
||||
_, err := rand.Read(testData)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Compress the data
|
||||
var compressedBuf bytes.Buffer
|
||||
compressor, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
require.NoError(t, err)
|
||||
_, err = compressor.Write(testData)
|
||||
require.NoError(t, err)
|
||||
err = compressor.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Encrypt the compressed data
|
||||
testRecipient := "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv"
|
||||
testSecretKey := "AGE-SECRET-KEY-1C77PYNTHXSHNNC6EYR2W52UWYXACXA5JT00J9CCW9986M3XY87PSGP89AQ"
|
||||
|
||||
encryptor, err := crypto.NewEncryptor([]string{testRecipient})
|
||||
require.NoError(t, err)
|
||||
|
||||
var encryptedBuf bytes.Buffer
|
||||
err = encryptor.EncryptStream(&encryptedBuf, bytes.NewReader(compressedBuf.Bytes()))
|
||||
require.NoError(t, err)
|
||||
|
||||
encryptedData := encryptedBuf.Bytes()
|
||||
|
||||
// Calculate the expected hash of the encrypted data directly
|
||||
expectedHash := sha256.Sum256(encryptedData)
|
||||
expectedHashStr := hex.EncodeToString(expectedHash[:])
|
||||
|
||||
t.Logf("Encrypted data size: %d bytes", len(encryptedData))
|
||||
t.Logf("Expected hash: %s", expectedHashStr)
|
||||
|
||||
// Now simulate what verifyBlob does: use TeeReader to hash while decrypting
|
||||
decryptor, err := crypto.NewDecryptor(testSecretKey)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create hasher and tee reader
|
||||
hasher := sha256.New()
|
||||
reader := bytes.NewReader(encryptedData)
|
||||
teeReader := io.TeeReader(reader, hasher)
|
||||
|
||||
// Decrypt through the tee reader
|
||||
decryptedReader, err := decryptor.DecryptStream(teeReader)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Decompress
|
||||
decompressor, err := zstd.NewReader(decryptedReader)
|
||||
require.NoError(t, err)
|
||||
defer decompressor.Close()
|
||||
|
||||
// Read all decompressed data (simulating chunk verification)
|
||||
decompressedData, err := io.ReadAll(decompressor)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify we got the original data back
|
||||
assert.Equal(t, testData, decompressedData, "Decompressed data should match original")
|
||||
|
||||
// Drain remaining decompressed data (should be 0)
|
||||
remaining, err := io.Copy(io.Discard, decompressor)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), remaining, "No remaining decompressed data")
|
||||
|
||||
// Calculate hash from tee reader
|
||||
calculatedHashStr := hex.EncodeToString(hasher.Sum(nil))
|
||||
t.Logf("Calculated hash (before drain): %s", calculatedHashStr)
|
||||
|
||||
// Verify the hash matches the direct hash of encrypted data
|
||||
assert.Equal(t, expectedHashStr, calculatedHashStr,
|
||||
"Hash calculated via TeeReader should match direct hash of encrypted data")
|
||||
}
|
||||
Reference in New Issue
Block a user