Compare commits
5
Commits
main
..
0c468504dc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c468504dc | ||
|
|
b30e79ee45 | ||
|
|
d886a9026f | ||
|
|
6e1f499048 | ||
|
|
d24f5dc33c |
+70
-2
@@ -10,14 +10,20 @@ run:
|
||||
|
||||
linters:
|
||||
default: all
|
||||
enable:
|
||||
# Successor to the deprecated gomodguard. Named explicitly, rather than
|
||||
# left to `default: all`, because it carries the module policy below.
|
||||
- gomodguard_v2
|
||||
disable:
|
||||
# Genuinely incompatible with project patterns
|
||||
- exhaustruct # Requires all struct fields
|
||||
- depguard # Dependency allow/block lists
|
||||
- godot # Requires comments to end with periods
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- wrapcheck # Too verbose for internal packages
|
||||
- varnamelen # Short names like db, id are idiomatic Go
|
||||
# Deprecated: the warning is attached to the old name, so it is
|
||||
# silenced by disabling that name, not by enabling the successor.
|
||||
- wsl # Deprecated, replaced by wsl_v5
|
||||
- gomodguard # Deprecated, replaced by gomodguard_v2
|
||||
settings:
|
||||
lll:
|
||||
line-length: 88
|
||||
@@ -28,6 +34,68 @@ linters:
|
||||
max-complexity: 15
|
||||
dupl:
|
||||
threshold: 100
|
||||
depguard:
|
||||
# Test-support code must not be compiled into the shipped binary. A
|
||||
# test-support package exists to hand a test privileges the program
|
||||
# itself must never have, so a file that is not a test must not import
|
||||
# one. Test files, and the files inside a package whose directory name
|
||||
# ends in `test`, are where that code belongs, and are exempt.
|
||||
#
|
||||
# The deny list below is the one part of this file a repository is
|
||||
# expected to extend, and the only part it may. depguard matches an
|
||||
# import path against a list of prefixes, so it cannot be told "any path
|
||||
# whose last segment ends in test"; a repository's own test-support
|
||||
# packages have to be named here one at a time, by full import path,
|
||||
# under a module path that differs from repository to repository. Add
|
||||
# them; change nothing else.
|
||||
rules:
|
||||
test-support:
|
||||
list-mode: lax
|
||||
files:
|
||||
- "$all"
|
||||
- "!$test"
|
||||
- "!**/*test/**"
|
||||
deny:
|
||||
- pkg: net/http/httptest
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
- pkg: sneak.berlin/go/vaultik/internal/storage/faultstore
|
||||
desc: >-
|
||||
Test-support code belongs in test files and in packages whose
|
||||
directory name ends in test, not in the shipped binary.
|
||||
# Only decisions already recorded in the Go package defaults are
|
||||
# listed here. Every entry matches the module path exactly.
|
||||
gomodguard_v2:
|
||||
blocked:
|
||||
- module: github.com/rs/zerolog
|
||||
recommendations:
|
||||
- log/slog
|
||||
reason: "Structured logging is stdlib log/slog."
|
||||
# One entry per pre-fork module path, because the later releases
|
||||
# are separate paths. A prefix match would be shorter but would
|
||||
# also reach github.com/go-redis/redismock, the test double for
|
||||
# the successor these entries recommend.
|
||||
- module: github.com/go-redis/redis
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v7
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/go-redis/redis/v8
|
||||
recommendations:
|
||||
- github.com/redis/go-redis/v9
|
||||
reason: "Pre-fork module; use the maintained go-redis v9."
|
||||
- module: github.com/sergi/go-diff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "No unified diff output; use go-udiff."
|
||||
- module: github.com/hexops/gotextdiff
|
||||
recommendations:
|
||||
- github.com/aymanbagabas/go-udiff
|
||||
reason: "Unmaintained fork; use go-udiff."
|
||||
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
|
||||
@@ -102,14 +102,12 @@ Version: 2025-06-08
|
||||
build files are acceptable in the root, but source code and other files
|
||||
should be organized in appropriate subdirectories.
|
||||
|
||||
13. Pre-1.0: NEVER write database migrations. There are no live databases
|
||||
anywhere — every user's local index can be rebuilt from a fresh full
|
||||
backup. To change the schema, edit `internal/database/schema/001.sql`
|
||||
(and any code that touches the affected tables) directly; do not add new
|
||||
numbered schema files. Those numbered files and the `schema_migrations`
|
||||
table they populate only bootstrap a fresh database — they are not an
|
||||
upgrade path. The local index is disposable until 1.0 ships and is
|
||||
tagged; once 1.0 is tagged that clause expires and the question of
|
||||
upgrading existing indexes returns. See [`docs/DATAMODEL.md`](docs/DATAMODEL.md)
|
||||
for the full explanation.
|
||||
13. Pre-1.0: NEVER add a database migration. Migrations are supported, but
|
||||
nothing is installed anywhere yet, so there is nothing to migrate. To
|
||||
change the schema, edit `internal/database/schema/001.sql` (and any
|
||||
code that touches the affected tables) directly. After 1.0, each schema
|
||||
change is a new numbered file in that directory and a released file is
|
||||
never edited; an existing local database is then migrated when vaultik
|
||||
is updated. See
|
||||
[`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations).
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ Read the rules in AGENTS.md and follow them.
|
||||
done provided to you in the initial instruction. Don't do part or most of
|
||||
the work, do all of the work until the criteria for done are met.
|
||||
|
||||
* We do not need to support migrations; schema upgrades can be handled by
|
||||
* We do not add migrations before 1.0; schema upgrades can be handled by
|
||||
deleting the local state file and doing a full backup to re-create it.
|
||||
|
||||
* When testing on a 2.5Gbit/s ethernet to an s3 server backed by 2000MB/sec SSD,
|
||||
|
||||
@@ -559,13 +559,13 @@ complete annotated example also lives in
|
||||
sequentially. Restore speed is bound by single-stream throughput.
|
||||
* **Device nodes, named pipes, and sockets are silently skipped.** Only
|
||||
regular files, directories, and symlinks are backed up.
|
||||
* **No upgrade path between versions.** There is no supported way to carry
|
||||
an existing local index across a schema change; if the local SQLite
|
||||
schema changes between versions, delete the local database (`vaultik
|
||||
database delete`) and run a full backup. Remote storage is unaffected.
|
||||
(The binary does embed numbered schema files and a `schema_migrations`
|
||||
table to bootstrap a fresh database — see [`docs/DATAMODEL.md`](docs/DATAMODEL.md)
|
||||
— but that is not an upgrade path.)
|
||||
* **Before 1.0, an update can make the local index unusable.** Vaultik
|
||||
supports schema migrations, but none are added before 1.0 because
|
||||
there is no installed base yet. If an update leaves your local index
|
||||
unusable, run `vaultik database delete` and then a full backup; remote
|
||||
storage is unaffected. After 1.0, the local index is migrated when
|
||||
vaultik is updated. See
|
||||
[`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations).
|
||||
* **Files that change during backup may be inconsistent.** There is no
|
||||
filesystem snapshot or freeze. If a file is modified between the scan
|
||||
and chunk phases, the backed-up copy may reflect a partial write.
|
||||
@@ -577,22 +577,16 @@ complete annotated example also lives in
|
||||
|
||||
## roadmap
|
||||
|
||||
Items still to do before / shortly after 1.0. Loosely ordered by
|
||||
priority.
|
||||
Work planned after 1.0. Loosely ordered by priority.
|
||||
|
||||
### correctness and operability
|
||||
|
||||
* **Security audit of the encryption implementation.** Pre-1.0
|
||||
blocker if we're advertising "secure" at the top of this README.
|
||||
age + zstd + content-defined chunking is mostly off-the-shelf
|
||||
pieces, but the seams (key handling, recipient parsing, manifest
|
||||
trust boundary, restore-time identity validation) need an outside
|
||||
read.
|
||||
* **Error-condition tests.** Today's coverage is the happy path
|
||||
plus a few specific regressions. Need fault-injection coverage:
|
||||
network failures mid-blob, disk-full during restore, corrupted /
|
||||
truncated / missing blobs, partial uploads, kill -9 between
|
||||
manifest and db.zst.age writes.
|
||||
* **Outside security audit.** Before 1.0 the encryption and
|
||||
blob-generation code was reviewed and every finding fixed; no
|
||||
outside audit has been done. age + zstd + content-defined chunking
|
||||
is mostly off-the-shelf pieces, but the seams (key handling,
|
||||
recipient parsing, manifest trust boundary, restore-time identity
|
||||
validation) need an outside read.
|
||||
* **Verify restored content end-to-end in CI.** The current
|
||||
integration test does this for a small synthetic snapshot but
|
||||
not at scale. A nightly job against a multi-GB representative
|
||||
@@ -615,13 +609,15 @@ priority.
|
||||
doesn't resume from where it stopped or skip already-present
|
||||
files. A `--resume` mode that checks targets before fetching
|
||||
blobs would matter for very large restores.
|
||||
* **Daemon mode.** A long-running mode that watches for file
|
||||
changes so frequent backups, such as hourly, skip the full scan.
|
||||
It adds little for the usual runs from cron every 12 to 36 hours.
|
||||
See [issue #204](https://git.eeqj.de/sneak/vaultik/issues/204).
|
||||
|
||||
### usability
|
||||
|
||||
* **Man pages and richer `--help` examples.** Cobra generates
|
||||
basic help; man pages would be a separate target.
|
||||
* **`--bwlimit` style human-readable size flags** across the
|
||||
command surface where they're currently raw integers.
|
||||
* **`vaultik snapshot diff <a> <b>`** — show which files changed
|
||||
between two snapshots without restoring either.
|
||||
* **Status reporting hook for `--cron`.** When a backup fails
|
||||
@@ -631,12 +627,11 @@ priority.
|
||||
|
||||
### infrastructure
|
||||
|
||||
* **Cross-version schema upgrades.** There is no upgrade path between
|
||||
released versions — pre-1.0 schema changes are handled by `vaultik
|
||||
database delete` plus a full re-scan (see
|
||||
[`docs/DATAMODEL.md`](docs/DATAMODEL.md)). Post-1.0 we'll need a
|
||||
migration story to keep existing index databases usable across
|
||||
upgrades.
|
||||
* **Schema migrations after 1.0.** Migrations are supported, but none
|
||||
are added before 1.0 because there is no installed base yet. After
|
||||
1.0, each schema change is a new migration, so an existing local
|
||||
index is migrated when vaultik is updated (see
|
||||
[`docs/DATAMODEL.md`](docs/DATAMODEL.md#schema-migrations)).
|
||||
* **Storage backend coverage tests.** S3, file://, and rclone://
|
||||
all share the Storer interface but the rclone path is the least
|
||||
exercised in CI.
|
||||
|
||||
@@ -14,14 +14,11 @@ pre-1.0
|
||||
|
||||
# Next Step
|
||||
|
||||
Define the remaining scope for the first tagged release under the 1.0.0
|
||||
milestone, then cut that tag. The mechanism to cut it now exists and is
|
||||
exercised; what is left is the scope decision, which is the owner's.
|
||||
This step deliberately names one version number: it previously said
|
||||
"cut v0.1.0" while the `Makefile` baked in `1.0.0-rc.1` and the issue
|
||||
milestone said 1.0.0, and three different answers to "what is the next
|
||||
release" is exactly the contradiction
|
||||
[issue #65](https://git.eeqj.de/sneak/vaultik/issues/65) was filed over.
|
||||
The 1.0 work is complete on `next`: the scope settled on
|
||||
[issue #125](https://git.eeqj.de/sneak/vaultik/issues/125) was the work
|
||||
already planned for 1.0, and all of it has landed. The mechanism to cut
|
||||
the tag exists and is exercised; what is left is merging `next` to
|
||||
`main` and tagging, both the owner's.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
@@ -693,4 +690,5 @@ release" is exactly the contradiction
|
||||
|
||||
# Future Steps
|
||||
|
||||
None queued; the release-scoping item is now the Next Step.
|
||||
Work planned after 1.0 is listed in the README
|
||||
[roadmap](README.md#roadmap).
|
||||
|
||||
+18
-23
@@ -6,33 +6,28 @@ Vaultik uses a local SQLite database to track file metadata, chunk mappings, and
|
||||
|
||||
**Important Notes:**
|
||||
|
||||
This section is the authoritative explanation of the schema/migration story;
|
||||
other documents (the README and `AGENTS.md`) link here.
|
||||
|
||||
- **No upgrade path between versions (pre-1.0)**: Vaultik has no supported way to
|
||||
carry an existing local index across a schema change. The index is disposable
|
||||
— if the on-disk schema changes between versions, delete the local SQLite
|
||||
database (`vaultik database delete`) and run a full backup. Remote storage is
|
||||
unaffected; the new index re-deduplicates against existing remote blobs. This
|
||||
is the standing project policy, and it is separate from the schema bootstrap
|
||||
described next.
|
||||
- **Schema bootstrap**: a fresh database is populated from numbered SQL files
|
||||
embedded in the binary under `internal/database/schema/`. `000.sql` creates the
|
||||
`schema_migrations` table; `001.sql` creates the application tables. On opening
|
||||
a database the code applies each numbered file that has not yet run and records
|
||||
its version in `schema_migrations`. This bootstraps a new database; it does not
|
||||
upgrade an existing one between released versions.
|
||||
- **Changing the schema (pre-1.0)**: edit `internal/database/schema/001.sql` (and
|
||||
the code that touches the affected tables) directly. Do not add new numbered
|
||||
files — there is no installed base to migrate.
|
||||
- **Disposability expires at 1.0**: the index is treated as disposable only until
|
||||
1.0 ships and is tagged. Once 1.0 is tagged that clause expires and the
|
||||
question of upgrading existing indexes returns. It is deliberately left open
|
||||
here.
|
||||
- **Version Compatibility**: In rare cases, you may need to use the same version
|
||||
of Vaultik to restore a backup as was used to create it. This ensures
|
||||
compatibility with the metadata format stored in S3.
|
||||
|
||||
## Schema Migrations
|
||||
|
||||
Vaultik supports schema migrations. They are the numbered SQL files in
|
||||
`internal/database/schema/`, embedded in the binary: `000.sql` creates the
|
||||
`schema_migrations` table, which records each migration that has run, and
|
||||
`001.sql` creates the application tables. `database.New` opens a database and
|
||||
applies, in order, every migration that database has not yet recorded.
|
||||
|
||||
**Before 1.0** no migrations are added, because nothing is installed anywhere
|
||||
yet. A schema change edits `001.sql` (and the code that uses the affected
|
||||
tables) directly. A local database created before the change has already
|
||||
recorded `001.sql` as run, so it keeps the old schema and can become unusable;
|
||||
`vaultik database delete` followed by a full backup rebuilds it.
|
||||
|
||||
**After 1.0** each schema change is a new numbered file, so an existing local
|
||||
database is migrated the first time an updated vaultik opens it. A file that
|
||||
has shipped in a release is never edited.
|
||||
|
||||
## Database Tables
|
||||
|
||||
### 1. `files`
|
||||
|
||||
@@ -680,18 +680,10 @@ func faultScannerFactory(
|
||||
|
||||
// Scenario 5: the restore target runs out of space mid-file. Restore
|
||||
// must fail with an out-of-space error, and must not leave a truncated
|
||||
// file at the target path presenting as a complete restore. Restore
|
||||
// today writes each file straight to its final path and does not remove
|
||||
// it when a write fails, so the truncated file survives; deleting it is
|
||||
// tracked by https://git.eeqj.de/sneak/vaultik/issues/163. Skipped until
|
||||
// that lands, so the destination assertion below is recorded rather than
|
||||
// dropped.
|
||||
// file at the target path presenting as a complete restore.
|
||||
//
|
||||
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||
func TestRestoreReportsDiskFull(t *testing.T) {
|
||||
t.Skip("blocked on https://git.eeqj.de/sneak/vaultik/issues/163: " +
|
||||
"a disk-full write leaves a truncated file at the target path " +
|
||||
"instead of removing it")
|
||||
log.Initialize(log.Config{})
|
||||
|
||||
osFS := afero.NewOsFs()
|
||||
@@ -717,10 +709,10 @@ func TestRestoreReportsDiskFull(t *testing.T) {
|
||||
id := fullFaultBackup(ctx, t, osFS, inner, cfg, repos, dataDir, dbPath, "diskfull")
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Restore onto a filesystem that allows only a few bytes of file
|
||||
// content: enough to create files, far too little to hold them.
|
||||
// Restore onto a target that allows only a few bytes of file content:
|
||||
// enough to create files, far too little to hold them.
|
||||
budget := int64(8)
|
||||
quota := "aFS{Fs: osFS, remaining: &budget}
|
||||
quota := "aFS{Fs: osFS, dir: restoreDir, remaining: &budget}
|
||||
|
||||
v := newReaderVaultik(ctx, cfg, inner, nil, quota)
|
||||
err = v.Restore(&vaultik.RestoreOptions{SnapshotID: id, TargetDir: restoreDir})
|
||||
@@ -754,21 +746,26 @@ func assertRestoredTree(
|
||||
// budget is exhausted, mirroring a real ENOSPC.
|
||||
var errNoSpace = errors.New("no space left on device")
|
||||
|
||||
// quotaFS is an afero.Fs whose files may write only a fixed total number
|
||||
// of content bytes before failing, simulating a full restore target. It
|
||||
// wraps the interface so every method except Create delegates to the
|
||||
// real filesystem; only file writes are capped.
|
||||
// quotaFS is an afero.Fs on which files opened under dir may write only
|
||||
// a fixed total number of content bytes before failing, simulating a full
|
||||
// restore target. Every other method, and every file outside dir, goes
|
||||
// straight to the real filesystem: restore also writes the decrypted
|
||||
// metadata database under $TMPDIR through this filesystem, and capping
|
||||
// that would fail the restore before it wrote anything to the target.
|
||||
type quotaFS struct {
|
||||
afero.Fs
|
||||
|
||||
dir string
|
||||
remaining *int64
|
||||
}
|
||||
|
||||
//nolint:ireturn // afero.Fs.Create's signature requires returning afero.File.
|
||||
func (q *quotaFS) Create(name string) (afero.File, error) {
|
||||
f, err := q.Fs.Create(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
//nolint:ireturn // afero.Fs.OpenFile's signature requires returning afero.File.
|
||||
func (q *quotaFS) OpenFile(
|
||||
name string, flag int, perm os.FileMode,
|
||||
) (afero.File, error) {
|
||||
f, err := q.Fs.OpenFile(name, flag, perm)
|
||||
if err != nil || !strings.HasPrefix(name, q.dir) {
|
||||
return f, err
|
||||
}
|
||||
|
||||
return "aFile{File: f, remaining: q.remaining}, nil
|
||||
|
||||
Reference in New Issue
Block a user