Compare commits
2
Commits
d27e276032
...
b98822a5ab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b98822a5ab | ||
|
|
cdc60c4dfa |
@@ -352,8 +352,9 @@ may hold snapshots this host doesn't know about), which is what
|
|||||||
prune` invocation to run as a follow-up. Local row cleanup (files,
|
prune` invocation to run as a follow-up. Local row cleanup (files,
|
||||||
chunks, blobs the snapshot was the last referrer for) runs
|
chunks, blobs the snapshot was the last referrer for) runs
|
||||||
automatically. If the destination store is unreachable, the local-DB
|
automatically. If the destination store is unreachable, the local-DB
|
||||||
removal still completes and a warning is emitted; rerun `vaultik prune`
|
removal still completes and a warning is emitted; run `vaultik snapshot
|
||||||
once the store is reachable to finish remote cleanup. To wipe everything
|
remove <snapshot-id>` again once the store is reachable to remove the
|
||||||
|
snapshot's metadata from it (`vaultik prune` does not). To wipe everything
|
||||||
on the destination in one go, use `vaultik remote nuke --force`.
|
on the destination in one go, use `vaultik remote nuke --force`.
|
||||||
* `--local-only`: Skip remote cleanup; only touch the local index
|
* `--local-only`: Skip remote cleanup; only touch the local index
|
||||||
* `--dry-run`: Show what would be deleted without deleting
|
* `--dry-run`: Show what would be deleted without deleting
|
||||||
|
|||||||
@@ -22,6 +22,27 @@ the tag exists and is exercised; what is left is merging `next` to
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-07: Made a backup notice a file rewritten with its size
|
||||||
|
unchanged and a new mtime in the same second as the one in the index
|
||||||
|
([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The
|
||||||
|
`files` table held mtime in whole seconds and the scanner compared
|
||||||
|
whole seconds, so every later snapshot kept the old content. A new
|
||||||
|
`mtime_nsec` column holds the nanoseconds within the second that
|
||||||
|
`mtime` holds, and the scanner compares the full mtime. A local index
|
||||||
|
created before the change lacks the column and is rebuilt with
|
||||||
|
`vaultik database delete` and a full backup.
|
||||||
|
|
||||||
|
- 2026-10-06: Made `snapshot remove --json` write only its document to
|
||||||
|
stdout when the destination store cannot be reached
|
||||||
|
([issue #251](https://git.eeqj.de/sneak/vaultik/issues/251)). Its
|
||||||
|
warning that the snapshot's metadata was left on the destination store
|
||||||
|
went to stdout ahead of the document, breaking `| jq` on a command that
|
||||||
|
exited 0. Under `--json` the warning now reaches stderr only, through
|
||||||
|
the logger. The warning, the README and the command's help said
|
||||||
|
`vaultik prune` would finish the cleanup, but `prune` never removes
|
||||||
|
snapshot metadata; they now say to run `vaultik snapshot remove` for the
|
||||||
|
snapshot again once the destination store is reachable.
|
||||||
|
|
||||||
- 2026-10-06: Made the backup summary and the `snapshots` row count each
|
- 2026-10-06: Made the backup summary and the `snapshots` row count each
|
||||||
file, byte and upload once
|
file, byte and upload once
|
||||||
([issue #225](https://git.eeqj.de/sneak/vaultik/issues/225)). The
|
([issue #225](https://git.eeqj.de/sneak/vaultik/issues/225)). The
|
||||||
|
|||||||
+2
-1
@@ -36,7 +36,8 @@ Stores metadata about files in the filesystem being backed up.
|
|||||||
**Columns:**
|
**Columns:**
|
||||||
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
||||||
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
||||||
- `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp
|
- `mtime` (INTEGER NOT NULL) - Modification time, whole seconds since the Unix epoch
|
||||||
|
- `mtime_nsec` (INTEGER NOT NULL) - Nanoseconds within that second, 0 to 999999999
|
||||||
- `size` (INTEGER NOT NULL) - File size in bytes
|
- `size` (INTEGER NOT NULL) - File size in bytes
|
||||||
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
||||||
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ package cli //nolint:testpackage // shares hermeticConfig and the capture helper
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -87,12 +89,45 @@ func TestEntryJSONFailureIsReportedOnStderr(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeUnusableDestinationConfig builds a config whose destination
|
// TestEntrySnapshotRemoveJSONWarningIsOnStderr runs `snapshot remove
|
||||||
// directory does not exist, which fails `remote info`, and whose local
|
// --json` on a snapshot in the local index, against a destination
|
||||||
// index is bound to another destination, which fails `prune` and
|
// directory that does not exist. The command removes the snapshot from
|
||||||
// `snapshot remove` (a missing destination alone only makes `snapshot
|
// the local index and still exits 0. Its stdout must hold the document
|
||||||
// remove` warn). Returns the config path.
|
// alone, with the warning about the destination store on stderr: the
|
||||||
func writeUnusableDestinationConfig(t *testing.T) string {
|
// command to run again once it is reachable, and the snapshot's ID in
|
||||||
|
// the record's snapshot_id field.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntrySnapshotRemoveJSONWarningIsOnStderr(t *testing.T) {
|
||||||
|
configPath, indexPath := writeMissingDestinationConfig(t)
|
||||||
|
seedStaleSnapshotRecord(t, indexPath)
|
||||||
|
|
||||||
|
code, stdout, stderr := runEntry(t, flagConfig, configPath,
|
||||||
|
cmdSnapshot, cmdRemove, stalePruneSnapshotID, flagJSON)
|
||||||
|
|
||||||
|
require.Equal(t, 0, code)
|
||||||
|
requireExactlyOneJSONDocument(t, stdout)
|
||||||
|
|
||||||
|
// stderr is a pipe here, so the logger writes one JSON record a line.
|
||||||
|
var warning map[string]any
|
||||||
|
|
||||||
|
for line := range strings.Lines(stderr) {
|
||||||
|
if strings.Contains(line,
|
||||||
|
"Could not remove snapshot metadata from remote storage") {
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(line), &warning))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NotNil(t, warning, "the warning must reach stderr")
|
||||||
|
assert.Contains(t, warning[slog.MessageKey],
|
||||||
|
"run 'vaultik snapshot remove' with the snapshot's ID again")
|
||||||
|
assert.Equal(t, stalePruneSnapshotID, warning["snapshot_id"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeMissingDestinationConfig builds a config whose destination
|
||||||
|
// directory does not exist. Returns the config path and the path of
|
||||||
|
// its local index, which is not created here.
|
||||||
|
func writeMissingDestinationConfig(t *testing.T) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
@@ -114,6 +149,19 @@ func writeUnusableDestinationConfig(t *testing.T) string {
|
|||||||
xdg.Reload()
|
xdg.Reload()
|
||||||
t.Cleanup(xdg.Reload)
|
t.Cleanup(xdg.Reload)
|
||||||
|
|
||||||
|
return configPath, indexPath
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeUnusableDestinationConfig builds a config whose destination
|
||||||
|
// directory does not exist, which fails `remote info`, and whose local
|
||||||
|
// index is bound to another destination, which fails `prune` and
|
||||||
|
// `snapshot remove` (a missing destination alone only makes `snapshot
|
||||||
|
// remove` warn). Returns the config path.
|
||||||
|
func writeUnusableDestinationConfig(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
configPath, indexPath := writeMissingDestinationConfig(t)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
db, err := database.New(ctx, indexPath)
|
db, err := database.New(ctx, indexPath)
|
||||||
@@ -122,7 +170,7 @@ func writeUnusableDestinationConfig(t *testing.T) string {
|
|||||||
defer func() { require.NoError(t, db.Close()) }()
|
defer func() { require.NoError(t, db.Close()) }()
|
||||||
|
|
||||||
require.NoError(t, database.NewRepositories(db).LocalMeta.Set(ctx,
|
require.NoError(t, database.NewRepositories(db).LocalMeta.Set(ctx,
|
||||||
database.LocalMetaKeyStorageURL, "file://"+filepath.Join(dir, "other")))
|
database.LocalMetaKeyStorageURL, "file://"+t.TempDir()))
|
||||||
|
|
||||||
return configPath
|
return configPath
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -258,8 +258,9 @@ Use --local-only to skip the remote half (e.g. when you want to forget a
|
|||||||
snapshot locally without touching the destination store).
|
snapshot locally without touching the destination store).
|
||||||
|
|
||||||
If the remote is unreachable, the local-database removal still completes
|
If the remote is unreachable, the local-database removal still completes
|
||||||
and a warning is emitted; rerun 'vaultik prune' once the destination store
|
and a warning is emitted; run 'vaultik snapshot remove <snapshot-id>' again
|
||||||
is reachable to finish remote cleanup.
|
once the destination store is reachable to remove the snapshot's metadata
|
||||||
|
from it ('vaultik prune' does not).
|
||||||
|
|
||||||
To wipe the entire destination store and start over, use 'vaultik remote
|
To wipe the entire destination store and start over, use 'vaultik remote
|
||||||
nuke --force' — it is the single supported entry point for that.`,
|
nuke --force' — it is the single supported entry point for that.`,
|
||||||
|
|||||||
+28
-19
@@ -33,11 +33,13 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
query := `
|
query := `
|
||||||
INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target)
|
INSERT INTO files
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(path) DO UPDATE SET
|
ON CONFLICT(path) DO UPDATE SET
|
||||||
source_path = excluded.source_path,
|
source_path = excluded.source_path,
|
||||||
mtime = excluded.mtime,
|
mtime = excluded.mtime,
|
||||||
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
size = excluded.size,
|
size = excluded.size,
|
||||||
mode = excluded.mode,
|
mode = excluded.mode,
|
||||||
uid = excluded.uid,
|
uid = excluded.uid,
|
||||||
@@ -54,16 +56,19 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
if tx != nil {
|
if tx != nil {
|
||||||
LogSQL("Execute", query,
|
LogSQL("Execute", query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String())
|
file.LinkTarget.String())
|
||||||
err = tx.QueryRowContext(ctx, query,
|
err = tx.QueryRowContext(ctx, query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String()).Scan(&idStr)
|
file.LinkTarget.String()).Scan(&idStr)
|
||||||
} else {
|
} else {
|
||||||
err = r.db.QueryRowWithLog(ctx, query,
|
err = r.db.QueryRowWithLog(ctx, query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String()).Scan(&idStr)
|
file.LinkTarget.String()).Scan(&idStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +89,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
// in the index.
|
// in the index.
|
||||||
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ?
|
WHERE path = ?
|
||||||
`
|
`
|
||||||
@@ -104,7 +109,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
|
|||||||
// GetByID retrieves a file by its UUID
|
// GetByID retrieves a file by its UUID
|
||||||
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
|
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE id = ?
|
WHERE id = ?
|
||||||
`
|
`
|
||||||
@@ -127,7 +132,7 @@ func (r *FileRepository) GetByPathTx(
|
|||||||
ctx context.Context, tx *sql.Tx, path string,
|
ctx context.Context, tx *sql.Tx, path string,
|
||||||
) (*File, error) {
|
) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ?
|
WHERE path = ?
|
||||||
`
|
`
|
||||||
@@ -158,13 +163,14 @@ func (r *FileRepository) ListModifiedSince(
|
|||||||
ctx context.Context, since time.Time,
|
ctx context.Context, since time.Time,
|
||||||
) ([]*File, error) {
|
) ([]*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE mtime >= ?
|
WHERE (mtime, mtime_nsec) >= (?, ?)
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
`
|
`
|
||||||
|
|
||||||
rows, err := r.db.conn.QueryContext(ctx, query, since.Unix())
|
rows, err := r.db.conn.QueryContext(ctx, query,
|
||||||
|
since.Unix(), since.Nanosecond())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying files: %w", err)
|
return nil, fmt.Errorf("querying files: %w", err)
|
||||||
}
|
}
|
||||||
@@ -239,7 +245,7 @@ func (r *FileRepository) ListUnderPath(
|
|||||||
|
|
||||||
// LIKE would ignore ASCII case and treat _ and % in path as wildcards.
|
// LIKE would ignore ASCII case and treat _ and % in path as wildcards.
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ? OR substr(path, 1, length(?)) = ?
|
WHERE path = ? OR substr(path, 1, length(?)) = ?
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
@@ -324,7 +330,7 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
|
|||||||
// ListAll returns all files in the database
|
// ListAll returns all files in the database
|
||||||
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
`
|
`
|
||||||
@@ -365,7 +371,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Each files row binds this many SQL variables.
|
// Each files row binds this many SQL variables.
|
||||||
const fileCols = 9
|
const fileCols = 10
|
||||||
|
|
||||||
// Batch at 100 rows to be safe with SQLite's variable limit.
|
// Batch at 100 rows to be safe with SQLite's variable limit.
|
||||||
const batchSize = 100
|
const batchSize = 100
|
||||||
@@ -376,7 +382,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
batch := files[i:end]
|
batch := files[i:end]
|
||||||
|
|
||||||
query := `INSERT INTO files
|
query := `INSERT INTO files
|
||||||
(id, path, source_path, mtime, size, mode, uid, gid, link_target)
|
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target)
|
||||||
VALUES `
|
VALUES `
|
||||||
|
|
||||||
args := make([]any, 0, len(batch)*fileCols)
|
args := make([]any, 0, len(batch)*fileCols)
|
||||||
@@ -388,11 +394,12 @@ func (r *FileRepository) CreateBatch(
|
|||||||
querySb325.WriteString(", ")
|
querySb325.WriteString(", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||||
|
|
||||||
args = append(args,
|
args = append(args,
|
||||||
f.ID.String(), f.Path.String(), f.SourcePath.String(),
|
f.ID.String(), f.Path.String(), f.SourcePath.String(),
|
||||||
f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
|
f.MTime.Unix(), f.MTime.Nanosecond(),
|
||||||
|
f.Size, f.Mode, f.UID, f.GID,
|
||||||
f.LinkTarget.String())
|
f.LinkTarget.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -401,6 +408,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
query += ` ON CONFLICT(path) DO UPDATE SET
|
query += ` ON CONFLICT(path) DO UPDATE SET
|
||||||
source_path = excluded.source_path,
|
source_path = excluded.source_path,
|
||||||
mtime = excluded.mtime,
|
mtime = excluded.mtime,
|
||||||
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
size = excluded.size,
|
size = excluded.size,
|
||||||
mode = excluded.mode,
|
mode = excluded.mode,
|
||||||
uid = excluded.uid,
|
uid = excluded.uid,
|
||||||
@@ -460,7 +468,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
var (
|
var (
|
||||||
file File
|
file File
|
||||||
idStr, pathStr, sourcePathStr string
|
idStr, pathStr, sourcePathStr string
|
||||||
mtimeUnix int64
|
mtimeUnix, mtimeNsec int64
|
||||||
linkTarget sql.NullString
|
linkTarget sql.NullString
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -469,6 +477,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
&pathStr,
|
&pathStr,
|
||||||
&sourcePathStr,
|
&sourcePathStr,
|
||||||
&mtimeUnix,
|
&mtimeUnix,
|
||||||
|
&mtimeNsec,
|
||||||
&file.Size,
|
&file.Size,
|
||||||
&file.Mode,
|
&file.Mode,
|
||||||
&file.UID,
|
&file.UID,
|
||||||
@@ -487,7 +496,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
file.Path = types.FilePath(pathStr)
|
file.Path = types.FilePath(pathStr)
|
||||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||||
|
|
||||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
file.MTime = time.Unix(mtimeUnix, mtimeNsec).UTC()
|
||||||
if linkTarget.Valid {
|
if linkTarget.Valid {
|
||||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -252,6 +252,58 @@ func TestFileRepositorySymlink(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An mtime after 2262 or before 1678 does not fit in int64 nanoseconds
|
||||||
|
// since the epoch, and must still come back from the database unchanged.
|
||||||
|
func TestFileRepositoryMTimeOutsideInt64NanosecondRange(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db, cleanup := setupTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
repo := database.NewFileRepository(db)
|
||||||
|
|
||||||
|
mtimes := []time.Time{
|
||||||
|
time.Date(2300, time.January, 1, 0, 0, 0, 123456789, time.UTC),
|
||||||
|
time.Date(1601, time.January, 1, 0, 0, 0, 987654321, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, mtime := range mtimes {
|
||||||
|
created := &database.File{
|
||||||
|
Path: types.FilePath("/created-" + mtime.Format(time.RFC3339Nano)),
|
||||||
|
MTime: mtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := repo.Create(ctx, nil, created)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
batched := &database.File{
|
||||||
|
ID: types.NewFileID(),
|
||||||
|
Path: types.FilePath("/batched-" + mtime.Format(time.RFC3339Nano)),
|
||||||
|
MTime: mtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = repo.CreateBatch(ctx, nil, []*database.File{batched})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to batch create file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range []types.FilePath{created.Path, batched.Path} {
|
||||||
|
retrieved, err := repo.GetByPath(ctx, path.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !retrieved.MTime.Equal(mtime) {
|
||||||
|
t.Errorf("%s: mtime got %v, want %v",
|
||||||
|
path, retrieved.MTime, mtime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileRepositoryTransaction(t *testing.T) {
|
func TestFileRepositoryTransaction(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -606,8 +606,7 @@ func TestTimezoneHandling(t *testing.T) {
|
|||||||
t.Skip("timezone not available")
|
t.Skip("timezone not available")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use Truncate to remove sub-second precision since we store as Unix timestamps
|
nyTime := time.Now().In(loc)
|
||||||
nyTime := time.Now().In(loc).Truncate(time.Second)
|
|
||||||
file := &File{
|
file := &File{
|
||||||
Path: "/timezone-test.txt",
|
Path: "/timezone-test.txt",
|
||||||
MTime: nyTime,
|
MTime: nyTime,
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS files (
|
|||||||
id TEXT PRIMARY KEY, -- UUID
|
id TEXT PRIMARY KEY, -- UUID
|
||||||
path TEXT NOT NULL UNIQUE,
|
path TEXT NOT NULL UNIQUE,
|
||||||
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
|
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
|
||||||
mtime INTEGER NOT NULL,
|
mtime INTEGER NOT NULL, -- whole seconds since the Unix epoch
|
||||||
|
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
mode INTEGER NOT NULL,
|
mode INTEGER NOT NULL,
|
||||||
uid INTEGER NOT NULL,
|
uid INTEGER NOT NULL,
|
||||||
|
|||||||
@@ -1221,7 +1221,7 @@ func (s *Scanner) checkFileInMemory(
|
|||||||
|
|
||||||
// Check if file has changed
|
// Check if file has changed
|
||||||
if existingFile.Size != file.Size ||
|
if existingFile.Size != file.Size ||
|
||||||
existingFile.MTime.Unix() != file.MTime.Unix() ||
|
!existingFile.MTime.Equal(file.MTime) ||
|
||||||
existingFile.Mode != file.Mode ||
|
existingFile.Mode != file.Mode ||
|
||||||
existingFile.UID != file.UID ||
|
existingFile.UID != file.UID ||
|
||||||
existingFile.GID != file.GID {
|
existingFile.GID != file.GID {
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A file rewritten with its size unchanged and a new mtime in the same
|
||||||
|
// second as the mtime the index holds must still be backed up. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/226.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
restoreDir := filepath.Join(tempDir, "restored")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
rewrittenPath := filepath.Join(dataDir, "small.txt")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
files := writeFaultSourceTree(t, fs, dataDir)
|
||||||
|
cfg := changedFileConfig(dataDir, dbPath)
|
||||||
|
|
||||||
|
firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC).
|
||||||
|
Add(100 * time.Millisecond)
|
||||||
|
secondMTime := firstMTime.Add(800 * time.Millisecond)
|
||||||
|
|
||||||
|
require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime))
|
||||||
|
|
||||||
|
store, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||||
|
|
||||||
|
require.NoError(t, backUp(v, "first"))
|
||||||
|
|
||||||
|
// Upper-casing ASCII text keeps its size.
|
||||||
|
files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath])
|
||||||
|
require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644))
|
||||||
|
require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime))
|
||||||
|
|
||||||
|
require.NoError(t, backUp(v, "second"))
|
||||||
|
|
||||||
|
id := localSnapshotID(ctx, t, repos, "second")
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||||
|
require.NoError(t, reader.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: id,
|
||||||
|
TargetDir: restoreDir,
|
||||||
|
Verify: true,
|
||||||
|
}))
|
||||||
|
|
||||||
|
assertRestoredTree(t, fs, restoreDir, files)
|
||||||
|
}
|
||||||
@@ -1110,6 +1110,12 @@ type RemoveResult struct {
|
|||||||
// just-removed snapshot left behind on the destination store.
|
// just-removed snapshot left behind on the destination store.
|
||||||
const pruneCommandHint = "vaultik prune"
|
const pruneCommandHint = "vaultik prune"
|
||||||
|
|
||||||
|
// snapshotRemoveCommandHint is the command suggested, with the
|
||||||
|
// snapshot's ID, when a remove could not reach the destination store:
|
||||||
|
// running it again removes the snapshot's metadata there, which
|
||||||
|
// `vaultik prune` never does.
|
||||||
|
const snapshotRemoveCommandHint = "vaultik snapshot remove"
|
||||||
|
|
||||||
// RemoveSnapshot removes a snapshot from the local index database and,
|
// RemoveSnapshot removes a snapshot from the local index database and,
|
||||||
// unless LocalOnly is set, also strips the snapshot's metadata from the
|
// unless LocalOnly is set, also strips the snapshot's metadata from the
|
||||||
// destination store. Blobs are NOT touched: removing a snapshot's
|
// destination store. Blobs are NOT touched: removing a snapshot's
|
||||||
@@ -1146,7 +1152,7 @@ func (v *Vaultik) RemoveSnapshot(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !opts.LocalOnly {
|
if !opts.LocalOnly {
|
||||||
result.RemoteRemoved = v.removeSnapshotRemote(snapshotID)
|
result.RemoteRemoved = v.removeSnapshotRemote(snapshotID, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
if v.SnapshotManager != nil {
|
if v.SnapshotManager != nil {
|
||||||
@@ -1229,9 +1235,11 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
|
|||||||
// removeSnapshotRemote strips the snapshot's metadata from the
|
// removeSnapshotRemote strips the snapshot's metadata from the
|
||||||
// destination store, warning and proceeding on failure: the local-DB
|
// destination store, warning and proceeding on failure: the local-DB
|
||||||
// removal has already happened, so the user is told the remote half
|
// removal has already happened, so the user is told the remote half
|
||||||
// didn't finish and can retry with `vaultik prune` once the destination
|
// didn't finish and to run `vaultik snapshot remove` for the snapshot
|
||||||
// store is reachable. Returns true when the remote removal succeeded.
|
// again once the destination store is reachable (`vaultik prune` never
|
||||||
func (v *Vaultik) removeSnapshotRemote(snapshotID string) bool {
|
// removes snapshot metadata). Returns true when the remote removal
|
||||||
|
// succeeded.
|
||||||
|
func (v *Vaultik) removeSnapshotRemote(snapshotID string, opts *RemoveOptions) bool {
|
||||||
log.Info("Removing snapshot metadata from remote storage",
|
log.Info("Removing snapshot metadata from remote storage",
|
||||||
"snapshot_id", snapshotID)
|
"snapshot_id", snapshotID)
|
||||||
|
|
||||||
@@ -1239,13 +1247,17 @@ func (v *Vaultik) removeSnapshotRemote(snapshotID string) bool {
|
|||||||
|
|
||||||
err := v.deleteRemoteSnapshotByKey(remoteKey)
|
err := v.deleteRemoteSnapshotByKey(remoteKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("Could not remove snapshot metadata from remote storage",
|
log.Warn("Could not remove snapshot metadata from remote storage; "+
|
||||||
"error", err)
|
"run '"+snapshotRemoveCommandHint+"' with the snapshot's ID "+
|
||||||
|
"again once the remote is reachable",
|
||||||
|
"snapshot_id", snapshotID, "error", err)
|
||||||
|
|
||||||
if v.UI != nil {
|
// The UI writes to stdout, which under --json holds only the
|
||||||
|
// document; the log record above is the warning on stderr.
|
||||||
|
if v.UI != nil && !opts.JSON {
|
||||||
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
|
v.UI.Warningf("Could not remove snapshot metadata from remote: "+
|
||||||
"%v. Run '%s' once the remote is reachable to finish cleanup.",
|
"%v. Run '%s %s' again once the remote is reachable.",
|
||||||
err, pruneCommandHint)
|
err, snapshotRemoveCommandHint, snapshotID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return false
|
return false
|
||||||
|
|||||||
Reference in New Issue
Block a user