Open the downloaded snapshot database read-only, on a private temp dir #186
@@ -17,6 +17,7 @@ import (
|
|||||||
"embed"
|
"embed"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
@@ -219,6 +220,135 @@ func openWithRecovery(ctx context.Context, path string) (*DB, error) {
|
|||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// errUntrustedSnapshotSchema is returned when a downloaded snapshot
|
||||||
|
// database carries schema objects the real schema never defines, or is
|
||||||
|
// missing a table the restore and deep-verify queries read.
|
||||||
|
var errUntrustedSnapshotSchema = errors.New(
|
||||||
|
"downloaded snapshot database has an untrusted schema")
|
||||||
|
|
||||||
|
// snapshotReadOnlyDSN builds the driver DSN that opens a materialized
|
||||||
|
// snapshot database file read-only. mode=ro opens the file read-only at
|
||||||
|
// the OS level, query_only rejects any write the engine is asked to make,
|
||||||
|
// and trusted_schema=OFF refuses to run application code named in the
|
||||||
|
// schema. The file: URI form is required for the driver to honour the
|
||||||
|
// mode parameter.
|
||||||
|
func snapshotReadOnlyDSN(path string) string {
|
||||||
|
u := url.URL{
|
||||||
|
Scheme: "file",
|
||||||
|
Path: path,
|
||||||
|
RawQuery: "mode=ro&_pragma=query_only(true)&_pragma=trusted_schema(false)",
|
||||||
|
}
|
||||||
|
|
||||||
|
return u.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenReadOnly opens an already-materialized SQLite file for read-only
|
||||||
|
// querying of a snapshot database downloaded from the store, used by
|
||||||
|
// restore and deep verify. Unlike New it never applies schema migrations
|
||||||
|
// and never writes: the connection is opened read-only with query_only
|
||||||
|
// and trusted_schema=OFF. It refuses any file whose schema carries a
|
||||||
|
// trigger, view or virtual table, or lacks an expected table, so a forged
|
||||||
|
// file cannot redefine what the restore queries return. The caller owns
|
||||||
|
// the file and must remove it.
|
||||||
|
func OpenReadOnly(ctx context.Context, path string) (*DB, error) {
|
||||||
|
conn, err := sql.Open("sqlite", snapshotReadOnlyDSN(path))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("opening read-only database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
configureConnPool(conn)
|
||||||
|
|
||||||
|
err = conn.PingContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("opening read-only database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = verifySnapshotSchema(ctx, conn)
|
||||||
|
if err != nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &DB{conn: conn, path: path}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifySnapshotSchema rejects a downloaded database whose schema is not
|
||||||
|
// the plain table set the real schema defines. Any trigger, view or
|
||||||
|
// virtual table, or a missing expected table, fails the open.
|
||||||
|
func verifySnapshotSchema(ctx context.Context, conn *sql.DB) error {
|
||||||
|
// expectedSnapshotTables are the tables the restore and deep-verify
|
||||||
|
// queries read. A downloaded database missing any of them is not a
|
||||||
|
// genuine snapshot database and is refused.
|
||||||
|
expectedSnapshotTables := []string{
|
||||||
|
"blob_chunks",
|
||||||
|
"blobs",
|
||||||
|
"chunks",
|
||||||
|
"file_chunks",
|
||||||
|
"files",
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := conn.QueryContext(
|
||||||
|
ctx, "SELECT type, name, sql FROM sqlite_master")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
|
present := make(map[string]struct{})
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var objType, name string
|
||||||
|
|
||||||
|
var objSQL sql.NullString
|
||||||
|
|
||||||
|
err = rows.Scan(&objType, &name, &objSQL)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch objType {
|
||||||
|
case "trigger", "view":
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: unexpected %s %q", errUntrustedSnapshotSchema, objType, name)
|
||||||
|
case "table":
|
||||||
|
if isVirtualTableSQL(objSQL.String) {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: unexpected virtual table %q",
|
||||||
|
errUntrustedSnapshotSchema, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
present[name] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = rows.Err()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading snapshot schema: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, table := range expectedSnapshotTables {
|
||||||
|
if _, ok := present[table]; !ok {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w: missing table %q", errUntrustedSnapshotSchema, table)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// isVirtualTableSQL reports whether a sqlite_master row's SQL defines a
|
||||||
|
// virtual table. Virtual tables are recorded with type 'table' but a
|
||||||
|
// "CREATE VIRTUAL TABLE" definition and can run module code, so they are
|
||||||
|
// refused alongside triggers and views.
|
||||||
|
func isVirtualTableSQL(createSQL string) bool {
|
||||||
|
return strings.HasPrefix(
|
||||||
|
strings.ToUpper(strings.TrimSpace(createSQL)), "CREATE VIRTUAL TABLE")
|
||||||
|
}
|
||||||
|
|
||||||
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
||||||
// The database is automatically initialized with the schema and is ready
|
// The database is automatically initialized with the schema and is ready
|
||||||
// for use. Each call creates a new independent database instance.
|
// for use. Each call creates a new independent database instance.
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
//nolint:testpackage // exercises unexported read-only open internals
|
||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// genuineSnapshotDB writes a real snapshot database (the full schema
|
||||||
|
// applied) to a fresh file and returns its path.
|
||||||
|
func genuineSnapshotDB(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "snapshot.db")
|
||||||
|
|
||||||
|
db, err := New(context.Background(), path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("creating snapshot database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("closing snapshot database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgedDB creates an empty database file and runs the given statements
|
||||||
|
// against it read-write, so a test can plant schema objects the real
|
||||||
|
// schema never defines.
|
||||||
|
func forgedDB(t *testing.T, stmts ...string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "forged.db")
|
||||||
|
|
||||||
|
db, err := sql.Open("sqlite", path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("opening forged database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, stmt := range stmts {
|
||||||
|
_, err = db.ExecContext(context.Background(), stmt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("executing %q: %v", stmt, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Close()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("closing forged database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenReadOnlyAcceptsGenuineSnapshot(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db, err := OpenReadOnly(context.Background(), genuineSnapshotDB(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenReadOnly refused a genuine snapshot database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenReadOnlyRefusesWrites(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db, err := OpenReadOnly(context.Background(), genuineSnapshotDB(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpenReadOnly: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
// A schema write depends on no table columns, so the only reason it
|
||||||
|
// can fail is that the database is open read-only.
|
||||||
|
_, err = db.Conn().ExecContext(context.Background(),
|
||||||
|
"CREATE TABLE probe_readonly (x)")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a write to a read-only snapshot database to fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenReadOnlyRejectsView(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := forgedDB(t, "CREATE VIEW files AS SELECT 1 AS path")
|
||||||
|
|
||||||
|
_, err := OpenReadOnly(context.Background(), path)
|
||||||
|
if !errors.Is(err, errUntrustedSnapshotSchema) {
|
||||||
|
t.Fatalf("expected a view named files to be refused, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenReadOnlyRejectsTrigger(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := forgedDB(t,
|
||||||
|
"CREATE TABLE files (path TEXT)",
|
||||||
|
"CREATE TRIGGER t AFTER INSERT ON files BEGIN SELECT 1; END")
|
||||||
|
|
||||||
|
_, err := OpenReadOnly(context.Background(), path)
|
||||||
|
if !errors.Is(err, errUntrustedSnapshotSchema) {
|
||||||
|
t.Fatalf("expected a trigger to be refused, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenReadOnlyRejectsMissingTable(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// Only one of the expected tables is present.
|
||||||
|
path := forgedDB(t, "CREATE TABLE files (path TEXT)")
|
||||||
|
|
||||||
|
_, err := OpenReadOnly(context.Background(), path)
|
||||||
|
if !errors.Is(err, errUntrustedSnapshotSchema) {
|
||||||
|
t.Fatalf("expected a missing expected table to be refused, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsVirtualTableSQL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
sql string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"CREATE VIRTUAL TABLE t USING fts5(x)", true},
|
||||||
|
{" create virtual table t using fts5(x)", true},
|
||||||
|
{"CREATE TABLE t (x)", false},
|
||||||
|
{"CREATE VIEW t AS SELECT 1", false},
|
||||||
|
{"", false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := isVirtualTableSQL(c.sql); got != c.want {
|
||||||
|
t.Errorf("isVirtualTableSQL(%q) = %v, want %v", c.sql, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+55
-33
@@ -39,8 +39,14 @@ var (
|
|||||||
"refusing to restore path outside the target directory")
|
"refusing to restore path outside the target directory")
|
||||||
errTrailingRestoreData = errors.New(
|
errTrailingRestoreData = errors.New(
|
||||||
"restored file has trailing data after its last chunk")
|
"restored file has trailing data after its last chunk")
|
||||||
|
errRestoreIncomplete = errors.New(
|
||||||
|
"restore loop ended with files still pending")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// snapshotDBFilename is the name the decrypted snapshot database is
|
||||||
|
// written under inside its private temp directory.
|
||||||
|
const snapshotDBFilename = "snapshot.db"
|
||||||
|
|
||||||
// restoreDirMode is the permission mode for directories created while
|
// restoreDirMode is the permission mode for directories created while
|
||||||
// restoring (parent directories and the target root; restored
|
// restoring (parent directories and the target root; restored
|
||||||
// directories themselves get their stored mode).
|
// directories themselves get their stored mode).
|
||||||
@@ -102,7 +108,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
// Step 1: Download and decrypt the snapshot metadata database
|
// Step 1: Download and decrypt the snapshot metadata database
|
||||||
log.Info("Downloading snapshot metadata...")
|
log.Info("Downloading snapshot metadata...")
|
||||||
|
|
||||||
tempDB, err := v.downloadSnapshotDB(opts.SnapshotID, identity)
|
tempDB, tempDir, err := v.downloadSnapshotDB(opts.SnapshotID, identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("downloading snapshot database: %w", err)
|
return fmt.Errorf("downloading snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
@@ -112,10 +118,11 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug("Failed to close temp database", "error", err)
|
log.Debug("Failed to close temp database", "error", err)
|
||||||
}
|
}
|
||||||
// Clean up temp file
|
// Remove the whole private directory, so the decrypted database
|
||||||
err = v.Fs.Remove(tempDB.Path())
|
// and any SQLite side files it produced are gone on every path.
|
||||||
|
err = v.Fs.RemoveAll(tempDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug("Failed to remove temp database", "error", err)
|
log.Debug("Failed to remove temp database directory", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -368,6 +375,13 @@ func (v *Vaultik) runRestoreLoop(
|
|||||||
totalBytesExpected, startTime, &lastStatusTime)
|
totalBytesExpected, startTime, &lastStatusTime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The loop above stops as soon as nothing is ready and nothing more
|
||||||
|
// can be downloaded. If files still remain, they were abandoned
|
||||||
|
// rather than restored; fail loudly instead of reporting success.
|
||||||
|
if plan.hasPending() {
|
||||||
|
return errRestoreIncomplete
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,8 +396,8 @@ func (v *Vaultik) runRestoreLoop(
|
|||||||
func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
|
func (s *restoreSession) downloadNextBlobSet(plan *restorePlan) (bool, error) {
|
||||||
s.sweeper.sweep()
|
s.sweeper.sweep()
|
||||||
|
|
||||||
next := plan.pickNextDownload()
|
next, ok := plan.pickNextDownload()
|
||||||
if next.IsZero() {
|
if !ok {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -594,10 +608,10 @@ func (v *Vaultik) handleRestoreVerification(
|
|||||||
// index can restore the snapshots it can only see on the store.
|
// index can restore the snapshots it can only see on the store.
|
||||||
func (v *Vaultik) downloadSnapshotDB(
|
func (v *Vaultik) downloadSnapshotDB(
|
||||||
snapshotID string, identity age.Identity,
|
snapshotID string, identity age.Identity,
|
||||||
) (*database.DB, error) {
|
) (*database.DB, string, error) {
|
||||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download encrypted database from storage
|
// Download encrypted database from storage
|
||||||
@@ -605,7 +619,7 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
|
|
||||||
reader, err := v.Storage.Get(v.ctx, dbKey)
|
reader, err := v.Storage.Get(v.ctx, dbKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("downloading %s: %w", dbKey, err)
|
return nil, "", fmt.Errorf("downloading %s: %w", dbKey, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = reader.Close() }()
|
defer func() { _ = reader.Close() }()
|
||||||
@@ -613,7 +627,7 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
// Read all data
|
// Read all data
|
||||||
encryptedData, err := io.ReadAll(reader)
|
encryptedData, err := io.ReadAll(reader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading encrypted data: %w", err)
|
return nil, "", fmt.Errorf("reading encrypted data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Downloaded encrypted database",
|
log.Debug("Downloaded encrypted database",
|
||||||
@@ -622,7 +636,7 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
// Decrypt and decompress using blobgen.Reader
|
// Decrypt and decompress using blobgen.Reader
|
||||||
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
|
blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating decryption reader: %w", err)
|
return nil, "", fmt.Errorf("creating decryption reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = blobReader.Close() }()
|
defer func() { _ = blobReader.Close() }()
|
||||||
@@ -630,44 +644,52 @@ func (v *Vaultik) downloadSnapshotDB(
|
|||||||
// Read the binary SQLite database
|
// Read the binary SQLite database
|
||||||
dbData, err := io.ReadAll(blobReader)
|
dbData, err := io.ReadAll(blobReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("decrypting and decompressing: %w", err)
|
return nil, "", fmt.Errorf("decrypting and decompressing: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
|
log.Debug("Decrypted database", "size", ubytes(int64(len(dbData))))
|
||||||
|
|
||||||
// Create a temporary database file and write the binary SQLite data directly
|
return v.materializeSnapshotDB(dbData)
|
||||||
tempFile, err := afero.TempFile(v.Fs, "", "vaultik-restore-*.db")
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("creating temp file: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tempPath := tempFile.Name()
|
// materializeSnapshotDB writes the decrypted snapshot database bytes into
|
||||||
|
// a fresh private (0700) temp directory and opens the file read-only. On
|
||||||
// Write the binary SQLite database directly
|
// any failure it removes the directory before returning, so no decrypted
|
||||||
_, err = tempFile.Write(dbData)
|
// metadata is left on disk when the open is interrupted or the payload is
|
||||||
|
// damaged. On success the returned directory is the caller's to remove.
|
||||||
|
func (v *Vaultik) materializeSnapshotDB(
|
||||||
|
dbData []byte,
|
||||||
|
) (*database.DB, string, error) {
|
||||||
|
tempDir, err := afero.TempDir(v.Fs, "", "vaultik-restore-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tempFile.Close()
|
return nil, "", fmt.Errorf("creating temp directory: %w", err)
|
||||||
_ = v.Fs.Remove(tempPath)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("writing database file: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = tempFile.Close()
|
success := false
|
||||||
if err != nil {
|
|
||||||
_ = v.Fs.Remove(tempPath)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("closing temp file: %w", err)
|
defer func() {
|
||||||
|
if !success {
|
||||||
|
_ = v.Fs.RemoveAll(tempDir)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
dbPath := filepath.Join(tempDir, snapshotDBFilename)
|
||||||
|
|
||||||
|
err = afero.WriteFile(v.Fs, dbPath, dbData, restoreFileMode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("writing database file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Created restore database", "path", tempPath)
|
log.Debug("Created restore database", "path", dbPath)
|
||||||
|
|
||||||
// Open the database
|
db, err := database.OpenReadOnly(v.ctx, dbPath)
|
||||||
db, err := database.New(v.ctx, tempPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("opening restore database: %w", err)
|
return nil, "", fmt.Errorf("opening restore database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return db, nil
|
success = true
|
||||||
|
|
||||||
|
return db, tempDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getFilesToRestore returns the list of files to restore based on path filters
|
// getFilesToRestore returns the list of files to restore based on path filters
|
||||||
|
|||||||
@@ -171,10 +171,13 @@ func (p *restorePlan) finishFile(fileID types.FileID) {
|
|||||||
// downloaded next, after which it — together with any other pending
|
// downloaded next, after which it — together with any other pending
|
||||||
// files whose blob sets become empty — moves to the ready queue.
|
// files whose blob sets become empty — moves to the ready queue.
|
||||||
//
|
//
|
||||||
// The zero FileID return means nothing is pending.
|
// The second return value is false when no file needs a download, so a
|
||||||
func (p *restorePlan) pickNextDownload() types.FileID {
|
// genuine file carrying the nil UUID is picked rather than mistaken for
|
||||||
|
// "nothing left".
|
||||||
|
func (p *restorePlan) pickNextDownload() (types.FileID, bool) {
|
||||||
var best types.FileID
|
var best types.FileID
|
||||||
|
|
||||||
|
found := false
|
||||||
bestCount := math.MaxInt
|
bestCount := math.MaxInt
|
||||||
|
|
||||||
var bestID string
|
var bestID string
|
||||||
@@ -188,14 +191,15 @@ func (p *restorePlan) pickNextDownload() types.FileID {
|
|||||||
}
|
}
|
||||||
|
|
||||||
idStr := id.String()
|
idStr := id.String()
|
||||||
if n < bestCount || (n == bestCount && (best.IsZero() || idStr < bestID)) {
|
if !found || n < bestCount || (n == bestCount && idStr < bestID) {
|
||||||
best = id
|
best = id
|
||||||
|
found = true
|
||||||
bestCount = n
|
bestCount = n
|
||||||
bestID = idStr
|
bestID = idStr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return best
|
return best, found
|
||||||
}
|
}
|
||||||
|
|
||||||
// blobsNeeded returns the uncached blob hashes for fileID in any order.
|
// blobsNeeded returns the uncached blob hashes for fileID in any order.
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package vaultik //nolint:testpackage // inspects unexported restore plan internals
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestPickNextDownloadReturnsNilUUIDFile proves a genuine pending file
|
||||||
|
// carrying the nil UUID is picked for download rather than mistaken for
|
||||||
|
// "nothing left" — the bug that could abandon every remaining file.
|
||||||
|
func TestPickNextDownloadReturnsNilUUIDFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var nilID types.FileID // zero value is the nil UUID
|
||||||
|
|
||||||
|
plan := &restorePlan{
|
||||||
|
fileBlobs: map[types.FileID]map[string]struct{}{
|
||||||
|
nilID: {"blobhash": {}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
id, ok := plan.pickNextDownload()
|
||||||
|
require.True(t, ok,
|
||||||
|
"pickNextDownload treated a pending nil-UUID file as nothing to do")
|
||||||
|
require.True(t, id.IsZero(), "expected the nil-UUID file to be picked")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPickNextDownloadEmptyPlan confirms the second return value is false
|
||||||
|
// only when no file needs a download.
|
||||||
|
func TestPickNextDownloadEmptyPlan(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
plan := &restorePlan{
|
||||||
|
fileBlobs: map[types.FileID]map[string]struct{}{},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, ok := plan.pickNextDownload()
|
||||||
|
require.False(t, ok, "pickNextDownload reported work on an empty plan")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRunRestoreLoopFailsOnAbandonedFiles proves the loop returns an
|
||||||
|
// error rather than silent success when files remain pending after it
|
||||||
|
// can make no further progress.
|
||||||
|
func TestRunRestoreLoopFailsOnAbandonedFiles(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
db, err := database.NewTestDB()
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
|
||||||
|
cache, err := newBlobDiskCache(math.MaxInt64)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = cache.Close() })
|
||||||
|
|
||||||
|
v := &Vaultik{ctx: ctx}
|
||||||
|
session := &restoreSession{
|
||||||
|
v: v,
|
||||||
|
ctx: ctx,
|
||||||
|
repos: repos,
|
||||||
|
sweeper: newRestoreSweeper(ctx, repos, cache, 1),
|
||||||
|
result: &RestoreResult{},
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file that is still pending but whose uncached-blob set is empty
|
||||||
|
// and which was never queued as ready: the loop can neither restore
|
||||||
|
// nor download it. This is the abandonment the guard must catch.
|
||||||
|
var stuck types.FileID
|
||||||
|
|
||||||
|
plan := &restorePlan{
|
||||||
|
fileBlobs: map[types.FileID]map[string]struct{}{stuck: {}},
|
||||||
|
blobFiles: map[string]map[types.FileID]struct{}{},
|
||||||
|
cached: map[string]struct{}{},
|
||||||
|
}
|
||||||
|
|
||||||
|
err = v.runRestoreLoop(session, plan, map[types.FileID]*database.File{}, 0)
|
||||||
|
require.ErrorIs(t, err, errRestoreIncomplete)
|
||||||
|
}
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package vaultik //nolint:testpackage // inspects unexported snapshot-db materialization
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// genuineSnapshotDBBytes returns the on-disk bytes of a real snapshot
|
||||||
|
// database (the full schema applied).
|
||||||
|
func genuineSnapshotDBBytes(t *testing.T) []byte {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "snapshot.db")
|
||||||
|
|
||||||
|
db, err := database.New(context.Background(), path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
data, err := os.ReadFile(path) //nolint:gosec // G304: test-controlled temp path
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMaterializeSnapshotDBPrivateDir proves the decrypted database lands
|
||||||
|
// in a private (0700) directory and opens read-only.
|
||||||
|
func TestMaterializeSnapshotDBPrivateDir(t *testing.T) {
|
||||||
|
dbData := genuineSnapshotDBBytes(t)
|
||||||
|
|
||||||
|
t.Setenv("TMPDIR", t.TempDir())
|
||||||
|
|
||||||
|
v := &Vaultik{ctx: context.Background(), Fs: afero.NewOsFs()}
|
||||||
|
|
||||||
|
db, dir, err := v.materializeSnapshotDB(dbData)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = db.Close()
|
||||||
|
_ = os.RemoveAll(dir)
|
||||||
|
})
|
||||||
|
|
||||||
|
info, err := os.Stat(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, os.FileMode(0o700), info.Mode().Perm(),
|
||||||
|
"snapshot database directory must not be world-readable")
|
||||||
|
|
||||||
|
_, err = db.Conn().ExecContext(context.Background(),
|
||||||
|
"CREATE TABLE probe_readonly (x)")
|
||||||
|
require.Error(t, err, "materialized snapshot database must be read-only")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMaterializeSnapshotDBRemovesDirOnOpenFailure proves a failed open
|
||||||
|
// leaves no temp directory behind.
|
||||||
|
func TestMaterializeSnapshotDBRemovesDirOnOpenFailure(t *testing.T) {
|
||||||
|
base := t.TempDir()
|
||||||
|
|
||||||
|
t.Setenv("TMPDIR", base)
|
||||||
|
|
||||||
|
v := &Vaultik{ctx: context.Background(), Fs: afero.NewOsFs()}
|
||||||
|
|
||||||
|
_, _, err := v.materializeSnapshotDB([]byte("this is not a sqlite database"))
|
||||||
|
require.Error(t, err)
|
||||||
|
|
||||||
|
entries, rerr := os.ReadDir(base)
|
||||||
|
require.NoError(t, rerr)
|
||||||
|
require.Empty(t, entries, "temp directory left behind after open failure")
|
||||||
|
}
|
||||||
+41
-24
@@ -9,12 +9,12 @@ import (
|
|||||||
"hash"
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
|
|
||||||
// Blank import registers the pure-Go sqlite driver for database/sql.
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/vaultik/internal/log"
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
)
|
)
|
||||||
@@ -195,7 +195,7 @@ func (v *Vaultik) loadVerificationData(
|
|||||||
fmt.Errorf("failed to decrypt database: %w", err))
|
fmt.Errorf("failed to decrypt database: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
dbBlobs, err := v.getBlobsFromDatabase(tdb.DB)
|
dbBlobs, err := v.getBlobsFromDatabase(tdb.db.Conn())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tdb.Close()
|
_ = tdb.Close()
|
||||||
|
|
||||||
@@ -256,7 +256,7 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
len(dbBlobs), ubytes(totalSize))
|
len(dbBlobs), ubytes(totalSize))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts)
|
err = v.performDeepVerificationFromDB(dbBlobs, tdb.db.Conn(), opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
return v.deepVerifyFailure(result, opts, err.Error(), err)
|
||||||
}
|
}
|
||||||
@@ -264,16 +264,18 @@ func (v *Vaultik) runVerificationSteps(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// tempDB wraps sql.DB with cleanup
|
// tempDB is the downloaded snapshot database opened read-only for deep
|
||||||
|
// verify, held in a private temp directory removed in full on Close.
|
||||||
type tempDB struct {
|
type tempDB struct {
|
||||||
*sql.DB
|
db *database.DB
|
||||||
|
tempDir string
|
||||||
tempPath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *tempDB) Close() error {
|
func (t *tempDB) Close() error {
|
||||||
err := t.DB.Close()
|
err := t.db.Close()
|
||||||
_ = os.Remove(t.tempPath)
|
// Remove the whole private directory so the decrypted database and
|
||||||
|
// any SQLite side files are gone on every path.
|
||||||
|
_ = os.RemoveAll(t.tempDir)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -300,41 +302,56 @@ func (v *Vaultik) decryptAndLoadDatabase(reader io.ReadCloser) (*tempDB, error)
|
|||||||
}
|
}
|
||||||
defer decompressor.Close()
|
defer decompressor.Close()
|
||||||
|
|
||||||
// Create temporary file for the database
|
// Materialize the decrypted database inside a private (0700) temp
|
||||||
tempFile, err := os.CreateTemp("", "vaultik-verify-*.db")
|
// directory so it is never world-readable, and remove the whole
|
||||||
|
// directory on any failure below.
|
||||||
|
tempDir, err := os.MkdirTemp("", "vaultik-verify-")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create temp directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
success := false
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if !success {
|
||||||
|
_ = os.RemoveAll(tempDir)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
dbPath := filepath.Join(tempDir, snapshotDBFilename)
|
||||||
|
|
||||||
|
//nolint:gosec // G304: dbPath is our MkdirTemp dir plus a constant filename
|
||||||
|
tempFile, err := os.OpenFile(
|
||||||
|
dbPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, restoreFileMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
return nil, fmt.Errorf("failed to create temp file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
tempPath := tempFile.Name()
|
|
||||||
|
|
||||||
// Stream decompress directly to file
|
// Stream decompress directly to file
|
||||||
log.Info("Decompressing database...")
|
log.Info("Decompressing database...")
|
||||||
|
|
||||||
written, err := io.Copy(tempFile, decompressor)
|
written, err := io.Copy(tempFile, decompressor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = tempFile.Close()
|
_ = tempFile.Close()
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("failed to decompress database: %w", err)
|
return nil, fmt.Errorf("failed to decompress database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = tempFile.Close()
|
err = tempFile.Close()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to close temp database file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
log.Info("Database decompressed", "size", ubytes(written))
|
log.Info("Database decompressed", "size", ubytes(written))
|
||||||
|
|
||||||
// Open the database
|
db, err := database.OpenReadOnly(v.ctx, dbPath)
|
||||||
db, err := sql.Open("sqlite", tempPath)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = os.Remove(tempPath)
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("failed to open database: %w", err)
|
return nil, fmt.Errorf("failed to open database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &tempDB{
|
success = true
|
||||||
DB: db,
|
|
||||||
tempPath: tempPath,
|
return &tempDB{db: db, tempDir: tempDir}, nil
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// verifyBlob downloads and verifies a single blob
|
// verifyBlob downloads and verifies a single blob
|
||||||
|
|||||||
Reference in New Issue
Block a user