List remote snapshots without requiring the private key (closes #64)

ListSnapshots built its table entirely from the local SQLite index. The only remote access, reportRemoteDrift, was gated on AgeSecretKey != "", so on a correctly configured host - which by design holds no private key - snapshot list never contacted the destination store at all. A user who lost their local index could not see their own backups, and the "<remote only>" cell the README documents was unreachable dead code. The listing is now the union of the local index and the destination store, with no age_secret_key gate. Remote-only snapshots cannot have their hostname or name recovered - RemoteSnapshotKey is one-way and the manifest stores the hash - so they are listed by abbreviated remote key with the real timestamp and compressed size from the manifest, and "<remote only>" in the two columns that require the local index. Nothing new is written to remote storage and the human ID is never fabricated. An unreachable destination degrades to local-only with a warning and a zero exit code. remote_present is null rather than false in that case, so "absent" and "unknown" stay distinguishable and no drift is claimed from a listing that never happened. Also: - Snapshot timestamps are normalised to UTC in scanSnapshotRows, the single point where they enter the domain. Previously one of three scanners omitted .UTC(), so on a non-UTC host the same snapshot rendered a different time depending on whether it was locally tracked. - The 1000-row cap and the unreadable-manifest count are reported in --json mode as well as table mode, so machine consumers cannot be silently truncated. The JSON shape is unchanged. - Warnings raised while listing are routed to stderr rather than the logger, which writes to stdout and would corrupt the JSON document. This is a local workaround for the logger bug tracked in #82 and should be removed when that lands. - downloadManifestByKey is now the only remote manifest reader, so the manifest privacy question in #81 has a single call site to change. - The orphaned "vaultik snapshot cleanup" hint now names vaultik prune; that command was folded into prune by the 2026-07-02 consolidation.
2026-08-09 07:34:15 +02:00
parent af607e3597
commit 50e20b460e
10 changed files with 1637 additions and 314 deletions
@@ -0,0 +1,608 @@
+package vaultik
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// remoteOnlyCell fills the table columns that can only be derived from
+// the local index. A snapshot present only on the destination store has
+// no local rows to derive them from.
+const remoteOnlyCell = "<remote only>"
+
+// remoteKeyDisplayLen is how many hex characters of a remote key are
+// shown in the identifier column for a remote-only snapshot. Twelve
+// matches the abbreviation length used elsewhere in the UI and is far
+// past the point of ambiguity for a SHA256 digest.
+const remoteKeyDisplayLen = 12
+
+// maxRemoteOnlyRows caps how many remote-only snapshots a single
+// `snapshot list` will describe. Each one costs a manifest read, so an
+// uncapped listing against a destination holding many thousands of
+// unknown snapshots would be both slow and unbounded in memory. Beyond
+// the cap the table is truncated and the count of omitted snapshots is
+// reported.
+const maxRemoteOnlyRows = 1000
+
+// remoteManifestFetchConcurrency bounds how many manifest reads are in
+// flight at once while describing remote-only snapshots. The listing
+// itself is a single streamed prefix request; only the per-snapshot
+// manifest reads need throttling.
+const remoteManifestFetchConcurrency = 8
+
+// tabPadding is the tabwriter cell padding for the snapshot table.
+const tabPadding = 3
+
+// ListSnapshots prints the table of snapshots known to this host: the
+// union of the local index database and the backup destination store.
+//
+// Remote listing needs no age secret key. A snapshot's manifest
+// (metadata/<remote-key>/manifest.json.zst) is compressed but not
+// encrypted, so a host holding only the public key — the configuration
+// vaultik is designed for — can still enumerate what it has backed up
+// and see each snapshot's timestamp and compressed size.
+//
+// What that host cannot see is a remote-only snapshot's human ID.
+// snapshot.RemoteSnapshotKey is one-way and the manifest stores the
+// hashed key rather than the ID, so hostname and snapshot name live
+// only in the local index and in the encrypted db.zst.age. Remote-only
+// rows are therefore identified by an abbreviation of their remote key,
+// and the two columns that genuinely require the local index
+// (uncompressed size, new chunk size) render as "<remote only>". No
+// attempt is made to recover or fabricate the human ID.
+//
+// Snapshots in the local index with no counterpart on the destination
+// store are reported as drift below the table.
+//
+// If remote listing fails (unmounted volume, permission denied,
+// network), we degrade to local-only with a warning. List never fails
+// just because the destination is unreachable.
+func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
+	log.Info("Listing snapshots")
+
+	localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
+	if err != nil {
+		return fmt.Errorf("listing local snapshots: %w", err)
+	}
+
+	snapshots := make([]SnapshotInfo, 0, len(localSnaps))
+	localKeys := make(map[string]bool, len(localSnaps))
+
+	for _, ls := range localSnaps {
+		if ls.CompletedAt == nil {
+			continue
+		}
+
+		info := v.snapshotInfoFromLocal(ls)
+		localKeys[info.RemoteKey] = true
+		snapshots = append(snapshots, info)
+	}
+
+	listing, remoteErr := v.collectRemoteSnapshots(localKeys, jsonOutput)
+	if remoteErr != nil {
+		v.warnRemoteListingFailed(remoteErr, jsonOutput)
+	} else {
+		snapshots = append(snapshots, listing.remoteOnly...)
+		markRemotePresence(snapshots, listing.keys)
+	}
+
+	// Stable so that rows sharing a timestamp keep the order they were
+	// merged in, rather than depending on the sort's pivot choices. The
+	// unparseable-timestamp fallback in remoteSnapshotInfo makes ties
+	// realistic: every such row carries the zero time.
+	sort.SliceStable(snapshots, func(i, j int) bool {
+		return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
+	})
+
+	if jsonOutput {
+		if remoteErr == nil {
+			v.reportJSONListingLimits(listing)
+		}
+
+		encoder := json.NewEncoder(v.Stdout)
+		encoder.SetIndent("", "  ")
+
+		return encoder.Encode(snapshots)
+	}
+
+	err = v.printSnapshotTable(snapshots)
+	if err != nil {
+		return err
+	}
+
+	if remoteErr == nil {
+		v.reportListDrift(snapshots, listing)
+	}
+
+	return nil
+}
+
+// warnRemoteListingFailed reports an unreachable or unreadable
+// destination store without failing the command: the local index is
+// still worth printing, and `snapshot list` exiting non-zero because a
+// volume is unmounted would be worse than useless.
+//
+// In --json mode the warning goes to stderr rather than through the
+// logger or the UI writer, both of which emit on stdout — the JSON
+// document has to be the only thing on stdout for `snapshot list --json
+// | jq` to work. The failure is also representable in the document
+// itself: every row's remote_present is null when the destination could
+// not be listed.
+func (v *Vaultik) warnRemoteListingFailed(err error, jsonOutput bool) {
+	if jsonOutput {
+		_, _ = fmt.Fprintf(v.Stderr,
+			"Warning: could not list backup destination store: %v. "+
+				"Showing snapshots from the local index only.\n", err)
+
+		return
+	}
+
+	// Once only: the logger also writes to stdout, so emitting through
+	// both it and the UI would print the same sentence to the user twice.
+	v.UI.Warningf("Could not list backup destination store: %v.", err)
+	v.UI.Infof("Showing snapshots from the local index only.")
+}
+
+// reportJSONListingLimits tells a --json consumer that the document it
+// is about to read is incomplete: manifests that could not be read, and
+// remote-only snapshots dropped by the maxRemoteOnlyRows cap.
+//
+// Table mode reports both below the table (see reportListDrift). In
+// --json mode they cannot go on stdout — the document has to be the
+// only thing there for `snapshot list --json | jq` to work — and the
+// document's shape is deliberately left alone so existing consumers
+// keep parsing. So they go to stderr, the same place the
+// unreachable-destination warning already goes. A consumer that must
+// react to truncation can treat any output on this stream as "this
+// listing is not the whole picture"; silent truncation of a listing
+// whose whole purpose is disaster recovery is the worse failure.
+func (v *Vaultik) reportJSONListingLimits(listing *remoteSnapshotListing) {
+	if listing.unreadable > 0 {
+		_, _ = fmt.Fprintf(v.Stderr,
+			"Warning: %d remote snapshot(s) could not be described: "+
+				"manifest missing or unreadable. They are missing from "+
+				"this listing.\n", listing.unreadable)
+	}
+
+	if listing.omitted > 0 {
+		_, _ = fmt.Fprintf(v.Stderr,
+			"Warning: listing truncated: %d further remote-only "+
+				"snapshot(s) not shown (limit %d per listing).\n",
+			listing.omitted, maxRemoteOnlyRows)
+	}
+}
+
+// kvPairSize is the number of variadic arguments that make up one
+// structured logging key/value pair.
+const kvPairSize = 2
+
+// warnWhileListing reports a per-snapshot problem found while
+// describing the destination store, through a writer that is safe for
+// the current output mode.
+//
+// In --json mode it writes to v.Stderr rather than calling log.Warn,
+// for the same reason warnRemoteListingFailed does: internal/log builds
+// its logger over os.Stdout and defaults to level Warn, so one warning
+// there would put a log line on stdout ahead of the JSON document and
+// break `snapshot list --json | jq`. A single corrupt manifest is
+// precisely the degradation this listing is built to survive, so it
+// must not be the thing that corrupts the output.
+//
+// This is a local workaround. Remove it, and the branch in
+// warnRemoteListingFailed, once issue #82 makes the logger's sink
+// configurable.
+func (v *Vaultik) warnWhileListing(jsonOutput bool, msg string, args ...any) {
+	if !jsonOutput {
+		log.Warn(msg, args...)
+
+		return
+	}
+
+	var line strings.Builder
+
+	_, _ = fmt.Fprintf(&line, "Warning: %s", msg)
+
+	for i := 0; i+kvPairSize <= len(args); i += kvPairSize {
+		pair := args[i : i+kvPairSize]
+		_, _ = fmt.Fprintf(&line, " %v=%v", pair[0], pair[1])
+	}
+
+	_, _ = fmt.Fprintln(v.Stderr, line.String())
+}
+
+// remoteSnapshotListing is the result of one pass over the destination
+// store's metadata/ prefix.
+type remoteSnapshotListing struct {
+	// keys holds every remote snapshot key present on the destination
+	// store, whether or not it is known locally.
+	keys map[string]bool
+
+	// remoteOnly holds one row per remote key with no local
+	// counterpart, built from that snapshot's manifest.
+	remoteOnly []SnapshotInfo
+
+	// omitted counts remote-only keys dropped because describing them
+	// all would have exceeded maxRemoteOnlyRows.
+	omitted int
+
+	// unreadable counts remote-only keys whose manifest could not be
+	// read or decoded.
+	unreadable int
+}
+
+// collectRemoteSnapshots enumerates the destination store and describes
+// every snapshot on it that localKeys does not already account for.
+//
+// The enumeration is a single streamed listing of the metadata/ prefix,
+// so the request count does not scale with the number of snapshots.
+// Manifest reads scale only with the number of snapshots the local
+// index does not already know about, and are capped at
+// maxRemoteOnlyRows.
+//
+// jsonOutput only selects where per-snapshot warnings are written; see
+// warnWhileListing.
+func (v *Vaultik) collectRemoteSnapshots(
+	localKeys map[string]bool, jsonOutput bool,
+) (*remoteSnapshotListing, error) {
+	keys, err := v.listAllRemoteSnapshotKeys()
+	if err != nil {
+		return nil, err
+	}
+
+	listing := &remoteSnapshotListing{
+		keys: make(map[string]bool, len(keys)),
+	}
+
+	unknown := make([]string, 0, len(keys))
+
+	for _, key := range keys {
+		listing.keys[key] = true
+
+		if !localKeys[key] {
+			unknown = append(unknown, key)
+		}
+	}
+
+	// Sorted so both the truncation point and the fetch order are
+	// deterministic run to run.
+	sort.Strings(unknown)
+
+	if len(unknown) > maxRemoteOnlyRows {
+		listing.omitted = len(unknown) - maxRemoteOnlyRows
+		unknown = unknown[:maxRemoteOnlyRows]
+	}
+
+	listing.remoteOnly, listing.unreadable = v.describeRemoteOnlySnapshots(
+		unknown, jsonOutput)
+
+	return listing, nil
+}
+
+// listingWarning is a problem found with one remote snapshot, recorded
+// rather than emitted on the spot. Manifest reads run concurrently and
+// the writer chosen by warnWhileListing is not guaranteed to be safe
+// for concurrent use, so warnings are held until every read has
+// finished and then emitted in key order from a single goroutine. That
+// also makes the warning order deterministic run to run.
+type listingWarning struct {
+	msg  string
+	args []any
+}
+
+// describeRemoteOnlySnapshots reads the manifest for each supplied
+// remote key and turns it into a table row, returning the rows and the
+// number of keys whose manifest could not be read.
+//
+// A key whose manifest is missing or corrupt is skipped rather than
+// failing the listing: one bad snapshot directory must not hide every
+// other snapshot the user has.
+func (v *Vaultik) describeRemoteOnlySnapshots(
+	keys []string, jsonOutput bool,
+) ([]SnapshotInfo, int) {
+	found := make([]SnapshotInfo, len(keys))
+	ok := make([]bool, len(keys))
+	warnings := make([]*listingWarning, len(keys))
+
+	var group errgroup.Group
+
+	group.SetLimit(remoteManifestFetchConcurrency)
+
+	for i, key := range keys {
+		group.Go(func() error {
+			info, warning, err := v.remoteSnapshotInfo(key)
+			if err != nil {
+				warnings[i] = &listingWarning{
+					msg:  "Could not describe remote snapshot",
+					args: []any{"remote_key", key, "error", err},
+				}
+
+				// Deliberately not returned: the failure is carried in
+				// warnings/ok and reported as a count. Returning it
+				// would cancel the group and let one bad snapshot
+				// directory hide every other snapshot the user has.
+				return nil //nolint:nilerr // see above
+			}
+
+			found[i] = info
+			warnings[i] = warning
+			ok[i] = true
+
+			return nil
+		})
+	}
+
+	// No goroutine above ever returns an error; failures are recorded
+	// in ok and reported as a count.
+	_ = group.Wait()
+
+	infos := make([]SnapshotInfo, 0, len(keys))
+	unreadable := 0
+
+	for i := range keys {
+		if warnings[i] != nil {
+			v.warnWhileListing(jsonOutput, warnings[i].msg, warnings[i].args...)
+		}
+
+		if !ok[i] {
+			unreadable++
+
+			continue
+		}
+
+		infos = append(infos, found[i])
+	}
+
+	return infos, unreadable
+}
+
+// remoteSnapshotInfo builds a table row for a snapshot that exists on
+// the destination store but not in the local index, from the only
+// source available without the private key: the unencrypted manifest.
+//
+// ID is deliberately left zero. Recovering it would mean inverting
+// snapshot.RemoteSnapshotKey, which is not possible, or writing the
+// human ID somewhere unencrypted on the destination, which would undo
+// the privacy property that hashing the key exists to provide (see
+// issue #81). The renderer marks the row as unnamed rather than
+// guessing.
+//
+// The returned warning, when non-nil, is a problem worth telling the
+// user about that was not bad enough to drop the row. It is returned
+// rather than logged because this runs on a worker goroutine; see
+// listingWarning.
+func (v *Vaultik) remoteSnapshotInfo(
+	remoteKey string,
+) (SnapshotInfo, *listingWarning, error) {
+	manifest, err := v.downloadManifestByKey(remoteKey)
+	if err != nil {
+		return SnapshotInfo{}, nil, err
+	}
+
+	var warning *listingWarning
+
+	timestamp, err := time.Parse(time.RFC3339, manifest.Timestamp)
+	if err != nil {
+		// The snapshot is really there; an unparseable timestamp is not
+		// reason enough to hide it. It sorts to the bottom as the zero
+		// time.
+		warning = &listingWarning{
+			msg: "Remote manifest has an unparseable timestamp",
+			args: []any{
+				"remote_key", remoteKey,
+				"timestamp", manifest.Timestamp,
+				"error", err,
+			},
+		}
+
+		timestamp = time.Time{}
+	}
+
+	return SnapshotInfo{
+		RemoteKey:      remoteKey,
+		Timestamp:      timestamp.UTC(),
+		CompressedSize: manifest.TotalCompressedSize,
+		LocallyTracked: false,
+	}, warning, nil
+}
+
+// markRemotePresence records, for every row, whether its remote key was
+// seen on the destination store during this listing. Only called when
+// the listing succeeded: when it did not, presence stays nil ("not
+// known") rather than being reported as absence.
+func markRemotePresence(snapshots []SnapshotInfo, remoteKeys map[string]bool) {
+	for i := range snapshots {
+		present := remoteKeys[snapshots[i].RemoteKey]
+		snapshots[i].RemotePresent = &present
+	}
+}
+
+// snapshotInfoFromLocal builds a SnapshotInfo row from a local snapshot
+// record. Failures from any per-snapshot stat query degrade that
+// column to its snapshot-row fallback but never fail the listing.
+func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
+	idStr := ls.ID.String()
+
+	totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(
+		v.ctx, idStr)
+	if err != nil {
+		log.Warn("Failed to get total compressed size", "id", idStr, "error", err)
+
+		totalSize = ls.BlobSize
+	}
+
+	uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(
+		v.ctx, idStr)
+	if err != nil {
+		log.Warn("Failed to get uncompressed chunk size", "id", idStr, "error", err)
+	}
+
+	newChunkSize, err := v.Repositories.Snapshots.GetSnapshotNewChunkSize(v.ctx, idStr)
+	if err != nil {
+		log.Warn("Failed to get new chunk size", "id", idStr, "error", err)
+	}
+
+	return SnapshotInfo{
+		ID:               ls.ID,
+		RemoteKey:        snapshot.RemoteSnapshotKey(idStr),
+		Timestamp:        ls.StartedAt,
+		CompressedSize:   totalSize,
+		UncompressedSize: uncompressedSize,
+		NewChunkSize:     newChunkSize,
+		LocallyTracked:   true,
+	}
+}
+
+// reportListDrift prints the reconciliation notes the merged table
+// cannot express on its own: local records with no counterpart on the
+// destination store, plus counts of remote snapshots that were
+// unreadable or omitted.
+//
+// This is what remains of the old reportRemoteDrift, and it no longer
+// touches the destination store. Its remote-only half collapsed into
+// the table — those snapshots are rows now, not a footnote count — and
+// its local-only half reads the merge ListSnapshots already computed,
+// so `snapshot list` lists the destination exactly once per invocation.
+func (v *Vaultik) reportListDrift(
+	snapshots []SnapshotInfo, listing *remoteSnapshotListing,
+) {
+	var localOnly []string
+
+	for _, snap := range snapshots {
+		if snap.LocallyTracked && !listing.keys[snap.RemoteKey] {
+			localOnly = append(localOnly, snap.ID.String())
+		}
+	}
+
+	if len(localOnly) > 0 {
+		v.UI.Warningf("%d local snapshot record(s) not found in backup "+
+			"destination store:", len(localOnly))
+
+		for _, id := range localOnly {
+			v.UI.Infof("%s", v.UI.Snapshot(id))
+		}
+
+		v.UI.Infof("Run '%s' to remove stale local records.", pruneCommandHint)
+	}
+
+	if len(listing.remoteOnly) > 0 {
+		v.UI.Noticef("NOTE: %d snapshot(s) on the backup destination store "+
+			"are not in the local index. Their hostname and snapshot name "+
+			"cannot be recovered without the age secret key, so they are "+
+			"listed by remote key.", len(listing.remoteOnly))
+	}
+
+	if listing.unreadable > 0 {
+		v.UI.Warningf("%d remote snapshot(s) could not be described: "+
+			"manifest missing or unreadable.", listing.unreadable)
+	}
+
+	if listing.omitted > 0 {
+		v.UI.Warningf("%d further remote-only snapshot(s) not shown "+
+			"(limit %d per listing).", listing.omitted, maxRemoteOnlyRows)
+	}
+}
+
+// formatRemoteOnlyID renders the identifier cell for a snapshot absent
+// from the local index. Its human ID cannot be recovered without the
+// private key, so the cell shows an abbreviation of the remote key
+// instead. The angle brackets make it obvious this is not a snapshot
+// name, which matters more than compactness: a bare hex string would
+// read as a name the user simply doesn't recognize.
+func formatRemoteOnlyID(remoteKey string) string {
+	short := remoteKey
+	if len(short) > remoteKeyDisplayLen {
+		short = short[:remoteKeyDisplayLen]
+	}
+
+	return "<remote only:" + short + ">"
+}
+
+// printSnapshotTable renders the snapshot list as a formatted table
+func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
+	w := tabwriter.NewWriter(v.Stdout, 0, 0, tabPadding, ' ', 0)
+
+	_, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:")
+	if err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintln(w, "NAME\tPATHS")
+	if err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintln(w, "────\t─────")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range v.Config.SnapshotNames() {
+		snap := v.Config.Snapshots[name]
+
+		paths := strings.Join(snap.Paths, ", ")
+
+		_, err = fmt.Fprintf(w, "%s\t%s\n", name, paths)
+		if err != nil {
+			return err
+		}
+	}
+
+	_, err = fmt.Fprintln(w)
+	if err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintln(w, "REMOTE SNAPSHOTS:")
+	if err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintln(w,
+		"SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\t"+
+			"UNCOMPRESSED SIZE\tNEW CHUNK SIZE")
+	if err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintln(w,
+		"───────────\t─────────\t───────────────\t"+
+			"─────────────────\t──────────────")
+	if err != nil {
+		return err
+	}
+
+	for _, snap := range snapshots {
+		var id, uncompressed, newChunks string
+
+		if snap.LocallyTracked {
+			id = snap.ID.String()
+			uncompressed = formatBytes(snap.UncompressedSize)
+			newChunks = formatBytes(snap.NewChunkSize)
+		} else {
+			id = formatRemoteOnlyID(snap.RemoteKey)
+			uncompressed = remoteOnlyCell
+			newChunks = remoteOnlyCell
+		}
+
+		_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
+			id,
+			snap.Timestamp.Format("2006-01-02 15:04:05"),
+			formatBytes(snap.CompressedSize),
+			uncompressed,
+			newChunks)
+		if err != nil {
+			return err
+		}
+	}
+
+	return w.Flush()
+}