vaultik/internal/vaultik/snapshot_list.go

package vaultik

import (
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"text/tabwriter"
	"time"

	"golang.org/x/sync/errgroup"
	"sneak.berlin/go/vaultik/internal/database"
	"sneak.berlin/go/vaultik/internal/log"
	"sneak.berlin/go/vaultik/internal/snapshot"
)

// remoteOnlyCell fills the table columns that can only be derived from
// the local index. A snapshot present only on the destination store has
// no local rows to derive them from.
const remoteOnlyCell = "<remote only>"

// remoteKeyDisplayLen is how many hex characters of a remote key are
// shown in the identifier column for a remote-only snapshot. Twelve
// matches the abbreviation length used elsewhere in the UI and is far
// past the point of ambiguity for a SHA256 digest.
const remoteKeyDisplayLen = 12

// maxRemoteOnlyRows caps how many remote-only snapshots a single
// `snapshot list` will describe. Each one costs a manifest read, so an
// uncapped listing against a destination holding many thousands of
// unknown snapshots would be both slow and unbounded in memory. Beyond
// the cap the table is truncated and the count of omitted snapshots is
// reported.
const maxRemoteOnlyRows = 1000

// remoteManifestFetchConcurrency bounds how many manifest reads are in
// flight at once while describing remote-only snapshots. The listing
// itself is a single streamed prefix request; only the per-snapshot
// manifest reads need throttling.
const remoteManifestFetchConcurrency = 8

// tabPadding is the tabwriter cell padding for the snapshot table.
const tabPadding = 3

// ListSnapshots prints the table of snapshots known to this host: the
// union of the local index database and the backup destination store.
//
// Remote listing needs no age secret key. A snapshot's manifest
// (metadata/<remote-key>/manifest.json.zst) is compressed but not
// encrypted, so a host holding only the public key — the configuration
// vaultik is designed for — can still enumerate what it has backed up
// and see each snapshot's timestamp and compressed size.
//
// What that host cannot see is a remote-only snapshot's human ID.
// snapshot.RemoteSnapshotKey is one-way and the manifest stores the
// hashed key rather than the ID, so hostname and snapshot name live
// only in the local index and in the encrypted db.zst.age. Remote-only
// rows are therefore identified by an abbreviation of their remote key,
// and the two columns that genuinely require the local index
// (uncompressed size, new chunk size) render as "<remote only>". No
// attempt is made to recover or fabricate the human ID.
//
// Snapshots in the local index with no counterpart on the destination
// store are reported as drift below the table.
//
// If remote listing fails (unmounted volume, permission denied,
// network), we degrade to local-only with a warning. List never fails
// just because the destination is unreachable.
func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
	log.Info("Listing snapshots")

	localSnaps, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
	if err != nil {
		return fmt.Errorf("listing local snapshots: %w", err)
	}

	snapshots := make([]SnapshotInfo, 0, len(localSnaps))
	localKeys := make(map[string]bool, len(localSnaps))

	for _, ls := range localSnaps {
		if ls.CompletedAt == nil {
			continue
		}

		info := v.snapshotInfoFromLocal(ls)
		localKeys[info.RemoteKey] = true
		snapshots = append(snapshots, info)
	}

	listing, remoteErr := v.collectRemoteSnapshots(localKeys)
	if remoteErr != nil {
		v.warnRemoteListingFailed(remoteErr, jsonOutput)
	} else {
		snapshots = append(snapshots, listing.remoteOnly...)
		markRemotePresence(snapshots, listing.keys)
	}

	// Stable so that rows sharing a timestamp keep the order they were
	// merged in, rather than depending on the sort's pivot choices. The
	// unparseable-timestamp fallback in remoteSnapshotInfo makes ties
	// realistic: every such row carries the zero time.
	sort.SliceStable(snapshots, func(i, j int) bool {
		return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
	})

	if jsonOutput {
		if remoteErr == nil {
			v.reportJSONListingLimits(listing)
		}

		encoder := json.NewEncoder(v.Stdout)
		encoder.SetIndent("", "  ")

		return encoder.Encode(snapshots)
	}

	err = v.printSnapshotTable(snapshots)
	if err != nil {
		return err
	}

	if remoteErr == nil {
		v.reportListDrift(snapshots, listing)
	}

	return nil
}

// warnRemoteListingFailed reports an unreachable or unreadable
// destination store without failing the command: the local index is
// still worth printing, and `snapshot list` exiting non-zero because a
// volume is unmounted would be worse than useless.
//
// The two output modes report it through different channels. Table mode
// uses the UI writer, whose prose and color match the table it sits
// under. The UI writer emits on stdout, though, so --json mode uses the
// logger instead: stdout has to hold nothing but the JSON document for
// `snapshot list --json | jq` to work. Both channels are chosen once,
// never both, so the user is not told the same thing twice.
//
// The failure is also representable in the document itself: every row's
// remote_present is null when the destination could not be listed.
func (v *Vaultik) warnRemoteListingFailed(err error, jsonOutput bool) {
	if jsonOutput {
		log.Warn("Could not list backup destination store; "+
			"showing snapshots from the local index only", "error", err)

		return
	}

	v.UI.Warningf("Could not list backup destination store: %v.", err)
	v.UI.Infof("Showing snapshots from the local index only.")
}

// reportJSONListingLimits tells a --json consumer that the document it
// is about to read is incomplete: manifests that could not be read, and
// remote-only snapshots dropped by the maxRemoteOnlyRows cap.
//
// Table mode reports both below the table (see reportListDrift) through
// the UI writer, which emits on stdout. In --json mode stdout has to
// hold nothing but the document for `snapshot list --json | jq` to
// work, and the document's shape is deliberately left alone so existing
// consumers keep parsing — so these go to the logger, which writes to
// stderr. A consumer that must react to truncation can treat any output
// on that stream as "this listing is not the whole picture"; silent
// truncation of a listing whose whole purpose is disaster recovery is
// the worse failure.
func (v *Vaultik) reportJSONListingLimits(listing *remoteSnapshotListing) {
	if listing.unreadable > 0 {
		log.Warn("Some remote snapshot(s) could not be described: "+
			"manifest missing or unreadable; they are missing from "+
			"this listing", "unreadable", listing.unreadable)
	}

	if listing.omitted > 0 {
		log.Warn("Listing truncated: further remote-only snapshot(s) "+
			"not shown", "omitted", listing.omitted,
			"limit", maxRemoteOnlyRows)
	}
}

// remoteSnapshotListing is the result of one pass over the destination
// store's metadata/ prefix.
type remoteSnapshotListing struct {
	// keys holds every remote snapshot key present on the destination
	// store, whether or not it is known locally.
	keys map[string]bool

	// remoteOnly holds one row per remote key with no local
	// counterpart, built from that snapshot's manifest.
	remoteOnly []SnapshotInfo

	// omitted counts remote-only keys dropped because describing them
	// all would have exceeded maxRemoteOnlyRows.
	omitted int

	// unreadable counts remote-only keys whose manifest could not be
	// read or decoded.
	unreadable int
}

// collectRemoteSnapshots enumerates the destination store and describes
// every snapshot on it that localKeys does not already account for.
//
// The enumeration is a single streamed listing of the metadata/ prefix,
// so the request count does not scale with the number of snapshots.
// Manifest reads scale only with the number of snapshots the local
// index does not already know about, and are capped at
// maxRemoteOnlyRows.
func (v *Vaultik) collectRemoteSnapshots(
	localKeys map[string]bool,
) (*remoteSnapshotListing, error) {
	keys, err := v.listAllRemoteSnapshotKeys()
	if err != nil {
		return nil, err
	}

	listing := &remoteSnapshotListing{
		keys: make(map[string]bool, len(keys)),
	}

	unknown := make([]string, 0, len(keys))

	for _, key := range keys {
		listing.keys[key] = true

		if !localKeys[key] {
			unknown = append(unknown, key)
		}
	}

	// Sorted so both the truncation point and the fetch order are
	// deterministic run to run.
	sort.Strings(unknown)

	if len(unknown) > maxRemoteOnlyRows {
		listing.omitted = len(unknown) - maxRemoteOnlyRows
		unknown = unknown[:maxRemoteOnlyRows]
	}

	listing.remoteOnly, listing.unreadable = v.describeRemoteOnlySnapshots(
		unknown)

	return listing, nil
}

// listingWarning is a problem found with one remote snapshot, recorded
// rather than emitted on the spot. Manifest reads run concurrently, so
// emitting from the worker that found the problem would order the
// warnings by fetch completion — which varies run to run with network
// timing and tells the reader nothing. Holding them and emitting in key
// order from a single goroutine after every read has finished makes two
// runs over the same damaged store produce the same diagnostics in the
// same order.
//
// Concurrency safety is no longer part of the reason: these are emitted
// through log.Warn, and slog handlers are safe for concurrent use.
type listingWarning struct {
	msg  string
	args []any
}

// describeRemoteOnlySnapshots reads the manifest for each supplied
// remote key and turns it into a table row, returning the rows and the
// number of keys whose manifest could not be read.
//
// A key whose manifest is missing or corrupt is skipped rather than
// failing the listing: one bad snapshot directory must not hide every
// other snapshot the user has.
func (v *Vaultik) describeRemoteOnlySnapshots(
	keys []string,
) ([]SnapshotInfo, int) {
	found := make([]SnapshotInfo, len(keys))
	ok := make([]bool, len(keys))
	warnings := make([]*listingWarning, len(keys))

	var group errgroup.Group

	group.SetLimit(remoteManifestFetchConcurrency)

	for i, key := range keys {
		group.Go(func() error {
			info, warning, err := v.remoteSnapshotInfo(key)
			if err != nil {
				warnings[i] = &listingWarning{
					msg:  "Could not describe remote snapshot",
					args: []any{"remote_key", key, "error", err},
				}

				// Deliberately not returned: the failure is carried in
				// warnings/ok and reported as a count. Returning it
				// would cancel the group and let one bad snapshot
				// directory hide every other snapshot the user has.
				return nil //nolint:nilerr // see above
			}

			found[i] = info
			warnings[i] = warning
			ok[i] = true

			return nil
		})
	}

	// No goroutine above ever returns an error; failures are recorded
	// in ok and reported as a count.
	_ = group.Wait()

	infos := make([]SnapshotInfo, 0, len(keys))
	unreadable := 0

	for i := range keys {
		if warnings[i] != nil {
			log.Warn(warnings[i].msg, warnings[i].args...)
		}

		if !ok[i] {
			unreadable++

			continue
		}

		infos = append(infos, found[i])
	}

	return infos, unreadable
}

// remoteSnapshotInfo builds a table row for a snapshot that exists on
// the destination store but not in the local index, from the only
// source available without the private key: the unencrypted manifest.
//
// ID is deliberately left zero. Recovering it would mean inverting
// snapshot.RemoteSnapshotKey, which is not possible, or writing the
// human ID somewhere unencrypted on the destination, which would undo
// the privacy property that hashing the key exists to provide (see
// issue #81). The renderer marks the row as unnamed rather than
// guessing.
//
// The returned warning, when non-nil, is a problem worth telling the
// user about that was not bad enough to drop the row. It is returned
// rather than logged because this runs on a worker goroutine; see
// listingWarning.
func (v *Vaultik) remoteSnapshotInfo(
	remoteKey string,
) (SnapshotInfo, *listingWarning, error) {
	manifest, err := v.downloadManifestByKey(remoteKey)
	if err != nil {
		return SnapshotInfo{}, nil, err
	}

	var warning *listingWarning

	timestamp, err := time.Parse(time.RFC3339, manifest.Timestamp)
	if err != nil {
		// The snapshot is really there; an unparseable timestamp is not
		// reason enough to hide it. It sorts to the bottom as the zero
		// time.
		warning = &listingWarning{
			msg: "Remote manifest has an unparseable timestamp",
			args: []any{
				"remote_key", remoteKey,
				"timestamp", manifest.Timestamp,
				"error", err,
			},
		}

		timestamp = time.Time{}
	}

	return SnapshotInfo{
		RemoteKey:      remoteKey,
		Timestamp:      timestamp.UTC(),
		CompressedSize: manifest.TotalCompressedSize,
		LocallyTracked: false,
	}, warning, nil
}

// markRemotePresence records, for every row, whether its remote key was
// seen on the destination store during this listing. Only called when
// the listing succeeded: when it did not, presence stays nil ("not
// known") rather than being reported as absence.
func markRemotePresence(snapshots []SnapshotInfo, remoteKeys map[string]bool) {
	for i := range snapshots {
		present := remoteKeys[snapshots[i].RemoteKey]
		snapshots[i].RemotePresent = &present
	}
}

// snapshotInfoFromLocal builds a SnapshotInfo row from a local snapshot
// record. Failures from any per-snapshot stat query degrade that
// column to its snapshot-row fallback but never fail the listing.
func (v *Vaultik) snapshotInfoFromLocal(ls *database.Snapshot) SnapshotInfo {
	idStr := ls.ID.String()

	totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(
		v.ctx, idStr)
	if err != nil {
		log.Warn("Failed to get total compressed size", "id", idStr, "error", err)

		totalSize = ls.BlobSize
	}

	uncompressedSize, err := v.Repositories.Snapshots.GetSnapshotUncompressedChunkSize(
		v.ctx, idStr)
	if err != nil {
		log.Warn("Failed to get uncompressed chunk size", "id", idStr, "error", err)
	}

	newChunkSize, err := v.Repositories.Snapshots.GetSnapshotNewChunkSize(v.ctx, idStr)
	if err != nil {
		log.Warn("Failed to get new chunk size", "id", idStr, "error", err)
	}

	return SnapshotInfo{
		ID:               ls.ID,
		RemoteKey:        snapshot.RemoteSnapshotKey(idStr),
		Timestamp:        ls.StartedAt,
		CompressedSize:   totalSize,
		UncompressedSize: uncompressedSize,
		NewChunkSize:     newChunkSize,
		LocallyTracked:   true,
	}
}

// reportListDrift prints the reconciliation notes the merged table
// cannot express on its own: local records with no counterpart on the
// destination store, plus counts of remote snapshots that were
// unreadable or omitted.
//
// This is what remains of the old reportRemoteDrift, and it no longer
// touches the destination store. Its remote-only half collapsed into
// the table — those snapshots are rows now, not a footnote count — and
// its local-only half reads the merge ListSnapshots already computed,
// so `snapshot list` lists the destination exactly once per invocation.
func (v *Vaultik) reportListDrift(
	snapshots []SnapshotInfo, listing *remoteSnapshotListing,
) {
	var localOnly []string

	for _, snap := range snapshots {
		if snap.LocallyTracked && !listing.keys[snap.RemoteKey] {
			localOnly = append(localOnly, snap.ID.String())
		}
	}

	if len(localOnly) > 0 {
		v.UI.Warningf("%d local snapshot record(s) not found in backup "+
			"destination store:", len(localOnly))

		for _, id := range localOnly {
			v.UI.Infof("%s", v.UI.Snapshot(id))
		}

		v.UI.Infof("Run '%s' to remove stale local records.", pruneCommandHint)
	}

	if len(listing.remoteOnly) > 0 {
		v.UI.Noticef("NOTE: %d snapshot(s) on the backup destination store "+
			"are not in the local index. Their hostname and snapshot name "+
			"cannot be recovered without the age secret key, so they are "+
			"listed by remote key.", len(listing.remoteOnly))
	}

	if listing.unreadable > 0 {
		v.UI.Warningf("%d remote snapshot(s) could not be described: "+
			"manifest missing or unreadable.", listing.unreadable)
	}

	if listing.omitted > 0 {
		v.UI.Warningf("%d further remote-only snapshot(s) not shown "+
			"(limit %d per listing).", listing.omitted, maxRemoteOnlyRows)
	}
}

// formatRemoteOnlyID renders the identifier cell for a snapshot absent
// from the local index. Its human ID cannot be recovered without the
// private key, so the cell shows an abbreviation of the remote key
// instead. The angle brackets make it obvious this is not a snapshot
// name, which matters more than compactness: a bare hex string would
// read as a name the user simply doesn't recognize.
func formatRemoteOnlyID(remoteKey string) string {
	short := remoteKey
	if len(short) > remoteKeyDisplayLen {
		short = short[:remoteKeyDisplayLen]
	}

	return "<remote only:" + short + ">"
}

// printSnapshotTable renders the snapshot list as a formatted table
func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
	w := tabwriter.NewWriter(v.Stdout, 0, 0, tabPadding, ' ', 0)

	_, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintln(w, "NAME\tPATHS")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintln(w, "────\t─────")
	if err != nil {
		return err
	}

	for _, name := range v.Config.SnapshotNames() {
		snap := v.Config.Snapshots[name]

		paths := strings.Join(snap.Paths, ", ")

		_, err = fmt.Fprintf(w, "%s\t%s\n", name, paths)
		if err != nil {
			return err
		}
	}

	_, err = fmt.Fprintln(w)
	if err != nil {
		return err
	}

	_, err = fmt.Fprintln(w, "REMOTE SNAPSHOTS:")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintln(w,
		"SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE\t"+
			"UNCOMPRESSED SIZE\tNEW CHUNK SIZE")
	if err != nil {
		return err
	}

	_, err = fmt.Fprintln(w,
		"───────────\t─────────\t───────────────\t"+
			"─────────────────\t──────────────")
	if err != nil {
		return err
	}

	for _, snap := range snapshots {
		var id, uncompressed, newChunks string

		if snap.LocallyTracked {
			id = snap.ID.String()
			uncompressed = formatBytes(snap.UncompressedSize)
			newChunks = formatBytes(snap.NewChunkSize)
		} else {
			id = formatRemoteOnlyID(snap.RemoteKey)
			uncompressed = remoteOnlyCell
			newChunks = remoteOnlyCell
		}

		_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
			id,
			snap.Timestamp.Format("2006-01-02 15:04:05"),
			formatBytes(snap.CompressedSize),
			uncompressed,
			newChunks)
		if err != nil {
			return err
		}
	}

	return w.Flush()
}