vaultik/internal/vaultik/restore_plan.go

package vaultik

import (
	"context"
	"errors"
	"fmt"
	"math"
	"os"

	"sneak.berlin/go/vaultik/internal/database"
	"sneak.berlin/go/vaultik/internal/types"
)

// Sentinel errors for restore planning index lookups.
var (
	errPlanChunkMissing  = errors.New("chunk missing from blob map")
	errPlanBlobIDMissing = errors.New("blob id missing from id-to-hash map")
)

// restorePlan orders restore-time file processing by blob locality. The
// goal is to keep the blob disk cache occupancy as small as possible:
// download one blob, drain every file referencing only that blob, let
// the sweeper free the blob, then move on. Files that span multiple
// blobs are processed when their full blob set is on disk.
//
// The plan keeps two indexes:
//
//   - fileBlobs: for each pending file, the set of blob hashes it
//     still needs that are NOT yet in the cache. Files with an empty
//     set are "ready" — they can be restored from the current cache
//     with no further downloads.
//   - blobFiles: for each blob, the set of pending files referencing
//     it. Used to short-circuit "when this blob lands, which files
//     become ready" without a global scan, and by the sweeper to tell
//     whether a cached blob is still needed.
type restorePlan struct {
	fileBlobs map[types.FileID]map[string]struct{}
	blobFiles map[string]map[types.FileID]struct{}
	ready     []types.FileID
	cached    map[string]struct{}
}

// newRestorePlan builds the file→blob index for the given files. Files
// whose chunks reference no blobs (symlinks, directories) start in the
// ready queue immediately. A file with a chunk that no blob holds cannot
// be restored: it is left out of the plan and returned, by ID, with the
// reason.
func newRestorePlan(
	ctx context.Context,
	repos *database.Repositories,
	files []*database.File,
	chunkToBlobMap map[string]*database.BlobChunk,
	blobIDToHash map[string]string,
) (*restorePlan, map[types.FileID]error, error) {
	p := &restorePlan{
		fileBlobs: make(map[types.FileID]map[string]struct{}, len(files)),
		blobFiles: make(map[string]map[types.FileID]struct{}),
		ready:     make([]types.FileID, 0, len(files)),
		cached:    make(map[string]struct{}),
	}
	unplanned := make(map[types.FileID]error)

nextFile:
	for _, f := range files {
		if f.IsSymlink() || f.Mode&uint32(os.ModeDir) != 0 {
			// No chunks to fetch — restore can run immediately.
			p.fileBlobs[f.ID] = nil
			p.ready = append(p.ready, f.ID)

			continue
		}

		fileChunks, err := repos.FileChunks.GetByFileID(ctx, f.ID)
		if err != nil {
			return nil, nil, fmt.Errorf("planning %s: %w", f.Path, err)
		}

		blobs := make(map[string]struct{})

		for _, fc := range fileChunks {
			bc, ok := chunkToBlobMap[fc.ChunkHash.String()]
			if !ok {
				unplanned[f.ID] = fmt.Errorf("%w: %s",
					errPlanChunkMissing, shortHash(fc.ChunkHash.String()))

				continue nextFile
			}

			hash, ok := blobIDToHash[bc.BlobID.String()]
			if !ok {
				unplanned[f.ID] = fmt.Errorf("%w: %s",
					errPlanBlobIDMissing, bc.BlobID)

				continue nextFile
			}

			blobs[hash] = struct{}{}
		}

		p.fileBlobs[f.ID] = blobs
		for hash := range blobs {
			set, ok := p.blobFiles[hash]
			if !ok {
				set = make(map[types.FileID]struct{})
				p.blobFiles[hash] = set
			}

			set[f.ID] = struct{}{}
		}

		if len(blobs) == 0 {
			p.ready = append(p.ready, f.ID)
		}
	}

	return p, unplanned, nil
}

// markBlobCached records that the named blob is now resident in the
// disk cache and moves any pending file whose remaining-uncached-blobs
// set just dropped to empty onto the ready queue.
func (p *restorePlan) markBlobCached(blobHash string) {
	if _, already := p.cached[blobHash]; already {
		return
	}

	p.cached[blobHash] = struct{}{}
	for fileID := range p.blobFiles[blobHash] {
		blobs := p.fileBlobs[fileID]
		delete(blobs, blobHash)

		if len(blobs) == 0 {
			p.ready = append(p.ready, fileID)
		}
	}
}

// popReady returns the next ready file, removing it from the queue. If
// no file is ready, the second return value is false.
func (p *restorePlan) popReady() (types.FileID, bool) {
	if len(p.ready) == 0 {
		return types.FileID{}, false
	}

	id := p.ready[0]
	p.ready = p.ready[1:]

	return id, true
}

// finishFile drops a restored file from both indexes so subsequent
// planning calls don't reconsider it.
func (p *restorePlan) finishFile(fileID types.FileID) {
	for hash := range p.fileBlobs[fileID] {
		if set, ok := p.blobFiles[hash]; ok {
			delete(set, fileID)

			if len(set) == 0 {
				delete(p.blobFiles, hash)
			}
		}
	}

	delete(p.fileBlobs, fileID)
	// Also scrub the file from any blobFiles entries where it might
	// still appear even after its uncached-blob set was emptied.
	for hash, set := range p.blobFiles {
		if _, ok := set[fileID]; ok {
			delete(set, fileID)

			if len(set) == 0 {
				delete(p.blobFiles, hash)
			}
		}
	}
}

// pickNextDownload returns the pending file whose remaining-uncached
// blob set is smallest (with ties broken by FileID string compare so
// the choice is deterministic across runs). This file's blobs are
// downloaded next, after which it — together with any other pending
// files whose blob sets become empty — moves to the ready queue.
//
// The second return value is false when no file needs a download, so a
// genuine file carrying the nil UUID is picked rather than mistaken for
// "nothing left".
func (p *restorePlan) pickNextDownload() (types.FileID, bool) {
	var best types.FileID

	found := false
	bestCount := math.MaxInt

	var bestID string

	for id, blobs := range p.fileBlobs {
		n := len(blobs)
		if n == 0 {
			// Already-ready files should have been popped via
			// popReady; ignore here just in case.
			continue
		}

		idStr := id.String()
		if !found || n < bestCount || (n == bestCount && idStr < bestID) {
			best = id
			found = true
			bestCount = n
			bestID = idStr
		}
	}

	return best, found
}

// blobsNeeded returns the uncached blob hashes for fileID in any order.
func (p *restorePlan) blobsNeeded(fileID types.FileID) []string {
	blobs := p.fileBlobs[fileID]

	out := make([]string, 0, len(blobs))
	for h := range blobs {
		out = append(out, h)
	}

	return out
}

// filesReferencingBlob returns the pending files that reference the
// named blob, in any order. The result is a copy, so the caller may
// finishFile each of them while ranging over it.
func (p *restorePlan) filesReferencingBlob(blobHash string) []types.FileID {
	files := p.blobFiles[blobHash]

	out := make([]types.FileID, 0, len(files))
	for id := range files {
		out = append(out, id)
	}

	return out
}

// pendingFileUsesBlob reports whether any pending file references the
// named blob, whether or not the blob is cached.
func (p *restorePlan) pendingFileUsesBlob(blobHash string) bool {
	return len(p.blobFiles[blobHash]) > 0
}

// hasPending reports whether any unfinished files remain.
func (p *restorePlan) hasPending() bool {
	return len(p.fileBlobs) > 0
}