mfer/mfer/checker.go

package mfer

import (
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"io"
	"os"
	"path/filepath"
	"time"

	"github.com/multiformats/go-multihash"
	"github.com/spf13/afero"
)

var (
	errManifestPathEmpty = errors.New("manifest path cannot be empty")
	errBasePathEmpty     = errors.New("base path cannot be empty")
)

// Result represents the outcome of checking a single file.
type Result struct {
	Path    RelFilePath // Relative path from manifest
	Status  Status      // Verification result status
	Message string      // Human-readable description of the result
}

// Status represents the verification status of a file.
type Status int

// Verification result statuses reported for each checked file.
const (
	StatusOK           Status = iota // File matches manifest (size and hash verified)
	StatusMissing                    // File not found on disk
	StatusSizeMismatch               // File size differs from manifest
	StatusHashMismatch               // File hash differs from manifest
	StatusModeMismatch               // File permission bits differ from a recorded mode
	StatusExtra                      // File exists on disk but not in manifest
	StatusError                      // Error occurred during verification
)

func (s Status) String() string {
	switch s {
	case StatusOK:
		return "OK"
	case StatusMissing:
		return "MISSING"
	case StatusSizeMismatch:
		return "SIZE_MISMATCH"
	case StatusHashMismatch:
		return "HASH_MISMATCH"
	case StatusModeMismatch:
		return "MODE_MISMATCH"
	case StatusExtra:
		return "EXTRA"
	case StatusError:
		return "ERROR"
	default:
		return "UNKNOWN"
	}
}

// CheckStatus contains progress information for the check operation.
type CheckStatus struct {
	TotalFiles   FileCount     // Total number of files in manifest
	CheckedFiles FileCount     // Number of files checked so far
	TotalBytes   FileSize      // Total bytes to verify (sum of all file sizes)
	CheckedBytes FileSize      // Bytes verified so far
	BytesPerSec  float64       // Current throughput rate
	ETA          time.Duration // Estimated time to completion
	Failures     FileCount     // Number of verification failures encountered
}

// Checker verifies files against a manifest.
type Checker struct {
	basePath AbsFilePath
	files    []*MFFilePath
	fs       afero.Fs
	// manifestPaths is a set of paths in the manifest for quick lookup
	manifestPaths map[RelFilePath]struct{}
	// manifestInfo is the manifest file, which FindExtraFiles leaves out,
	// matched with os.SameFile. It is nil for a manifest read from
	// CheckerOptions.ManifestReader.
	manifestInfo os.FileInfo
	// signature info from the manifest
	signature     []byte
	signer        []byte
	signingPubKey []byte
}

// CheckerOptions configures a Checker.
type CheckerOptions struct {
	// ManifestPath is the manifest file to check against. It is required
	// unless ManifestReader is given.
	ManifestPath string
	// ManifestReader, if not nil, is read for the manifest to check against
	// in place of ManifestPath, such as a manifest downloaded into memory.
	// FindExtraFiles then has no manifest file to leave out.
	ManifestReader io.Reader
	// BasePath is the directory relative to which manifest paths are
	// resolved (required).
	BasePath string
	// Fs is the filesystem to use, defaults to OsFs if nil.
	Fs afero.Fs
}

// NewChecker creates a new Checker with the given options. It returns an
// error if opts is nil, the base path is empty, or neither a manifest path
// nor a manifest reader is given.
func NewChecker(opts *CheckerOptions) (*Checker, error) {
	if opts == nil || (opts.ManifestPath == "" && opts.ManifestReader == nil) {
		return nil, errManifestPathEmpty
	}

	if opts.BasePath == "" {
		return nil, errBasePathEmpty
	}

	fs := opts.Fs
	if fs == nil {
		fs = afero.NewOsFs()
	}

	var (
		m            *manifest
		manifestInfo os.FileInfo
		err          error
	)

	if opts.ManifestReader != nil {
		m, err = NewManifestFromReader(opts.ManifestReader)
		if err != nil {
			return nil, err
		}
	} else {
		m, err = NewManifestFromFile(&ManifestFromFileOptions{
			Path: opts.ManifestPath,
			Fs:   fs,
		})
		if err != nil {
			return nil, err
		}

		manifestInfo, err = fs.Stat(opts.ManifestPath)
		if err != nil {
			return nil, err
		}
	}

	abs, err := filepath.Abs(opts.BasePath)
	if err != nil {
		return nil, err
	}

	files := m.Files()

	manifestPaths := make(map[RelFilePath]struct{}, len(files))
	for _, f := range files {
		manifestPaths[RelFilePath(f.GetPath())] = struct{}{}
	}

	return &Checker{
		basePath:      AbsFilePath(abs),
		files:         files,
		fs:            fs,
		manifestPaths: manifestPaths,
		manifestInfo:  manifestInfo,
		signature:     m.pbOuter.GetSignature(),
		signer:        m.pbOuter.GetSigner(),
		signingPubKey: m.pbOuter.GetSigningPubKey(),
	}, nil
}

// FileCount returns the number of files in the manifest.
func (c *Checker) FileCount() FileCount {
	return FileCount(len(c.files))
}

// TotalBytes returns the total size of all files in the manifest.
func (c *Checker) TotalBytes() FileSize {
	var total FileSize
	for _, f := range c.files {
		total += FileSize(f.GetSize())
	}

	return total
}

// IsSigned returns true if the manifest has a signature.
func (c *Checker) IsSigned() bool {
	return len(c.signature) > 0
}

// Signer returns the fingerprint of the key that made the manifest's
// signature, which loading the manifest checked, or nil if the manifest is
// not signed.
func (c *Checker) Signer() []byte {
	if !c.IsSigned() {
		return nil
	}

	return c.signer
}

// SigningPubKey returns the signing public key if the manifest is signed,
// nil otherwise.
func (c *Checker) SigningPubKey() []byte {
	return c.signingPubKey
}

// Check verifies all files against the manifest.
// Results are sent to the results channel as files are checked.
// Progress updates are sent to the progress channel approximately once per second.
// Both channels are closed when the method returns.
func (c *Checker) Check(
	ctx context.Context,
	results chan<- Result,
	progress chan<- CheckStatus,
) error {
	if results != nil {
		defer close(results)
	}

	if progress != nil {
		defer close(progress)
	}

	totalFiles := FileCount(len(c.files))
	totalBytes := c.TotalBytes()

	var (
		checkedFiles FileCount
		checkedBytes FileSize
		failures     FileCount
	)

	startTime := time.Now()
	lastProgressTime := time.Now()

	for _, entry := range c.files {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		result := c.checkFile(entry, &checkedBytes)
		if result.Status != StatusOK {
			failures++
		}

		checkedFiles++

		if results != nil {
			results <- result
		}

		// Send progress at most once per second (rate-limited)
		if progress != nil {
			now := time.Now()

			isLast := checkedFiles == totalFiles
			if isLast || now.Sub(lastProgressTime) >= time.Second {
				bytesPerSec, eta := computeRateETA(
					time.Since(startTime), checkedBytes, totalBytes,
				)

				sendCheckStatus(progress, CheckStatus{
					TotalFiles:   totalFiles,
					CheckedFiles: checkedFiles,
					TotalBytes:   totalBytes,
					CheckedBytes: checkedBytes,
					BytesPerSec:  bytesPerSec,
					ETA:          eta,
					Failures:     failures,
				})

				lastProgressTime = now
			}
		}
	}

	return nil
}

// FindExtraFiles walks the filesystem and reports the regular files, and
// symlinks to them, that are not in the manifest, hidden files and
// directories included. The manifest file itself is not reported. Anything
// the search cannot read, such as a directory that cannot be listed, is
// reported with StatusError and the search goes on. Results are sent to the
// results channel. The channel is closed when done.
func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) error {
	if results != nil {
		defer close(results)
	}

	// The search does not follow symlinks, so a base directory named
	// through one is resolved first. If that fails, the base is searched as
	// named and the search reports the problem.
	root := string(c.basePath)

	resolved, err := filepath.EvalSymlinks(root)
	if err == nil {
		root = resolved
	}

	walkFn := func(walkPath string, info os.FileInfo, walkErr error) error {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		// Get relative path
		rel, err := filepath.Rel(root, walkPath)
		if err != nil {
			return err
		}

		relPath := RelFilePath(rel)

		// Report what cannot be read, such as a directory that cannot be
		// listed, and go on with the rest.
		if walkErr != nil {
			if results != nil {
				results <- Result{
					Path:    relPath,
					Status:  StatusError,
					Message: walkErr.Error(),
				}
			}

			return nil
		}

		// Skip directories
		if info.IsDir() {
			return nil
		}

		// A symlink is compared by what it points to, so a manifest reached
		// through one is not reported either.
		if info.Mode()&os.ModeSymlink != 0 {
			target, statErr := c.fs.Stat(walkPath)
			if statErr == nil {
				info = target
			}
		}

		// gen lists regular files and symlinks to them only, so a named
		// pipe, a socket, a device, a symlink to a directory and a broken
		// symlink are never reported.
		if !info.Mode().IsRegular() {
			return nil
		}

		// Skip the manifest file itself, however its path is spelled
		if os.SameFile(info, c.manifestInfo) {
			return nil
		}

		// Check if path is in manifest
		if _, exists := c.manifestPaths[relPath]; !exists {
			if results != nil {
				results <- Result{
					Path:    relPath,
					Status:  StatusExtra,
					Message: "not in manifest",
				}
			}
		}

		return nil
	}

	return afero.Walk(c.fs, root, walkFn)
}

func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
	// entry.GetPath() is safe to join here: a manifest's entry paths are
	// validated against the path invariants when it is loaded (see
	// deserializeInner) or built (see Builder.AddFile), so a traversal or
	// absolute path can never reach this point.
	absPath := filepath.Join(string(c.basePath), entry.GetPath())
	relPath := RelFilePath(entry.GetPath())

	// Check if file exists
	info, err := c.fs.Stat(absPath)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) || errors.Is(err, afero.ErrFileNotFound) {
			return Result{
				Path:    relPath,
				Status:  StatusMissing,
				Message: "file not found",
			}
		}

		return Result{Path: relPath, Status: StatusError, Message: err.Error()}
	}

	// Opening a named pipe can block, and reading a device may never end.
	if !info.Mode().IsRegular() {
		return Result{
			Path:    relPath,
			Status:  StatusError,
			Message: "not a regular file",
		}
	}

	// Check size
	if info.Size() != entry.GetSize() {
		*checkedBytes += FileSize(info.Size())

		return Result{
			Path:    relPath,
			Status:  StatusSizeMismatch,
			Message: "size mismatch",
		}
	}

	// Open and hash file
	f, err := c.fs.Open(absPath)
	if err != nil {
		return Result{Path: relPath, Status: StatusError, Message: err.Error()}
	}

	defer func() { _ = f.Close() }()

	h := sha256.New()

	n, err := io.Copy(h, f)
	if err != nil {
		return Result{Path: relPath, Status: StatusError, Message: err.Error()}
	}

	*checkedBytes += FileSize(n)

	// Encode as multihash and compare
	computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
	if err != nil {
		return Result{Path: relPath, Status: StatusError, Message: err.Error()}
	}

	// Check against all hashes in manifest (at least one must match),
	// then against the recorded mode, where one is: 0 means none was.
	for _, hash := range entry.GetHashes() {
		if !bytes.Equal(computed, hash.GetMultiHash()) {
			continue
		}

		if entry.GetMode() != 0 && info.Mode().Perm() != os.FileMode(entry.GetMode()) {
			return Result{
				Path:    relPath,
				Status:  StatusModeMismatch,
				Message: "mode mismatch",
			}
		}

		return Result{Path: relPath, Status: StatusOK}
	}

	return Result{
		Path:    relPath,
		Status:  StatusHashMismatch,
		Message: "hash mismatch",
	}
}

// sendCheckStatus sends a status update without blocking.
func sendCheckStatus(ch chan<- CheckStatus, status CheckStatus) {
	if ch == nil {
		return
	}

	select {
	case ch <- status:
	default:
	}
}