// Package mfer implements the mfer manifest file format: building, // serializing, verifying, and checking manifests of file trees. package mfer import ( "crypto/sha256" "errors" "fmt" "io" "sort" "strings" "sync" "time" "unicode/utf8" "github.com/multiformats/go-multihash" ) // readChunkSize is the buffer size used when reading file contents for // hashing. const readChunkSize = 64 * 1024 // The errPath* sentinels below are worded as the trailing fragment of the // message ValidatePath renders, because the offending path is quoted // before them (`path %q ...`). Wrapping them mid-sentence keeps the // rendered text exactly as mfer has always printed it. Match them with // errors.Is rather than by reading their messages. var ( errPathEmpty = errors.New("path cannot be empty") errPathNotUTF8 = errors.New("is not valid UTF-8") errPathBackslash = errors.New("contains backslash; use forward slashes only") errPathAbsolute = errors.New("is absolute; must be relative") errPathEmptySegment = errors.New("contains empty segment") errPathDotDot = errors.New("contains '..' segment") errSizeMismatch = errors.New("size mismatch") errNegativeSize = errors.New("size cannot be negative") errEmptyHash = errors.New("hash cannot be nil or empty") ) // ValidatePath checks that a file path conforms to manifest path invariants: // - Must be valid UTF-8 // - Must use forward slashes only (no backslashes) // - Must be relative (no leading /) // - Must not contain ".." segments // - Must not contain empty segments (no "//") // - Must not be empty func ValidatePath(p string) error { if p == "" { return errPathEmpty } if !utf8.ValidString(p) { return fmt.Errorf("path %q %w", p, errPathNotUTF8) } if strings.ContainsRune(p, '\\') { return fmt.Errorf("path %q %w", p, errPathBackslash) } if strings.HasPrefix(p, "/") { return fmt.Errorf("path %q %w", p, errPathAbsolute) } for _, seg := range strings.Split(p, "/") { if seg == "" { return fmt.Errorf("path %q %w", p, errPathEmptySegment) } if seg == ".." { return fmt.Errorf("path %q %w", p, errPathDotDot) } } return nil } // RelFilePath represents a relative file path within a manifest. type RelFilePath string // AbsFilePath represents an absolute file path on the filesystem. type AbsFilePath string // FileSize represents the size of a file in bytes. type FileSize int64 // FileCount represents a count of files. type FileCount int64 // ModTime represents a file's modification time. type ModTime time.Time // UnixSeconds represents seconds since Unix epoch. type UnixSeconds int64 // UnixNanos represents the nanosecond component of a timestamp (0-999999999). type UnixNanos int32 // Timestamp converts ModTime to a protobuf Timestamp. func (m ModTime) Timestamp() *Timestamp { return newTimestampFromTime(time.Time(m)) } // Multihash represents a multihash-encoded file hash (typically SHA2-256). type Multihash []byte // FileHashProgress reports progress during file hashing. type FileHashProgress struct { BytesRead FileSize // Total bytes read so far for the current file } // Builder constructs a manifest by adding files one at a time. type Builder struct { mu sync.Mutex files []*MFFilePath createdAt time.Time includeTimestamps bool signingOptions *SigningOptions fixedUUID []byte // if set, use this UUID instead of generating one } // NewBuilder creates a new Builder. func NewBuilder() *Builder { return &Builder{ files: make([]*MFFilePath, 0), createdAt: time.Now(), } } // SetSeed derives a deterministic UUID from the given seed string. // The seed is hashed once with SHA-256 and the first 16 bytes are used // as a fixed UUID for the manifest. func (b *Builder) SetSeed(seed string) { hash := sha256.Sum256([]byte(seed)) b.fixedUUID = hash[:uuidLength] } // AddFile reads file content from reader, computes hashes, and adds to manifest. // Progress updates are sent to the progress channel (if non-nil) without blocking. // Returns the number of bytes read. func (b *Builder) AddFile( path RelFilePath, size FileSize, mtime ModTime, reader io.Reader, progress chan<- FileHashProgress, ) (FileSize, error) { err := ValidatePath(string(path)) if err != nil { return 0, err } // Create hash writer h := sha256.New() // Read file in chunks, updating hash and progress var totalRead FileSize buf := make([]byte, readChunkSize) for { n, err := reader.Read(buf) if n > 0 { h.Write(buf[:n]) totalRead += FileSize(n) sendFileHashProgress(progress, FileHashProgress{BytesRead: totalRead}) } if err == io.EOF { break } if err != nil { return totalRead, err } } // Verify actual bytes read matches declared size if totalRead != size { return totalRead, fmt.Errorf( "%w for %q: declared %d bytes but read %d bytes", errSizeMismatch, path, size, totalRead, ) } // Encode hash as multihash (SHA2-256) mh, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) if err != nil { return totalRead, err } // Create file entry entry := &MFFilePath{ Path: string(path), Size: int64(size), Hashes: []*MFFileChecksum{ {MultiHash: mh}, }, Mtime: mtime.Timestamp(), } b.mu.Lock() b.files = append(b.files, entry) b.mu.Unlock() return totalRead, nil } // sendFileHashProgress sends a progress update without blocking. func sendFileHashProgress(ch chan<- FileHashProgress, p FileHashProgress) { if ch == nil { return } select { case ch <- p: default: } } // FileCount returns the number of files added to the builder. func (b *Builder) FileCount() int { b.mu.Lock() defer b.mu.Unlock() return len(b.files) } // AddFileWithHash adds a file entry with a pre-computed hash. // This is useful when the hash is already known (e.g., from an existing manifest). // Returns an error if path is empty, size is negative, or hash is nil/empty. func (b *Builder) AddFileWithHash( path RelFilePath, size FileSize, mtime ModTime, hash Multihash, ) error { err := ValidatePath(string(path)) if err != nil { return fmt.Errorf("add file: %w", err) } if size < 0 { return errNegativeSize } if len(hash) == 0 { return errEmptyHash } entry := &MFFilePath{ Path: string(path), Size: int64(size), Hashes: []*MFFileChecksum{ {MultiHash: hash}, }, Mtime: mtime.Timestamp(), } b.mu.Lock() b.files = append(b.files, entry) b.mu.Unlock() return nil } // SetIncludeTimestamps controls whether the manifest includes a createdAt timestamp. // By default timestamps are omitted for deterministic output. func (b *Builder) SetIncludeTimestamps(include bool) { b.mu.Lock() defer b.mu.Unlock() b.includeTimestamps = include } // SetSigningOptions sets the GPG signing options for the manifest. // If opts is non-nil, the manifest will be signed when Build() is called. func (b *Builder) SetSigningOptions(opts *SigningOptions) { b.mu.Lock() defer b.mu.Unlock() b.signingOptions = opts } // Build finalizes the manifest and writes it to the writer. func (b *Builder) Build(w io.Writer) error { b.mu.Lock() defer b.mu.Unlock() // Sort files by path for deterministic output sort.Slice(b.files, func(i, j int) bool { return b.files[i].GetPath() < b.files[j].GetPath() }) // Create inner manifest inner := &MFFile{ Version: MFFile_VERSION_ONE, Files: b.files, } if b.includeTimestamps { inner.CreatedAt = newTimestampFromTime(b.createdAt) } // Create a temporary manifest to use existing serialization m := &manifest{ pbInner: inner, signingOptions: b.signingOptions, fixedUUID: b.fixedUUID, } // Generate outer wrapper err := m.generateOuter() if err != nil { return fmt.Errorf("build: generate outer: %w", err) } // Generate final output err = m.generate() if err != nil { return fmt.Errorf("build: generate: %w", err) } // Write to output _, err = w.Write(m.output.Bytes()) if err != nil { return fmt.Errorf("build: write output: %w", err) } return nil }