vaultik/internal/chunker/chunker.go

// Package chunker splits input data into content-defined chunks using the
// FastCDC algorithm so that identical data sequences produce identical
// chunks regardless of their position in the file.
package chunker

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
)

// Chunk represents a single chunk of data produced by the content-defined
// chunking algorithm. Each chunk is identified by its SHA256 hash and
// contains the raw data along with its position and size information from
// the original file.
type Chunk struct {
	Hash   string // Content hash of the chunk
	Data   []byte // Chunk data
	Offset int64  // Offset in the original file
	Size   int64  // Size of the chunk
}

// Chunker provides content-defined chunking using the FastCDC algorithm.
// It splits data into variable-sized chunks based on content patterns, ensuring
// that identical data sequences produce identical chunks regardless of their
// position in the file. This enables efficient deduplication.
type Chunker struct {
	avgChunkSize int
	minChunkSize int
	maxChunkSize int
}

// chunkSizeSpread is the FastCDC-recommended factor between the average
// chunk size and the minimum (avg/spread) and maximum (avg*spread) sizes.
const chunkSizeSpread = 4

// NewChunker creates a new chunker with the specified average chunk size.
// The actual chunk sizes will vary between avgChunkSize/4 and avgChunkSize*4
// as recommended by the FastCDC algorithm. Typical values for avgChunkSize
// are 64KB (65536), 256KB (262144), or 1MB (1048576).
func NewChunker(avgChunkSize int64) *Chunker {
	// FastCDC recommends min = avg/4 and max = avg*4
	return &Chunker{
		avgChunkSize: int(avgChunkSize),
		minChunkSize: int(avgChunkSize / chunkSizeSpread),
		maxChunkSize: int(avgChunkSize * chunkSizeSpread),
	}
}

// ChunkReader splits the reader into content-defined chunks and returns all
// chunks at once. This method loads all chunk data into memory, so it should
// only be used for reasonably sized inputs. For large files or streams, use
// ChunkReaderStreaming instead.
// Returns an error if chunking fails or if reading from the input fails.
func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
	chunker := AcquireReusableChunker(
		r, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
	defer chunker.Release()

	var chunks []Chunk

	offset := int64(0)

	for {
		chunk, err := chunker.Next()
		if errors.Is(err, io.EOF) {
			break
		}

		if err != nil {
			return nil, fmt.Errorf("reading chunk: %w", err)
		}

		// Calculate hash
		hash := sha256.Sum256(chunk.Data)

		// Make a copy of the data since the chunker reuses the buffer
		chunkData := make([]byte, len(chunk.Data))
		copy(chunkData, chunk.Data)

		chunks = append(chunks, Chunk{
			Hash:   hex.EncodeToString(hash[:]),
			Data:   chunkData,
			Offset: offset,
			Size:   int64(len(chunk.Data)),
		})

		offset += int64(len(chunk.Data))
	}

	return chunks, nil
}

// ChunkCallback is a function called for each chunk as it's processed.
// The callback receives a Chunk containing the hash, data, offset, and size.
// If the callback returns an error, chunk processing stops and the error is
// propagated.
type ChunkCallback func(chunk Chunk) error

// ChunkReaderStreaming splits the reader into chunks and calls the callback
// for each chunk. This is the preferred method for processing large files or
// streams as it doesn't accumulate all chunks in memory. The callback is
// invoked for each chunk as it's produced, allowing for streaming processing
// and immediate storage or transmission.
// Returns the SHA256 hash of the entire file content and an error if
// chunking fails, reading fails, or if the callback returns an error.
func (c *Chunker) ChunkReaderStreaming(
	r io.Reader, callback ChunkCallback,
) (string, error) {
	// Create a tee reader to calculate full file hash while chunking
	fileHasher := sha256.New()
	teeReader := io.TeeReader(r, fileHasher)

	chunker := AcquireReusableChunker(
		teeReader, c.minChunkSize, c.avgChunkSize, c.maxChunkSize)
	defer chunker.Release()

	offset := int64(0)

	for {
		chunk, err := chunker.Next()
		if errors.Is(err, io.EOF) {
			break
		}

		if err != nil {
			return "", fmt.Errorf("reading chunk: %w", err)
		}

		// Calculate chunk hash
		hash := sha256.Sum256(chunk.Data)

		// Pass the data directly - caller must process it before we call
		// Next() again (chunker reuses its internal buffer, but since we
		// process synchronously and completely before continuing, no copy
		// is needed)
		err = callback(Chunk{
			Hash:   hex.EncodeToString(hash[:]),
			Data:   chunk.Data,
			Offset: offset,
			Size:   int64(len(chunk.Data)),
		})
		if err != nil {
			return "", fmt.Errorf("callback error: %w", err)
		}

		offset += int64(len(chunk.Data))
	}

	// Return the full file hash
	return hex.EncodeToString(fileHasher.Sum(nil)), nil
}

// ChunkFile splits a file into content-defined chunks by reading the entire file.
// This is a convenience method that opens the file and passes it to ChunkReader.
// For large files, consider using ChunkReaderStreaming with a file handle instead.
// Returns an error if the file cannot be opened or if chunking fails.
func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
	file, err := os.Open(path) //nolint:gosec // G304: path is caller-supplied by design
	if err != nil {
		return nil, fmt.Errorf("opening file: %w", err)
	}
	defer func() {
		err := file.Close()
		if err != nil && err.Error() != "invalid argument" {
			// Log error or handle as needed
			_ = err
		}
	}()

	return c.ChunkReader(file)
}