diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index 3995990..1615899 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -3,6 +3,7 @@ package chunker import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" @@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) { defer chunker.Release() var chunks []Chunk + offset := int64(0) for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return nil, fmt.Errorf("reading chunk: %w", err) } @@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str for { chunk, err := chunker.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } + if err != nil { return "", fmt.Errorf("reading chunk: %w", err) } @@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) { return nil, fmt.Errorf("opening file: %w", err) } defer func() { - if err := file.Close(); err != nil && err.Error() != "invalid argument" { + err := file.Close() + if err != nil && err.Error() != "invalid argument" { // Log error or handle as needed _ = err } diff --git a/internal/chunker/chunker_isolated_test.go b/internal/chunker/chunker_isolated_test.go index 8e3fb0f..b5f86bf 100644 --- a/internal/chunker/chunker_isolated_test.go +++ b/internal/chunker/chunker_isolated_test.go @@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { // Create data with some variation to trigger chunk boundaries data := make([]byte, tt.fileSize) - for i := 0; i < len(data); i++ { + for i := range data { // Use a pattern that should create boundaries data[i] = byte((i * 17) ^ (i >> 5)) } @@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { t.Errorf("too few chunks: got %d, expected at least %d", len(chunks), tt.minExpected) } + if len(chunks) > tt.maxExpected { t.Errorf("too many chunks: got %d, expected at most %d", len(chunks), tt.maxExpected) @@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) { for _, chunk := range chunks { reconstructed = append(reconstructed, chunk.Data...) } + if !bytes.Equal(data, reconstructed) { t.Error("reconstructed data doesn't match original") } diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go index a13e143..fca59f2 100644 --- a/internal/chunker/chunker_test.go +++ b/internal/chunker/chunker_test.go @@ -60,6 +60,7 @@ func TestChunker(t *testing.T) { if chunk.Offset != expectedOffset { t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset) } + expectedOffset += chunk.Size } }) @@ -90,6 +91,7 @@ func TestChunker(t *testing.T) { if chunks1[i].Hash != chunks2[i].Hash { t.Errorf("chunk %d: different hashes", i) } + if chunks1[i].Size != chunks2[i].Size { t.Errorf("chunk %d: different sizes", i) } @@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) { if i < len(chunks)-1 && chunk.Size < minSize { t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize) } + if chunk.Size > maxSize { t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize) } diff --git a/internal/chunker/fastcdc.go b/internal/chunker/fastcdc.go index 54c87c7..526c4a0 100644 --- a/internal/chunker/fastcdc.go +++ b/internal/chunker/fastcdc.go @@ -1,6 +1,7 @@ package chunker import ( + "errors" "io" "math" "sync" @@ -28,7 +29,7 @@ type ReusableChunker struct { // reusableChunkerPool pools ReusableChunker instances to avoid allocations. var reusableChunkerPool = sync.Pool{ - New: func() interface{} { + New: func() any { return &ReusableChunker{} }, } @@ -39,17 +40,20 @@ var bufferPools = sync.Map{} func getBuffer(size int) []byte { poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{ - New: func() interface{} { + New: func() any { buf := make([]byte, size) + return &buf }, }) pool := poolI.(*sync.Pool) + return *pool.Get().(*[]byte) } func putBuffer(buf []byte) { size := cap(buf) + poolI, ok := bufferPools.Load(size) if ok { pool := poolI.(*sync.Pool) @@ -77,6 +81,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab if c.buf != nil { putBuffer(c.buf) } + c.buf = getBuffer(bufSize) } else { // Restore buffer to full capacity (may have been truncated by previous EOF) @@ -120,6 +125,7 @@ func (c *ReusableChunker) fillBuffer() error { if c.eof { c.buf = c.buf[:n] + return nil } @@ -128,21 +134,24 @@ func (c *ReusableChunker) fillBuffer() error { // Fill the rest of the buffer m, err := io.ReadFull(c.rd, c.buf[n:]) - if err == io.EOF || err == io.ErrUnexpectedEOF { + if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) { c.buf = c.buf[:n+m] c.eof = true } else if err != nil { return err } + return nil } // Next returns the next chunk or io.EOF when done. // The returned Data slice is only valid until the next call to Next. func (c *ReusableChunker) Next() (FastCDCChunk, error) { - if err := c.fillBuffer(); err != nil { + err := c.fillBuffer() + if err != nil { return FastCDCChunk{}, err } + if len(c.buf) == 0 { return FastCDCChunk{}, io.EOF } @@ -189,13 +198,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) { return i, fp } -func min(a, b int) int { - if a < b { - return a - } - return b -} - // 256 random uint64s for the rolling hash function (from FastCDC paper) var table = [256]uint64{ 0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,