Apply linter autofixes: internal/chunker (refs #61)
This commit is contained in:
@@ -3,6 +3,7 @@ package chunker
|
|||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
@@ -50,13 +51,15 @@ func (c *Chunker) ChunkReader(r io.Reader) ([]Chunk, error) {
|
|||||||
defer chunker.Release()
|
defer chunker.Release()
|
||||||
|
|
||||||
var chunks []Chunk
|
var chunks []Chunk
|
||||||
|
|
||||||
offset := int64(0)
|
offset := int64(0)
|
||||||
|
|
||||||
for {
|
for {
|
||||||
chunk, err := chunker.Next()
|
chunk, err := chunker.Next()
|
||||||
if err == io.EOF {
|
if errors.Is(err, io.EOF) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("reading chunk: %w", err)
|
return nil, fmt.Errorf("reading chunk: %w", err)
|
||||||
}
|
}
|
||||||
@@ -104,9 +107,10 @@ func (c *Chunker) ChunkReaderStreaming(r io.Reader, callback ChunkCallback) (str
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
chunk, err := chunker.Next()
|
chunk, err := chunker.Next()
|
||||||
if err == io.EOF {
|
if errors.Is(err, io.EOF) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("reading chunk: %w", err)
|
return "", fmt.Errorf("reading chunk: %w", err)
|
||||||
}
|
}
|
||||||
@@ -143,7 +147,8 @@ func (c *Chunker) ChunkFile(path string) ([]Chunk, error) {
|
|||||||
return nil, fmt.Errorf("opening file: %w", err)
|
return nil, fmt.Errorf("opening file: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := file.Close(); err != nil && err.Error() != "invalid argument" {
|
err := file.Close()
|
||||||
|
if err != nil && err.Error() != "invalid argument" {
|
||||||
// Log error or handle as needed
|
// Log error or handle as needed
|
||||||
_ = err
|
_ = err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
|
|
||||||
// Create data with some variation to trigger chunk boundaries
|
// Create data with some variation to trigger chunk boundaries
|
||||||
data := make([]byte, tt.fileSize)
|
data := make([]byte, tt.fileSize)
|
||||||
for i := 0; i < len(data); i++ {
|
for i := range data {
|
||||||
// Use a pattern that should create boundaries
|
// Use a pattern that should create boundaries
|
||||||
data[i] = byte((i * 17) ^ (i >> 5))
|
data[i] = byte((i * 17) ^ (i >> 5))
|
||||||
}
|
}
|
||||||
@@ -59,6 +59,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
t.Errorf("too few chunks: got %d, expected at least %d",
|
t.Errorf("too few chunks: got %d, expected at least %d",
|
||||||
len(chunks), tt.minExpected)
|
len(chunks), tt.minExpected)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(chunks) > tt.maxExpected {
|
if len(chunks) > tt.maxExpected {
|
||||||
t.Errorf("too many chunks: got %d, expected at most %d",
|
t.Errorf("too many chunks: got %d, expected at most %d",
|
||||||
len(chunks), tt.maxExpected)
|
len(chunks), tt.maxExpected)
|
||||||
@@ -69,6 +70,7 @@ func TestChunkerExpectedChunkCount(t *testing.T) {
|
|||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
reconstructed = append(reconstructed, chunk.Data...)
|
reconstructed = append(reconstructed, chunk.Data...)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !bytes.Equal(data, reconstructed) {
|
if !bytes.Equal(data, reconstructed) {
|
||||||
t.Error("reconstructed data doesn't match original")
|
t.Error("reconstructed data doesn't match original")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -60,6 +60,7 @@ func TestChunker(t *testing.T) {
|
|||||||
if chunk.Offset != expectedOffset {
|
if chunk.Offset != expectedOffset {
|
||||||
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
t.Errorf("chunk %d: expected offset %d, got %d", i, expectedOffset, chunk.Offset)
|
||||||
}
|
}
|
||||||
|
|
||||||
expectedOffset += chunk.Size
|
expectedOffset += chunk.Size
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -90,6 +91,7 @@ func TestChunker(t *testing.T) {
|
|||||||
if chunks1[i].Hash != chunks2[i].Hash {
|
if chunks1[i].Hash != chunks2[i].Hash {
|
||||||
t.Errorf("chunk %d: different hashes", i)
|
t.Errorf("chunk %d: different hashes", i)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunks1[i].Size != chunks2[i].Size {
|
if chunks1[i].Size != chunks2[i].Size {
|
||||||
t.Errorf("chunk %d: different sizes", i)
|
t.Errorf("chunk %d: different sizes", i)
|
||||||
}
|
}
|
||||||
@@ -121,6 +123,7 @@ func TestChunkBoundaries(t *testing.T) {
|
|||||||
if i < len(chunks)-1 && chunk.Size < minSize {
|
if i < len(chunks)-1 && chunk.Size < minSize {
|
||||||
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
t.Errorf("chunk %d size %d is below minimum %d", i, chunk.Size, minSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk.Size > maxSize {
|
if chunk.Size > maxSize {
|
||||||
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
t.Errorf("chunk %d size %d exceeds maximum %d", i, chunk.Size, maxSize)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package chunker
|
package chunker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -28,7 +29,7 @@ type ReusableChunker struct {
|
|||||||
|
|
||||||
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
// reusableChunkerPool pools ReusableChunker instances to avoid allocations.
|
||||||
var reusableChunkerPool = sync.Pool{
|
var reusableChunkerPool = sync.Pool{
|
||||||
New: func() interface{} {
|
New: func() any {
|
||||||
return &ReusableChunker{}
|
return &ReusableChunker{}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -39,17 +40,20 @@ var bufferPools = sync.Map{}
|
|||||||
|
|
||||||
func getBuffer(size int) []byte {
|
func getBuffer(size int) []byte {
|
||||||
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
poolI, _ := bufferPools.LoadOrStore(size, &sync.Pool{
|
||||||
New: func() interface{} {
|
New: func() any {
|
||||||
buf := make([]byte, size)
|
buf := make([]byte, size)
|
||||||
|
|
||||||
return &buf
|
return &buf
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
pool := poolI.(*sync.Pool)
|
pool := poolI.(*sync.Pool)
|
||||||
|
|
||||||
return *pool.Get().(*[]byte)
|
return *pool.Get().(*[]byte)
|
||||||
}
|
}
|
||||||
|
|
||||||
func putBuffer(buf []byte) {
|
func putBuffer(buf []byte) {
|
||||||
size := cap(buf)
|
size := cap(buf)
|
||||||
|
|
||||||
poolI, ok := bufferPools.Load(size)
|
poolI, ok := bufferPools.Load(size)
|
||||||
if ok {
|
if ok {
|
||||||
pool := poolI.(*sync.Pool)
|
pool := poolI.(*sync.Pool)
|
||||||
@@ -77,6 +81,7 @@ func AcquireReusableChunker(rd io.Reader, minSize, avgSize, maxSize int) *Reusab
|
|||||||
if c.buf != nil {
|
if c.buf != nil {
|
||||||
putBuffer(c.buf)
|
putBuffer(c.buf)
|
||||||
}
|
}
|
||||||
|
|
||||||
c.buf = getBuffer(bufSize)
|
c.buf = getBuffer(bufSize)
|
||||||
} else {
|
} else {
|
||||||
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
// Restore buffer to full capacity (may have been truncated by previous EOF)
|
||||||
@@ -120,6 +125,7 @@ func (c *ReusableChunker) fillBuffer() error {
|
|||||||
|
|
||||||
if c.eof {
|
if c.eof {
|
||||||
c.buf = c.buf[:n]
|
c.buf = c.buf[:n]
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,21 +134,24 @@ func (c *ReusableChunker) fillBuffer() error {
|
|||||||
|
|
||||||
// Fill the rest of the buffer
|
// Fill the rest of the buffer
|
||||||
m, err := io.ReadFull(c.rd, c.buf[n:])
|
m, err := io.ReadFull(c.rd, c.buf[n:])
|
||||||
if err == io.EOF || err == io.ErrUnexpectedEOF {
|
if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||||
c.buf = c.buf[:n+m]
|
c.buf = c.buf[:n+m]
|
||||||
c.eof = true
|
c.eof = true
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Next returns the next chunk or io.EOF when done.
|
// Next returns the next chunk or io.EOF when done.
|
||||||
// The returned Data slice is only valid until the next call to Next.
|
// The returned Data slice is only valid until the next call to Next.
|
||||||
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
func (c *ReusableChunker) Next() (FastCDCChunk, error) {
|
||||||
if err := c.fillBuffer(); err != nil {
|
err := c.fillBuffer()
|
||||||
|
if err != nil {
|
||||||
return FastCDCChunk{}, err
|
return FastCDCChunk{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(c.buf) == 0 {
|
if len(c.buf) == 0 {
|
||||||
return FastCDCChunk{}, io.EOF
|
return FastCDCChunk{}, io.EOF
|
||||||
}
|
}
|
||||||
@@ -189,13 +198,6 @@ func (c *ReusableChunker) nextChunk(data []byte) (int, uint64) {
|
|||||||
return i, fp
|
return i, fp
|
||||||
}
|
}
|
||||||
|
|
||||||
func min(a, b int) int {
|
|
||||||
if a < b {
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
// 256 random uint64s for the rolling hash function (from FastCDC paper)
|
||||||
var table = [256]uint64{
|
var table = [256]uint64{
|
||||||
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
0xe80e8d55032474b3, 0x11b25b61f5924e15, 0x03aa5bd82a9eb669, 0xc45a153ef107a38c,
|
||||||
|
|||||||
Reference in New Issue
Block a user