sneak/mfer
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Releases Wiki Activity

Add custom types for type safety throughout codebase

Browse Source 
- Add FileCount, FileSize, RelFilePath, AbsFilePath, ModTime, Multihash types
- Add UnixSeconds and UnixNanos types for timestamp handling
- Add URL types (ManifestURL, FileURL, BaseURL) with safe path joining
- Consolidate scanner package into mfer package
- Update checker to use custom types in Result and CheckStatus
- Add ModTime.Timestamp() method for protobuf conversion
- Update all tests to use proper custom types
This commit is contained in:
Jeffrey Paul
2025-12-18 01:01:18 -08:00
parent a9f0d2abe4
commit dc115c5ba2
9 changed files with 428 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
internal/checker/checker.go
View File

@@ -17,9 +17,9 @@ import (
// Result represents the outcome of checking a single file.
type Result struct {
Path string // Relative path from manifest
Status Status // Verification result status
Message string // Human-readable description of the result
Path mfer.RelFilePath // Relative path from manifest
Status Status // Verification result status
Message string // Human-readable description of the result
}
// Status represents the verification status of a file.
@@ -55,22 +55,22 @@ func (s Status) String() string {
// CheckStatus contains progress information for the check operation.
type CheckStatus struct {
TotalFiles int64 // Total number of files in manifest
CheckedFiles int64 // Number of files checked so far
TotalBytes int64 // Total bytes to verify (sum of all file sizes)
CheckedBytes int64 // Bytes verified so far
BytesPerSec float64 // Current throughput rate
ETA time.Duration // Estimated time to completion
Failures int64 // Number of verification failures encountered
TotalFiles mfer.FileCount // Total number of files in manifest
CheckedFiles mfer.FileCount // Number of files checked so far
TotalBytes mfer.FileSize // Total bytes to verify (sum of all file sizes)
CheckedBytes mfer.FileSize // Bytes verified so far
BytesPerSec float64 // Current throughput rate
ETA time.Duration // Estimated time to completion
Failures mfer.FileCount // Number of verification failures encountered
}
// Checker verifies files against a manifest.
type Checker struct {
basePath string
basePath mfer.AbsFilePath
files []*mfer.MFFilePath
fs afero.Fs
// manifestPaths is a set of paths in the manifest for quick lookup
manifestPaths map[string]struct{}
manifestPaths map[mfer.RelFilePath]struct{}
}
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
@@ -92,13 +92,13 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
}
files := m.Files()
manifestPaths := make(map[string]struct{}, len(files))
manifestPaths := make(map[mfer.RelFilePath]struct{}, len(files))
for _, f := range files {
manifestPaths[f.Path] = struct{}{}
manifestPaths[mfer.RelFilePath(f.Path)] = struct{}{}
}
return &Checker{
basePath: abs,
basePath: mfer.AbsFilePath(abs),
files: files,
fs: fs,
manifestPaths: manifestPaths,
@@ -106,15 +106,15 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
}
// FileCount returns the number of files in the manifest.
func (c *Checker) FileCount() int64 {
return int64(len(c.files))
func (c *Checker) FileCount() mfer.FileCount {
return mfer.FileCount(len(c.files))
}
// TotalBytes returns the total size of all files in the manifest.
func (c *Checker) TotalBytes() int64 {
var total int64
func (c *Checker) TotalBytes() mfer.FileSize {
var total mfer.FileSize
for _, f := range c.files {
total += f.Size
total += mfer.FileSize(f.Size)
}
return total
}
@@ -131,12 +131,12 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
defer close(progress)
}
totalFiles := int64(len(c.files))
totalFiles := mfer.FileCount(len(c.files))
totalBytes := c.TotalBytes()
var checkedFiles int64
var checkedBytes int64
var failures int64
var checkedFiles mfer.FileCount
var checkedBytes mfer.FileSize
var failures mfer.FileCount
startTime := time.Now()
@@ -186,28 +186,29 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
return nil
}
func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *int64) Result {
absPath := filepath.Join(c.basePath, entry.Path)
func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize) Result {
absPath := filepath.Join(string(c.basePath), entry.Path)
relPath := mfer.RelFilePath(entry.Path)
// Check if file exists
info, err := c.fs.Stat(absPath)
if err != nil {
if errors.Is(err, afero.ErrFileNotFound) || errors.Is(err, errors.New("file does not exist")) {
return Result{Path: entry.Path, Status: StatusMissing, Message: "file not found"}
return Result{Path: relPath, Status: StatusMissing, Message: "file not found"}
}
// Check for "file does not exist" style errors
exists, _ := afero.Exists(c.fs, absPath)
if !exists {
return Result{Path: entry.Path, Status: StatusMissing, Message: "file not found"}
return Result{Path: relPath, Status: StatusMissing, Message: "file not found"}
}
return Result{Path: entry.Path, Status: StatusError, Message: err.Error()}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check size
if info.Size() != entry.Size {
*checkedBytes += info.Size()
*checkedBytes += mfer.FileSize(info.Size())
return Result{
Path: entry.Path,
Path: relPath,
Status: StatusSizeMismatch,
Message: "size mismatch",
}
@@ -216,31 +217,31 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *int64) Result
// Open and hash file
f, err := c.fs.Open(absPath)
if err != nil {
return Result{Path: entry.Path, Status: StatusError, Message: err.Error()}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
defer func() { _ = f.Close() }()
h := sha256.New()
n, err := io.Copy(h, f)
if err != nil {
return Result{Path: entry.Path, Status: StatusError, Message: err.Error()}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
*checkedBytes += n
*checkedBytes += mfer.FileSize(n)
// Encode as multihash and compare
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
if err != nil {
return Result{Path: entry.Path, Status: StatusError, Message: err.Error()}
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
}
// Check against all hashes in manifest (at least one must match)
for _, hash := range entry.Hashes {
if bytes.Equal(computed, hash.MultiHash) {
return Result{Path: entry.Path, Status: StatusOK}
return Result{Path: relPath, Status: StatusOK}
}
}
return Result{Path: entry.Path, Status: StatusHashMismatch, Message: "hash mismatch"}
return Result{Path: relPath, Status: StatusHashMismatch, Message: "hash mismatch"}
}
// FindExtraFiles walks the filesystem and reports files not in the manifest.
@@ -250,7 +251,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
defer close(results)
}
return afero.Walk(c.fs, c.basePath, func(path string, info os.FileInfo, err error) error {
return afero.Walk(c.fs, string(c.basePath), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -267,10 +268,11 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
}
// Get relative path
relPath, err := filepath.Rel(c.basePath, path)
rel, err := filepath.Rel(string(c.basePath), path)
if err != nil {
return err
}
relPath := mfer.RelFilePath(rel)
// Check if path is in manifest
if _, exists := c.manifestPaths[relPath]; !exists {

30
internal/checker/checker_test.go
View File

@@ -40,7 +40,7 @@ func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files ma
builder := mfer.NewBuilder()
for path, content := range files {
reader := bytes.NewReader(content)
_, err := builder.AddFile(path, int64(len(content)), time.Now(), reader, nil)
_, err := builder.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)), mfer.ModTime(time.Now()), reader, nil)
require.NoError(t, err)
}
@@ -72,7 +72,7 @@ func TestNewChecker(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/", fs)
require.NoError(t, err)
assert.NotNil(t, chk)
assert.Equal(t, int64(2), chk.FileCount())
assert.Equal(t, mfer.FileCount(2), chk.FileCount())
})
t.Run("missing manifest", func(t *testing.T) {
@@ -101,8 +101,8 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/", fs)
require.NoError(t, err)
assert.Equal(t, int64(3), chk.FileCount())
assert.Equal(t, int64(2+11+1000), chk.TotalBytes())
assert.Equal(t, mfer.FileCount(3), chk.FileCount())
assert.Equal(t, mfer.FileSize(2+11+1000), chk.TotalBytes())
}
func TestCheckAllFilesOK(t *testing.T) {
@@ -158,7 +158,7 @@ func TestCheckMissingFile(t *testing.T) {
okCount++
case StatusMissing:
missingCount++
assert.Equal(t, "missing.txt", r.Path)
assert.Equal(t, mfer.RelFilePath("missing.txt"), r.Path)
}
}
@@ -186,7 +186,7 @@ func TestCheckSizeMismatch(t *testing.T) {
r := <-results
assert.Equal(t, StatusSizeMismatch, r.Status)
assert.Equal(t, "file.txt", r.Path)
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path)
}
func TestCheckHashMismatch(t *testing.T) {
@@ -212,7 +212,7 @@ func TestCheckHashMismatch(t *testing.T) {
r := <-results
assert.Equal(t, StatusHashMismatch, r.Status)
assert.Equal(t, "file.txt", r.Path)
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path)
}
func TestCheckWithProgress(t *testing.T) {
@@ -246,11 +246,11 @@ func TestCheckWithProgress(t *testing.T) {
assert.NotEmpty(t, progressUpdates)
// Final progress should show all files checked
final := progressUpdates[len(progressUpdates)-1]
assert.Equal(t, int64(2), final.TotalFiles)
assert.Equal(t, int64(2), final.CheckedFiles)
assert.Equal(t, int64(300), final.TotalBytes)
assert.Equal(t, int64(300), final.CheckedBytes)
assert.Equal(t, int64(0), final.Failures)
assert.Equal(t, mfer.FileCount(2), final.TotalFiles)
assert.Equal(t, mfer.FileCount(2), final.CheckedFiles)
assert.Equal(t, mfer.FileSize(300), final.TotalBytes)
assert.Equal(t, mfer.FileSize(300), final.CheckedBytes)
assert.Equal(t, mfer.FileCount(0), final.Failures)
}
func TestCheckContextCancellation(t *testing.T) {
@@ -301,7 +301,7 @@ func TestFindExtraFiles(t *testing.T) {
}
assert.Len(t, extras, 1)
assert.Equal(t, "file2.txt", extras[0].Path)
assert.Equal(t, mfer.RelFilePath("file2.txt"), extras[0].Path)
assert.Equal(t, StatusExtra, extras[0].Status)
assert.Equal(t, "not in manifest", extras[0].Message)
}
@@ -390,8 +390,8 @@ func TestCheckEmptyManifest(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err)
assert.Equal(t, int64(0), chk.FileCount())
assert.Equal(t, int64(0), chk.TotalBytes())
assert.Equal(t, mfer.FileCount(0), chk.FileCount())
assert.Equal(t, mfer.FileSize(0), chk.TotalBytes())
results := make(chan Result, 10)
err = chk.Check(context.Background(), results, nil)