sneak/mfer
1
0
Fork 0
Code Issues 4 Pull Requests 1 Releases Wiki Activity

Move checker package into mfer package

Browse Source 
Consolidate checker functionality into the mfer package alongside
scanner, removing the need for a separate internal/checker package.
This commit is contained in:
Jeffrey Paul 2025-12-18 01:28:35 -08:00
parent dc115c5ba2
commit e25e309581
3 changed files with 62 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
internal/cli/check.go
View File

@ -8,8 +8,8 @@ import (
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
"sneak.berlin/go/mfer/internal/checker"
"sneak.berlin/go/mfer/internal/log" "sneak.berlin/go/mfer/internal/log"
"sneak.berlin/go/mfer/mfer"
) )
// findManifest looks for a manifest file in the given directory. // findManifest looks for a manifest file in the given directory.
@ -63,7 +63,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Infof("checking manifest %s with base %s", manifestPath, basePath) log.Infof("checking manifest %s with base %s", manifestPath, basePath)
// Create checker // Create checker
chk, err := checker.NewChecker(manifestPath, basePath, mfa.Fs) chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
if err != nil { if err != nil {
return fmt.Errorf("failed to load manifest: %w", err) return fmt.Errorf("failed to load manifest: %w", err)
} }
@ -71,12 +71,12 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
log.Infof("manifest contains %d files, %s", chk.FileCount(), humanize.IBytes(uint64(chk.TotalBytes()))) log.Infof("manifest contains %d files, %s", chk.FileCount(), humanize.IBytes(uint64(chk.TotalBytes())))
// Set up results channel // Set up results channel
results := make(chan checker.Result, 1) results := make(chan mfer.Result, 1)
// Set up progress channel // Set up progress channel
var progress chan checker.CheckStatus var progress chan mfer.CheckStatus
if showProgress { if showProgress {
progress = make(chan checker.CheckStatus, 1) progress = make(chan mfer.CheckStatus, 1)
go func() { go func() {
for status := range progress { for status := range progress {
if status.ETA > 0 { if status.ETA > 0 {
@ -103,7 +103,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
for result := range results { for result := range results {
if result.Status != checker.StatusOK { if result.Status != mfer.StatusOK {
failures++ failures++
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message) log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
} else { } else {
@ -124,7 +124,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
// Check for extra files if requested // Check for extra files if requested
if ctx.Bool("no-extra-files") { if ctx.Bool("no-extra-files") {
extraResults := make(chan checker.Result, 1) extraResults := make(chan mfer.Result, 1)
extraDone := make(chan struct{}) extraDone := make(chan struct{})
go func() { go func() {
for result := range extraResults { for result := range extraResults {

65
internal/checker/checker.go → mfer/checker.go
View File

@ -1,4 +1,4 @@
package checker package mfer
import ( import (
"bytes" "bytes"
@ -12,14 +12,13 @@ import (
"github.com/multiformats/go-multihash" "github.com/multiformats/go-multihash"
"github.com/spf13/afero" "github.com/spf13/afero"
"sneak.berlin/go/mfer/mfer"
) )
// Result represents the outcome of checking a single file. // Result represents the outcome of checking a single file.
type Result struct { type Result struct {
Path mfer.RelFilePath // Relative path from manifest Path RelFilePath // Relative path from manifest
Status Status // Verification result status Status Status // Verification result status
Message string // Human-readable description of the result Message string // Human-readable description of the result
} }
// Status represents the verification status of a file. // Status represents the verification status of a file.
@ -55,22 +54,22 @@ func (s Status) String() string {
// CheckStatus contains progress information for the check operation. // CheckStatus contains progress information for the check operation.
type CheckStatus struct { type CheckStatus struct {
TotalFiles mfer.FileCount // Total number of files in manifest TotalFiles FileCount // Total number of files in manifest
CheckedFiles mfer.FileCount // Number of files checked so far CheckedFiles FileCount // Number of files checked so far
TotalBytes mfer.FileSize // Total bytes to verify (sum of all file sizes) TotalBytes FileSize // Total bytes to verify (sum of all file sizes)
CheckedBytes mfer.FileSize // Bytes verified so far CheckedBytes FileSize // Bytes verified so far
BytesPerSec float64 // Current throughput rate BytesPerSec float64 // Current throughput rate
ETA time.Duration // Estimated time to completion ETA time.Duration // Estimated time to completion
Failures mfer.FileCount // Number of verification failures encountered Failures FileCount // Number of verification failures encountered
} }
// Checker verifies files against a manifest. // Checker verifies files against a manifest.
type Checker struct { type Checker struct {
basePath mfer.AbsFilePath basePath AbsFilePath
files []*mfer.MFFilePath files []*MFFilePath
fs afero.Fs fs afero.Fs
// manifestPaths is a set of paths in the manifest for quick lookup // manifestPaths is a set of paths in the manifest for quick lookup
manifestPaths map[mfer.RelFilePath]struct{} manifestPaths map[RelFilePath]struct{}
} }
// NewChecker creates a new Checker for the given manifest, base path, and filesystem. // NewChecker creates a new Checker for the given manifest, base path, and filesystem.
@ -81,7 +80,7 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
fs = afero.NewOsFs() fs = afero.NewOsFs()
} }
m, err := mfer.NewManifestFromFile(fs, manifestPath) m, err := NewManifestFromFile(fs, manifestPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -92,13 +91,13 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
} }
files := m.Files() files := m.Files()
manifestPaths := make(map[mfer.RelFilePath]struct{}, len(files)) manifestPaths := make(map[RelFilePath]struct{}, len(files))
for _, f := range files { for _, f := range files {
manifestPaths[mfer.RelFilePath(f.Path)] = struct{}{} manifestPaths[RelFilePath(f.Path)] = struct{}{}
} }
return &Checker{ return &Checker{
basePath: mfer.AbsFilePath(abs), basePath: AbsFilePath(abs),
files: files, files: files,
fs: fs, fs: fs,
manifestPaths: manifestPaths, manifestPaths: manifestPaths,
@ -106,15 +105,15 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
} }
// FileCount returns the number of files in the manifest. // FileCount returns the number of files in the manifest.
func (c *Checker) FileCount() mfer.FileCount { func (c *Checker) FileCount() FileCount {
return mfer.FileCount(len(c.files)) return FileCount(len(c.files))
} }
// TotalBytes returns the total size of all files in the manifest. // TotalBytes returns the total size of all files in the manifest.
func (c *Checker) TotalBytes() mfer.FileSize { func (c *Checker) TotalBytes() FileSize {
var total mfer.FileSize var total FileSize
for _, f := range c.files { for _, f := range c.files {
total += mfer.FileSize(f.Size) total += FileSize(f.Size)
} }
return total return total
} }
@ -131,12 +130,12 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
defer close(progress) defer close(progress)
} }
totalFiles := mfer.FileCount(len(c.files)) totalFiles := FileCount(len(c.files))
totalBytes := c.TotalBytes() totalBytes := c.TotalBytes()
var checkedFiles mfer.FileCount var checkedFiles FileCount
var checkedBytes mfer.FileSize var checkedBytes FileSize
var failures mfer.FileCount var failures FileCount
startTime := time.Now() startTime := time.Now()
@ -186,9 +185,9 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
return nil return nil
} }
func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize) Result { func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
absPath := filepath.Join(string(c.basePath), entry.Path) absPath := filepath.Join(string(c.basePath), entry.Path)
relPath := mfer.RelFilePath(entry.Path) relPath := RelFilePath(entry.Path)
// Check if file exists // Check if file exists
info, err := c.fs.Stat(absPath) info, err := c.fs.Stat(absPath)
@ -206,7 +205,7 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize)
// Check size // Check size
if info.Size() != entry.Size { if info.Size() != entry.Size {
*checkedBytes += mfer.FileSize(info.Size()) *checkedBytes += FileSize(info.Size())
return Result{ return Result{
Path: relPath, Path: relPath,
Status: StatusSizeMismatch, Status: StatusSizeMismatch,
@ -226,7 +225,7 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize)
if err != nil { if err != nil {
return Result{Path: relPath, Status: StatusError, Message: err.Error()} return Result{Path: relPath, Status: StatusError, Message: err.Error()}
} }
*checkedBytes += mfer.FileSize(n) *checkedBytes += FileSize(n)
// Encode as multihash and compare // Encode as multihash and compare
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256) computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
@ -272,7 +271,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
if err != nil { if err != nil {
return err return err
} }
relPath := mfer.RelFilePath(rel) relPath := RelFilePath(rel)
// Check if path is in manifest // Check if path is in manifest
if _, exists := c.manifestPaths[relPath]; !exists { if _, exists := c.manifestPaths[relPath]; !exists {

47
internal/checker/checker_test.go → mfer/checker_test.go
View File

@ -1,4 +1,4 @@
package checker package mfer
import ( import (
"bytes" "bytes"
@ -9,7 +9,6 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"sneak.berlin/go/mfer/mfer"
) )
func TestStatusString(t *testing.T) { func TestStatusString(t *testing.T) {
@ -37,16 +36,16 @@ func TestStatusString(t *testing.T) {
func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte) { func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte) {
t.Helper() t.Helper()
builder := mfer.NewBuilder() builder := NewBuilder()
for path, content := range files { for path, content := range files {
reader := bytes.NewReader(content) reader := bytes.NewReader(content)
_, err := builder.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)), mfer.ModTime(time.Now()), reader, nil) _, err := builder.AddFile(RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil)
require.NoError(t, err) require.NoError(t, err)
} }
var buf bytes.Buffer var buf bytes.Buffer
require.NoError(t, builder.Build(&buf)) require.NoError(t, builder.Build(&buf))
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0644)) require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
} }
// createFilesOnDisk creates the given files on the filesystem. // createFilesOnDisk creates the given files on the filesystem.
@ -55,8 +54,8 @@ func createFilesOnDisk(t *testing.T, fs afero.Fs, basePath string, files map[str
for path, content := range files { for path, content := range files {
fullPath := basePath + "/" + path fullPath := basePath + "/" + path
require.NoError(t, fs.MkdirAll(basePath, 0755)) require.NoError(t, fs.MkdirAll(basePath, 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
} }
@ -72,7 +71,7 @@ func TestNewChecker(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/", fs) chk, err := NewChecker("/manifest.mf", "/", fs)
require.NoError(t, err) require.NoError(t, err)
assert.NotNil(t, chk) assert.NotNil(t, chk)
assert.Equal(t, mfer.FileCount(2), chk.FileCount()) assert.Equal(t, FileCount(2), chk.FileCount())
}) })
t.Run("missing manifest", func(t *testing.T) { t.Run("missing manifest", func(t *testing.T) {
@ -83,7 +82,7 @@ func TestNewChecker(t *testing.T) {
t.Run("invalid manifest", func(t *testing.T) { t.Run("invalid manifest", func(t *testing.T) {
fs := afero.NewMemMapFs() fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0644)) require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
_, err := NewChecker("/bad.mf", "/", fs) _, err := NewChecker("/bad.mf", "/", fs)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -101,8 +100,8 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/", fs) chk, err := NewChecker("/manifest.mf", "/", fs)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, mfer.FileCount(3), chk.FileCount()) assert.Equal(t, FileCount(3), chk.FileCount())
assert.Equal(t, mfer.FileSize(2+11+1000), chk.TotalBytes()) assert.Equal(t, FileSize(2+11+1000), chk.TotalBytes())
} }
func TestCheckAllFilesOK(t *testing.T) { func TestCheckAllFilesOK(t *testing.T) {
@ -158,7 +157,7 @@ func TestCheckMissingFile(t *testing.T) {
okCount++ okCount++
case StatusMissing: case StatusMissing:
missingCount++ missingCount++
assert.Equal(t, mfer.RelFilePath("missing.txt"), r.Path) assert.Equal(t, RelFilePath("missing.txt"), r.Path)
} }
} }
@ -186,7 +185,7 @@ func TestCheckSizeMismatch(t *testing.T) {
r := <-results r := <-results
assert.Equal(t, StatusSizeMismatch, r.Status) assert.Equal(t, StatusSizeMismatch, r.Status)
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path) assert.Equal(t, RelFilePath("file.txt"), r.Path)
} }
func TestCheckHashMismatch(t *testing.T) { func TestCheckHashMismatch(t *testing.T) {
@ -212,7 +211,7 @@ func TestCheckHashMismatch(t *testing.T) {
r := <-results r := <-results
assert.Equal(t, StatusHashMismatch, r.Status) assert.Equal(t, StatusHashMismatch, r.Status)
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path) assert.Equal(t, RelFilePath("file.txt"), r.Path)
} }
func TestCheckWithProgress(t *testing.T) { func TestCheckWithProgress(t *testing.T) {
@ -246,11 +245,11 @@ func TestCheckWithProgress(t *testing.T) {
assert.NotEmpty(t, progressUpdates) assert.NotEmpty(t, progressUpdates)
// Final progress should show all files checked // Final progress should show all files checked
final := progressUpdates[len(progressUpdates)-1] final := progressUpdates[len(progressUpdates)-1]
assert.Equal(t, mfer.FileCount(2), final.TotalFiles) assert.Equal(t, FileCount(2), final.TotalFiles)
assert.Equal(t, mfer.FileCount(2), final.CheckedFiles) assert.Equal(t, FileCount(2), final.CheckedFiles)
assert.Equal(t, mfer.FileSize(300), final.TotalBytes) assert.Equal(t, FileSize(300), final.TotalBytes)
assert.Equal(t, mfer.FileSize(300), final.CheckedBytes) assert.Equal(t, FileSize(300), final.CheckedBytes)
assert.Equal(t, mfer.FileCount(0), final.Failures) assert.Equal(t, FileCount(0), final.Failures)
} }
func TestCheckContextCancellation(t *testing.T) { func TestCheckContextCancellation(t *testing.T) {
@ -301,7 +300,7 @@ func TestFindExtraFiles(t *testing.T) {
} }
assert.Len(t, extras, 1) assert.Len(t, extras, 1)
assert.Equal(t, mfer.RelFilePath("file2.txt"), extras[0].Path) assert.Equal(t, RelFilePath("file2.txt"), extras[0].Path)
assert.Equal(t, StatusExtra, extras[0].Status) assert.Equal(t, StatusExtra, extras[0].Status)
assert.Equal(t, "not in manifest", extras[0].Message) assert.Equal(t, "not in manifest", extras[0].Message)
} }
@ -363,8 +362,8 @@ func TestCheckSubdirectories(t *testing.T) {
// Create files with full directory structure // Create files with full directory structure
for path, content := range files { for path, content := range files {
fullPath := "/data/" + path fullPath := "/data/" + path
require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0755)) require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0o755))
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0644)) require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
} }
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker("/manifest.mf", "/data", fs)
@ -390,8 +389,8 @@ func TestCheckEmptyManifest(t *testing.T) {
chk, err := NewChecker("/manifest.mf", "/data", fs) chk, err := NewChecker("/manifest.mf", "/data", fs)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, mfer.FileCount(0), chk.FileCount()) assert.Equal(t, FileCount(0), chk.FileCount())
assert.Equal(t, mfer.FileSize(0), chk.TotalBytes()) assert.Equal(t, FileSize(0), chk.TotalBytes())
results := make(chan Result, 10) results := make(chan Result, 10)
err = chk.Check(context.Background(), results, nil) err = chk.Check(context.Background(), results, nil)