Move checker package into mfer package
Consolidate checker functionality into the mfer package alongside scanner, removing the need for a separate internal/checker package.
This commit is contained in:
parent
dc115c5ba2
commit
e25e309581
@ -8,8 +8,8 @@ import (
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/urfave/cli/v2"
|
||||
"sneak.berlin/go/mfer/internal/checker"
|
||||
"sneak.berlin/go/mfer/internal/log"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
// findManifest looks for a manifest file in the given directory.
|
||||
@ -63,7 +63,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
log.Infof("checking manifest %s with base %s", manifestPath, basePath)
|
||||
|
||||
// Create checker
|
||||
chk, err := checker.NewChecker(manifestPath, basePath, mfa.Fs)
|
||||
chk, err := mfer.NewChecker(manifestPath, basePath, mfa.Fs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load manifest: %w", err)
|
||||
}
|
||||
@ -71,12 +71,12 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
log.Infof("manifest contains %d files, %s", chk.FileCount(), humanize.IBytes(uint64(chk.TotalBytes())))
|
||||
|
||||
// Set up results channel
|
||||
results := make(chan checker.Result, 1)
|
||||
results := make(chan mfer.Result, 1)
|
||||
|
||||
// Set up progress channel
|
||||
var progress chan checker.CheckStatus
|
||||
var progress chan mfer.CheckStatus
|
||||
if showProgress {
|
||||
progress = make(chan checker.CheckStatus, 1)
|
||||
progress = make(chan mfer.CheckStatus, 1)
|
||||
go func() {
|
||||
for status := range progress {
|
||||
if status.ETA > 0 {
|
||||
@ -103,7 +103,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for result := range results {
|
||||
if result.Status != checker.StatusOK {
|
||||
if result.Status != mfer.StatusOK {
|
||||
failures++
|
||||
log.Infof("%s: %s (%s)", result.Status, result.Path, result.Message)
|
||||
} else {
|
||||
@ -124,7 +124,7 @@ func (mfa *CLIApp) checkManifestOperation(ctx *cli.Context) error {
|
||||
|
||||
// Check for extra files if requested
|
||||
if ctx.Bool("no-extra-files") {
|
||||
extraResults := make(chan checker.Result, 1)
|
||||
extraResults := make(chan mfer.Result, 1)
|
||||
extraDone := make(chan struct{})
|
||||
go func() {
|
||||
for result := range extraResults {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package checker
|
||||
package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -12,12 +12,11 @@ import (
|
||||
|
||||
"github.com/multiformats/go-multihash"
|
||||
"github.com/spf13/afero"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
// Result represents the outcome of checking a single file.
|
||||
type Result struct {
|
||||
Path mfer.RelFilePath // Relative path from manifest
|
||||
Path RelFilePath // Relative path from manifest
|
||||
Status Status // Verification result status
|
||||
Message string // Human-readable description of the result
|
||||
}
|
||||
@ -55,22 +54,22 @@ func (s Status) String() string {
|
||||
|
||||
// CheckStatus contains progress information for the check operation.
|
||||
type CheckStatus struct {
|
||||
TotalFiles mfer.FileCount // Total number of files in manifest
|
||||
CheckedFiles mfer.FileCount // Number of files checked so far
|
||||
TotalBytes mfer.FileSize // Total bytes to verify (sum of all file sizes)
|
||||
CheckedBytes mfer.FileSize // Bytes verified so far
|
||||
TotalFiles FileCount // Total number of files in manifest
|
||||
CheckedFiles FileCount // Number of files checked so far
|
||||
TotalBytes FileSize // Total bytes to verify (sum of all file sizes)
|
||||
CheckedBytes FileSize // Bytes verified so far
|
||||
BytesPerSec float64 // Current throughput rate
|
||||
ETA time.Duration // Estimated time to completion
|
||||
Failures mfer.FileCount // Number of verification failures encountered
|
||||
Failures FileCount // Number of verification failures encountered
|
||||
}
|
||||
|
||||
// Checker verifies files against a manifest.
|
||||
type Checker struct {
|
||||
basePath mfer.AbsFilePath
|
||||
files []*mfer.MFFilePath
|
||||
basePath AbsFilePath
|
||||
files []*MFFilePath
|
||||
fs afero.Fs
|
||||
// manifestPaths is a set of paths in the manifest for quick lookup
|
||||
manifestPaths map[mfer.RelFilePath]struct{}
|
||||
manifestPaths map[RelFilePath]struct{}
|
||||
}
|
||||
|
||||
// NewChecker creates a new Checker for the given manifest, base path, and filesystem.
|
||||
@ -81,7 +80,7 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
|
||||
fs = afero.NewOsFs()
|
||||
}
|
||||
|
||||
m, err := mfer.NewManifestFromFile(fs, manifestPath)
|
||||
m, err := NewManifestFromFile(fs, manifestPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -92,13 +91,13 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
|
||||
}
|
||||
|
||||
files := m.Files()
|
||||
manifestPaths := make(map[mfer.RelFilePath]struct{}, len(files))
|
||||
manifestPaths := make(map[RelFilePath]struct{}, len(files))
|
||||
for _, f := range files {
|
||||
manifestPaths[mfer.RelFilePath(f.Path)] = struct{}{}
|
||||
manifestPaths[RelFilePath(f.Path)] = struct{}{}
|
||||
}
|
||||
|
||||
return &Checker{
|
||||
basePath: mfer.AbsFilePath(abs),
|
||||
basePath: AbsFilePath(abs),
|
||||
files: files,
|
||||
fs: fs,
|
||||
manifestPaths: manifestPaths,
|
||||
@ -106,15 +105,15 @@ func NewChecker(manifestPath string, basePath string, fs afero.Fs) (*Checker, er
|
||||
}
|
||||
|
||||
// FileCount returns the number of files in the manifest.
|
||||
func (c *Checker) FileCount() mfer.FileCount {
|
||||
return mfer.FileCount(len(c.files))
|
||||
func (c *Checker) FileCount() FileCount {
|
||||
return FileCount(len(c.files))
|
||||
}
|
||||
|
||||
// TotalBytes returns the total size of all files in the manifest.
|
||||
func (c *Checker) TotalBytes() mfer.FileSize {
|
||||
var total mfer.FileSize
|
||||
func (c *Checker) TotalBytes() FileSize {
|
||||
var total FileSize
|
||||
for _, f := range c.files {
|
||||
total += mfer.FileSize(f.Size)
|
||||
total += FileSize(f.Size)
|
||||
}
|
||||
return total
|
||||
}
|
||||
@ -131,12 +130,12 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
|
||||
defer close(progress)
|
||||
}
|
||||
|
||||
totalFiles := mfer.FileCount(len(c.files))
|
||||
totalFiles := FileCount(len(c.files))
|
||||
totalBytes := c.TotalBytes()
|
||||
|
||||
var checkedFiles mfer.FileCount
|
||||
var checkedBytes mfer.FileSize
|
||||
var failures mfer.FileCount
|
||||
var checkedFiles FileCount
|
||||
var checkedBytes FileSize
|
||||
var failures FileCount
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
@ -186,9 +185,9 @@ func (c *Checker) Check(ctx context.Context, results chan<- Result, progress cha
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize) Result {
|
||||
func (c *Checker) checkFile(entry *MFFilePath, checkedBytes *FileSize) Result {
|
||||
absPath := filepath.Join(string(c.basePath), entry.Path)
|
||||
relPath := mfer.RelFilePath(entry.Path)
|
||||
relPath := RelFilePath(entry.Path)
|
||||
|
||||
// Check if file exists
|
||||
info, err := c.fs.Stat(absPath)
|
||||
@ -206,7 +205,7 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize)
|
||||
|
||||
// Check size
|
||||
if info.Size() != entry.Size {
|
||||
*checkedBytes += mfer.FileSize(info.Size())
|
||||
*checkedBytes += FileSize(info.Size())
|
||||
return Result{
|
||||
Path: relPath,
|
||||
Status: StatusSizeMismatch,
|
||||
@ -226,7 +225,7 @@ func (c *Checker) checkFile(entry *mfer.MFFilePath, checkedBytes *mfer.FileSize)
|
||||
if err != nil {
|
||||
return Result{Path: relPath, Status: StatusError, Message: err.Error()}
|
||||
}
|
||||
*checkedBytes += mfer.FileSize(n)
|
||||
*checkedBytes += FileSize(n)
|
||||
|
||||
// Encode as multihash and compare
|
||||
computed, err := multihash.Encode(h.Sum(nil), multihash.SHA2_256)
|
||||
@ -272,7 +271,7 @@ func (c *Checker) FindExtraFiles(ctx context.Context, results chan<- Result) err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relPath := mfer.RelFilePath(rel)
|
||||
relPath := RelFilePath(rel)
|
||||
|
||||
// Check if path is in manifest
|
||||
if _, exists := c.manifestPaths[relPath]; !exists {
|
||||
@ -1,4 +1,4 @@
|
||||
package checker
|
||||
package mfer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -9,7 +9,6 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/mfer/mfer"
|
||||
)
|
||||
|
||||
func TestStatusString(t *testing.T) {
|
||||
@ -37,16 +36,16 @@ func TestStatusString(t *testing.T) {
|
||||
func createTestManifest(t *testing.T, fs afero.Fs, manifestPath string, files map[string][]byte) {
|
||||
t.Helper()
|
||||
|
||||
builder := mfer.NewBuilder()
|
||||
builder := NewBuilder()
|
||||
for path, content := range files {
|
||||
reader := bytes.NewReader(content)
|
||||
_, err := builder.AddFile(mfer.RelFilePath(path), mfer.FileSize(len(content)), mfer.ModTime(time.Now()), reader, nil)
|
||||
_, err := builder.AddFile(RelFilePath(path), FileSize(len(content)), ModTime(time.Now()), reader, nil)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
require.NoError(t, builder.Build(&buf))
|
||||
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0644))
|
||||
require.NoError(t, afero.WriteFile(fs, manifestPath, buf.Bytes(), 0o644))
|
||||
}
|
||||
|
||||
// createFilesOnDisk creates the given files on the filesystem.
|
||||
@ -55,8 +54,8 @@ func createFilesOnDisk(t *testing.T, fs afero.Fs, basePath string, files map[str
|
||||
|
||||
for path, content := range files {
|
||||
fullPath := basePath + "/" + path
|
||||
require.NoError(t, fs.MkdirAll(basePath, 0755))
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0644))
|
||||
require.NoError(t, fs.MkdirAll(basePath, 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||
}
|
||||
}
|
||||
|
||||
@ -72,7 +71,7 @@ func TestNewChecker(t *testing.T) {
|
||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, chk)
|
||||
assert.Equal(t, mfer.FileCount(2), chk.FileCount())
|
||||
assert.Equal(t, FileCount(2), chk.FileCount())
|
||||
})
|
||||
|
||||
t.Run("missing manifest", func(t *testing.T) {
|
||||
@ -83,7 +82,7 @@ func TestNewChecker(t *testing.T) {
|
||||
|
||||
t.Run("invalid manifest", func(t *testing.T) {
|
||||
fs := afero.NewMemMapFs()
|
||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0644))
|
||||
require.NoError(t, afero.WriteFile(fs, "/bad.mf", []byte("not a manifest"), 0o644))
|
||||
_, err := NewChecker("/bad.mf", "/", fs)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
@ -101,8 +100,8 @@ func TestCheckerFileCountAndTotalBytes(t *testing.T) {
|
||||
chk, err := NewChecker("/manifest.mf", "/", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, mfer.FileCount(3), chk.FileCount())
|
||||
assert.Equal(t, mfer.FileSize(2+11+1000), chk.TotalBytes())
|
||||
assert.Equal(t, FileCount(3), chk.FileCount())
|
||||
assert.Equal(t, FileSize(2+11+1000), chk.TotalBytes())
|
||||
}
|
||||
|
||||
func TestCheckAllFilesOK(t *testing.T) {
|
||||
@ -158,7 +157,7 @@ func TestCheckMissingFile(t *testing.T) {
|
||||
okCount++
|
||||
case StatusMissing:
|
||||
missingCount++
|
||||
assert.Equal(t, mfer.RelFilePath("missing.txt"), r.Path)
|
||||
assert.Equal(t, RelFilePath("missing.txt"), r.Path)
|
||||
}
|
||||
}
|
||||
|
||||
@ -186,7 +185,7 @@ func TestCheckSizeMismatch(t *testing.T) {
|
||||
|
||||
r := <-results
|
||||
assert.Equal(t, StatusSizeMismatch, r.Status)
|
||||
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path)
|
||||
assert.Equal(t, RelFilePath("file.txt"), r.Path)
|
||||
}
|
||||
|
||||
func TestCheckHashMismatch(t *testing.T) {
|
||||
@ -212,7 +211,7 @@ func TestCheckHashMismatch(t *testing.T) {
|
||||
|
||||
r := <-results
|
||||
assert.Equal(t, StatusHashMismatch, r.Status)
|
||||
assert.Equal(t, mfer.RelFilePath("file.txt"), r.Path)
|
||||
assert.Equal(t, RelFilePath("file.txt"), r.Path)
|
||||
}
|
||||
|
||||
func TestCheckWithProgress(t *testing.T) {
|
||||
@ -246,11 +245,11 @@ func TestCheckWithProgress(t *testing.T) {
|
||||
assert.NotEmpty(t, progressUpdates)
|
||||
// Final progress should show all files checked
|
||||
final := progressUpdates[len(progressUpdates)-1]
|
||||
assert.Equal(t, mfer.FileCount(2), final.TotalFiles)
|
||||
assert.Equal(t, mfer.FileCount(2), final.CheckedFiles)
|
||||
assert.Equal(t, mfer.FileSize(300), final.TotalBytes)
|
||||
assert.Equal(t, mfer.FileSize(300), final.CheckedBytes)
|
||||
assert.Equal(t, mfer.FileCount(0), final.Failures)
|
||||
assert.Equal(t, FileCount(2), final.TotalFiles)
|
||||
assert.Equal(t, FileCount(2), final.CheckedFiles)
|
||||
assert.Equal(t, FileSize(300), final.TotalBytes)
|
||||
assert.Equal(t, FileSize(300), final.CheckedBytes)
|
||||
assert.Equal(t, FileCount(0), final.Failures)
|
||||
}
|
||||
|
||||
func TestCheckContextCancellation(t *testing.T) {
|
||||
@ -301,7 +300,7 @@ func TestFindExtraFiles(t *testing.T) {
|
||||
}
|
||||
|
||||
assert.Len(t, extras, 1)
|
||||
assert.Equal(t, mfer.RelFilePath("file2.txt"), extras[0].Path)
|
||||
assert.Equal(t, RelFilePath("file2.txt"), extras[0].Path)
|
||||
assert.Equal(t, StatusExtra, extras[0].Status)
|
||||
assert.Equal(t, "not in manifest", extras[0].Message)
|
||||
}
|
||||
@ -363,8 +362,8 @@ func TestCheckSubdirectories(t *testing.T) {
|
||||
// Create files with full directory structure
|
||||
for path, content := range files {
|
||||
fullPath := "/data/" + path
|
||||
require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0755))
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0644))
|
||||
require.NoError(t, fs.MkdirAll("/data/dir1/dir2/dir3", 0o755))
|
||||
require.NoError(t, afero.WriteFile(fs, fullPath, content, 0o644))
|
||||
}
|
||||
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
@ -390,8 +389,8 @@ func TestCheckEmptyManifest(t *testing.T) {
|
||||
chk, err := NewChecker("/manifest.mf", "/data", fs)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, mfer.FileCount(0), chk.FileCount())
|
||||
assert.Equal(t, mfer.FileSize(0), chk.TotalBytes())
|
||||
assert.Equal(t, FileCount(0), chk.FileCount())
|
||||
assert.Equal(t, FileSize(0), chk.TotalBytes())
|
||||
|
||||
results := make(chan Result, 10)
|
||||
err = chk.Check(context.Background(), results, nil)
|
||||
Loading…
Reference in New Issue
Block a user