Abort the run when packing fails, even under --skip-errors (closes #161)
A chunk is registered as pending (known, scanner-pending, packer pending-row) before it is packed. Under --skip-errors the scanner skipped a file on any processing error, including a failure inside addChunkToPacker (packing, database, encryption, upload). The pending chunk then stayed queued and a later blob finalize inserted it into the chunks table with no blob_chunks row, so a snapshot could complete holding a file whose chunk is in no blob and cannot be restored. Errors from addChunkToPacker are now marked and abort the run regardless of --skip-errors; only open and read errors are skipped. The bookkeeping order is unchanged. Flag help and comments now say only unreadable files are skipped. Model: opus-4-8
This commit was merged in pull request #185.
This commit is contained in:
@@ -63,7 +63,9 @@ type Scanner struct {
|
||||
exclude []string // Glob patterns for files/directories to exclude
|
||||
compiledExclude []compiledPattern // Compiled glob patterns
|
||||
progress *ProgressReporter
|
||||
skipErrors bool // Skip file read errors (log loudly but continue)
|
||||
// skipErrors skips files that cannot be opened or read (logged loudly);
|
||||
// packer, database, encryption, and upload errors still abort the run.
|
||||
skipErrors bool
|
||||
// ui is the user-facing output; never nil (defaults to a discarding writer).
|
||||
ui *ui.Writer
|
||||
|
||||
@@ -121,7 +123,9 @@ type ScannerConfig struct {
|
||||
EnableProgress bool // Enable the live progress reporter (ETAs, throughput)
|
||||
UI *ui.Writer // Where user-facing scanner messages go; nil = discard
|
||||
Exclude []string // Glob patterns for files/directories to exclude
|
||||
SkipErrors bool // Skip file read errors (log loudly but continue)
|
||||
// SkipErrors skips files that cannot be opened or read (log loudly but
|
||||
// continue); packer, database, encryption, and upload errors still abort.
|
||||
SkipErrors bool
|
||||
}
|
||||
|
||||
// ScanResult contains the results of a scan operation
|
||||
@@ -1336,6 +1340,15 @@ func (s *Scanner) processFileWithErrorHandling(
|
||||
) (bool, error) {
|
||||
err := s.processFileStreaming(ctx, fileToProcess, result)
|
||||
if err != nil {
|
||||
// A packer/database/encryption/upload failure means the chunk's data
|
||||
// may not have been stored. Skipping the file would let the snapshot
|
||||
// record a file whose chunk is in no blob and cannot be restored, so
|
||||
// abort the run even under --skip-errors. Only open and read errors
|
||||
// are skipped below.
|
||||
var pErr *packerError
|
||||
if errors.As(err, &pErr) {
|
||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||
}
|
||||
// Handle files that were deleted between scan and process phases
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.Warn("File was deleted during backup, skipping",
|
||||
@@ -1345,7 +1358,7 @@ func (s *Scanner) processFileWithErrorHandling(
|
||||
|
||||
return true, nil
|
||||
}
|
||||
// Skip file read errors if --skip-errors is enabled
|
||||
// Skip open/read errors if --skip-errors is enabled
|
||||
if s.skipErrors {
|
||||
log.Error("Failed to process file (skipping due to --skip-errors)",
|
||||
"path", fileToProcess.Path, "error", err)
|
||||
@@ -1712,6 +1725,20 @@ type streamingChunkInfo struct {
|
||||
size int64
|
||||
}
|
||||
|
||||
// packerError marks an error that came from adding a chunk to the packer
|
||||
// (packing, database, encryption, or upload). Such an error means the chunk's
|
||||
// data may not have been stored, so the run must abort even under --skip-errors:
|
||||
// skipping the file would leave the chunk recorded as backed up while it lives
|
||||
// in no blob, and a later snapshot could record a file that cannot be restored.
|
||||
// Only open and read errors are safe to skip.
|
||||
type packerError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *packerError) Error() string { return e.err.Error() }
|
||||
|
||||
func (e *packerError) Unwrap() error { return e.err }
|
||||
|
||||
// processFileStreaming processes a file by streaming chunks directly to the packer
|
||||
func (s *Scanner) processFileStreaming(
|
||||
ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
|
||||
@@ -1762,7 +1789,11 @@ func (s *Scanner) processFileStreaming(
|
||||
if !chunkExists {
|
||||
err := s.addChunkToPacker(ctx, chunk)
|
||||
if err != nil {
|
||||
return err
|
||||
// Mark as a packer error so --skip-errors cannot swallow it:
|
||||
// the chunk was registered as pending before packing, so a
|
||||
// skipped file here would be recorded as backed up while its
|
||||
// data was never stored.
|
||||
return &packerError{err: err}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user