Improve processing progress output with bytes and blob messages

- Show bytes processed/total instead of just files
- Display data rate in bytes/sec
- Calculate ETA based on bytes (more accurate than files)
- Print message when each blob is stored with size and speed
This commit is contained in:
Jeffrey Paul 2025-12-19 12:24:55 +07:00
parent 88e2508dc7
commit a1d559c30d

View File

@ -464,11 +464,18 @@ func (s *Scanner) batchInsertFiles(ctx context.Context, files []*database.File)
// processPhase processes the files that need backing up // processPhase processes the files that need backing up
func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error { func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error {
// Calculate total bytes to process
var totalBytes int64
for _, f := range filesToProcess {
totalBytes += f.FileInfo.Size()
}
// Set up periodic status output // Set up periodic status output
lastStatusTime := time.Now() lastStatusTime := time.Now()
statusInterval := 15 * time.Second statusInterval := 15 * time.Second
startTime := time.Now() startTime := time.Now()
filesProcessed := 0 filesProcessed := 0
var bytesProcessed int64
totalFiles := len(filesToProcess) totalFiles := len(filesToProcess)
// Process each file // Process each file
@ -489,18 +496,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
} }
filesProcessed++ filesProcessed++
bytesProcessed += fileToProcess.FileInfo.Size()
// Output periodic status // Output periodic status
if time.Since(lastStatusTime) >= statusInterval { if time.Since(lastStatusTime) >= statusInterval {
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
remaining := totalFiles - filesProcessed pct := float64(bytesProcessed) / float64(totalBytes) * 100
rate := float64(bytesProcessed) / elapsed.Seconds()
// Calculate ETA based on bytes (more accurate than files)
remainingBytes := totalBytes - bytesProcessed
var eta time.Duration var eta time.Duration
if filesProcessed > 0 { if rate > 0 {
eta = elapsed / time.Duration(filesProcessed) * time.Duration(remaining) eta = time.Duration(float64(remainingBytes)/rate) * time.Second
} }
fmt.Printf("Progress: %s/%s files", formatNumber(filesProcessed), formatNumber(totalFiles)) fmt.Printf("Progress: %s/%s (%s/%s files, %.1f%%), %s/sec",
if remaining > 0 && eta > 0 { humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalBytes)),
formatNumber(filesProcessed),
formatNumber(totalFiles),
pct,
humanize.Bytes(uint64(rate)))
if eta > 0 {
fmt.Printf(", ETA: %s", eta.Round(time.Second)) fmt.Printf(", ETA: %s", eta.Round(time.Second))
} }
fmt.Println() fmt.Println()
@ -589,13 +607,23 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
uploadDuration := time.Since(startTime) uploadDuration := time.Since(startTime)
// Calculate upload speed
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
// Print blob stored message
fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n",
finishedBlob.Hash[:12]+"...",
humanize.Bytes(uint64(finishedBlob.Compressed)),
humanize.Bytes(uint64(uploadSpeedBps)),
uploadDuration.Round(time.Millisecond))
// Log upload stats // Log upload stats
uploadSpeed := float64(finishedBlob.Compressed) * 8 / uploadDuration.Seconds() // bits per second uploadSpeedBits := uploadSpeedBps * 8 // bits per second
log.Info("Successfully uploaded blob to storage", log.Info("Successfully uploaded blob to storage",
"path", blobPath, "path", blobPath,
"size", humanize.Bytes(uint64(finishedBlob.Compressed)), "size", humanize.Bytes(uint64(finishedBlob.Compressed)),
"duration", uploadDuration, "duration", uploadDuration,
"speed", humanize.SI(uploadSpeed, "bps")) "speed", humanize.SI(uploadSpeedBits, "bps"))
// Report upload complete // Report upload complete
if s.progress != nil { if s.progress != nil {