Improve processing progress output with bytes and blob messages
- Show bytes processed/total instead of just files - Display data rate in bytes/sec - Calculate ETA based on bytes (more accurate than files) - Print message when each blob is stored with size and speed
This commit is contained in:
parent
88e2508dc7
commit
a1d559c30d
@ -464,11 +464,18 @@ func (s *Scanner) batchInsertFiles(ctx context.Context, files []*database.File)
|
||||
|
||||
// processPhase processes the files that need backing up
|
||||
func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error {
|
||||
// Calculate total bytes to process
|
||||
var totalBytes int64
|
||||
for _, f := range filesToProcess {
|
||||
totalBytes += f.FileInfo.Size()
|
||||
}
|
||||
|
||||
// Set up periodic status output
|
||||
lastStatusTime := time.Now()
|
||||
statusInterval := 15 * time.Second
|
||||
startTime := time.Now()
|
||||
filesProcessed := 0
|
||||
var bytesProcessed int64
|
||||
totalFiles := len(filesToProcess)
|
||||
|
||||
// Process each file
|
||||
@ -489,18 +496,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
||||
}
|
||||
|
||||
filesProcessed++
|
||||
bytesProcessed += fileToProcess.FileInfo.Size()
|
||||
|
||||
// Output periodic status
|
||||
if time.Since(lastStatusTime) >= statusInterval {
|
||||
elapsed := time.Since(startTime)
|
||||
remaining := totalFiles - filesProcessed
|
||||
pct := float64(bytesProcessed) / float64(totalBytes) * 100
|
||||
rate := float64(bytesProcessed) / elapsed.Seconds()
|
||||
|
||||
// Calculate ETA based on bytes (more accurate than files)
|
||||
remainingBytes := totalBytes - bytesProcessed
|
||||
var eta time.Duration
|
||||
if filesProcessed > 0 {
|
||||
eta = elapsed / time.Duration(filesProcessed) * time.Duration(remaining)
|
||||
if rate > 0 {
|
||||
eta = time.Duration(float64(remainingBytes)/rate) * time.Second
|
||||
}
|
||||
|
||||
fmt.Printf("Progress: %s/%s files", formatNumber(filesProcessed), formatNumber(totalFiles))
|
||||
if remaining > 0 && eta > 0 {
|
||||
fmt.Printf("Progress: %s/%s (%s/%s files, %.1f%%), %s/sec",
|
||||
humanize.Bytes(uint64(bytesProcessed)),
|
||||
humanize.Bytes(uint64(totalBytes)),
|
||||
formatNumber(filesProcessed),
|
||||
formatNumber(totalFiles),
|
||||
pct,
|
||||
humanize.Bytes(uint64(rate)))
|
||||
if eta > 0 {
|
||||
fmt.Printf(", ETA: %s", eta.Round(time.Second))
|
||||
}
|
||||
fmt.Println()
|
||||
@ -589,13 +607,23 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
|
||||
|
||||
uploadDuration := time.Since(startTime)
|
||||
|
||||
// Calculate upload speed
|
||||
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
|
||||
|
||||
// Print blob stored message
|
||||
fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n",
|
||||
finishedBlob.Hash[:12]+"...",
|
||||
humanize.Bytes(uint64(finishedBlob.Compressed)),
|
||||
humanize.Bytes(uint64(uploadSpeedBps)),
|
||||
uploadDuration.Round(time.Millisecond))
|
||||
|
||||
// Log upload stats
|
||||
uploadSpeed := float64(finishedBlob.Compressed) * 8 / uploadDuration.Seconds() // bits per second
|
||||
uploadSpeedBits := uploadSpeedBps * 8 // bits per second
|
||||
log.Info("Successfully uploaded blob to storage",
|
||||
"path", blobPath,
|
||||
"size", humanize.Bytes(uint64(finishedBlob.Compressed)),
|
||||
"duration", uploadDuration,
|
||||
"speed", humanize.SI(uploadSpeed, "bps"))
|
||||
"speed", humanize.SI(uploadSpeedBits, "bps"))
|
||||
|
||||
// Report upload complete
|
||||
if s.progress != nil {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user