Improve processing progress output with bytes and blob messages
- Show bytes processed/total instead of just files - Display data rate in bytes/sec - Calculate ETA based on bytes (more accurate than files) - Print message when each blob is stored with size and speed
This commit is contained in:
parent
88e2508dc7
commit
a1d559c30d
@ -464,11 +464,18 @@ func (s *Scanner) batchInsertFiles(ctx context.Context, files []*database.File)
|
|||||||
|
|
||||||
// processPhase processes the files that need backing up
|
// processPhase processes the files that need backing up
|
||||||
func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error {
|
func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProcess, result *ScanResult) error {
|
||||||
|
// Calculate total bytes to process
|
||||||
|
var totalBytes int64
|
||||||
|
for _, f := range filesToProcess {
|
||||||
|
totalBytes += f.FileInfo.Size()
|
||||||
|
}
|
||||||
|
|
||||||
// Set up periodic status output
|
// Set up periodic status output
|
||||||
lastStatusTime := time.Now()
|
lastStatusTime := time.Now()
|
||||||
statusInterval := 15 * time.Second
|
statusInterval := 15 * time.Second
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
filesProcessed := 0
|
filesProcessed := 0
|
||||||
|
var bytesProcessed int64
|
||||||
totalFiles := len(filesToProcess)
|
totalFiles := len(filesToProcess)
|
||||||
|
|
||||||
// Process each file
|
// Process each file
|
||||||
@ -489,18 +496,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
|||||||
}
|
}
|
||||||
|
|
||||||
filesProcessed++
|
filesProcessed++
|
||||||
|
bytesProcessed += fileToProcess.FileInfo.Size()
|
||||||
|
|
||||||
// Output periodic status
|
// Output periodic status
|
||||||
if time.Since(lastStatusTime) >= statusInterval {
|
if time.Since(lastStatusTime) >= statusInterval {
|
||||||
elapsed := time.Since(startTime)
|
elapsed := time.Since(startTime)
|
||||||
remaining := totalFiles - filesProcessed
|
pct := float64(bytesProcessed) / float64(totalBytes) * 100
|
||||||
|
rate := float64(bytesProcessed) / elapsed.Seconds()
|
||||||
|
|
||||||
|
// Calculate ETA based on bytes (more accurate than files)
|
||||||
|
remainingBytes := totalBytes - bytesProcessed
|
||||||
var eta time.Duration
|
var eta time.Duration
|
||||||
if filesProcessed > 0 {
|
if rate > 0 {
|
||||||
eta = elapsed / time.Duration(filesProcessed) * time.Duration(remaining)
|
eta = time.Duration(float64(remainingBytes)/rate) * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Printf("Progress: %s/%s files", formatNumber(filesProcessed), formatNumber(totalFiles))
|
fmt.Printf("Progress: %s/%s (%s/%s files, %.1f%%), %s/sec",
|
||||||
if remaining > 0 && eta > 0 {
|
humanize.Bytes(uint64(bytesProcessed)),
|
||||||
|
humanize.Bytes(uint64(totalBytes)),
|
||||||
|
formatNumber(filesProcessed),
|
||||||
|
formatNumber(totalFiles),
|
||||||
|
pct,
|
||||||
|
humanize.Bytes(uint64(rate)))
|
||||||
|
if eta > 0 {
|
||||||
fmt.Printf(", ETA: %s", eta.Round(time.Second))
|
fmt.Printf(", ETA: %s", eta.Round(time.Second))
|
||||||
}
|
}
|
||||||
fmt.Println()
|
fmt.Println()
|
||||||
@ -589,13 +607,23 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
|
|||||||
|
|
||||||
uploadDuration := time.Since(startTime)
|
uploadDuration := time.Since(startTime)
|
||||||
|
|
||||||
|
// Calculate upload speed
|
||||||
|
uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds()
|
||||||
|
|
||||||
|
// Print blob stored message
|
||||||
|
fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n",
|
||||||
|
finishedBlob.Hash[:12]+"...",
|
||||||
|
humanize.Bytes(uint64(finishedBlob.Compressed)),
|
||||||
|
humanize.Bytes(uint64(uploadSpeedBps)),
|
||||||
|
uploadDuration.Round(time.Millisecond))
|
||||||
|
|
||||||
// Log upload stats
|
// Log upload stats
|
||||||
uploadSpeed := float64(finishedBlob.Compressed) * 8 / uploadDuration.Seconds() // bits per second
|
uploadSpeedBits := uploadSpeedBps * 8 // bits per second
|
||||||
log.Info("Successfully uploaded blob to storage",
|
log.Info("Successfully uploaded blob to storage",
|
||||||
"path", blobPath,
|
"path", blobPath,
|
||||||
"size", humanize.Bytes(uint64(finishedBlob.Compressed)),
|
"size", humanize.Bytes(uint64(finishedBlob.Compressed)),
|
||||||
"duration", uploadDuration,
|
"duration", uploadDuration,
|
||||||
"speed", humanize.SI(uploadSpeed, "bps"))
|
"speed", humanize.SI(uploadSpeedBits, "bps"))
|
||||||
|
|
||||||
// Report upload complete
|
// Report upload complete
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user