Send fetch progress at most once per second (closes #206) #215

Merged
clawbot merged 1 commits from issue-206-fetch-progress-rate into next 2026-10-14 00:01:23 +02:00
2 changed files with 34 additions and 15 deletions
+14 -4
View File
@@ -679,13 +679,15 @@ func resolveManifestURL(inputURL string) (string, error) {
return parsed.JoinPath(defaultManifestName).String(), nil return parsed.JoinPath(defaultManifestName).String(), nil
} }
// progressWriter wraps an io.Writer and reports progress to a channel. // progressWriter wraps an io.Writer and reports progress to a channel,
// at most once per second and once more when the last byte is written.
type progressWriter struct { type progressWriter struct {
w io.Writer w io.Writer
path string path string
total int64 total int64
written int64 written int64
startTime time.Time startTime time.Time
lastProgress time.Time
progress chan<- DownloadProgress progress chan<- DownloadProgress
} }
@@ -693,7 +695,10 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
n, err := pw.w.Write(p) n, err := pw.w.Write(p)
pw.written += int64(n) pw.written += int64(n)
if pw.progress != nil {
isLast := pw.written == pw.total
if pw.progress != nil &&
(isLast || time.Since(pw.lastProgress) >= time.Second) {
var ( var (
bytesPerSec float64 bytesPerSec float64
eta time.Duration eta time.Duration
@@ -715,6 +720,8 @@ func (pw *progressWriter) Write(p []byte) (int, error) {
BytesPerSec: bytesPerSec, BytesPerSec: bytesPerSec,
ETA: eta, ETA: eta,
}) })
pw.lastProgress = time.Now()
} }
return n, err return n, err
@@ -899,12 +906,15 @@ func saveResponse(
// Set up hash computation // Set up hash computation
h := sha256.New() h := sha256.New()
// Create progress-reporting writer that also computes hash // Create progress-reporting writer that also computes hash. The first
// update waits a second, as gen's and check's do.
start := time.Now()
pw := &progressWriter{ pw := &progressWriter{
w: io.MultiWriter(out, h), w: io.MultiWriter(out, h),
path: localPath, path: localPath,
total: totalBytes, total: totalBytes,
startTime: time.Now(), startTime: start,
lastProgress: start,
progress: progress, progress: progress,
} }
+10 -1
View File
@@ -516,7 +516,7 @@ func TestManifestDownloadStopsAtLimit(t *testing.T) {
func TestFetchProgress(t *testing.T) { func TestFetchProgress(t *testing.T) {
// Create source filesystem with a larger test file // Create source filesystem with a larger test file
sourceFs := afero.NewMemMapFs() sourceFs := afero.NewMemMapFs()
// Create content large enough to trigger multiple progress updates // Content large enough that io.Copy writes it in at least four chunks
content := bytes.Repeat([]byte("x"), 100*1024) // 100KB content := bytes.Repeat([]byte("x"), 100*1024) // 100KB
require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644)) require.NoError(t, afero.WriteFile(sourceFs, "/large.txt", content, 0o644))
@@ -559,8 +559,11 @@ func TestFetchProgress(t *testing.T) {
}() }()
// Download // Download
start := time.Now()
err = downloadFile(context.Background(), testClient(), err = downloadFile(context.Background(), testClient(),
server.URL+"/large.txt", ".", "large.txt", files[0], progress) server.URL+"/large.txt", ".", "large.txt", files[0], progress)
elapsed := time.Since(start)
close(progress) close(progress)
<-done <-done
@@ -569,6 +572,12 @@ func TestFetchProgress(t *testing.T) {
// Verify we got progress updates // Verify we got progress updates
assert.NotEmpty(t, progressUpdates, "should have received progress updates") assert.NotEmpty(t, progressUpdates, "should have received progress updates")
// At most one update per second, plus the one reporting the file
// complete: a download that takes under a second sends two at most,
// however many chunks it writes.
assert.LessOrEqual(t, len(progressUpdates), 2+int(elapsed/time.Second),
"one progress update per second at most, plus the last")
// Verify final progress shows complete // Verify final progress shows complete
if len(progressUpdates) > 0 { if len(progressUpdates) > 0 {
last := progressUpdates[len(progressUpdates)-1] last := progressUpdates[len(progressUpdates)-1]