add retry logic with exponential backoff, up to 5 tries

client.go

@@ -8,7 +8,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"net/http"
+	"time"
+)
+
+const (
+	MaxRetries = 5
 )
 
 type Client struct {
@@ -57,33 +63,50 @@ func (c *Client) Scrape(ctx context.Context, url, selector string) (ScrapeRespon
 		return ScrapeResponse{}, fmt.Errorf("failed to marshal request: %v", err)
 	}
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/scrape", bytes.NewBuffer(requestBody))
-	if err != nil {
-		return ScrapeResponse{}, fmt.Errorf("failed to create request: %v", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
 	client := &http.Client{}
-	resp, err := client.Do(req)
+	var resp *http.Response
+	var body []byte
+	startTime := time.Now()
+
+	for attempt := 0; attempt < MaxRetries; attempt++ {
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/scrape", bytes.NewBuffer(requestBody))
+		if err != nil {
+			return ScrapeResponse{}, fmt.Errorf("failed to create request: %v", err)
+		}
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, err = client.Do(req)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			defer resp.Body.Close()
+			body, err = ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err)
+			}
+			content := string(body)
+			return ScrapeResponse{
+				URL:      url,
+				Selector: selector,
+				Content:  content,
+			}, nil
+		}
+
+		if resp != nil {
+			resp.Body.Close()
+		}
+
+		select {
+		case <-ctx.Done():
+			totalDuration := time.Since(startTime)
+			return ScrapeResponse{}, fmt.Errorf("context cancelled after %d retries and %v: %v", attempt+1, totalDuration, ctx.Err())
+		case <-time.After(time.Duration(math.Pow(2, float64(attempt))) * time.Second):
+			// continue to next retry
+		}
+	}
+
+	totalDuration := time.Since(startTime)
 	if err != nil {
-		return ScrapeResponse{}, fmt.Errorf("failed to send request: %v", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return ScrapeResponse{}, fmt.Errorf("received non-OK response: %s", resp.Status)
+		return ScrapeResponse{}, fmt.Errorf("failed to send request after %d retries and %v: %v", MaxRetries, totalDuration, err)
 	}
 
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err)
-	}
-
-	content := string(body)
-
-	return ScrapeResponse{
-		URL:      url,
-		Selector: selector,
-		Content:  content,
-	}, nil
+	return ScrapeResponse{}, fmt.Errorf("received non-OK response after %d retries and %v: %s", MaxRetries, totalDuration, resp.Status)
 }