diff --git a/client.go b/client.go index dce1914..748ecc6 100644 --- a/client.go +++ b/client.go @@ -8,7 +8,13 @@ import ( "encoding/json" "fmt" "io/ioutil" + "math" "net/http" + "time" +) + +const ( + MaxRetries = 5 ) type Client struct { @@ -57,33 +63,50 @@ func (c *Client) Scrape(ctx context.Context, url, selector string) (ScrapeRespon return ScrapeResponse{}, fmt.Errorf("failed to marshal request: %v", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/scrape", bytes.NewBuffer(requestBody)) - if err != nil { - return ScrapeResponse{}, fmt.Errorf("failed to create request: %v", err) - } - req.Header.Set("Content-Type", "application/json") - client := &http.Client{} - resp, err := client.Do(req) + var resp *http.Response + var body []byte + startTime := time.Now() + + for attempt := 0; attempt < MaxRetries; attempt++ { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.BaseURL+"/scrape", bytes.NewBuffer(requestBody)) + if err != nil { + return ScrapeResponse{}, fmt.Errorf("failed to create request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err = client.Do(req) + if err == nil && resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + body, err = ioutil.ReadAll(resp.Body) + if err != nil { + return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err) + } + content := string(body) + return ScrapeResponse{ + URL: url, + Selector: selector, + Content: content, + }, nil + } + + if resp != nil { + resp.Body.Close() + } + + select { + case <-ctx.Done(): + totalDuration := time.Since(startTime) + return ScrapeResponse{}, fmt.Errorf("context cancelled after %d retries and %v: %v", attempt+1, totalDuration, ctx.Err()) + case <-time.After(time.Duration(math.Pow(2, float64(attempt))) * time.Second): + // continue to next retry + } + } + + totalDuration := time.Since(startTime) if err != nil { - return ScrapeResponse{}, fmt.Errorf("failed to send request: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return ScrapeResponse{}, fmt.Errorf("received non-OK response: %s", resp.Status) + return ScrapeResponse{}, fmt.Errorf("failed to send request after %d retries and %v: %v", MaxRetries, totalDuration, err) } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err) - } - - content := string(body) - - return ScrapeResponse{ - URL: url, - Selector: selector, - Content: content, - }, nil + return ScrapeResponse{}, fmt.Errorf("received non-OK response after %d retries and %v: %s", MaxRetries, totalDuration, resp.Status) }