sneak/puppeteerapiclient
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

handle http errors better

Browse Source
This commit is contained in:
Jeffrey Paul 2024-06-02 12:06:43 -07:00
parent 421b3d8f12
commit e496a4b65b
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
client.go
View File

@ -36,7 +36,6 @@ func NewClient(baseURL, salt string) *Client {
} }
func (c *Client) CalculateHash(url string) string { func (c *Client) CalculateHash(url string) string {
// The hash should be calculated by appending the salt to the URL with a colon separator.
data := url + ":" + c.Salt data := url + ":" + c.Salt
hash := md5.Sum([]byte(data)) hash := md5.Sum([]byte(data))
return hex.EncodeToString(hash[:]) return hex.EncodeToString(hash[:])
@ -71,12 +70,15 @@ func (c *Client) Scrape(ctx context.Context, url, selector string) (ScrapeRespon
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ScrapeResponse{}, fmt.Errorf("received non-OK response: %s", resp.Status)
}
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err) return ScrapeResponse{}, fmt.Errorf("failed to read response body: %v", err)
} }
// Since the response is HTML, we don't need to unmarshal JSON.
content := string(body) content := string(body)
return ScrapeResponse{ return ScrapeResponse{