This commit is contained in:
Jeffrey Paul 2024-05-20 04:16:17 -07:00
parent df646a5ccf
commit 413bd25d22
4 changed files with 107 additions and 38 deletions

4
Makefile Normal file
View File

@ -0,0 +1,4 @@
default: run
run:
go run main.go

70
fetch_api.sh Executable file
View File

@ -0,0 +1,70 @@
#!/bin/bash
# Ensure both cursor and limit arguments are provided
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <cursor> <limit>"
exit 1
fi
cursor=$1
limit=$2
# Construct the JSON payload with the provided cursor and limit
json_payload=$(cat <<EOF
{
"0": {
"json": {
"query": "",
"sort": "relevance",
"set": "bet",
"filters": [],
"limit": $limit,
"variantLimit": false,
"collectionLimit": false,
"cursor": $cursor,
"direction": "forward"
}
}
}
EOF
)
# Print the JSON payload for debugging
echo "JSON Payload:"
echo "$json_payload"
# URL encode the JSON payload
url_encoded_json=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''$json_payload'''))")
# Print the URL-encoded JSON payload for debugging
echo "URL-encoded JSON Payload:"
echo "$url_encoded_json"
# Construct the full URL
api_url="https://curiosa.io/api/trpc/card.search?batch=1&input=$url_encoded_json"
# Print the full API URL for debugging
echo "Full API URL:"
echo "$api_url"
# Run the curl command and output the response as JSON
curl "$api_url" \
--compressed \
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0' \
-H 'Accept: */*' \
-H 'Accept-Language: en-US,en;q=0.5' \
-H 'Accept-Encoding: gzip, deflate' \
-H 'Referer: https://curiosa.io/cards?set=bet' \
-H 'content-type: application/json' \
-H 'x-build-id: a68dd8c6f6e3b605951c534ad22e788557cabd0d' \
-H 'x-trpc-source: nextjs-react' \
-H 'DNT: 1' \
-H 'Sec-GPC: 1' \
-H 'Connection: keep-alive' \
-H 'Sec-Fetch-Dest: empty' \
-H 'Sec-Fetch-Mode: cors' \
-H 'Sec-Fetch-Site: same-origin' \
-H 'Priority: u=4' \
-H 'TE: trailers'

3
go.mod Normal file
View File

@ -0,0 +1,3 @@
module git.eeqj.de/sneak/curiosa-scr-scrape
go 1.22.2

52
main.go
View File

@ -8,8 +8,8 @@ import (
"os"
)
// SearchRequest represents the structure of the input JSON for the API request.
type SearchRequest struct {
// JSONPayload represents the structure of the nested JSON for the API request.
type JSONPayload struct {
Query string `json:"query"`
Sort string `json:"sort"`
Set string `json:"set"`
@ -17,37 +17,39 @@ type SearchRequest struct {
Limit int `json:"limit"`
VariantLimit bool `json:"variantLimit"`
CollectionLimit bool `json:"collectionLimit"`
Cursor int `json:"cursor"`
Direction string `json:"direction"`
}
// Input represents the top-level structure of the input JSON for the API request.
type Input struct {
Part0 SearchRequest `json:"0"`
Part1 SearchRequest `json:"1"`
Part0 struct {
JSON JSONPayload `json:"json"`
} `json:"0"`
}
func main() {
// Parameterize the Limit value
// Parameterize the Limit and Cursor values
limit := 35
cursor := 35
set := "bet"
// Define the input data structure
input := Input{
Part0: SearchRequest{
Part0: struct {
JSON JSONPayload `json:"json"`
}{
JSON: JSONPayload{
Query: "",
Sort: "relevance",
Set: "*",
Set: set,
Filters: []interface{}{},
Limit: limit,
VariantLimit: false,
CollectionLimit: false,
Cursor: cursor,
Direction: "forward",
},
Part1: SearchRequest{
Query: "",
Sort: "relevance",
Set: "*",
Filters: []interface{}{},
CollectionLimit: false,
},
}
@ -60,11 +62,14 @@ func main() {
// URL encode the JSON data
urlEncodedInput := url.QueryEscape(string(jsonData))
// Print the URL-encoded JSON input for comparison
// Print the URL-encoded JSON input for debugging
fmt.Fprintln(os.Stderr, "URL-encoded JSON input:", urlEncodedInput)
// Construct the full URL
apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search,card.count?batch=1&input=%s", urlEncodedInput)
apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search?batch=1&input=%s", urlEncodedInput)
// Print the full API URL for debugging
fmt.Fprintln(os.Stderr, "Full API URL:", apiURL)
// Create a new HTTP request
req, err := http.NewRequest("GET", apiURL, nil)
@ -72,7 +77,7 @@ func main() {
logErrorAndExit(fmt.Errorf("failed to create request: %w", err))
}
// Set headers
// Set essential headers
setHeaders(req)
// Perform the HTTP request
@ -108,20 +113,7 @@ func main() {
func setHeaders(req *http.Request) {
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0")
req.Header.Set("Accept", "*/*")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
req.Header.Set("Accept-Encoding", "gzip, deflate")
req.Header.Set("Referer", "https://curiosa.io/cards")
req.Header.Set("content-type", "application/json") // Match exact casing as bash script
req.Header.Set("x-build-id", "a68dd8c6f6e3b605951c534ad22e788557cabd0d")
req.Header.Set("x-trpc-source", "nextjs-react")
req.Header.Set("DNT", "1")
req.Header.Set("Sec-GPC", "1")
req.Header.Set("Connection", "keep-alive")
req.Header.Set("Sec-Fetch-Dest", "empty")
req.Header.Set("Sec-Fetch-Mode", "cors")
req.Header.Set("Sec-Fetch-Site", "same-origin")
req.Header.Set("Priority", "u=4")
req.Header.Set("TE", "trailers")
req.Header.Set("content-type", "application/json")
}
// logErrorAndExit logs the error to stderr and exits with a non-zero status code.