latest

Makefile

@@ -0,0 +1,4 @@
+default: run
+
+run:
+	go run main.go

fetch_api.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Ensure both cursor and limit arguments are provided
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <cursor> <limit>"
+    exit 1
+fi
+
+cursor=$1
+limit=$2
+
+# Construct the JSON payload with the provided cursor and limit
+json_payload=$(cat <<EOF
+{
+  "0": {
+    "json": {
+      "query": "",
+      "sort": "relevance",
+      "set": "bet",
+      "filters": [],
+      "limit": $limit,
+      "variantLimit": false,
+      "collectionLimit": false,
+      "cursor": $cursor,
+      "direction": "forward"
+    }
+  }
+}
+EOF
+)
+
+# Print the JSON payload for debugging
+echo "JSON Payload:"
+echo "$json_payload"
+
+# URL encode the JSON payload
+url_encoded_json=$(python3 -c "import urllib.parse; print(urllib.parse.quote('''$json_payload'''))")
+
+# Print the URL-encoded JSON payload for debugging
+echo "URL-encoded JSON Payload:"
+echo "$url_encoded_json"
+
+# Construct the full URL
+api_url="https://curiosa.io/api/trpc/card.search?batch=1&input=$url_encoded_json"
+
+# Print the full API URL for debugging
+echo "Full API URL:"
+echo "$api_url"
+
+# Run the curl command and output the response as JSON
+curl "$api_url" \
+  --compressed \
+  -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0' \
+  -H 'Accept: */*' \
+  -H 'Accept-Language: en-US,en;q=0.5' \
+  -H 'Accept-Encoding: gzip, deflate' \
+  -H 'Referer: https://curiosa.io/cards?set=bet' \
+  -H 'content-type: application/json' \
+  -H 'x-build-id: a68dd8c6f6e3b605951c534ad22e788557cabd0d' \
+  -H 'x-trpc-source: nextjs-react' \
+  -H 'DNT: 1' \
+  -H 'Sec-GPC: 1' \
+  -H 'Connection: keep-alive' \
+  -H 'Sec-Fetch-Dest: empty' \
+  -H 'Sec-Fetch-Mode: cors' \
+  -H 'Sec-Fetch-Site: same-origin' \
+  -H 'Priority: u=4' \
+  -H 'TE: trailers'
+
+

go.mod

@@ -0,0 +1,3 @@
+module git.eeqj.de/sneak/curiosa-scr-scrape
+
+go 1.22.2

main.go

@@ -8,8 +8,8 @@ import (
 	"os"
 )
 
-// SearchRequest represents the structure of the input JSON for the API request.
-type SearchRequest struct {
+// JSONPayload represents the structure of the nested JSON for the API request.
+type JSONPayload struct {
 	Query           string        `json:"query"`
 	Sort            string        `json:"sort"`
 	Set             string        `json:"set"`
@@ -17,37 +17,39 @@ type SearchRequest struct {
 	Limit           int           `json:"limit"`
 	VariantLimit    bool          `json:"variantLimit"`
 	CollectionLimit bool          `json:"collectionLimit"`
+	Cursor          int           `json:"cursor"`
 	Direction       string        `json:"direction"`
 }
 
 // Input represents the top-level structure of the input JSON for the API request.
 type Input struct {
-	Part0 SearchRequest `json:"0"`
-	Part1 SearchRequest `json:"1"`
+	Part0 struct {
+		JSON JSONPayload `json:"json"`
+	} `json:"0"`
 }
 
 func main() {
-	// Parameterize the Limit value
+	// Parameterize the Limit and Cursor values
 	limit := 35
+	cursor := 35
+	set := "bet"
 
 	// Define the input data structure
 	input := Input{
-		Part0: SearchRequest{
+		Part0: struct {
+			JSON JSONPayload `json:"json"`
+		}{
+			JSON: JSONPayload{
 				Query:           "",
 				Sort:            "relevance",
-			Set:             "*",
+				Set:             set,
 				Filters:         []interface{}{},
 				Limit:           limit,
 				VariantLimit:    false,
 				CollectionLimit: false,
+				Cursor:          cursor,
 				Direction:       "forward",
 			},
-		Part1: SearchRequest{
-			Query:           "",
-			Sort:            "relevance",
-			Set:             "*",
-			Filters:         []interface{}{},
-			CollectionLimit: false,
 		},
 	}
 
@@ -60,11 +62,14 @@ func main() {
 	// URL encode the JSON data
 	urlEncodedInput := url.QueryEscape(string(jsonData))
 
-	// Print the URL-encoded JSON input for comparison
+	// Print the URL-encoded JSON input for debugging
 	fmt.Fprintln(os.Stderr, "URL-encoded JSON input:", urlEncodedInput)
 
 	// Construct the full URL
-	apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search,card.count?batch=1&input=%s", urlEncodedInput)
+	apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search?batch=1&input=%s", urlEncodedInput)
+
+	// Print the full API URL for debugging
+	fmt.Fprintln(os.Stderr, "Full API URL:", apiURL)
 
 	// Create a new HTTP request
 	req, err := http.NewRequest("GET", apiURL, nil)
@@ -72,7 +77,7 @@ func main() {
 		logErrorAndExit(fmt.Errorf("failed to create request: %w", err))
 	}
 
-	// Set headers
+	// Set essential headers
 	setHeaders(req)
 
 	// Perform the HTTP request
@@ -108,20 +113,7 @@ func main() {
 func setHeaders(req *http.Request) {
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0")
 	req.Header.Set("Accept", "*/*")
-	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
-	req.Header.Set("Accept-Encoding", "gzip, deflate")
-	req.Header.Set("Referer", "https://curiosa.io/cards")
-	req.Header.Set("content-type", "application/json") // Match exact casing as bash script
-	req.Header.Set("x-build-id", "a68dd8c6f6e3b605951c534ad22e788557cabd0d")
-	req.Header.Set("x-trpc-source", "nextjs-react")
-	req.Header.Set("DNT", "1")
-	req.Header.Set("Sec-GPC", "1")
-	req.Header.Set("Connection", "keep-alive")
-	req.Header.Set("Sec-Fetch-Dest", "empty")
-	req.Header.Set("Sec-Fetch-Mode", "cors")
-	req.Header.Set("Sec-Fetch-Site", "same-origin")
-	req.Header.Set("Priority", "u=4")
-	req.Header.Set("TE", "trailers")
+	req.Header.Set("content-type", "application/json")
 }
 
 // logErrorAndExit logs the error to stderr and exits with a non-zero status code.