latest

Makefile

@@ -1,4 +1,4 @@
 default: run
 
 run:
-	go run main.go
+	go run *.go

go.mod

@@ -1,3 +1,5 @@
 module git.eeqj.de/sneak/curiosa-scr-scrape
 
 go 1.22.2
+
+require github.com/dustin/go-humanize v1.0.1

go.sum

@@ -0,0 +1,2 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

main.go

@@ -3,119 +3,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
+
+	humanize "github.com/dustin/go-humanize"
 )
 
-// Card represents the structure of a card in the API response.
-type Card struct {
-	ID       string    `json:"id"`
-	Slug     string    `json:"slug"`
-	Name     string    `json:"name"`
-	Hotscore int       `json:"hotscore"`
-	Guardian Guardian  `json:"guardian"`
-	Elements []Element `json:"elements"`
-	Variants []Variant `json:"variants"`
-}
-
-// Guardian represents the structure of a guardian in the card details.
-type Guardian struct {
-	ID              string `json:"id"`
-	Type            string `json:"type"`
-	Rarity          string `json:"rarity"`
-	TypeText        string `json:"typeText"`
-	SubType         string `json:"subType"`
-	RulesText       string `json:"rulesText"`
-	Cost            int    `json:"cost"`
-	Attack          *int   `json:"attack"`
-	Defense         *int   `json:"defense"`
-	Life            *int   `json:"life"`
-	WaterThreshold  int    `json:"waterThreshold"`
-	EarthThreshold  int    `json:"earthThreshold"`
-	FireThreshold   int    `json:"fireThreshold"`
-	AirThreshold    int    `json:"airThreshold"`
-	CardID          string `json:"cardId"`
-}
-
-// Element represents the structure of an element in the card details.
-type Element struct {
-	ID   string `json:"id"`
-	Name string `json:"name"`
-}
-
-// Variant represents the structure of a variant in the card details.
-type Variant struct {
-	ID         string  `json:"id"`
-	Slug       string  `json:"slug"`
-	Src        string  `json:"src"`
-	Finish     string  `json:"finish"`
-	Product    string  `json:"product"`
-	Artist     string  `json:"artist"`
-	FlavorText string  `json:"flavorText"`
-	CardID     string  `json:"cardId"`
-	SetCardID  string  `json:"setCardId"`
-	SetCard    SetCard `json:"setCard"`
-}
-
-// SetCard represents the structure of a set card in the card details.
-type SetCard struct {
-	ID         string    `json:"id"`
-	Slug       string    `json:"slug"`
-	SetID      string    `json:"setId"`
-	CardID     string    `json:"cardId"`
-	Meta       MetaData  `json:"meta"`
-	SetDetails SetDetails `json:"set"`
-}
-
-// MetaData represents the structure of meta data in the set card details.
-type MetaData struct {
-	ID              string `json:"id"`
-	Type            string `json:"type"`
-	Rarity          string `json:"rarity"`
-	TypeText        string `json:"typeText"`
-	SubType         string `json:"subType"`
-	RulesText       string `json:"rulesText"`
-	Cost            int    `json:"cost"`
-	Attack          *int   `json:"attack"`
-	Defense         *int   `json:"defense"`
-	Life            *int   `json:"life"`
-	WaterThreshold  int    `json:"waterThreshold"`
-	EarthThreshold  int    `json:"earthThreshold"`
-	FireThreshold   int    `json:"fireThreshold"`
-	AirThreshold    int    `json:"airThreshold"`
-	SetCardID       string `json:"setCardId"`
-}
-
-// SetDetails represents the structure of set details in the set card.
-type SetDetails struct {
-	ID          string `json:"id"`
-	Name        string `json:"name"`
-	ReleaseDate string `json:"releaseDate"`
-}
-
-// JSONPayload represents the structure of the nested JSON for the API request.
-type JSONPayload struct {
-	Query           string        `json:"query"`
-	Sort            string        `json:"sort"`
-	Set             string        `json:"set"`
-	Filters         []interface{} `json:"filters"`
-	Limit           int           `json:"limit"`
-	VariantLimit    bool          `json:"variantLimit"`
-	CollectionLimit bool          `json:"collectionLimit"`
-	Cursor          int           `json:"cursor"`
-	Direction       string        `json:"direction"`
-}
-
-// Input represents the top-level structure of the input JSON for the API request.
-type Input struct {
-	Part0 struct {
-		JSON JSONPayload `json:"json"`
-	} `json:"0"`
-}
-
 // fetchCards fetches cards from the API using the provided limit, cursor, and set type.
 func fetchCards(limit, cursor int, setType string) ([]*Card, error) {
 	// Define the input data structure
@@ -225,25 +123,66 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
 	return allCards, nil
 }
 
+// downloadImage downloads an image from the given URL and saves it to the specified path.
+func downloadImage(url, filepath, setType string) error {
+	if _, err := os.Stat(filepath); err == nil {
+		fmt.Printf("File already exists: %s\n", filepath)
+		return nil
+	}
+
+	start := time.Now()
+	resp, err := http.Get(url)
+	duration := time.Since(start)
+	if err != nil {
+		return fmt.Errorf("failed to fetch image: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("non-200 response: %s", resp.Status)
+	}
+
+	file, err := os.Create(filepath)
+	if err != nil {
+		return fmt.Errorf("failed to create image file: %w", err)
+	}
+	defer file.Close()
+
+	bodySize, err := io.Copy(file, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to save image: %w", err)
+	}
+
+	fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n",
+		filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
+	return nil
+}
+
 func main() {
 	const limit = 100
 	const dataDir = "./data"
+	const imageDir = "./images"
 
-	// Ensure the data directory exists
+	// Ensure the data and images directories exist
 	err := os.MkdirAll(dataDir, os.ModePerm)
 	if err != nil {
 		logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err))
 	}
 
-	// Fetch and save beta cards
+	err = os.MkdirAll(imageDir, os.ModePerm)
-	saveCards("bet", "beta", limit, dataDir)
+	if err != nil {
-
+		logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
-	// Fetch and save alpha cards
-	saveCards("alp", "alpha", limit, dataDir)
 	}
 
-// saveCards fetches all cards for the given set type and saves them to files.
+	// Fetch and save beta cards and images
-func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
+	saveCards("bet", "beta", limit, dataDir, imageDir)
+
+	// Fetch and save alpha cards and images
+	saveCards("alp", "alpha", limit, dataDir, imageDir)
+}
+
+// saveCards fetches all cards for the given set type and saves them to files and images.
+func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
 	allCards, err := fetchAllCards(limit, apiSetType)
 	if err != nil {
 		logErrorAndExit(err)
@@ -253,6 +192,24 @@ func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
 	filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
 	writeCardsToFile(filename, allCards)
 
+	// Download images for each variant and save to the appropriate directory
+	for _, card := range allCards {
+		for _, variant := range card.Variants {
+			imageFilename := getImageFilename(variant.Slug)
+			imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
+
+			err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
+			if err != nil {
+				logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
+			}
+
+			err = downloadImage(variant.Src, imagePath, apiSetType)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
+			}
+		}
+	}
+
 	// Group cards by element and write to files
 	groupedByElement := groupCardsByElement(allCards)
 	for element, cards := range groupedByElement {
@@ -327,3 +284,20 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
 	return grouped
 }
 
+// getImageFilename constructs a human-readable image filename from the slug.
+func getImageFilename(slug string) string {
+	parts := strings.Split(slug, "_")
+	name := strings.Join(parts[1:len(parts)-1], "_")
+	variant := parts[len(parts)-1]
+
+	switch variant {
+	case "s":
+		return fmt.Sprintf("%s_standard.jpg", name)
+	case "f":
+		return fmt.Sprintf("%s_foil.jpg", name)
+	case "p":
+		return fmt.Sprintf("%s_promo.jpg", name)
+	default:
+		return fmt.Sprintf("%s.jpg", name)
+	}
+}

types.go

@@ -0,0 +1,107 @@
+package main
+
+// Card represents the structure of a card in the API response.
+type Card struct {
+	ID       string    `json:"id"`
+	Slug     string    `json:"slug"`
+	Name     string    `json:"name"`
+	Hotscore int       `json:"hotscore"`
+	Guardian Guardian  `json:"guardian"`
+	Elements []Element `json:"elements"`
+	Variants []Variant `json:"variants"`
+}
+
+// Guardian represents the structure of a guardian in the card details.
+type Guardian struct {
+	ID              string `json:"id"`
+	Type            string `json:"type"`
+	Rarity          string `json:"rarity"`
+	TypeText        string `json:"typeText"`
+	SubType         string `json:"subType"`
+	RulesText       string `json:"rulesText"`
+	Cost            int    `json:"cost"`
+	Attack          *int   `json:"attack"`
+	Defense         *int   `json:"defense"`
+	Life            *int   `json:"life"`
+	WaterThreshold  int    `json:"waterThreshold"`
+	EarthThreshold  int    `json:"earthThreshold"`
+	FireThreshold   int    `json:"fireThreshold"`
+	AirThreshold    int    `json:"airThreshold"`
+	CardID          string `json:"cardId"`
+}
+
+// Element represents the structure of an element in the card details.
+type Element struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+// Variant represents the structure of a variant in the card details.
+type Variant struct {
+	ID         string  `json:"id"`
+	Slug       string  `json:"slug"`
+	Src        string  `json:"src"`
+	Finish     string  `json:"finish"`
+	Product    string  `json:"product"`
+	Artist     string  `json:"artist"`
+	FlavorText string  `json:"flavorText"`
+	CardID     string  `json:"cardId"`
+	SetCardID  string  `json:"setCardId"`
+	SetCard    SetCard `json:"setCard"`
+}
+
+// SetCard represents the structure of a set card in the card details.
+type SetCard struct {
+	ID         string    `json:"id"`
+	Slug       string    `json:"slug"`
+	SetID      string    `json:"setId"`
+	CardID     string    `json:"cardId"`
+	Meta       MetaData  `json:"meta"`
+	SetDetails SetDetails `json:"set"`
+}
+
+// MetaData represents the structure of meta data in the set card details.
+type MetaData struct {
+	ID              string `json:"id"`
+	Type            string `json:"type"`
+	Rarity          string `json:"rarity"`
+	TypeText        string `json:"typeText"`
+	SubType         string `json:"subType"`
+	RulesText       string `json:"rulesText"`
+	Cost            int    `json:"cost"`
+	Attack          *int   `json:"attack"`
+	Defense         *int   `json:"defense"`
+	Life            *int   `json:"life"`
+	WaterThreshold  int    `json:"waterThreshold"`
+	EarthThreshold  int    `json:"earthThreshold"`
+	FireThreshold   int    `json:"fireThreshold"`
+	AirThreshold    int    `json:"airThreshold"`
+	SetCardID       string `json:"setCardId"`
+}
+
+// SetDetails represents the structure of set details in the set card.
+type SetDetails struct {
+	ID          string `json:"id"`
+	Name        string `json:"name"`
+	ReleaseDate string `json:"releaseDate"`
+}
+
+// JSONPayload represents the structure of the nested JSON for the API request.
+type JSONPayload struct {
+	Query           string        `json:"query"`
+	Sort            string        `json:"sort"`
+	Set             string        `json:"set"`
+	Filters         []interface{} `json:"filters"`
+	Limit           int           `json:"limit"`
+	VariantLimit    bool          `json:"variantLimit"`
+	CollectionLimit bool          `json:"collectionLimit"`
+	Cursor          int           `json:"cursor"`
+	Direction       string        `json:"direction"`
+}
+
+// Input represents the top-level structure of the input JSON for the API request.
+type Input struct {
+	Part0 struct {
+		JSON JSONPayload `json:"json"`
+	} `json:"0"`
+}