diff --git a/Makefile b/Makefile index 7f06f77..483138d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ default: run run: - go run main.go + go run *.go diff --git a/go.mod b/go.mod index eef6095..df3b907 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module git.eeqj.de/sneak/curiosa-scr-scrape go 1.22.2 + +require github.com/dustin/go-humanize v1.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..2f95363 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= diff --git a/main.go b/main.go index 7bd5672..7abec00 100644 --- a/main.go +++ b/main.go @@ -3,119 +3,17 @@ package main import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "os" + "path/filepath" "strings" "time" + + humanize "github.com/dustin/go-humanize" ) -// Card represents the structure of a card in the API response. -type Card struct { - ID string `json:"id"` - Slug string `json:"slug"` - Name string `json:"name"` - Hotscore int `json:"hotscore"` - Guardian Guardian `json:"guardian"` - Elements []Element `json:"elements"` - Variants []Variant `json:"variants"` -} - -// Guardian represents the structure of a guardian in the card details. -type Guardian struct { - ID string `json:"id"` - Type string `json:"type"` - Rarity string `json:"rarity"` - TypeText string `json:"typeText"` - SubType string `json:"subType"` - RulesText string `json:"rulesText"` - Cost int `json:"cost"` - Attack *int `json:"attack"` - Defense *int `json:"defense"` - Life *int `json:"life"` - WaterThreshold int `json:"waterThreshold"` - EarthThreshold int `json:"earthThreshold"` - FireThreshold int `json:"fireThreshold"` - AirThreshold int `json:"airThreshold"` - CardID string `json:"cardId"` -} - -// Element represents the structure of an element in the card details. -type Element struct { - ID string `json:"id"` - Name string `json:"name"` -} - -// Variant represents the structure of a variant in the card details. -type Variant struct { - ID string `json:"id"` - Slug string `json:"slug"` - Src string `json:"src"` - Finish string `json:"finish"` - Product string `json:"product"` - Artist string `json:"artist"` - FlavorText string `json:"flavorText"` - CardID string `json:"cardId"` - SetCardID string `json:"setCardId"` - SetCard SetCard `json:"setCard"` -} - -// SetCard represents the structure of a set card in the card details. -type SetCard struct { - ID string `json:"id"` - Slug string `json:"slug"` - SetID string `json:"setId"` - CardID string `json:"cardId"` - Meta MetaData `json:"meta"` - SetDetails SetDetails `json:"set"` -} - -// MetaData represents the structure of meta data in the set card details. -type MetaData struct { - ID string `json:"id"` - Type string `json:"type"` - Rarity string `json:"rarity"` - TypeText string `json:"typeText"` - SubType string `json:"subType"` - RulesText string `json:"rulesText"` - Cost int `json:"cost"` - Attack *int `json:"attack"` - Defense *int `json:"defense"` - Life *int `json:"life"` - WaterThreshold int `json:"waterThreshold"` - EarthThreshold int `json:"earthThreshold"` - FireThreshold int `json:"fireThreshold"` - AirThreshold int `json:"airThreshold"` - SetCardID string `json:"setCardId"` -} - -// SetDetails represents the structure of set details in the set card. -type SetDetails struct { - ID string `json:"id"` - Name string `json:"name"` - ReleaseDate string `json:"releaseDate"` -} - -// JSONPayload represents the structure of the nested JSON for the API request. -type JSONPayload struct { - Query string `json:"query"` - Sort string `json:"sort"` - Set string `json:"set"` - Filters []interface{} `json:"filters"` - Limit int `json:"limit"` - VariantLimit bool `json:"variantLimit"` - CollectionLimit bool `json:"collectionLimit"` - Cursor int `json:"cursor"` - Direction string `json:"direction"` -} - -// Input represents the top-level structure of the input JSON for the API request. -type Input struct { - Part0 struct { - JSON JSONPayload `json:"json"` - } `json:"0"` -} - // fetchCards fetches cards from the API using the provided limit, cursor, and set type. func fetchCards(limit, cursor int, setType string) ([]*Card, error) { // Define the input data structure @@ -225,25 +123,66 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) { return allCards, nil } +// downloadImage downloads an image from the given URL and saves it to the specified path. +func downloadImage(url, filepath, setType string) error { + if _, err := os.Stat(filepath); err == nil { + fmt.Printf("File already exists: %s\n", filepath) + return nil + } + + start := time.Now() + resp, err := http.Get(url) + duration := time.Since(start) + if err != nil { + return fmt.Errorf("failed to fetch image: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("non-200 response: %s", resp.Status) + } + + file, err := os.Create(filepath) + if err != nil { + return fmt.Errorf("failed to create image file: %w", err) + } + defer file.Close() + + bodySize, err := io.Copy(file, resp.Body) + if err != nil { + return fmt.Errorf("failed to save image: %w", err) + } + + fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n", + filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration) + return nil +} + func main() { const limit = 100 const dataDir = "./data" + const imageDir = "./images" - // Ensure the data directory exists + // Ensure the data and images directories exist err := os.MkdirAll(dataDir, os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err)) } - // Fetch and save beta cards - saveCards("bet", "beta", limit, dataDir) + err = os.MkdirAll(imageDir, os.ModePerm) + if err != nil { + logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err)) + } - // Fetch and save alpha cards - saveCards("alp", "alpha", limit, dataDir) + // Fetch and save beta cards and images + saveCards("bet", "beta", limit, dataDir, imageDir) + + // Fetch and save alpha cards and images + saveCards("alp", "alpha", limit, dataDir, imageDir) } -// saveCards fetches all cards for the given set type and saves them to files. -func saveCards(apiSetType, filePrefix string, limit int, dataDir string) { +// saveCards fetches all cards for the given set type and saves them to files and images. +func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) { allCards, err := fetchAllCards(limit, apiSetType) if err != nil { logErrorAndExit(err) @@ -253,6 +192,24 @@ func saveCards(apiSetType, filePrefix string, limit int, dataDir string) { filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix) writeCardsToFile(filename, allCards) + // Download images for each variant and save to the appropriate directory + for _, card := range allCards { + for _, variant := range card.Variants { + imageFilename := getImageFilename(variant.Slug) + imagePath := filepath.Join(imageDir, apiSetType, imageFilename) + + err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm) + if err != nil { + logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err)) + } + + err = downloadImage(variant.Src, imagePath, apiSetType) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err) + } + } + } + // Group cards by element and write to files groupedByElement := groupCardsByElement(allCards) for element, cards := range groupedByElement { @@ -327,3 +284,20 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card { return grouped } +// getImageFilename constructs a human-readable image filename from the slug. +func getImageFilename(slug string) string { + parts := strings.Split(slug, "_") + name := strings.Join(parts[1:len(parts)-1], "_") + variant := parts[len(parts)-1] + + switch variant { + case "s": + return fmt.Sprintf("%s_standard.jpg", name) + case "f": + return fmt.Sprintf("%s_foil.jpg", name) + case "p": + return fmt.Sprintf("%s_promo.jpg", name) + default: + return fmt.Sprintf("%s.jpg", name) + } +} diff --git a/types.go b/types.go new file mode 100644 index 0000000..f3fa378 --- /dev/null +++ b/types.go @@ -0,0 +1,107 @@ +package main + +// Card represents the structure of a card in the API response. +type Card struct { + ID string `json:"id"` + Slug string `json:"slug"` + Name string `json:"name"` + Hotscore int `json:"hotscore"` + Guardian Guardian `json:"guardian"` + Elements []Element `json:"elements"` + Variants []Variant `json:"variants"` +} + +// Guardian represents the structure of a guardian in the card details. +type Guardian struct { + ID string `json:"id"` + Type string `json:"type"` + Rarity string `json:"rarity"` + TypeText string `json:"typeText"` + SubType string `json:"subType"` + RulesText string `json:"rulesText"` + Cost int `json:"cost"` + Attack *int `json:"attack"` + Defense *int `json:"defense"` + Life *int `json:"life"` + WaterThreshold int `json:"waterThreshold"` + EarthThreshold int `json:"earthThreshold"` + FireThreshold int `json:"fireThreshold"` + AirThreshold int `json:"airThreshold"` + CardID string `json:"cardId"` +} + +// Element represents the structure of an element in the card details. +type Element struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// Variant represents the structure of a variant in the card details. +type Variant struct { + ID string `json:"id"` + Slug string `json:"slug"` + Src string `json:"src"` + Finish string `json:"finish"` + Product string `json:"product"` + Artist string `json:"artist"` + FlavorText string `json:"flavorText"` + CardID string `json:"cardId"` + SetCardID string `json:"setCardId"` + SetCard SetCard `json:"setCard"` +} + +// SetCard represents the structure of a set card in the card details. +type SetCard struct { + ID string `json:"id"` + Slug string `json:"slug"` + SetID string `json:"setId"` + CardID string `json:"cardId"` + Meta MetaData `json:"meta"` + SetDetails SetDetails `json:"set"` +} + +// MetaData represents the structure of meta data in the set card details. +type MetaData struct { + ID string `json:"id"` + Type string `json:"type"` + Rarity string `json:"rarity"` + TypeText string `json:"typeText"` + SubType string `json:"subType"` + RulesText string `json:"rulesText"` + Cost int `json:"cost"` + Attack *int `json:"attack"` + Defense *int `json:"defense"` + Life *int `json:"life"` + WaterThreshold int `json:"waterThreshold"` + EarthThreshold int `json:"earthThreshold"` + FireThreshold int `json:"fireThreshold"` + AirThreshold int `json:"airThreshold"` + SetCardID string `json:"setCardId"` +} + +// SetDetails represents the structure of set details in the set card. +type SetDetails struct { + ID string `json:"id"` + Name string `json:"name"` + ReleaseDate string `json:"releaseDate"` +} + +// JSONPayload represents the structure of the nested JSON for the API request. +type JSONPayload struct { + Query string `json:"query"` + Sort string `json:"sort"` + Set string `json:"set"` + Filters []interface{} `json:"filters"` + Limit int `json:"limit"` + VariantLimit bool `json:"variantLimit"` + CollectionLimit bool `json:"collectionLimit"` + Cursor int `json:"cursor"` + Direction string `json:"direction"` +} + +// Input represents the top-level structure of the input JSON for the API request. +type Input struct { + Part0 struct { + JSON JSONPayload `json:"json"` + } `json:"0"` +}