latest
This commit is contained in:
parent
4553d0f6da
commit
27980498f9
2
go.mod
2
go.mod
|
@ -1,3 +1,5 @@
|
|||
module git.eeqj.de/sneak/curiosa-scr-scrape
|
||||
|
||||
go 1.22.2
|
||||
|
||||
require github.com/dustin/go-humanize v1.0.1
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
200
main.go
200
main.go
|
@ -3,119 +3,17 @@ package main
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
)
|
||||
|
||||
// Card represents the structure of a card in the API response.
|
||||
type Card struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Hotscore int `json:"hotscore"`
|
||||
Guardian Guardian `json:"guardian"`
|
||||
Elements []Element `json:"elements"`
|
||||
Variants []Variant `json:"variants"`
|
||||
}
|
||||
|
||||
// Guardian represents the structure of a guardian in the card details.
|
||||
type Guardian struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Rarity string `json:"rarity"`
|
||||
TypeText string `json:"typeText"`
|
||||
SubType string `json:"subType"`
|
||||
RulesText string `json:"rulesText"`
|
||||
Cost int `json:"cost"`
|
||||
Attack *int `json:"attack"`
|
||||
Defense *int `json:"defense"`
|
||||
Life *int `json:"life"`
|
||||
WaterThreshold int `json:"waterThreshold"`
|
||||
EarthThreshold int `json:"earthThreshold"`
|
||||
FireThreshold int `json:"fireThreshold"`
|
||||
AirThreshold int `json:"airThreshold"`
|
||||
CardID string `json:"cardId"`
|
||||
}
|
||||
|
||||
// Element represents the structure of an element in the card details.
|
||||
type Element struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Variant represents the structure of a variant in the card details.
|
||||
type Variant struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Src string `json:"src"`
|
||||
Finish string `json:"finish"`
|
||||
Product string `json:"product"`
|
||||
Artist string `json:"artist"`
|
||||
FlavorText string `json:"flavorText"`
|
||||
CardID string `json:"cardId"`
|
||||
SetCardID string `json:"setCardId"`
|
||||
SetCard SetCard `json:"setCard"`
|
||||
}
|
||||
|
||||
// SetCard represents the structure of a set card in the card details.
|
||||
type SetCard struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
SetID string `json:"setId"`
|
||||
CardID string `json:"cardId"`
|
||||
Meta MetaData `json:"meta"`
|
||||
SetDetails SetDetails `json:"set"`
|
||||
}
|
||||
|
||||
// MetaData represents the structure of meta data in the set card details.
|
||||
type MetaData struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Rarity string `json:"rarity"`
|
||||
TypeText string `json:"typeText"`
|
||||
SubType string `json:"subType"`
|
||||
RulesText string `json:"rulesText"`
|
||||
Cost int `json:"cost"`
|
||||
Attack *int `json:"attack"`
|
||||
Defense *int `json:"defense"`
|
||||
Life *int `json:"life"`
|
||||
WaterThreshold int `json:"waterThreshold"`
|
||||
EarthThreshold int `json:"earthThreshold"`
|
||||
FireThreshold int `json:"fireThreshold"`
|
||||
AirThreshold int `json:"airThreshold"`
|
||||
SetCardID string `json:"setCardId"`
|
||||
}
|
||||
|
||||
// SetDetails represents the structure of set details in the set card.
|
||||
type SetDetails struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ReleaseDate string `json:"releaseDate"`
|
||||
}
|
||||
|
||||
// JSONPayload represents the structure of the nested JSON for the API request.
|
||||
type JSONPayload struct {
|
||||
Query string `json:"query"`
|
||||
Sort string `json:"sort"`
|
||||
Set string `json:"set"`
|
||||
Filters []interface{} `json:"filters"`
|
||||
Limit int `json:"limit"`
|
||||
VariantLimit bool `json:"variantLimit"`
|
||||
CollectionLimit bool `json:"collectionLimit"`
|
||||
Cursor int `json:"cursor"`
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Input represents the top-level structure of the input JSON for the API request.
|
||||
type Input struct {
|
||||
Part0 struct {
|
||||
JSON JSONPayload `json:"json"`
|
||||
} `json:"0"`
|
||||
}
|
||||
|
||||
// fetchCards fetches cards from the API using the provided limit, cursor, and set type.
|
||||
func fetchCards(limit, cursor int, setType string) ([]*Card, error) {
|
||||
// Define the input data structure
|
||||
|
@ -225,25 +123,66 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
|
|||
return allCards, nil
|
||||
}
|
||||
|
||||
// downloadImage downloads an image from the given URL and saves it to the specified path.
|
||||
func downloadImage(url, filepath, setType string) error {
|
||||
if _, err := os.Stat(filepath); err == nil {
|
||||
fmt.Printf("File already exists: %s\n", filepath)
|
||||
return nil
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
resp, err := http.Get(url)
|
||||
duration := time.Since(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch image: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("non-200 response: %s", resp.Status)
|
||||
}
|
||||
|
||||
file, err := os.Create(filepath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create image file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
bodySize, err := io.Copy(file, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save image: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n",
|
||||
filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
const limit = 100
|
||||
const dataDir = "./data"
|
||||
const imageDir = "./images"
|
||||
|
||||
// Ensure the data directory exists
|
||||
// Ensure the data and images directories exist
|
||||
err := os.MkdirAll(dataDir, os.ModePerm)
|
||||
if err != nil {
|
||||
logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err))
|
||||
}
|
||||
|
||||
// Fetch and save beta cards
|
||||
saveCards("bet", "beta", limit, dataDir)
|
||||
err = os.MkdirAll(imageDir, os.ModePerm)
|
||||
if err != nil {
|
||||
logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
|
||||
}
|
||||
|
||||
// Fetch and save alpha cards
|
||||
saveCards("alp", "alpha", limit, dataDir)
|
||||
// Fetch and save beta cards and images
|
||||
saveCards("bet", "beta", limit, dataDir, imageDir)
|
||||
|
||||
// Fetch and save alpha cards and images
|
||||
saveCards("alp", "alpha", limit, dataDir, imageDir)
|
||||
}
|
||||
|
||||
// saveCards fetches all cards for the given set type and saves them to files.
|
||||
func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
|
||||
// saveCards fetches all cards for the given set type and saves them to files and images.
|
||||
func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
|
||||
allCards, err := fetchAllCards(limit, apiSetType)
|
||||
if err != nil {
|
||||
logErrorAndExit(err)
|
||||
|
@ -253,6 +192,24 @@ func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
|
|||
filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
|
||||
writeCardsToFile(filename, allCards)
|
||||
|
||||
// Download images for each variant and save to the appropriate directory
|
||||
for _, card := range allCards {
|
||||
for _, variant := range card.Variants {
|
||||
imageFilename := getImageFilename(variant.Slug)
|
||||
imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
|
||||
if err != nil {
|
||||
logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
|
||||
}
|
||||
|
||||
err = downloadImage(variant.Src, imagePath, apiSetType)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Group cards by element and write to files
|
||||
groupedByElement := groupCardsByElement(allCards)
|
||||
for element, cards := range groupedByElement {
|
||||
|
@ -327,3 +284,20 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
|
|||
return grouped
|
||||
}
|
||||
|
||||
// getImageFilename constructs a human-readable image filename from the slug.
|
||||
func getImageFilename(slug string) string {
|
||||
parts := strings.Split(slug, "_")
|
||||
name := strings.Join(parts[1:len(parts)-1], "_")
|
||||
variant := parts[len(parts)-1]
|
||||
|
||||
switch variant {
|
||||
case "s":
|
||||
return fmt.Sprintf("%s_standard.jpg", name)
|
||||
case "f":
|
||||
return fmt.Sprintf("%s_foil.jpg", name)
|
||||
case "p":
|
||||
return fmt.Sprintf("%s_promo.jpg", name)
|
||||
default:
|
||||
return fmt.Sprintf("%s.jpg", name)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,107 @@
|
|||
package main
|
||||
|
||||
// Card represents the structure of a card in the API response.
|
||||
type Card struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Hotscore int `json:"hotscore"`
|
||||
Guardian Guardian `json:"guardian"`
|
||||
Elements []Element `json:"elements"`
|
||||
Variants []Variant `json:"variants"`
|
||||
}
|
||||
|
||||
// Guardian represents the structure of a guardian in the card details.
|
||||
type Guardian struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Rarity string `json:"rarity"`
|
||||
TypeText string `json:"typeText"`
|
||||
SubType string `json:"subType"`
|
||||
RulesText string `json:"rulesText"`
|
||||
Cost int `json:"cost"`
|
||||
Attack *int `json:"attack"`
|
||||
Defense *int `json:"defense"`
|
||||
Life *int `json:"life"`
|
||||
WaterThreshold int `json:"waterThreshold"`
|
||||
EarthThreshold int `json:"earthThreshold"`
|
||||
FireThreshold int `json:"fireThreshold"`
|
||||
AirThreshold int `json:"airThreshold"`
|
||||
CardID string `json:"cardId"`
|
||||
}
|
||||
|
||||
// Element represents the structure of an element in the card details.
|
||||
type Element struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Variant represents the structure of a variant in the card details.
|
||||
type Variant struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
Src string `json:"src"`
|
||||
Finish string `json:"finish"`
|
||||
Product string `json:"product"`
|
||||
Artist string `json:"artist"`
|
||||
FlavorText string `json:"flavorText"`
|
||||
CardID string `json:"cardId"`
|
||||
SetCardID string `json:"setCardId"`
|
||||
SetCard SetCard `json:"setCard"`
|
||||
}
|
||||
|
||||
// SetCard represents the structure of a set card in the card details.
|
||||
type SetCard struct {
|
||||
ID string `json:"id"`
|
||||
Slug string `json:"slug"`
|
||||
SetID string `json:"setId"`
|
||||
CardID string `json:"cardId"`
|
||||
Meta MetaData `json:"meta"`
|
||||
SetDetails SetDetails `json:"set"`
|
||||
}
|
||||
|
||||
// MetaData represents the structure of meta data in the set card details.
|
||||
type MetaData struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Rarity string `json:"rarity"`
|
||||
TypeText string `json:"typeText"`
|
||||
SubType string `json:"subType"`
|
||||
RulesText string `json:"rulesText"`
|
||||
Cost int `json:"cost"`
|
||||
Attack *int `json:"attack"`
|
||||
Defense *int `json:"defense"`
|
||||
Life *int `json:"life"`
|
||||
WaterThreshold int `json:"waterThreshold"`
|
||||
EarthThreshold int `json:"earthThreshold"`
|
||||
FireThreshold int `json:"fireThreshold"`
|
||||
AirThreshold int `json:"airThreshold"`
|
||||
SetCardID string `json:"setCardId"`
|
||||
}
|
||||
|
||||
// SetDetails represents the structure of set details in the set card.
|
||||
type SetDetails struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
ReleaseDate string `json:"releaseDate"`
|
||||
}
|
||||
|
||||
// JSONPayload represents the structure of the nested JSON for the API request.
|
||||
type JSONPayload struct {
|
||||
Query string `json:"query"`
|
||||
Sort string `json:"sort"`
|
||||
Set string `json:"set"`
|
||||
Filters []interface{} `json:"filters"`
|
||||
Limit int `json:"limit"`
|
||||
VariantLimit bool `json:"variantLimit"`
|
||||
CollectionLimit bool `json:"collectionLimit"`
|
||||
Cursor int `json:"cursor"`
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Input represents the top-level structure of the input JSON for the API request.
|
||||
type Input struct {
|
||||
Part0 struct {
|
||||
JSON JSONPayload `json:"json"`
|
||||
} `json:"0"`
|
||||
}
|
Loading…
Reference in New Issue