sneak/scrcards
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
Jeffrey Paul 2024-05-20 05:31:58 -07:00
parent 4553d0f6da
commit 27980498f9
5 changed files with 199 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1,4 +1,4 @@
default: run
run:
go run main.go
go run *.go

2
go.mod
View File

@ -1,3 +1,5 @@
module git.eeqj.de/sneak/curiosa-scr-scrape
go 1.22.2
require github.com/dustin/go-humanize v1.0.1

2
go.sum Normal file
View File

@ -0,0 +1,2 @@
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=

200
main.go
View File

@ -3,119 +3,17 @@ package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
humanize "github.com/dustin/go-humanize"
)
// Card represents the structure of a card in the API response.
type Card struct {
ID string `json:"id"`
Slug string `json:"slug"`
Name string `json:"name"`
Hotscore int `json:"hotscore"`
Guardian Guardian `json:"guardian"`
Elements []Element `json:"elements"`
Variants []Variant `json:"variants"`
}
// Guardian represents the structure of a guardian in the card details.
type Guardian struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
CardID string `json:"cardId"`
}
// Element represents the structure of an element in the card details.
type Element struct {
ID string `json:"id"`
Name string `json:"name"`
}
// Variant represents the structure of a variant in the card details.
type Variant struct {
ID string `json:"id"`
Slug string `json:"slug"`
Src string `json:"src"`
Finish string `json:"finish"`
Product string `json:"product"`
Artist string `json:"artist"`
FlavorText string `json:"flavorText"`
CardID string `json:"cardId"`
SetCardID string `json:"setCardId"`
SetCard SetCard `json:"setCard"`
}
// SetCard represents the structure of a set card in the card details.
type SetCard struct {
ID string `json:"id"`
Slug string `json:"slug"`
SetID string `json:"setId"`
CardID string `json:"cardId"`
Meta MetaData `json:"meta"`
SetDetails SetDetails `json:"set"`
}
// MetaData represents the structure of meta data in the set card details.
type MetaData struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
SetCardID string `json:"setCardId"`
}
// SetDetails represents the structure of set details in the set card.
type SetDetails struct {
ID string `json:"id"`
Name string `json:"name"`
ReleaseDate string `json:"releaseDate"`
}
// JSONPayload represents the structure of the nested JSON for the API request.
type JSONPayload struct {
Query string `json:"query"`
Sort string `json:"sort"`
Set string `json:"set"`
Filters []interface{} `json:"filters"`
Limit int `json:"limit"`
VariantLimit bool `json:"variantLimit"`
CollectionLimit bool `json:"collectionLimit"`
Cursor int `json:"cursor"`
Direction string `json:"direction"`
}
// Input represents the top-level structure of the input JSON for the API request.
type Input struct {
Part0 struct {
JSON JSONPayload `json:"json"`
} `json:"0"`
}
// fetchCards fetches cards from the API using the provided limit, cursor, and set type.
func fetchCards(limit, cursor int, setType string) ([]*Card, error) {
// Define the input data structure
@ -225,25 +123,66 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
return allCards, nil
}
// downloadImage downloads an image from the given URL and saves it to the specified path.
func downloadImage(url, filepath, setType string) error {
if _, err := os.Stat(filepath); err == nil {
fmt.Printf("File already exists: %s\n", filepath)
return nil
}
start := time.Now()
resp, err := http.Get(url)
duration := time.Since(start)
if err != nil {
return fmt.Errorf("failed to fetch image: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("non-200 response: %s", resp.Status)
}
file, err := os.Create(filepath)
if err != nil {
return fmt.Errorf("failed to create image file: %w", err)
}
defer file.Close()
bodySize, err := io.Copy(file, resp.Body)
if err != nil {
return fmt.Errorf("failed to save image: %w", err)
}
fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n",
filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
return nil
}
func main() {
const limit = 100
const dataDir = "./data"
const imageDir = "./images"
// Ensure the data directory exists
// Ensure the data and images directories exist
err := os.MkdirAll(dataDir, os.ModePerm)
if err != nil {
logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err))
}
// Fetch and save beta cards
saveCards("bet", "beta", limit, dataDir)
err = os.MkdirAll(imageDir, os.ModePerm)
if err != nil {
logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
}
// Fetch and save alpha cards
saveCards("alp", "alpha", limit, dataDir)
// Fetch and save beta cards and images
saveCards("bet", "beta", limit, dataDir, imageDir)
// Fetch and save alpha cards and images
saveCards("alp", "alpha", limit, dataDir, imageDir)
}
// saveCards fetches all cards for the given set type and saves them to files.
func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
// saveCards fetches all cards for the given set type and saves them to files and images.
func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
allCards, err := fetchAllCards(limit, apiSetType)
if err != nil {
logErrorAndExit(err)
@ -253,6 +192,24 @@ func saveCards(apiSetType, filePrefix string, limit int, dataDir string) {
filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
writeCardsToFile(filename, allCards)
// Download images for each variant and save to the appropriate directory
for _, card := range allCards {
for _, variant := range card.Variants {
imageFilename := getImageFilename(variant.Slug)
imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
if err != nil {
logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
}
err = downloadImage(variant.Src, imagePath, apiSetType)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
}
}
}
// Group cards by element and write to files
groupedByElement := groupCardsByElement(allCards)
for element, cards := range groupedByElement {
@ -327,3 +284,20 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
return grouped
}
// getImageFilename constructs a human-readable image filename from the slug.
func getImageFilename(slug string) string {
parts := strings.Split(slug, "_")
name := strings.Join(parts[1:len(parts)-1], "_")
variant := parts[len(parts)-1]
switch variant {
case "s":
return fmt.Sprintf("%s_standard.jpg", name)
case "f":
return fmt.Sprintf("%s_foil.jpg", name)
case "p":
return fmt.Sprintf("%s_promo.jpg", name)
default:
return fmt.Sprintf("%s.jpg", name)
}
}

107
types.go Normal file
View File

@ -0,0 +1,107 @@
package main
// Card represents the structure of a card in the API response.
type Card struct {
ID string `json:"id"`
Slug string `json:"slug"`
Name string `json:"name"`
Hotscore int `json:"hotscore"`
Guardian Guardian `json:"guardian"`
Elements []Element `json:"elements"`
Variants []Variant `json:"variants"`
}
// Guardian represents the structure of a guardian in the card details.
type Guardian struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
CardID string `json:"cardId"`
}
// Element represents the structure of an element in the card details.
type Element struct {
ID string `json:"id"`
Name string `json:"name"`
}
// Variant represents the structure of a variant in the card details.
type Variant struct {
ID string `json:"id"`
Slug string `json:"slug"`
Src string `json:"src"`
Finish string `json:"finish"`
Product string `json:"product"`
Artist string `json:"artist"`
FlavorText string `json:"flavorText"`
CardID string `json:"cardId"`
SetCardID string `json:"setCardId"`
SetCard SetCard `json:"setCard"`
}
// SetCard represents the structure of a set card in the card details.
type SetCard struct {
ID string `json:"id"`
Slug string `json:"slug"`
SetID string `json:"setId"`
CardID string `json:"cardId"`
Meta MetaData `json:"meta"`
SetDetails SetDetails `json:"set"`
}
// MetaData represents the structure of meta data in the set card details.
type MetaData struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
SetCardID string `json:"setCardId"`
}
// SetDetails represents the structure of set details in the set card.
type SetDetails struct {
ID string `json:"id"`
Name string `json:"name"`
ReleaseDate string `json:"releaseDate"`
}
// JSONPayload represents the structure of the nested JSON for the API request.
type JSONPayload struct {
Query string `json:"query"`
Sort string `json:"sort"`
Set string `json:"set"`
Filters []interface{} `json:"filters"`
Limit int `json:"limit"`
VariantLimit bool `json:"variantLimit"`
CollectionLimit bool `json:"collectionLimit"`
Cursor int `json:"cursor"`
Direction string `json:"direction"`
}
// Input represents the top-level structure of the input JSON for the API request.
type Input struct {
Part0 struct {
JSON JSONPayload `json:"json"`
} `json:"0"`
}