sneak/scrcards
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
Jeffrey Paul 2024-05-20 06:32:15 -07:00
parent 27980498f9
commit 56961bda66
6 changed files with 70205 additions and 150624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
curiosa-scr-scrape
/images

3
Makefile
View File

@ -2,3 +2,6 @@ default: run
run: run:
go run *.go go run *.go
fmt:
go fmt *.go

140184
data/alpha_cards.json
View File

File diff suppressed because it is too large Load Diff

80450
data/cards.json
View File

File diff suppressed because it is too large Load Diff

120
main.go
View File

@ -123,13 +123,28 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
return allCards, nil return allCards, nil
} }
// generateImageFilename generates the image filename for each card variant.
func generateImageFilename(variant Variant) string {
setName := strings.ToLower(variant.SetCard.SetDetails.Name)
if setName == "" {
setName = "other"
}
ext := filepath.Ext(variant.Src)
if ext == "" {
ext = ".png" // Default to .png if no extension found
}
return fmt.Sprintf("./images/%s/%s.%s%s", setName, strings.ToLower(variant.ID), strings.ToLower(variant.Slug), ext)
}
// downloadImage downloads an image from the given URL and saves it to the specified path. // downloadImage downloads an image from the given URL and saves it to the specified path.
func downloadImage(url, filepath, setType string) error { func downloadImage(url, filepath string) error {
if _, err := os.Stat(filepath); err == nil { if _, err := os.Stat(filepath); err == nil {
fmt.Printf("File already exists: %s\n", filepath) fmt.Printf("Skipping download, file already exists: %s\n", filepath)
return nil return nil
} }
tempFilePath := filepath + ".tmp"
start := time.Now() start := time.Now()
resp, err := http.Get(url) resp, err := http.Get(url)
duration := time.Since(start) duration := time.Since(start)
@ -142,7 +157,7 @@ func downloadImage(url, filepath, setType string) error {
return fmt.Errorf("non-200 response: %s", resp.Status) return fmt.Errorf("non-200 response: %s", resp.Status)
} }
file, err := os.Create(filepath) file, err := os.Create(tempFilePath)
if err != nil { if err != nil {
return fmt.Errorf("failed to create image file: %w", err) return fmt.Errorf("failed to create image file: %w", err)
} }
@ -153,8 +168,13 @@ func downloadImage(url, filepath, setType string) error {
return fmt.Errorf("failed to save image: %w", err) return fmt.Errorf("failed to save image: %w", err)
} }
fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n", err = os.Rename(tempFilePath, filepath)
filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration) if err != nil {
return fmt.Errorf("failed to rename temp file: %w", err)
}
fmt.Printf("Downloaded %s | Status: %d | Size: %s | Duration: %s\n",
filepath, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
return nil return nil
} }
@ -174,57 +194,81 @@ func main() {
logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err)) logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
} }
// Fetch and save beta cards and images // Fetch all cards for both alpha and beta sets
saveCards("bet", "beta", limit, dataDir, imageDir) alphaCards, err := fetchAllCards(limit, "alp")
// Fetch and save alpha cards and images
saveCards("alp", "alpha", limit, dataDir, imageDir)
}
// saveCards fetches all cards for the given set type and saves them to files and images.
func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
allCards, err := fetchAllCards(limit, apiSetType)
if err != nil { if err != nil {
logErrorAndExit(err) logErrorAndExit(err)
} }
betaCards, err := fetchAllCards(limit, "bet")
if err != nil {
logErrorAndExit(err)
}
// Save alpha and beta cards separately and write combined cards.json
allCards := append(alphaCards, betaCards...)
saveCards(alphaCards, "alpha", dataDir, imageDir)
saveCards(betaCards, "beta", dataDir, imageDir)
saveCards(allCards, "all", dataDir, imageDir)
}
// saveCards saves all cards for the given set type and writes to files and images.
func saveCards(cards []*Card, filePrefix, dataDir, imageDir string) {
// Remove duplicates
uniqueCards := removeDuplicateCards(cards)
// Write all cards to cards.json in the data directory // Write all cards to cards.json in the data directory
filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix) filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
writeCardsToFile(filename, allCards) writeCardsToFile(filename, uniqueCards)
// Download images for each variant and save to the appropriate directory // Download images for each variant and save to the appropriate directory
for _, card := range allCards { for _, card := range uniqueCards {
for _, variant := range card.Variants { for _, variant := range card.Variants {
imageFilename := getImageFilename(variant.Slug) imagePath := generateImageFilename(variant)
imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm) err := os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
if err != nil { if err != nil {
logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err)) logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
} }
err = downloadImage(variant.Src, imagePath, apiSetType) err = downloadImage(variant.Src, imagePath)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err) fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
continue
} }
} }
} }
// Group cards by element and write to files // Group cards by element and write to files
groupedByElement := groupCardsByElement(allCards) groupedByElement := groupCardsByElement(uniqueCards)
for element, cards := range groupedByElement { for element, elementCards := range groupedByElement {
filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element)) filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element))
writeCardsToFile(filename, cards) writeCardsToFile(filename, elementCards)
} }
// Group cards by element, rarity, and type and write to files // Group cards by element, rarity, and type and write to files
groupedByElementRarityType := groupCardsByElementRarityType(allCards) groupedByElementRarityType := groupCardsByElementRarityType(uniqueCards)
for key, cards := range groupedByElementRarityType { for key, keyCards := range groupedByElementRarityType {
filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key)) filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key))
writeCardsToFile(filename, cards) writeCardsToFile(filename, keyCards)
} }
fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(allCards)) fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(uniqueCards))
}
// removeDuplicateCards removes duplicate cards by checking the 'id' field.
func removeDuplicateCards(cards []*Card) []*Card {
cardMap := make(map[string]*Card)
for _, card := range cards {
cardMap[card.ID] = card
}
uniqueCards := make([]*Card, 0, len(cardMap))
for _, card := range cardMap {
uniqueCards = append(uniqueCards, card)
}
return uniqueCards
} }
// setHeaders sets the necessary headers for the HTTP request. // setHeaders sets the necessary headers for the HTTP request.
@ -277,27 +321,9 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
grouped := make(map[string][]*Card) grouped := make(map[string][]*Card)
for _, card := range cards { for _, card := range cards {
for _, element := range card.Elements { for _, element := range card.Elements {
key := fmt.Sprintf("%s_%s_%s", element.Name, strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type)) key := fmt.Sprintf("%s_%s_%s", strings.ToLower(element.Name), strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
grouped[key] = append(grouped[key], card) grouped[key] = append(grouped[key], card)
} }
} }
return grouped return grouped
} }
// getImageFilename constructs a human-readable image filename from the slug.
func getImageFilename(slug string) string {
parts := strings.Split(slug, "_")
name := strings.Join(parts[1:len(parts)-1], "_")
variant := parts[len(parts)-1]
switch variant {
case "s":
return fmt.Sprintf("%s_standard.jpg", name)
case "f":
return fmt.Sprintf("%s_foil.jpg", name)
case "p":
return fmt.Sprintf("%s_promo.jpg", name)
default:
return fmt.Sprintf("%s.jpg", name)
}
}

70
types.go
View File

@ -13,21 +13,21 @@ type Card struct {
// Guardian represents the structure of a guardian in the card details. // Guardian represents the structure of a guardian in the card details.
type Guardian struct { type Guardian struct {
ID string `json:"id"` ID string `json:"id"`
Type string `json:"type"` Type string `json:"type"`
Rarity string `json:"rarity"` Rarity string `json:"rarity"`
TypeText string `json:"typeText"` TypeText string `json:"typeText"`
SubType string `json:"subType"` SubType string `json:"subType"`
RulesText string `json:"rulesText"` RulesText string `json:"rulesText"`
Cost int `json:"cost"` Cost int `json:"cost"`
Attack *int `json:"attack"` Attack *int `json:"attack"`
Defense *int `json:"defense"` Defense *int `json:"defense"`
Life *int `json:"life"` Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"` WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"` EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"` FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"` AirThreshold int `json:"airThreshold"`
CardID string `json:"cardId"` CardID string `json:"cardId"`
} }
// Element represents the structure of an element in the card details. // Element represents the structure of an element in the card details.
@ -52,31 +52,31 @@ type Variant struct {
// SetCard represents the structure of a set card in the card details. // SetCard represents the structure of a set card in the card details.
type SetCard struct { type SetCard struct {
ID string `json:"id"` ID string `json:"id"`
Slug string `json:"slug"` Slug string `json:"slug"`
SetID string `json:"setId"` SetID string `json:"setId"`
CardID string `json:"cardId"` CardID string `json:"cardId"`
Meta MetaData `json:"meta"` Meta MetaData `json:"meta"`
SetDetails SetDetails `json:"set"` SetDetails SetDetails `json:"set"`
} }
// MetaData represents the structure of meta data in the set card details. // MetaData represents the structure of meta data in the set card details.
type MetaData struct { type MetaData struct {
ID string `json:"id"` ID string `json:"id"`
Type string `json:"type"` Type string `json:"type"`
Rarity string `json:"rarity"` Rarity string `json:"rarity"`
TypeText string `json:"typeText"` TypeText string `json:"typeText"`
SubType string `json:"subType"` SubType string `json:"subType"`
RulesText string `json:"rulesText"` RulesText string `json:"rulesText"`
Cost int `json:"cost"` Cost int `json:"cost"`
Attack *int `json:"attack"` Attack *int `json:"attack"`
Defense *int `json:"defense"` Defense *int `json:"defense"`
Life *int `json:"life"` Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"` WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"` EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"` FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"` AirThreshold int `json:"airThreshold"`
SetCardID string `json:"setCardId"` SetCardID string `json:"setCardId"`
} }
// SetDetails represents the structure of set details in the set card. // SetDetails represents the structure of set details in the set card.