sneak
/
scrcards
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

latest

This commit is contained in:
Jeffrey Paul 2024-05-20 06:32:15 -07:00
parent 27980498f9
commit 56961bda66
6 changed files with 70205 additions and 150624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
curiosa-scr-scrape
/images

3
Makefile
View File

@ -2,3 +2,6 @@ default: run
run:
go run *.go
fmt:
go fmt *.go

140184
data/alpha_cards.json
View File

File diff suppressed because it is too large Load Diff

80450
data/cards.json
View File

File diff suppressed because it is too large Load Diff

120
main.go
View File

@ -123,13 +123,28 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
return allCards, nil
}
// generateImageFilename generates the image filename for each card variant.
func generateImageFilename(variant Variant) string {
setName := strings.ToLower(variant.SetCard.SetDetails.Name)
if setName == "" {
setName = "other"
}
ext := filepath.Ext(variant.Src)
if ext == "" {
ext = ".png" // Default to .png if no extension found
}
return fmt.Sprintf("./images/%s/%s.%s%s", setName, strings.ToLower(variant.ID), strings.ToLower(variant.Slug), ext)
}
// downloadImage downloads an image from the given URL and saves it to the specified path.
func downloadImage(url, filepath, setType string) error {
func downloadImage(url, filepath string) error {
if _, err := os.Stat(filepath); err == nil {
fmt.Printf("File already exists: %s\n", filepath)
fmt.Printf("Skipping download, file already exists: %s\n", filepath)
return nil
}
tempFilePath := filepath + ".tmp"
start := time.Now()
resp, err := http.Get(url)
duration := time.Since(start)
@ -142,7 +157,7 @@ func downloadImage(url, filepath, setType string) error {
return fmt.Errorf("non-200 response: %s", resp.Status)
}
file, err := os.Create(filepath)
file, err := os.Create(tempFilePath)
if err != nil {
return fmt.Errorf("failed to create image file: %w", err)
}
@ -153,8 +168,13 @@ func downloadImage(url, filepath, setType string) error {
return fmt.Errorf("failed to save image: %w", err)
}
fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n",
filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
err = os.Rename(tempFilePath, filepath)
if err != nil {
return fmt.Errorf("failed to rename temp file: %w", err)
}
fmt.Printf("Downloaded %s | Status: %d | Size: %s | Duration: %s\n",
filepath, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
return nil
}
@ -174,57 +194,81 @@ func main() {
logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
}
// Fetch and save beta cards and images
saveCards("bet", "beta", limit, dataDir, imageDir)
// Fetch and save alpha cards and images
saveCards("alp", "alpha", limit, dataDir, imageDir)
}
// saveCards fetches all cards for the given set type and saves them to files and images.
func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
allCards, err := fetchAllCards(limit, apiSetType)
// Fetch all cards for both alpha and beta sets
alphaCards, err := fetchAllCards(limit, "alp")
if err != nil {
logErrorAndExit(err)
}
betaCards, err := fetchAllCards(limit, "bet")
if err != nil {
logErrorAndExit(err)
}
// Save alpha and beta cards separately and write combined cards.json
allCards := append(alphaCards, betaCards...)
saveCards(alphaCards, "alpha", dataDir, imageDir)
saveCards(betaCards, "beta", dataDir, imageDir)
saveCards(allCards, "all", dataDir, imageDir)
}
// saveCards saves all cards for the given set type and writes to files and images.
func saveCards(cards []*Card, filePrefix, dataDir, imageDir string) {
// Remove duplicates
uniqueCards := removeDuplicateCards(cards)
// Write all cards to cards.json in the data directory
filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
writeCardsToFile(filename, allCards)
writeCardsToFile(filename, uniqueCards)
// Download images for each variant and save to the appropriate directory
for _, card := range allCards {
for _, card := range uniqueCards {
for _, variant := range card.Variants {
imageFilename := getImageFilename(variant.Slug)
imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
imagePath := generateImageFilename(variant)
err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
err := os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
if err != nil {
logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
}
err = downloadImage(variant.Src, imagePath, apiSetType)
err = downloadImage(variant.Src, imagePath)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
continue
}
}
}
// Group cards by element and write to files
groupedByElement := groupCardsByElement(allCards)
for element, cards := range groupedByElement {
groupedByElement := groupCardsByElement(uniqueCards)
for element, elementCards := range groupedByElement {
filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element))
writeCardsToFile(filename, cards)
writeCardsToFile(filename, elementCards)
}
// Group cards by element, rarity, and type and write to files
groupedByElementRarityType := groupCardsByElementRarityType(allCards)
for key, cards := range groupedByElementRarityType {
groupedByElementRarityType := groupCardsByElementRarityType(uniqueCards)
for key, keyCards := range groupedByElementRarityType {
filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key))
writeCardsToFile(filename, cards)
writeCardsToFile(filename, keyCards)
}
fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(allCards))
fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(uniqueCards))
}
// removeDuplicateCards removes duplicate cards by checking the 'id' field.
func removeDuplicateCards(cards []*Card) []*Card {
cardMap := make(map[string]*Card)
for _, card := range cards {
cardMap[card.ID] = card
}
uniqueCards := make([]*Card, 0, len(cardMap))
for _, card := range cardMap {
uniqueCards = append(uniqueCards, card)
}
return uniqueCards
}
// setHeaders sets the necessary headers for the HTTP request.
@ -277,27 +321,9 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
grouped := make(map[string][]*Card)
for _, card := range cards {
for _, element := range card.Elements {
key := fmt.Sprintf("%s_%s_%s", element.Name, strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
key := fmt.Sprintf("%s_%s_%s", strings.ToLower(element.Name), strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
grouped[key] = append(grouped[key], card)
}
}
return grouped
}
// getImageFilename constructs a human-readable image filename from the slug.
func getImageFilename(slug string) string {
parts := strings.Split(slug, "_")
name := strings.Join(parts[1:len(parts)-1], "_")
variant := parts[len(parts)-1]
switch variant {
case "s":
return fmt.Sprintf("%s_standard.jpg", name)
case "f":
return fmt.Sprintf("%s_foil.jpg", name)
case "p":
return fmt.Sprintf("%s_promo.jpg", name)
default:
return fmt.Sprintf("%s.jpg", name)
}
}

70
types.go
View File

@ -13,21 +13,21 @@ type Card struct {
// Guardian represents the structure of a guardian in the card details.
type Guardian struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
CardID string `json:"cardId"`
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
CardID string `json:"cardId"`
}
// Element represents the structure of an element in the card details.
@ -52,31 +52,31 @@ type Variant struct {
// SetCard represents the structure of a set card in the card details.
type SetCard struct {
ID string `json:"id"`
Slug string `json:"slug"`
SetID string `json:"setId"`
CardID string `json:"cardId"`
Meta MetaData `json:"meta"`
ID string `json:"id"`
Slug string `json:"slug"`
SetID string `json:"setId"`
CardID string `json:"cardId"`
Meta MetaData `json:"meta"`
SetDetails SetDetails `json:"set"`
}
// MetaData represents the structure of meta data in the set card details.
type MetaData struct {
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
SetCardID string `json:"setCardId"`
ID string `json:"id"`
Type string `json:"type"`
Rarity string `json:"rarity"`
TypeText string `json:"typeText"`
SubType string `json:"subType"`
RulesText string `json:"rulesText"`
Cost int `json:"cost"`
Attack *int `json:"attack"`
Defense *int `json:"defense"`
Life *int `json:"life"`
WaterThreshold int `json:"waterThreshold"`
EarthThreshold int `json:"earthThreshold"`
FireThreshold int `json:"fireThreshold"`
AirThreshold int `json:"airThreshold"`
SetCardID string `json:"setCardId"`
}
// SetDetails represents the structure of set details in the set card.