latest

.gitignore

@@ -0,0 +1,2 @@
+curiosa-scr-scrape
+/images

Makefile

@@ -2,3 +2,6 @@ default: run
 
 run:
 	go run *.go
+
+fmt:
+	go fmt *.go

main.go

@@ -123,13 +123,28 @@ func fetchAllCards(limit int, setType string) ([]*Card, error) {
 	return allCards, nil
 }
 
+// generateImageFilename generates the image filename for each card variant.
+func generateImageFilename(variant Variant) string {
+	setName := strings.ToLower(variant.SetCard.SetDetails.Name)
+	if setName == "" {
+		setName = "other"
+	}
+	ext := filepath.Ext(variant.Src)
+	if ext == "" {
+		ext = ".png" // Default to .png if no extension found
+	}
+	return fmt.Sprintf("./images/%s/%s.%s%s", setName, strings.ToLower(variant.ID), strings.ToLower(variant.Slug), ext)
+}
+
 // downloadImage downloads an image from the given URL and saves it to the specified path.
-func downloadImage(url, filepath, setType string) error {
+func downloadImage(url, filepath string) error {
 	if _, err := os.Stat(filepath); err == nil {
-		fmt.Printf("File already exists: %s\n", filepath)
+		fmt.Printf("Skipping download, file already exists: %s\n", filepath)
 		return nil
 	}
 
+	tempFilePath := filepath + ".tmp"
+
 	start := time.Now()
 	resp, err := http.Get(url)
 	duration := time.Since(start)
@@ -142,7 +157,7 @@ func downloadImage(url, filepath, setType string) error {
 		return fmt.Errorf("non-200 response: %s", resp.Status)
 	}
 
-	file, err := os.Create(filepath)
+	file, err := os.Create(tempFilePath)
 	if err != nil {
 		return fmt.Errorf("failed to create image file: %w", err)
 	}
@@ -153,8 +168,13 @@ func downloadImage(url, filepath, setType string) error {
 		return fmt.Errorf("failed to save image: %w", err)
 	}
 
-	fmt.Printf("Downloaded %s | Set: %s | Status: %d | Size: %s | Duration: %s\n",
-		filepath, setType, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
+	err = os.Rename(tempFilePath, filepath)
+	if err != nil {
+		return fmt.Errorf("failed to rename temp file: %w", err)
+	}
+
+	fmt.Printf("Downloaded %s | Status: %d | Size: %s | Duration: %s\n",
+		filepath, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
 	return nil
 }
 
@@ -174,57 +194,81 @@ func main() {
 		logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
 	}
 
-	// Fetch and save beta cards and images
-	saveCards("bet", "beta", limit, dataDir, imageDir)
-
-	// Fetch and save alpha cards and images
-	saveCards("alp", "alpha", limit, dataDir, imageDir)
-}
-
-// saveCards fetches all cards for the given set type and saves them to files and images.
-func saveCards(apiSetType, filePrefix string, limit int, dataDir, imageDir string) {
-	allCards, err := fetchAllCards(limit, apiSetType)
+	// Fetch all cards for both alpha and beta sets
+	alphaCards, err := fetchAllCards(limit, "alp")
 	if err != nil {
 		logErrorAndExit(err)
 	}
 
+	betaCards, err := fetchAllCards(limit, "bet")
+	if err != nil {
+		logErrorAndExit(err)
+	}
+
+	// Save alpha and beta cards separately and write combined cards.json
+	allCards := append(alphaCards, betaCards...)
+	saveCards(alphaCards, "alpha", dataDir, imageDir)
+	saveCards(betaCards, "beta", dataDir, imageDir)
+	saveCards(allCards, "all", dataDir, imageDir)
+}
+
+// saveCards saves all cards for the given set type and writes to files and images.
+func saveCards(cards []*Card, filePrefix, dataDir, imageDir string) {
+	// Remove duplicates
+	uniqueCards := removeDuplicateCards(cards)
+
 	// Write all cards to cards.json in the data directory
 	filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
-	writeCardsToFile(filename, allCards)
+	writeCardsToFile(filename, uniqueCards)
 
 	// Download images for each variant and save to the appropriate directory
-	for _, card := range allCards {
+	for _, card := range uniqueCards {
 		for _, variant := range card.Variants {
-			imageFilename := getImageFilename(variant.Slug)
-			imagePath := filepath.Join(imageDir, apiSetType, imageFilename)
+			imagePath := generateImageFilename(variant)
 
-			err = os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
+			err := os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
 			if err != nil {
 				logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
 			}
 
-			err = downloadImage(variant.Src, imagePath, apiSetType)
+			err = downloadImage(variant.Src, imagePath)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
+				continue
 			}
 		}
 	}
 
 	// Group cards by element and write to files
-	groupedByElement := groupCardsByElement(allCards)
-	for element, cards := range groupedByElement {
+	groupedByElement := groupCardsByElement(uniqueCards)
+	for element, elementCards := range groupedByElement {
 		filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element))
-		writeCardsToFile(filename, cards)
+		writeCardsToFile(filename, elementCards)
 	}
 
 	// Group cards by element, rarity, and type and write to files
-	groupedByElementRarityType := groupCardsByElementRarityType(allCards)
-	for key, cards := range groupedByElementRarityType {
+	groupedByElementRarityType := groupCardsByElementRarityType(uniqueCards)
+	for key, keyCards := range groupedByElementRarityType {
 		filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key))
-		writeCardsToFile(filename, cards)
+		writeCardsToFile(filename, keyCards)
 	}
 
-	fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(allCards))
+	fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(uniqueCards))
+}
+
+// removeDuplicateCards removes duplicate cards by checking the 'id' field.
+func removeDuplicateCards(cards []*Card) []*Card {
+	cardMap := make(map[string]*Card)
+	for _, card := range cards {
+		cardMap[card.ID] = card
+	}
+
+	uniqueCards := make([]*Card, 0, len(cardMap))
+	for _, card := range cardMap {
+		uniqueCards = append(uniqueCards, card)
+	}
+
+	return uniqueCards
 }
 
 // setHeaders sets the necessary headers for the HTTP request.
@@ -277,27 +321,9 @@ func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
 	grouped := make(map[string][]*Card)
 	for _, card := range cards {
 		for _, element := range card.Elements {
-			key := fmt.Sprintf("%s_%s_%s", element.Name, strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
+			key := fmt.Sprintf("%s_%s_%s", strings.ToLower(element.Name), strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
 			grouped[key] = append(grouped[key], card)
 		}
 	}
 	return grouped
 }
-
-// getImageFilename constructs a human-readable image filename from the slug.
-func getImageFilename(slug string) string {
-	parts := strings.Split(slug, "_")
-	name := strings.Join(parts[1:len(parts)-1], "_")
-	variant := parts[len(parts)-1]
-
-	switch variant {
-	case "s":
-		return fmt.Sprintf("%s_standard.jpg", name)
-	case "f":
-		return fmt.Sprintf("%s_foil.jpg", name)
-	case "p":
-		return fmt.Sprintf("%s_promo.jpg", name)
-	default:
-		return fmt.Sprintf("%s.jpg", name)
-	}
-}