scrcards/main.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
)

// fetchCards fetches cards from the API using the provided limit, cursor, and set type.
func fetchCards(limit, cursor int, setType string) ([]*Card, error) {
	// Define the input data structure
	input := Input{
		Part0: struct {
			JSON JSONPayload `json:"json"`
		}{
			JSON: JSONPayload{
				Query:           "",
				Sort:            "relevance",
				Set:             setType,
				Filters:         []interface{}{},
				Limit:           limit,
				VariantLimit:    false,
				CollectionLimit: false,
				Cursor:          cursor,
				Direction:       "forward",
			},
		},
	}

	// Serialize the input data structure to JSON
	jsonData, err := json.Marshal(input)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal input data: %w", err)
	}

	// URL encode the JSON data
	urlEncodedInput := url.QueryEscape(string(jsonData))

	// Construct the full URL
	apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search?batch=1&input=%s", urlEncodedInput)

	// Create a new HTTP request
	req, err := http.NewRequest("GET", apiURL, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	// Set essential headers
	setHeaders(req)

	// Perform the HTTP request
	start := time.Now()
	client := &http.Client{}
	resp, err := client.Do(req)
	duration := time.Since(start)
	if err != nil {
		return nil, fmt.Errorf("failed to perform request: %w", err)
	}
	defer resp.Body.Close()

	// Check for non-200 status codes
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("non-200 response: %s", resp.Status)
	}

	// Read and decode the JSON response
	var result []struct {
		Result struct {
			Data struct {
				JSON struct {
					Cards []*Card `json:"cards"`
				} `json:"json"`
			} `json:"data"`
		} `json:"result"`
	}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		return nil, fmt.Errorf("failed to decode response body: %w", err)
	}

	// Extract cards from the result
	var cards []*Card
	if len(result) > 0 {
		cards = result[0].Result.Data.JSON.Cards
	}

	fmt.Printf("Cursor: %d, Limit: %d, Duration: %s, Status: %d, Cards: %d\n",
		cursor, limit, duration, resp.StatusCode, len(cards))

	return cards, nil
}

// fetchAllCards fetches all cards by making multiple requests with different cursor values.
func fetchAllCards(limit int, setType string) ([]*Card, error) {
	var allCards []*Card
	cursor := 0

	for {
		cards, err := fetchCards(limit, cursor, setType)
		if err != nil {
			return nil, err
		}

		allCards = append(allCards, cards...)

		// If the number of returned cards is less than the limit, we've fetched all cards.
		if len(cards) < limit {
			break
		}

		// Update cursor for the next batch
		cursor += limit
	}

	return allCards, nil
}

// generateImageFilename generates the image filename for each card variant.
func generateImageFilename(variant Variant) string {
	setName := strings.ToLower(variant.SetCard.SetDetails.Name)
	if setName == "" {
		setName = "other"
	}
	ext := filepath.Ext(variant.Src)
	if ext == "" {
		ext = ".png" // Default to .png if no extension found
	}
	return fmt.Sprintf("./images/%s/%s.%s%s", setName, strings.ToLower(variant.ID), strings.ToLower(variant.Slug), ext)
}

// downloadImage downloads an image from the given URL and saves it to the specified path.
func downloadImage(url, filepath string) error {
	if _, err := os.Stat(filepath); err == nil {
		fmt.Printf("Skipping download, file already exists: %s\n", filepath)
		return nil
	}

	tempFilePath := filepath + ".tmp"

	start := time.Now()
	resp, err := http.Get(url)
	duration := time.Since(start)
	if err != nil {
		return fmt.Errorf("failed to fetch image: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("non-200 response: %s", resp.Status)
	}

	file, err := os.Create(tempFilePath)
	if err != nil {
		return fmt.Errorf("failed to create image file: %w", err)
	}
	defer file.Close()

	bodySize, err := io.Copy(file, resp.Body)
	if err != nil {
		return fmt.Errorf("failed to save image: %w", err)
	}

	err = os.Rename(tempFilePath, filepath)
	if err != nil {
		return fmt.Errorf("failed to rename temp file: %w", err)
	}

	fmt.Printf("Downloaded %s | Status: %d | Size: %s | Duration: %s\n",
		filepath, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration)
	return nil
}

func main() {
	const limit = 100
	const dataDir = "./data"
	const imageDir = "./images"

	// Ensure the data and images directories exist
	err := os.MkdirAll(dataDir, os.ModePerm)
	if err != nil {
		logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err))
	}

	err = os.MkdirAll(imageDir, os.ModePerm)
	if err != nil {
		logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err))
	}

	// Create foils and promos directories inside alpha and beta directories
	for _, setType := range []string{"alpha", "beta"} {
		err = os.MkdirAll(filepath.Join(imageDir, setType, "foils"), os.ModePerm)
		if err != nil {
			logErrorAndExit(fmt.Errorf("failed to create foils directory: %w", err))
		}
		err = os.MkdirAll(filepath.Join(imageDir, setType, "promos"), os.ModePerm)
		if err != nil {
			logErrorAndExit(fmt.Errorf("failed to create promos directory: %w", err))
		}
	}

	// Fetch all cards for both alpha and beta sets
	alphaCards, err := fetchAllCards(limit, "alp")
	if err != nil {
		logErrorAndExit(err)
	}

	betaCards, err := fetchAllCards(limit, "bet")
	if err != nil {
		logErrorAndExit(err)
	}

	// Save alpha and beta cards separately and write combined cards.json
	allCards := append(alphaCards, betaCards...)
	saveCardsAsJSON(alphaCards, "alpha", dataDir)
	saveCardsAsJSON(betaCards, "beta", dataDir)
	saveCardsAsJSON(allCards, "all", dataDir)
	fetchImagesForCards(allCards, imageDir)
}

// saveCardsAsJSON saves all cards for the given set type to JSON files.
func saveCardsAsJSON(cards []*Card, filePrefix, dataDir string) {
	// Remove duplicates
	uniqueCards := removeDuplicateCards(cards)

	// Write all cards to cards.json in the data directory
	filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix)
	writeCardsToFile(filename, uniqueCards)

	// Group cards by element and write to files
	groupedByElement := groupCardsByElement(uniqueCards)
	for element, elementCards := range groupedByElement {
		filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element))
		writeCardsToFile(filename, elementCards)
	}

	// Group cards by element, rarity, and type and write to files
	groupedByElementRarityType := groupCardsByElementRarityType(uniqueCards)
	for key, keyCards := range groupedByElementRarityType {
		filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key))
		writeCardsToFile(filename, keyCards)
	}

	fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(uniqueCards))
}

// fetchImagesForCards downloads images for each card variant and saves them to the appropriate directory.
func fetchImagesForCards(cards []*Card, imageDir string) {
	for _, card := range cards {
		for _, variant := range card.Variants {
			imagePath := generateImageFilename(variant)

			err := os.MkdirAll(filepath.Dir(imagePath), os.ModePerm)
			if err != nil {
				logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err))
			}

			// Download the image
			err = downloadImage(variant.Src, imagePath)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err)
				continue
			}

			// Determine set type
			setType := strings.ToLower(variant.SetCard.SetDetails.Name)
			if setType == "" {
				setType = "other"
			}

			// Ensure foils and promos directories exist
			err = os.MkdirAll(filepath.Join(imageDir, setType, "foils"), os.ModePerm)
			if err != nil {
				logErrorAndExit(fmt.Errorf("failed to create foils directory: %w", err))
			}
			err = os.MkdirAll(filepath.Join(imageDir, setType, "promos"), os.ModePerm)
			if err != nil {
				logErrorAndExit(fmt.Errorf("failed to create promos directory: %w", err))
			}

			// Hardlink to foils directory if the variant is a foil
			if variant.Finish == "Foil" {
				linkPath := filepath.Join(imageDir, setType, "foils", filepath.Base(imagePath))
				err := createHardlink(imagePath, linkPath)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Failed to create hardlink %s: %v\n", linkPath, err)
				}
			}

			// Hardlink to promos directory if the variant is a promo
			if variant.Product == "Promo" {
				linkPath := filepath.Join(imageDir, setType, "promos", filepath.Base(imagePath))
				err := createHardlink(imagePath, linkPath)
				if err != nil {
					fmt.Fprintf(os.Stderr, "Failed to create hardlink %s: %v\n", linkPath, err)
				}
			}
		}
	}
}

// createHardlink creates a hardlink from src to dst, if dst does not already exist.
func createHardlink(src, dst string) error {
	if _, err := os.Stat(dst); err == nil {
		return nil // Hardlink already exists
	}
	return os.Link(src, dst)
}

// removeDuplicateCards removes duplicate cards by checking the 'id' field.
func removeDuplicateCards(cards []*Card) []*Card {
	cardMap := make(map[string]*Card)
	for _, card := range cards {
		cardMap[card.ID] = card
	}

	uniqueCards := make([]*Card, 0, len(cardMap))
	for _, card := range cardMap {
		uniqueCards = append(uniqueCards, card)
	}

	return uniqueCards
}

// setHeaders sets the necessary headers for the HTTP request.
func setHeaders(req *http.Request) {
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0")
	req.Header.Set("Accept", "*/*")
	req.Header.Set("content-type", "application/json")
}

// logErrorAndExit logs the error to stderr and exits with a non-zero status code.
func logErrorAndExit(err error) {
	fmt.Fprintln(os.Stderr, err)
	os.Exit(1)
}

// writeCardsToFile writes the given cards to a JSON file with the given filename.
func writeCardsToFile(filename string, cards []*Card) {
	file, err := os.Create(filename)
	if err != nil {
		logErrorAndExit(fmt.Errorf("failed to create %s: %w", filename, err))
	}
	defer file.Close()

	prettyData, err := json.MarshalIndent(cards, "", "  ")
	if err != nil {
		logErrorAndExit(fmt.Errorf("failed to marshal pretty JSON: %w", err))
	}

	_, err = file.Write(prettyData)
	if err != nil {
		logErrorAndExit(fmt.Errorf("failed to write to %s: %w", filename, err))
	}

	fmt.Printf("%s has been written. Total cards: %d\n", filename, len(cards))
}

// groupCardsByElement groups the given cards by their element.
func groupCardsByElement(cards []*Card) map[string][]*Card {
	grouped := make(map[string][]*Card)
	for _, card := range cards {
		for _, element := range card.Elements {
			grouped[strings.ToLower(element.Name)] = append(grouped[strings.ToLower(element.Name)], card)
		}
	}
	return grouped
}

// groupCardsByElementRarityType groups the given cards by their element, rarity, and type.
func groupCardsByElementRarityType(cards []*Card) map[string][]*Card {
	grouped := make(map[string][]*Card)
	for _, card := range cards {
		for _, element := range card.Elements {
			key := fmt.Sprintf("%s_%s_%s", strings.ToLower(element.Name), strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type))
			grouped[key] = append(grouped[key], card)
		}
	}
	return grouped
}