package main import ( "encoding/json" "fmt" "io" "net/http" "net/url" "os" "path/filepath" "strings" "time" humanize "github.com/dustin/go-humanize" ) // fetchCards fetches cards from the API using the provided limit, cursor, and set type. func fetchCards(limit, cursor int, setType string) ([]*Card, error) { // Define the input data structure input := Input{ Part0: struct { JSON JSONPayload `json:"json"` }{ JSON: JSONPayload{ Query: "", Sort: "relevance", Set: setType, Filters: []interface{}{}, Limit: limit, VariantLimit: false, CollectionLimit: false, Cursor: cursor, Direction: "forward", }, }, } // Serialize the input data structure to JSON jsonData, err := json.Marshal(input) if err != nil { return nil, fmt.Errorf("failed to marshal input data: %w", err) } // URL encode the JSON data urlEncodedInput := url.QueryEscape(string(jsonData)) // Construct the full URL apiURL := fmt.Sprintf("https://curiosa.io/api/trpc/card.search?batch=1&input=%s", urlEncodedInput) // Create a new HTTP request req, err := http.NewRequest("GET", apiURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } // Set essential headers setHeaders(req) // Perform the HTTP request start := time.Now() client := &http.Client{} resp, err := client.Do(req) duration := time.Since(start) if err != nil { return nil, fmt.Errorf("failed to perform request: %w", err) } defer resp.Body.Close() // Check for non-200 status codes if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("non-200 response: %s", resp.Status) } // Read and decode the JSON response var result []struct { Result struct { Data struct { JSON struct { Cards []*Card `json:"cards"` } `json:"json"` } `json:"data"` } `json:"result"` } err = json.NewDecoder(resp.Body).Decode(&result) if err != nil { return nil, fmt.Errorf("failed to decode response body: %w", err) } // Extract cards from the result var cards []*Card if len(result) > 0 { cards = result[0].Result.Data.JSON.Cards } fmt.Printf("Cursor: %d, Limit: %d, Duration: %s, Status: %d, Cards: %d\n", cursor, limit, duration, resp.StatusCode, len(cards)) return cards, nil } // fetchAllCards fetches all cards by making multiple requests with different cursor values. func fetchAllCards(limit int, setType string) ([]*Card, error) { var allCards []*Card cursor := 0 for { cards, err := fetchCards(limit, cursor, setType) if err != nil { return nil, err } allCards = append(allCards, cards...) // If the number of returned cards is less than the limit, we've fetched all cards. if len(cards) < limit { break } // Update cursor for the next batch cursor += limit } return allCards, nil } // generateImageFilename generates the image filename for each card variant. func generateImageFilename(variant Variant) string { setName := strings.ToLower(variant.SetCard.SetDetails.Name) if setName == "" { setName = "other" } ext := filepath.Ext(variant.Src) if ext == "" { ext = ".png" // Default to .png if no extension found } return fmt.Sprintf("./images/%s/%s.%s%s", setName, strings.ToLower(variant.ID), strings.ToLower(variant.Slug), ext) } // downloadImage downloads an image from the given URL and saves it to the specified path. func downloadImage(url, filepath string) error { if _, err := os.Stat(filepath); err == nil { fmt.Printf("Skipping download, file already exists: %s\n", filepath) return nil } tempFilePath := filepath + ".tmp" start := time.Now() resp, err := http.Get(url) duration := time.Since(start) if err != nil { return fmt.Errorf("failed to fetch image: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("non-200 response: %s", resp.Status) } file, err := os.Create(tempFilePath) if err != nil { return fmt.Errorf("failed to create image file: %w", err) } defer file.Close() bodySize, err := io.Copy(file, resp.Body) if err != nil { return fmt.Errorf("failed to save image: %w", err) } err = os.Rename(tempFilePath, filepath) if err != nil { return fmt.Errorf("failed to rename temp file: %w", err) } fmt.Printf("Downloaded %s | Status: %d | Size: %s | Duration: %s\n", filepath, resp.StatusCode, humanize.Bytes(uint64(bodySize)), duration) return nil } func main() { const limit = 100 const dataDir = "./data" const imageDir = "./images" // Ensure the data and images directories exist err := os.MkdirAll(dataDir, os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create data directory: %w", err)) } err = os.MkdirAll(imageDir, os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create images directory: %w", err)) } // Create foils and promos directories inside alpha and beta directories for _, setType := range []string{"alpha", "beta"} { err = os.MkdirAll(filepath.Join(imageDir, setType, "foils"), os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create foils directory: %w", err)) } err = os.MkdirAll(filepath.Join(imageDir, setType, "promos"), os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create promos directory: %w", err)) } } // Fetch all cards for both alpha and beta sets alphaCards, err := fetchAllCards(limit, "alp") if err != nil { logErrorAndExit(err) } betaCards, err := fetchAllCards(limit, "bet") if err != nil { logErrorAndExit(err) } // Save alpha and beta cards separately and write combined cards.json allCards := append(alphaCards, betaCards...) saveCardsAsJSON(alphaCards, "alpha", dataDir) saveCardsAsJSON(betaCards, "beta", dataDir) saveCardsAsJSON(allCards, "all", dataDir) fetchImagesForCards(allCards, imageDir) } // saveCardsAsJSON saves all cards for the given set type to JSON files. func saveCardsAsJSON(cards []*Card, filePrefix, dataDir string) { // Remove duplicates uniqueCards := removeDuplicateCards(cards) // Write all cards to cards.json in the data directory filename := fmt.Sprintf("%s/%s_cards.json", dataDir, filePrefix) writeCardsToFile(filename, uniqueCards) // Group cards by element and write to files groupedByElement := groupCardsByElement(uniqueCards) for element, elementCards := range groupedByElement { filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(element)) writeCardsToFile(filename, elementCards) } // Group cards by element, rarity, and type and write to files groupedByElementRarityType := groupCardsByElementRarityType(uniqueCards) for key, keyCards := range groupedByElementRarityType { filename := fmt.Sprintf("%s/%s_%s_cards.json", dataDir, filePrefix, strings.ToLower(key)) writeCardsToFile(filename, keyCards) } fmt.Printf("All cards have been written to %s_cards.json. Total cards: %d\n", filePrefix, len(uniqueCards)) } // fetchImagesForCards downloads images for each card variant and saves them to the appropriate directory. func fetchImagesForCards(cards []*Card, imageDir string) { for _, card := range cards { for _, variant := range card.Variants { imagePath := generateImageFilename(variant) err := os.MkdirAll(filepath.Dir(imagePath), os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create image directory: %w", err)) } // Download the image err = downloadImage(variant.Src, imagePath) if err != nil { fmt.Fprintf(os.Stderr, "Failed to download image %s: %v\n", imagePath, err) continue } // Determine set type setType := strings.ToLower(variant.SetCard.SetDetails.Name) if setType == "" { setType = "other" } // Ensure foils and promos directories exist err = os.MkdirAll(filepath.Join(imageDir, setType, "foils"), os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create foils directory: %w", err)) } err = os.MkdirAll(filepath.Join(imageDir, setType, "promos"), os.ModePerm) if err != nil { logErrorAndExit(fmt.Errorf("failed to create promos directory: %w", err)) } // Hardlink to foils directory if the variant is a foil if variant.Finish == "Foil" { linkPath := filepath.Join(imageDir, setType, "foils", filepath.Base(imagePath)) err := createHardlink(imagePath, linkPath) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create hardlink %s: %v\n", linkPath, err) } } // Hardlink to promos directory if the variant is a promo if variant.Product == "Promo" { linkPath := filepath.Join(imageDir, setType, "promos", filepath.Base(imagePath)) err := createHardlink(imagePath, linkPath) if err != nil { fmt.Fprintf(os.Stderr, "Failed to create hardlink %s: %v\n", linkPath, err) } } } } } // createHardlink creates a hardlink from src to dst, if dst does not already exist. func createHardlink(src, dst string) error { if _, err := os.Stat(dst); err == nil { return nil // Hardlink already exists } return os.Link(src, dst) } // removeDuplicateCards removes duplicate cards by checking the 'id' field. func removeDuplicateCards(cards []*Card) []*Card { cardMap := make(map[string]*Card) for _, card := range cards { cardMap[card.ID] = card } uniqueCards := make([]*Card, 0, len(cardMap)) for _, card := range cardMap { uniqueCards = append(uniqueCards, card) } return uniqueCards } // setHeaders sets the necessary headers for the HTTP request. func setHeaders(req *http.Request) { req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0") req.Header.Set("Accept", "*/*") req.Header.Set("content-type", "application/json") } // logErrorAndExit logs the error to stderr and exits with a non-zero status code. func logErrorAndExit(err error) { fmt.Fprintln(os.Stderr, err) os.Exit(1) } // writeCardsToFile writes the given cards to a JSON file with the given filename. func writeCardsToFile(filename string, cards []*Card) { file, err := os.Create(filename) if err != nil { logErrorAndExit(fmt.Errorf("failed to create %s: %w", filename, err)) } defer file.Close() prettyData, err := json.MarshalIndent(cards, "", " ") if err != nil { logErrorAndExit(fmt.Errorf("failed to marshal pretty JSON: %w", err)) } _, err = file.Write(prettyData) if err != nil { logErrorAndExit(fmt.Errorf("failed to write to %s: %w", filename, err)) } fmt.Printf("%s has been written. Total cards: %d\n", filename, len(cards)) } // groupCardsByElement groups the given cards by their element. func groupCardsByElement(cards []*Card) map[string][]*Card { grouped := make(map[string][]*Card) for _, card := range cards { for _, element := range card.Elements { grouped[strings.ToLower(element.Name)] = append(grouped[strings.ToLower(element.Name)], card) } } return grouped } // groupCardsByElementRarityType groups the given cards by their element, rarity, and type. func groupCardsByElementRarityType(cards []*Card) map[string][]*Card { grouped := make(map[string][]*Card) for _, card := range cards { for _, element := range card.Elements { key := fmt.Sprintf("%s_%s_%s", strings.ToLower(element.Name), strings.ToLower(card.Guardian.Rarity), strings.ToLower(card.Guardian.Type)) grouped[key] = append(grouped[key], card) } } return grouped }