64 lines
1.2 KiB
Go
64 lines
1.2 KiB
Go
|
package puppeteerapiclient
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"log"
|
||
|
|
"os"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/joho/godotenv"
|
||
|
|
)
|
||
|
|
|
||
|
|
type TestCase struct {
|
||
|
|
URL string
|
||
|
|
Selector string
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestMain(m *testing.M) {
|
||
|
|
err := godotenv.Load()
|
||
|
|
if err != nil {
|
||
|
|
log.Fatalf("Error loading .env file")
|
||
|
|
}
|
||
|
|
|
||
|
|
os.Exit(m.Run())
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestScrape(t *testing.T) {
|
||
|
|
apiURL := os.Getenv("API_URL")
|
||
|
|
apiSalt := os.Getenv("API_SALT")
|
||
|
|
|
||
|
|
if apiURL == "" || apiSalt == "" {
|
||
|
|
t.Fatal("API_URL or API_SALT environment variables not set")
|
||
|
|
}
|
||
|
|
|
||
|
|
client := NewClient(apiURL, apiSalt)
|
||
|
|
|
||
|
|
testCases := []TestCase{
|
||
|
|
{"https://sneak.berlin", "body"},
|
||
|
|
{"https://nytimes.com", "h1"},
|
||
|
|
{"https://sneak.berlin", "#quoteCycler"},
|
||
|
|
{"https://news.ycombinator.com", ".athing .title"},
|
||
|
|
}
|
||
|
|
|
||
|
|
for _, tc := range testCases {
|
||
|
|
t.Run(tc.URL+" "+tc.Selector, func(t *testing.T) {
|
||
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
|
|
defer cancel()
|
||
|
|
|
||
|
|
response, err := client.Scrape(ctx, tc.URL, tc.Selector)
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("Error scraping content: %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
if response.Content == "" {
|
||
|
|
t.Fatalf("No content fetched")
|
||
|
|
}
|
||
|
|
|
||
|
|
t.Logf("Scraped Content: %s", response.Content)
|
||
|
|
t.Logf("URL: %s", response.URL)
|
||
|
|
t.Logf("Selector: %s", response.Selector)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
}
|