pixa/internal/imgcache/cache_usage_internal_test.go

package imgcache

import (
	"bytes"
	"strconv"
	"strings"
	"testing"

	"sneak.berlin/go/pixa/internal/imagerequest"
)

// TestEvictionReadsTheUsageTotal adds, stores again, resizes and evicts
// cache content, then drops both content tables. UsageBytes must still
// report what the tables held, and an eviction pass under the limit must
// still succeed: neither may sum the tables.
func TestEvictionReadsTheUsageTotal(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)
	ctx := t.Context()

	kept := storeEvictionTestSource(t, cache, "total.example.com", "/kept.jpg",
		bytes.Repeat([]byte{0x71}, 1000))
	evicted := storeEvictionTestSource(t, cache, "total.example.com", "/evicted.jpg",
		bytes.Repeat([]byte{0x72}, 700))

	storeEvictionTestVariant(t, cache, testVariantKeyOne,
		bytes.Repeat([]byte{0x73}, 500))
	storeEvictionTestVariant(t, cache, testVariantKeyOne,
		bytes.Repeat([]byte{0x74}, 300))
	storeEvictionTestVariant(t, cache, testVariantKeyTwo,
		bytes.Repeat([]byte{0x75}, 200))

	// pixa never changes a source's size, but a statement that does must
	// change the total too.
	_, err := cache.db.ExecContext(ctx,
		`UPDATE source_content SET size_bytes = 900 WHERE source_key = ?`,
		string(kept))
	if err != nil {
		t.Fatalf("failed to change the source's size: %v", err)
	}

	err = cache.evictSourceBlob(ctx, evicted)
	if err != nil {
		t.Fatalf("evictSourceBlob failed: %v", err)
	}

	err = cache.evictVariant(ctx, testVariantKeyTwo)
	if err != nil {
		t.Fatalf("evictVariant failed: %v", err)
	}

	_, err = cache.db.ExecContext(ctx,
		`DROP TABLE source_content; DROP TABLE variant_content`)
	if err != nil {
		t.Fatalf("failed to drop the content tables: %v", err)
	}

	usage, err := cache.UsageBytes(ctx)
	t.Logf("UsageBytes() = %d, error = %v", usage, err)

	if err != nil {
		t.Fatalf("UsageBytes() error = %v, want nil: it read a content table", err)
	}

	if usage != 1200 {
		t.Errorf("UsageBytes() = %d, want 1200 (900 + 300)", usage)
	}

	err = cache.EvictToLimit(ctx)
	if err != nil {
		t.Errorf("EvictToLimit() error = %v, want nil: it read a content table", err)
	}
}

// TestReconciliationCorrectsTheUsageTotal sets the total cache usage to a
// wrong value and checks that a reconciliation pass, which sums both
// content tables, puts it right.
func TestReconciliationCorrectsTheUsageTotal(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)
	ctx := t.Context()

	storeEvictionTestSource(t, cache, "total.example.com", "/a.jpg",
		bytes.Repeat([]byte{0x76}, 1000))
	storeEvictionTestVariant(t, cache, testVariantKeyOne,
		bytes.Repeat([]byte{0x77}, 500))

	_, err := cache.db.ExecContext(ctx,
		`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
	if err != nil {
		t.Fatalf("failed to set a wrong total: %v", err)
	}

	err = cache.reconcileAccounting(ctx)
	if err != nil {
		t.Fatalf("reconcileAccounting failed: %v", err)
	}

	usage, err := cache.UsageBytes(ctx)
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 1500 {
		t.Errorf("UsageBytes() after reconciliation = %d, want 1500 (1000 + 500)",
			usage)
	}
}

// TestUsageTotalNotCorrectedFromAnOlderSum stores a variant after the
// change count was read, then offers a sum taken before the store as the
// correction: the total must keep the stored variant, as that sum may
// have missed it.
func TestUsageTotalNotCorrectedFromAnOlderSum(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)
	ctx := t.Context()

	var changeCount int64

	err := cache.db.QueryRowContext(ctx,
		`SELECT change_count FROM cache_usage WHERE id = 1`,
	).Scan(&changeCount)
	if err != nil {
		t.Fatalf("failed to read the change count: %v", err)
	}

	storeEvictionTestVariant(t, cache, testVariantKeyOne,
		bytes.Repeat([]byte{0x78}, 500))

	corrected, err := cache.correctUsageTotal(ctx, 0, changeCount)
	if err != nil {
		t.Fatalf("correctUsageTotal failed: %v", err)
	}

	if corrected {
		t.Error("correctUsageTotal() = true, want false: content changed since the sum")
	}

	usage, err := cache.UsageBytes(ctx)
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 500 {
		t.Errorf("UsageBytes() = %d, want 500", usage)
	}
}

// TestReconciliationReadsAPageAtATime stores five source images and five
// variants, sets the page size to two rows and runs a reconciliation
// pass. No read the pass makes of a content table, to check its rows or
// to sum them, may cover more than two rows, so a request's query waits
// for one page at most. Between them the reads must still cover every
// variant row twice, once to check it and once to sum it, and every
// source row once, as source rows are read only to sum them, and the sum
// must put a wrong total right.
func TestReconciliationReadsAPageAtATime(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)
	ctx := t.Context()

	const pageSize = 2

	cache.reconciliationPageSize = pageSize

	// Sources of 100 bytes and variants of 10, five of each.
	for i := range 5 {
		content := []byte(strconv.Itoa(i))

		storeEvictionTestSource(t, cache, "pages.example.com",
			"/"+strconv.Itoa(i)+".jpg", bytes.Repeat(content, 100))
		storeEvictionTestVariant(t, cache,
			imagerequest.VariantKey("aabbccdd000"+strconv.Itoa(i)),
			bytes.Repeat(content, 10))
	}

	_, err := cache.db.ExecContext(ctx,
		`UPDATE cache_usage SET total_size_bytes = 1 WHERE id = 1`)
	if err != nil {
		t.Fatalf("failed to set a wrong total: %v", err)
	}

	var rowsPerRead []int

	cache.reconciliationReadTestHook = func(rows int) {
		rowsPerRead = append(rowsPerRead, rows)
	}

	err = cache.reconcileAccounting(ctx)
	if err != nil {
		t.Fatalf("reconcileAccounting failed: %v", err)
	}

	t.Logf("rows covered by each read, in order: %v", rowsPerRead)

	coveredRows := 0

	for _, rows := range rowsPerRead {
		if rows > pageSize {
			t.Errorf("a read covered %d rows, want at most %d", rows, pageSize)
		}

		coveredRows += rows
	}

	// Five variant rows, each read once to check it and once to sum it,
	// and five source rows, read only to sum them.
	if coveredRows != 15 {
		t.Errorf("the reads covered %d rows in all, want 15", coveredRows)
	}

	usage, err := cache.UsageBytes(ctx)
	if err != nil {
		t.Fatalf("UsageBytes failed: %v", err)
	}

	if usage != 550 {
		t.Errorf("UsageBytes() after reconciliation = %d, want 550 (5*100 + 5*10)",
			usage)
	}
}

// TestSourceEvictionCandidatesComeFromTheIndex checks that the query
// choosing source images to evict reads source_content in last access
// order from its index, instead of reading and sorting the whole table.
func TestSourceEvictionCandidatesComeFromTheIndex(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)

	rows, err := cache.db.QueryContext(t.Context(),
		`EXPLAIN QUERY PLAN `+sourceCandidatesQuery, evictionBatchSize)
	if err != nil {
		t.Fatalf("EXPLAIN QUERY PLAN failed: %v", err)
	}

	defer func() { _ = rows.Close() }()

	var plan []string

	for rows.Next() {
		var id, parent, unused int

		var detail string

		err := rows.Scan(&id, &parent, &unused, &detail)
		if err != nil {
			t.Fatalf("failed to scan the query plan: %v", err)
		}

		plan = append(plan, detail)
	}

	err = rows.Err()
	if err != nil {
		t.Fatalf("query plan iteration failed: %v", err)
	}

	t.Logf("query plan: %q", plan)

	if !strings.Contains(strings.Join(plan, "\n"), "idx_source_content_last_accessed") {
		t.Errorf("the source candidate query does not use "+
			"idx_source_content_last_accessed; plan: %q", plan)
	}
}