pixa/internal/imgcache/partial_cache_files_internal_test.go

package imgcache

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// getImage calls Get for req and returns the image it answered with and
// its cache status.
func getImage(t *testing.T, svc *Service, req *ImageRequest) ([]byte, CacheStatus) {
	t.Helper()

	resp, err := svc.Get(t.Context(), req)
	if err != nil {
		t.Fatalf("Get() of quality %d error = %v", req.Quality, err)
	}

	defer func() { _ = resp.Content.Close() }()

	body, err := io.ReadAll(resp.Content)
	if err != nil {
		t.Fatalf("reading the image of quality %d: %v", req.Quality, err)
	}

	return body, resp.CacheStatus
}

// fileSize returns the size of the file at path.
func fileSize(t *testing.T, path string) int64 {
	t.Helper()

	info, err := os.Stat(path)
	if err != nil {
		t.Fatalf("stat %s: %v", path, err)
	}

	return info.Size()
}

// truncateFile cuts the file at path down to size bytes.
func truncateFile(t *testing.T, path string, size int64) {
	t.Helper()

	t.Logf("cutting %s from %d to %d bytes",
		filepath.Base(path), fileSize(t, path), size)

	err := os.Truncate(path, size)
	if err != nil {
		t.Fatalf("truncating %s: %v", path, err)
	}
}

// cachedSourcePath returns the path of the cached source file req uses.
func cachedSourcePath(t *testing.T, svc *Service, req *ImageRequest) string {
	t.Helper()

	contentHash, _, err := svc.cache.LookupSource(t.Context(), req)
	if err != nil || contentHash == "" {
		t.Fatalf("LookupSource() = %q, %v; want the cached source", contentHash, err)
	}

	return svc.cache.srcContent.hashToPath(contentHash)
}

// reconcileAsAtStartup runs the cache's reconciliation pass, as pixa does
// when it starts, and returns what the pass logged.
func reconcileAsAtStartup(t *testing.T, cache *Cache) string {
	t.Helper()

	var logBuf bytes.Buffer

	cache.log = slog.New(slog.NewJSONHandler(&logBuf, nil))

	err := cache.reconcileAccounting(t.Context())
	if err != nil {
		t.Fatalf("reconcileAccounting() error = %v", err)
	}

	t.Logf("reconciliation log: %s", logBuf.String())

	return logBuf.String()
}

// assertWarningNames checks that logs hold a warning naming the file at
// path.
func assertWarningNames(t *testing.T, logs, path string) {
	t.Helper()

	for line := range strings.Lines(logs) {
		if strings.Contains(line, `"level":"WARN"`) &&
			strings.Contains(line, `"path":"`+path+`"`) {
			return
		}
	}

	t.Errorf("reconciliation logged no warning naming %s", path)
}

// TestService_Get_FetchesAgainForEmptyCachedSource empties the cached source
// file, as a crash or a restored snapshot could, and runs the reconciliation
// pass, which must remove it with a warning naming it. It then asks for two
// new sizes of the source: the first must fetch it again and cache it whole,
// so that the second is made from the cache without a third fetch.
func TestService_Get_FetchesAgainForEmptyCachedSource(t *testing.T) {
	t.Parallel()

	svc, fixtures := SetupTestService(t)

	getImage(t, svc, photoVariant(fixtures, 85, FitCover))

	path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover))
	wholeSize := fileSize(t, path)

	truncateFile(t, path, 0)
	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)

	getImage(t, svc, photoVariant(fixtures, 40, FitCover))
	getImage(t, svc, photoVariant(fixtures, 30, FitCover))

	fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount
	t.Logf("upstream fetches = %d", fetches)

	if fetches != 2 {
		t.Errorf("upstream fetches = %d, want 2: the empty cached source "+
			"was kept and fetched again on every new size", fetches)
	}

	size := fileSize(t, path)
	if size != wholeSize {
		t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
	}
}

// TestService_Get_FetchesAgainForPartialCachedSource cuts the cached source
// file to half its size, as a crash or a restored snapshot could, and runs
// the reconciliation pass, which must remove it with a warning naming it.
// It then asks for two new sizes of the source: the first must fetch it
// again rather than fail to decode the partial file, and the second is
// made from the cache.
func TestService_Get_FetchesAgainForPartialCachedSource(t *testing.T) {
	t.Parallel()

	svc, fixtures := SetupTestService(t)

	getImage(t, svc, photoVariant(fixtures, 85, FitCover))

	path := cachedSourcePath(t, svc, photoVariant(fixtures, 85, FitCover))
	wholeSize := fileSize(t, path)

	truncateFile(t, path, wholeSize/2)
	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)

	getImage(t, svc, photoVariant(fixtures, 40, FitCover))
	getImage(t, svc, photoVariant(fixtures, 30, FitCover))

	fetches := readCacheStatsCounters(t, svc.cache).upstreamFetchCount
	t.Logf("upstream fetches = %d", fetches)

	if fetches != 2 {
		t.Errorf("upstream fetches = %d, want 2", fetches)
	}

	size := fileSize(t, path)
	if size != wholeSize {
		t.Errorf("cached source is %d bytes, want %d", size, wholeSize)
	}
}

// assertConvertedAgain caches a variant of the photo, cuts its file to size
// bytes and runs the reconciliation pass, which must remove it with a
// warning naming it. It then checks that Get converts the image again
// instead of serving the file, and that the next Get serves the whole
// variant from the cache.
func assertConvertedAgain(t *testing.T, size int64) {
	t.Helper()

	svc, fixtures := SetupTestService(t)
	req := photoVariant(fixtures, 85, FitCover)

	want, _ := getImage(t, svc, req)

	path := svc.cache.variants.keyToPath(CacheKey(req))

	truncateFile(t, path, size)
	assertWarningNames(t, reconcileAsAtStartup(t, svc.cache), path)

	got, status := getImage(t, svc, req)
	t.Logf("after the cut: %d bytes, %s", len(got), status)

	if status != CacheMiss || !bytes.Equal(got, want) {
		t.Errorf("after the cut: Get() = %d bytes, %s; want the %d bytes "+
			"converted again, %s", len(got), status, len(want), CacheMiss)
	}

	got, status = getImage(t, svc, req)
	t.Logf("next request: %d bytes, %s", len(got), status)

	if status != CacheHit || !bytes.Equal(got, want) {
		t.Errorf("next request: Get() = %d bytes, %s; want the %d bytes, %s",
			len(got), status, len(want), CacheHit)
	}
}

// TestService_Get_ConvertsAgainForEmptyCachedVariant empties a cached
// variant file, as a crash or a restored snapshot could, and checks that it
// is converted again rather than served empty.
func TestService_Get_ConvertsAgainForEmptyCachedVariant(t *testing.T) {
	t.Parallel()

	assertConvertedAgain(t, 0)
}

// TestService_Get_ConvertsAgainForPartialCachedVariant cuts a cached
// variant file to ten bytes, as a crash or a restored snapshot could, and
// checks that it is converted again rather than served cut short.
func TestService_Get_ConvertsAgainForPartialCachedVariant(t *testing.T) {
	t.Parallel()

	assertConvertedAgain(t, 10)
}

// TestReconciliationRemovesEmptyVariantFileWithoutRow checks that the
// reconciliation pass removes an empty variant file that has no
// accounting row, with a warning naming it, instead of adding a row for
// it that would have it served empty.
func TestReconciliationRemovesEmptyVariantFileWithoutRow(t *testing.T) {
	t.Parallel()

	cache, _ := newEvictionTestCache(t, 1<<30)

	_, err := cache.variants.Store(testVariantKeyOne, bytes.NewReader(nil),
		testContentTypeWebP)
	if err != nil {
		t.Fatalf("failed to store an empty variant file: %v", err)
	}

	logs := reconcileAsAtStartup(t, cache)

	assertWarningNames(t, logs, cache.variants.keyToPath(testVariantKeyOne))

	if cache.variants.Exists(testVariantKeyOne) {
		t.Error("the empty variant file is still on disk")
	}

	rows := countRows(t, cache,
		`SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`,
		string(testVariantKeyOne))
	if rows != 0 {
		t.Errorf("the empty variant file got %d accounting rows, want 0", rows)
	}
}

// TestVariantStorage_StoreReplacesMetaFile stores a variant over one
// already stored and checks that Store puts a new .meta file in place of
// the old one, as it does the image, rather than writing into the old
// file, which a crash could leave empty. The old .meta file is given a
// second name before the store, so that it can be told apart from the new
// one after it.
func TestVariantStorage_StoreReplacesMetaFile(t *testing.T) {
	t.Parallel()

	storage, key := storeTestVariant(t)
	metaPath := storage.keyToPath(key) + variantMetaSuffix
	oldMetaPath := metaPath + ".old"

	err := os.Link(metaPath, oldMetaPath)
	if err != nil {
		t.Fatalf("giving the .meta file a second name: %v", err)
	}

	_, err = storage.Store(key, bytes.NewReader([]byte("new variant data")),
		testContentTypeJPEG)
	if err != nil {
		t.Fatalf("Store() error = %v", err)
	}

	oldMeta, err := os.Stat(oldMetaPath)
	if err != nil {
		t.Fatalf("stat of the old .meta file: %v", err)
	}

	newMeta, err := os.Stat(metaPath)
	if err != nil {
		t.Fatalf("stat of the new .meta file: %v", err)
	}

	if os.SameFile(oldMeta, newMeta) {
		t.Error("Store() wrote into the old .meta file instead of putting " +
			"a new one in its place")
	}

	reader, _, contentType, err := storage.LoadWithMeta(key)
	if err != nil {
		t.Fatalf("LoadWithMeta() error = %v", err)
	}

	_ = reader.Close()

	t.Logf("content type after the store: %s", contentType)

	if contentType != testContentTypeJPEG {
		t.Errorf("content type = %q, want %q", contentType, testContentTypeJPEG)
	}
}

// TestContentStorage_StoreHashedReplacesFileOfAnotherSize checks that a
// store over a file of another size at the content's path, such as a
// partial file a crash left, replaces it with the content.
func TestContentStorage_StoreHashedReplacesFileOfAnotherSize(t *testing.T) {
	t.Parallel()

	storage, err := NewContentStorage(t.TempDir())
	if err != nil {
		t.Fatalf("NewContentStorage() error = %v", err)
	}

	content := []byte("the whole source image")
	sum := sha256.Sum256(content)
	hash := ContentHash(hex.EncodeToString(sum[:]))
	path := storage.hashToPath(hash)

	err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
	if err != nil {
		t.Fatalf("creating the content's directory: %v", err)
	}

	err = os.WriteFile(path, content[:5], StorageFilePerm)
	if err != nil {
		t.Fatalf("writing a partial file: %v", err)
	}

	_, err = storage.StoreHashed(hash, content)
	if err != nil {
		t.Fatalf("StoreHashed() error = %v", err)
	}

	stored, err := storage.Load(hash)
	if err != nil {
		t.Fatalf("Load() error = %v", err)
	}

	defer func() { _ = stored.Close() }()

	got, err := io.ReadAll(stored)
	if err != nil {
		t.Fatalf("reading the stored content: %v", err)
	}

	if !bytes.Equal(got, content) {
		t.Errorf("stored file = %q, want %q", got, content)
	}
}