pixa/internal/imgcache/storage.go

package imgcache

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"sneak.berlin/go/pixa/internal/imagerequest"
	"sneak.berlin/go/pixa/internal/magic"
)

// Storage constants.
const (
	// StorageDirPerm is the permission mode for storage directories.
	StorageDirPerm = 0750
	// StorageFilePerm is the permission mode for storage files.
	StorageFilePerm = 0600
	// MinHashLength is the minimum hash length for path splitting.
	MinHashLength = 4
)

// Storage errors.
var (
	ErrNotFound = errors.New("content not found")
)

// ContentStorage handles source image file storage, one file per source
// URL. Files are stored at: <basedir>/<ab>/<cd>/<source key>
type ContentStorage struct {
	baseDir string
}

// NewContentStorage creates a new content storage at the given base directory.
func NewContentStorage(baseDir string) (*ContentStorage, error) {
	err := os.MkdirAll(baseDir, StorageDirPerm)
	if err != nil {
		return nil, fmt.Errorf("failed to create storage directory: %w", err)
	}

	return &ContentStorage{baseDir: baseDir}, nil
}

// Store writes data with writeFileSynced to the path derived from key,
// replacing any file already there, and returns its size.
func (s *ContentStorage) Store(
	key imagerequest.SourceKey, data []byte,
) (int64, error) {
	path := s.keyToPath(key)

	err := os.MkdirAll(filepath.Dir(path), StorageDirPerm)
	if err != nil {
		return 0, fmt.Errorf("failed to create directory: %w", err)
	}

	err = writeFileSynced(path, data)
	if err != nil {
		return 0, err
	}

	return int64(len(data)), nil
}

// Load returns a reader for the content with the given key.
func (s *ContentStorage) Load(key imagerequest.SourceKey) (io.ReadCloser, error) {
	path := s.keyToPath(key)

	f, err := os.Open(path) //nolint:gosec // path derived from source key
	if err != nil {
		if os.IsNotExist(err) {
			return nil, ErrNotFound
		}

		return nil, fmt.Errorf("failed to open content: %w", err)
	}

	return f, nil
}

// LoadWithSize returns a reader and file size for the content with the given key.
func (s *ContentStorage) LoadWithSize(
	key imagerequest.SourceKey,
) (io.ReadCloser, int64, error) {
	path := s.keyToPath(key)

	f, err := os.Open(path) //nolint:gosec // path derived from source key
	if err != nil {
		if os.IsNotExist(err) {
			return nil, 0, ErrNotFound
		}

		return nil, 0, fmt.Errorf("failed to open content: %w", err)
	}

	stat, err := f.Stat()
	if err != nil {
		_ = f.Close()

		return nil, 0, fmt.Errorf("failed to stat content: %w", err)
	}

	return f, stat.Size(), nil
}

// Delete removes content with the given key.
func (s *ContentStorage) Delete(key imagerequest.SourceKey) error {
	path := s.keyToPath(key)

	err := os.Remove(path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to delete content: %w", err)
	}

	return nil
}

// Exists checks if content with the given key exists.
func (s *ContentStorage) Exists(key imagerequest.SourceKey) bool {
	path := s.keyToPath(key)
	_, err := os.Stat(path)

	return err == nil
}

// writeFileSynced writes data to path through a temp file in the same
// directory: the temp file is synced to disk, renamed to path, and the
// directory synced after, so that neither a concurrent reader nor a
// crash finds a partial file under path. When the directory sync fails,
// the file is removed from path, so that a failed write leaves no file
// there.
func writeFileSynced(path string, data []byte) error {
	dir := filepath.Dir(path)

	tmpFile, err := os.CreateTemp(dir, ".tmp-*")
	if err != nil {
		return fmt.Errorf("failed to create temp file: %w", err)
	}

	tmpPath := tmpFile.Name()

	_, err = tmpFile.Write(data)
	if err != nil {
		_ = tmpFile.Close()
		_ = os.Remove(tmpPath)

		return fmt.Errorf("failed to write temp file: %w", err)
	}

	err = tmpFile.Sync()
	if err != nil {
		_ = tmpFile.Close()
		_ = os.Remove(tmpPath)

		return fmt.Errorf("failed to sync temp file: %w", err)
	}

	err = tmpFile.Close()
	if err != nil {
		_ = os.Remove(tmpPath)

		return fmt.Errorf("failed to close temp file: %w", err)
	}

	err = os.Rename(filepath.Clean(tmpPath), filepath.Clean(path))
	if err != nil {
		_ = os.Remove(tmpPath)

		return fmt.Errorf("failed to rename temp file: %w", err)
	}

	err = syncDirectory(dir)
	if err != nil {
		_ = os.Remove(path)

		return err
	}

	return nil
}

// syncDirectory syncs the directory dir to disk, so that a file just
// renamed into it is still there after a crash.
func syncDirectory(dir string) error {
	d, err := os.Open(filepath.Clean(dir))
	if err != nil {
		return fmt.Errorf("failed to open directory: %w", err)
	}

	err = d.Sync()
	_ = d.Close()

	if err != nil {
		return fmt.Errorf("failed to sync directory: %w", err)
	}

	return nil
}

// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
func (s *ContentStorage) keyToPath(key imagerequest.SourceKey) string {
	k := string(key)
	if len(k) < MinHashLength {
		return filepath.Clean(filepath.Join(s.baseDir, k))
	}

	return filepath.Clean(filepath.Join(s.baseDir, k[0:2], k[2:4], k))
}

// VariantStorage handles key-based file storage for processed image variants.
// Files are stored at: <basedir>/<ab>/<cd>/<cache_key>
type VariantStorage struct {
	baseDir string
}

// errNotAnOutputFormat is returned for a variant file whose first bytes are
// those of a format pixa reads but never writes, such as SVG.
var errNotAnOutputFormat = errors.New("variant file is not in an output format")

// NewVariantStorage creates a new variant storage at the given base directory.
func NewVariantStorage(baseDir string) (*VariantStorage, error) {
	err := os.MkdirAll(baseDir, StorageDirPerm)
	if err != nil {
		return nil, fmt.Errorf("failed to create variant storage directory: %w", err)
	}

	return &VariantStorage{baseDir: baseDir}, nil
}

// Store writes content to storage at the given key with writeFileSynced.
func (s *VariantStorage) Store(
	key imagerequest.VariantKey, r io.Reader,
) (int64, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return 0, fmt.Errorf("failed to read content: %w", err)
	}

	path := s.keyToPath(key)

	err = os.MkdirAll(filepath.Dir(path), StorageDirPerm)
	if err != nil {
		return 0, fmt.Errorf("failed to create directory: %w", err)
	}

	err = writeFileSynced(path, data)
	if err != nil {
		return 0, err
	}

	return int64(len(data)), nil
}

// Load returns a reader for the content at the given key.
func (s *VariantStorage) Load(key imagerequest.VariantKey) (io.ReadCloser, error) {
	path := s.keyToPath(key)

	f, err := os.Open(path) //nolint:gosec // path derived from cache key
	if err != nil {
		if os.IsNotExist(err) {
			return nil, ErrNotFound
		}

		return nil, fmt.Errorf("failed to open content: %w", err)
	}

	return f, nil
}

// LoadWithContentType opens the content at the given key and returns a
// reader, the file size and the content type, which is read from the
// file's first bytes. A missing file is ErrNotFound. A file whose first
// bytes are not those of an output format, an empty one included, is
// closed again and the reason returned.
func (s *VariantStorage) LoadWithContentType(
	key imagerequest.VariantKey,
) (io.ReadCloser, int64, string, error) {
	path := s.keyToPath(key)

	f, err := os.Open(path) //nolint:gosec // path derived from cache key
	if err != nil {
		if os.IsNotExist(err) {
			return nil, 0, "", ErrNotFound
		}

		return nil, 0, "", fmt.Errorf("failed to open content: %w", err)
	}

	stat, err := f.Stat()
	if err != nil {
		_ = f.Close()

		return nil, 0, "", fmt.Errorf("failed to stat content: %w", err)
	}

	contentType, err := outputContentType(f)
	if err != nil {
		_ = f.Close()

		return nil, 0, "", err
	}

	return f, stat.Size(), contentType, nil
}

// outputContentType reads the first bytes of content, without moving its
// offset, and returns the content type of the output format they show.
func outputContentType(content io.ReaderAt) (string, error) {
	firstBytes := make([]byte, magic.MinMagicBytes)

	_, err := content.ReadAt(firstBytes, 0)
	if err != nil {
		return "", fmt.Errorf("failed to read the first bytes: %w", err)
	}

	detected, err := magic.DetectFormat(firstBytes)
	if err != nil {
		return "", err
	}

	_, isOutputFormat := magic.MIMEToImageFormat(string(detected))
	if !isOutputFormat {
		return "", fmt.Errorf("%w: %s", errNotAnOutputFormat, detected)
	}

	return string(detected), nil
}

// Exists checks if content exists at the given key.
func (s *VariantStorage) Exists(key imagerequest.VariantKey) bool {
	path := s.keyToPath(key)
	_, err := os.Stat(path)

	return err == nil
}

// Delete removes the content at the given key. A missing file is not an
// error.
func (s *VariantStorage) Delete(key imagerequest.VariantKey) error {
	path := s.keyToPath(key)

	err := os.Remove(path)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("failed to delete content: %w", err)
	}

	return nil
}

// keyToPath converts a key to a file path: <basedir>/<ab>/<cd>/<key>
func (s *VariantStorage) keyToPath(key imagerequest.VariantKey) string {
	k := string(key)
	if len(k) < MinHashLength {
		return filepath.Join(s.baseDir, k)
	}

	return filepath.Join(s.baseDir, k[0:2], k[2:4], k)
}