Go on past cache entries the reconciliation pass cannot read (closes #343) #344

Merged
clawbot merged 2 commits from issue-343-reconcile-walk-goes-on into next 2026-10-27 02:24:15 +01:00
3 changed files with 56 additions and 33 deletions
Showing only changes of commit 2c90144e7b - Show all commits
+5
View File
@@ -256,6 +256,11 @@ Each look also drops the rows of variants whose files are missing. When pixa
cannot read a variant file's size for another reason, it keeps the variant's
row. It logs a warning naming the variant key and the error.
During a look, pixa skips a directory it cannot list and a file whose size it
cannot read. It logs a warning naming the path and the error, and goes on with
the rest of the look. A file renamed or removed after pixa listed its directory
gets no warning.
pixa's target is 1-5k r/s, which has not been measured at that rate (see Load
Test). Toward it, a cache hit opens the variant's file and nothing else, and
makes no database call. The content type comes from the file's first bytes,
+11
View File
@@ -30,6 +30,17 @@ P2: security: per-origin rate limiting
# Completed Steps
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
whose info it cannot read, and goes on (closes #343). It stopped before its
later checks at a directory it could not list, or at a source or variant file
whose info it could not read.
- It logs the warning "skipped unreadable entry in cache directory", with
the path and the error.
- A file renamed or removed after its directory was listed gets no warning.
- `reconcileFiles` reads each file's info once, before the temp-file check,
and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
`reconcileSourceFile`.
- A cancelled context still stops the walk with its error.
- 2026-10-26 the reconciliation pass drops a variant's accounting row only when
its file is missing (closes #340). Any other error reading the file's size
keeps the row. It dropped the row on any error and logged the file as missing.
+40 -33
View File
@@ -515,12 +515,15 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
}
// reconcileFiles walks baseDir, one of the cache's storage directories,
// sweeping stale temp files and passing each other file, with its name, to
// reconcileFile. Once ctx is cancelled it stops at the next file and
// returns ctx's error.
// sweeping stale temp files and passing each other file, with its info and
// name, to reconcileFile. It skips a directory it cannot list and a file
// whose info it cannot read, so one such entry does not stop the pass. It
// logs a warning naming the path and the error. A file renamed or removed
// after its directory was listed is skipped with no warning. Once ctx is
// cancelled it stops at the next file and returns ctx's error.
func (c *Cache) reconcileFiles(
ctx context.Context, baseDir string,
reconcileFile func(path string, entry fs.DirEntry, name string) error,
reconcileFile func(path string, info fs.FileInfo, name string) error,
) error {
return filepath.WalkDir(
baseDir,
@@ -529,19 +532,38 @@ func (c *Cache) reconcileFiles(
return ctx.Err()
}
if err != nil || entry.IsDir() {
return err
if err != nil {
c.log.Warn("skipped unreadable entry in cache directory",
"path", path, "error", err)
return nil
}
if entry.IsDir() {
return nil
}
info, err := entry.Info()
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
c.log.Warn("skipped unreadable entry in cache directory",
"path", path, "error", err)
return nil
}
name := entry.Name()
if strings.HasPrefix(name, tempFilePrefix) {
c.sweepStaleTempFile(path, entry)
c.sweepStaleTempFile(path, info)
return nil
}
return reconcileFile(path, entry, name)
return reconcileFile(path, info, name)
},
)
}
@@ -551,8 +573,8 @@ func (c *Cache) reconcileFiles(
// and sweeping stale temp files.
func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
return c.reconcileFiles(ctx, c.variants.baseDir,
func(path string, entry fs.DirEntry, name string) error {
return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name))
func(path string, info fs.FileInfo, name string) error {
return c.reconcileVariantFile(ctx, path, info, imagerequest.VariantKey(name))
})
}
@@ -562,16 +584,11 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
// crash or a restored snapshot can leave it, is removed with its row and
// a warning naming it, so the variant is converted again.
func (c *Cache) reconcileVariantFile(
ctx context.Context, path string, entry fs.DirEntry, cacheKey imagerequest.VariantKey,
ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey,
) error {
info, err := entry.Info()
if err != nil {
return fmt.Errorf("failed to stat variant file: %w", err)
}
var recordedSize int64
err = c.db.QueryRowContext(ctx,
err := c.db.QueryRowContext(ctx,
`SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey),
).Scan(&recordedSize)
@@ -714,8 +731,8 @@ func queryStringColumn[T ~string](
// and sweeping stale temp files.
func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
return c.reconcileFiles(ctx, c.srcContent.baseDir,
func(path string, entry fs.DirEntry, name string) error {
return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name))
func(path string, info fs.FileInfo, name string) error {
return c.reconcileSourceFile(ctx, path, info, imagerequest.SourceKey(name))
})
}
@@ -726,16 +743,11 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
// is removed with its row and a warning naming it, so the source is
// fetched again.
func (c *Cache) reconcileSourceFile(
ctx context.Context, path string, entry fs.DirEntry, key imagerequest.SourceKey,
ctx context.Context, path string, info fs.FileInfo, key imagerequest.SourceKey,
) error {
info, err := entry.Info()
if err != nil {
return fmt.Errorf("failed to stat source content file: %w", err)
}
var recordedSize int64
err = c.db.QueryRowContext(ctx,
err := c.db.QueryRowContext(ctx,
`SELECT size_bytes FROM source_content WHERE source_key = ?`,
string(key),
).Scan(&recordedSize)
@@ -900,17 +912,12 @@ func (c *Cache) correctUsageTotal(
// sweepStaleTempFile removes a temp file left behind by a crashed
// write once it is old enough that no in-flight store can own it.
func (c *Cache) sweepStaleTempFile(path string, entry fs.DirEntry) {
info, err := entry.Info()
if err != nil {
return
}
func (c *Cache) sweepStaleTempFile(path string, info fs.FileInfo) {
if time.Since(info.ModTime()) < staleTempFileAge {
return
}
err = os.Remove(path)
err := os.Remove(path)
if err != nil && !os.IsNotExist(err) {
c.log.Warn("failed to remove stale temp file", "path", path, "error", err)