Go on past cache entries the reconciliation pass cannot read (closes #343)
check / check (push) Canceled after 0s

The walk of each storage directory returned the error of an entry it
could not read. The pass then stopped before its later checks, and the
same entry stopped every later pass. Now reconcileFiles skips a
directory it cannot list and a file whose info it cannot read. It logs
a warning naming the path and the error, and the walk goes on. It reads
each file's info itself and hands it to reconcileVariantFile and
reconcileSourceFile. A cancelled context still stops the walk with its
error. README.md says so under Storage.

Model: opus-5-5
This commit is contained in:
2026-10-27 00:16:54 +00:00
parent b7a0d35732
commit 5779f4efcd
3 changed files with 44 additions and 25 deletions
+32 -25
View File
@@ -515,12 +515,14 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
}
// reconcileFiles walks baseDir, one of the cache's storage directories,
// sweeping stale temp files and passing each other file, with its name, to
// reconcileFile. Once ctx is cancelled it stops at the next file and
// returns ctx's error.
// sweeping stale temp files and passing each other file, with its info and
// name, to reconcileFile. It skips a directory it cannot list and a file
// whose info it cannot read. Each gets a warning naming its path and the
// error, and the walk goes on, so one such entry does not stop the pass.
// Once ctx is cancelled it stops at the next file and returns ctx's error.
func (c *Cache) reconcileFiles(
ctx context.Context, baseDir string,
reconcileFile func(path string, entry fs.DirEntry, name string) error,
reconcileFile func(path string, info fs.FileInfo, name string) error,
) error {
return filepath.WalkDir(
baseDir,
@@ -529,8 +531,15 @@ func (c *Cache) reconcileFiles(
return ctx.Err()
}
if err != nil || entry.IsDir() {
return err
if err != nil {
c.log.Warn("skipped unreadable entry in cache directory",
"path", path, "error", err)
return nil
}
if entry.IsDir() {
return nil
}
name := entry.Name()
@@ -541,7 +550,15 @@ func (c *Cache) reconcileFiles(
return nil
}
return reconcileFile(path, entry, name)
info, err := entry.Info()
if err != nil {
c.log.Warn("skipped unreadable entry in cache directory",
"path", path, "error", err)
return nil
}
return reconcileFile(path, info, name)
},
)
}
@@ -551,8 +568,8 @@ func (c *Cache) reconcileFiles(
// and sweeping stale temp files.
func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
return c.reconcileFiles(ctx, c.variants.baseDir,
func(path string, entry fs.DirEntry, name string) error {
return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name))
func(path string, info fs.FileInfo, name string) error {
return c.reconcileVariantFile(ctx, path, info, imagerequest.VariantKey(name))
})
}
@@ -562,16 +579,11 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
// crash or a restored snapshot can leave it, is removed with its row and
// a warning naming it, so the variant is converted again.
func (c *Cache) reconcileVariantFile(
ctx context.Context, path string, entry fs.DirEntry, cacheKey imagerequest.VariantKey,
ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey,
) error {
info, err := entry.Info()
if err != nil {
return fmt.Errorf("failed to stat variant file: %w", err)
}
var recordedSize int64
err = c.db.QueryRowContext(ctx,
err := c.db.QueryRowContext(ctx,
`SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey),
).Scan(&recordedSize)
@@ -714,8 +726,8 @@ func queryStringColumn[T ~string](
// and sweeping stale temp files.
func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
return c.reconcileFiles(ctx, c.srcContent.baseDir,
func(path string, entry fs.DirEntry, name string) error {
return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name))
func(path string, info fs.FileInfo, name string) error {
return c.reconcileSourceFile(ctx, path, info, imagerequest.SourceKey(name))
})
}
@@ -726,16 +738,11 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
// is removed with its row and a warning naming it, so the source is
// fetched again.
func (c *Cache) reconcileSourceFile(
ctx context.Context, path string, entry fs.DirEntry, key imagerequest.SourceKey,
ctx context.Context, path string, info fs.FileInfo, key imagerequest.SourceKey,
) error {
info, err := entry.Info()
if err != nil {
return fmt.Errorf("failed to stat source content file: %w", err)
}
var recordedSize int64
err = c.db.QueryRowContext(ctx,
err := c.db.QueryRowContext(ctx,
`SELECT size_bytes FROM source_content WHERE source_key = ?`,
string(key),
).Scan(&recordedSize)