Go on past cache entries the reconciliation pass cannot read (closes #343)

The walk of each storage directory returned the error of an entry it
could not read. The pass then stopped before its later checks, and the
same entry stopped every later pass. Now reconcileFiles skips a
directory it cannot list and a file whose info it cannot read, temp
files included. It logs a warning naming the path and the error, and the
walk goes on. It reads each file's info once, before the temp-file
check, and hands it to sweepStaleTempFile, reconcileVariantFile or
reconcileSourceFile. A cancelled context still stops the walk with its
error. README.md says so under Storage.

Model: opus-5-5

TODO.md

@@ -31,12 +31,10 @@ P2: security: per-origin rate limiting
 # Completed Steps
 
 - 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
-  whose info it cannot read, and goes on (closes #343). It stopped before its
-  later checks at a directory it could not list, or at a source or variant file
-  whose info it could not read.
-    - It logs the warning "skipped unreadable entry in cache directory", with
-      the path and the error.
-    - A file renamed or removed after its directory was listed gets no warning.
+  whose info it cannot read, and goes on (closes #343). It returned the error,
+  so one such entry stopped every pass before its later checks.
+    - Each skipped entry gets the warning "skipped unreadable entry in cache
+      directory", with its path and the error.
     - `reconcileFiles` reads each file's info once, before the temp-file check,
       and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
       `reconcileSourceFile`.

internal/imgcache/eviction.go

@@ -517,10 +517,9 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
 // reconcileFiles walks baseDir, one of the cache's storage directories,
 // sweeping stale temp files and passing each other file, with its info and
 // name, to reconcileFile. It skips a directory it cannot list and a file
-// whose info it cannot read, so one such entry does not stop the pass. It
-// logs a warning naming the path and the error. A file renamed or removed
-// after its directory was listed is skipped with no warning. Once ctx is
-// cancelled it stops at the next file and returns ctx's error.
+// whose info it cannot read. Each gets a warning naming its path and the
+// error, and the walk goes on, so one such entry does not stop the pass.
+// Once ctx is cancelled it stops at the next file and returns ctx's error.
 func (c *Cache) reconcileFiles(
 	ctx context.Context, baseDir string,
 	reconcileFile func(path string, info fs.FileInfo, name string) error,
@@ -544,10 +543,6 @@ func (c *Cache) reconcileFiles(
 			}
 
 			info, err := entry.Info()
-			if errors.Is(err, fs.ErrNotExist) {
-				return nil
-			}
-
 			if err != nil {
 				c.log.Warn("skipped unreadable entry in cache directory",
 					"path", path, "error", err)

internal/imgcache/eviction_internal_test.go

@@ -819,11 +819,10 @@ func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
 	return root
 }
 
-// makeEntriesTooLongToRead makes a new directory and a new file in the
-// deepest directory of the chain that openDirectoryNearPathLimit makes
-// inside dir. Even the root user can then neither list the new directory
-// nor read the new file's info. It returns the paths of the new directory
-// and the new file.
+// makeEntriesTooLongToRead makes a directory and a file in the directory
+// that openDirectoryNearPathLimit opens inside dir. Even the root user can
+// then neither list that directory nor read the file's info. It returns
+// their two paths.
 func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
 	t.Helper()
 
@@ -848,10 +847,9 @@ func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
 	}
 }
 
-// makeTempFileTooLongToRead makes a temp file in the deepest directory of
-// the chain that openDirectoryNearPathLimit makes inside dir. Even the
-// root user then cannot read the temp file's info. It returns the temp
-// file's path.
+// makeTempFileTooLongToRead makes a temp file in the directory that
+// openDirectoryNearPathLimit opens inside dir. Even the root user then
+// cannot read its info. It returns its path.
 func makeTempFileTooLongToRead(t *testing.T, dir string) string {
 	t.Helper()
 
@@ -866,25 +864,6 @@ func makeTempFileTooLongToRead(t *testing.T, dir string) string {
 	return filepath.Join(deepest.Name(), name)
 }
 
-// assertSkippedEntryWarning checks that logs hold the warning "skipped
-// unreadable entry in cache directory" naming path, with the error
-// ENAMETOOLONG for that path.
-func assertSkippedEntryWarning(t *testing.T, logs, path string) {
-	t.Helper()
-
-	for line := range strings.Lines(logs) {
-		if strings.Contains(line, `"level":"WARN"`) &&
-			strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) &&
-			strings.Contains(line, `"path":"`+path+`"`) &&
-			strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) {
-			return
-		}
-	}
-
-	t.Errorf("no warning %q names %s with the error %q",
-		"skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG)
-}
-
 // TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
 // directory, a directory the pass cannot list and a file whose info it
 // cannot read. The pass walks each directory in name order, so it reaches
@@ -936,7 +915,11 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
 	logs := reconcileAsAtStartup(t, cache)
 
 	for _, path := range unreadable {
-		assertSkippedEntryWarning(t, logs, path)
+		assertWarningNames(t, logs, path)
+
+		if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
+			t.Errorf("no log line gives the error reading %s", path)
+		}
 	}
 
 	rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
@@ -965,11 +948,11 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
 	}
 }
 
-// TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead puts, in each
-// storage directory, a temp file whose info the pass cannot read. The
-// pass must log a warning naming each one's path and the error, as it
-// does for any other file whose info it cannot read.
-func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
+// TestReconciliationWarnsOfTempFilesItCannotRead puts, in each storage
+// directory, a temp file whose info the pass cannot read. The pass must
+// log a warning naming each one's path and the error, as it does for any
+// other file whose info it cannot read.
+func TestReconciliationWarnsOfTempFilesItCannotRead(t *testing.T) {
 	t.Parallel()
 
 	cache, _ := newEvictionTestCache(t, 1<<30)
@@ -982,7 +965,11 @@ func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
 	logs := reconcileAsAtStartup(t, cache)
 
 	for _, path := range unreadable {
-		assertSkippedEntryWarning(t, logs, path)
+		assertWarningNames(t, logs, path)
+
+		if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
+			t.Errorf("no log line gives the error reading %s", path)
+		}
 	}
 }