Go on past cache entries the reconciliation pass cannot read (closes #343)

The walk of each storage directory returned the error of an entry it
could not read. The pass then stopped before its later checks, and the
same entry stopped every later pass. Now reconcileFiles skips a
directory it cannot list and a file whose info it cannot read. It logs
a warning naming the path and the error, and the walk goes on. It reads
each file's info itself and hands it to reconcileVariantFile and
reconcileSourceFile. A cancelled context still stops the walk with its
error. README.md says so under Storage.

Model: opus-5-5

TODO.md

@@ -31,15 +31,12 @@ P2: security: per-origin rate limiting
 # Completed Steps
 
 - 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
-  whose info it cannot read, and goes on (closes #343). It stopped before its
+  whose info it cannot read, and goes on (closes #343). It returned the error,
-  later checks at a directory it could not list, or at a source or variant file
+  so one such entry stopped every pass before its later checks.
-  whose info it could not read.
+    - Each skipped entry gets the warning "skipped unreadable entry in cache
-    - It logs the warning "skipped unreadable entry in cache directory", with
+      directory", with its path and the error.
-      the path and the error.
+    - `reconcileFiles` reads each file's info and hands it to
-    - A file renamed or removed after its directory was listed gets no warning.
+      `reconcileVariantFile` and `reconcileSourceFile`.
-    - `reconcileFiles` reads each file's info once, before the temp-file check,
-      and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
-      `reconcileSourceFile`.
     - A cancelled context still stops the walk with its error.
 - 2026-10-26 the reconciliation pass drops a variant's accounting row only when
   its file is missing (closes #340). Any other error reading the file's size

internal/imgcache/eviction.go

@@ -517,10 +517,9 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
 // reconcileFiles walks baseDir, one of the cache's storage directories,
 // sweeping stale temp files and passing each other file, with its info and
 // name, to reconcileFile. It skips a directory it cannot list and a file
-// whose info it cannot read, so one such entry does not stop the pass. It
+// whose info it cannot read. Each gets a warning naming its path and the
-// logs a warning naming the path and the error. A file renamed or removed
+// error, and the walk goes on, so one such entry does not stop the pass.
-// after its directory was listed is skipped with no warning. Once ctx is
+// Once ctx is cancelled it stops at the next file and returns ctx's error.
-// cancelled it stops at the next file and returns ctx's error.
 func (c *Cache) reconcileFiles(
 	ctx context.Context, baseDir string,
 	reconcileFile func(path string, info fs.FileInfo, name string) error,
@@ -543,22 +542,18 @@ func (c *Cache) reconcileFiles(
 				return nil
 			}
 
-			info, err := entry.Info()
-			if errors.Is(err, fs.ErrNotExist) {
-				return nil
-			}
-
-			if err != nil {
-				c.log.Warn("skipped unreadable entry in cache directory",
-					"path", path, "error", err)
-
-				return nil
-			}
-
 			name := entry.Name()
 
 			if strings.HasPrefix(name, tempFilePrefix) {
-				c.sweepStaleTempFile(path, info)
+				c.sweepStaleTempFile(path, entry)
+
+				return nil
+			}
+
+			info, err := entry.Info()
+			if err != nil {
+				c.log.Warn("skipped unreadable entry in cache directory",
+					"path", path, "error", err)
 
 				return nil
 			}
@@ -912,12 +907,17 @@ func (c *Cache) correctUsageTotal(
 
 // sweepStaleTempFile removes a temp file left behind by a crashed
 // write once it is old enough that no in-flight store can own it.
-func (c *Cache) sweepStaleTempFile(path string, info fs.FileInfo) {
+func (c *Cache) sweepStaleTempFile(path string, entry fs.DirEntry) {
+	info, err := entry.Info()
+	if err != nil {
+		return
+	}
+
 	if time.Since(info.ModTime()) < staleTempFileAge {
 		return
 	}
 
-	err := os.Remove(path)
+	err = os.Remove(path)
 	if err != nil && !os.IsNotExist(err) {
 		c.log.Warn("failed to remove stale temp file", "path", path, "error", err)
 

internal/imgcache/eviction_internal_test.go

@@ -787,14 +787,15 @@ func TestReconcileVariantRowsDropsOnlyRowsOfMissingFiles(t *testing.T) {
 	}
 }
 
-// openDirectoryNearPathLimit makes a chain of directories inside dir, each
+// makeEntriesTooLongToRead makes a chain of directories inside dir, each
 // named with 255 zeros, the longest name Linux allows. Linux refuses a path
 // of 4096 bytes or more, and the chain goes as deep as it can below that.
-// It returns the chain's deepest directory as an os.Root, closed when the
+// In its deepest directory it makes a directory and a file whose paths
-// test ends. An entry made through it with a 255-byte name has a path even
+// reach that length. Even root can then neither list that directory nor
-// the root user cannot read. The os.Root can make that entry because it
+// read the file's info. It returns their two paths. It makes them through
-// names the entry only by its name, not by the whole path.
+// an os.Root, which names one directory at a time, as no call may name
-func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
+// the whole path.
+func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
 	t.Helper()
 
 	name := strings.Repeat("0", 255)
@@ -804,85 +805,32 @@ func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
 		deepest = filepath.Join(deepest, name)
 	}
 
-	err := os.MkdirAll(deepest, StorageDirPerm)
+	chain, err := filepath.Rel(dir, deepest)
 	if err != nil {
-		t.Fatalf("failed to make a chain of directories: %v", err)
+		t.Fatalf("filepath.Rel() error = %v", err)
 	}
 
-	root, err := os.OpenRoot(deepest)
+	root, err := os.OpenRoot(dir)
 	if err != nil {
-		t.Fatalf("os.OpenRoot(%s) error = %v", deepest, err)
+		t.Fatalf("os.OpenRoot(%s) error = %v", dir, err)
 	}
 
-	t.Cleanup(func() { _ = root.Close() })
+	defer func() { _ = root.Close() }()
 
-	return root
+	err = root.MkdirAll(filepath.Join(chain, name), StorageDirPerm)
-}
-
-// makeEntriesTooLongToRead makes a new directory and a new file in the
-// deepest directory of the chain that openDirectoryNearPathLimit makes
-// inside dir. Even the root user can then neither list the new directory
-// nor read the new file's info. It returns the paths of the new directory
-// and the new file.
-func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
-	t.Helper()
-
-	deepest := openDirectoryNearPathLimit(t, dir)
-	dirName := strings.Repeat("0", 255)
-
-	err := deepest.Mkdir(dirName, StorageDirPerm)
 	if err != nil {
 		t.Fatalf("failed to make a directory too deep to list: %v", err)
 	}
 
 	fileName := strings.Repeat("1", 255)
 
-	err = deepest.WriteFile(fileName, []byte("unreadable"), StorageFilePerm)
+	err = root.WriteFile(filepath.Join(chain, fileName), []byte("unreadable"),
+		StorageFilePerm)
 	if err != nil {
 		t.Fatalf("failed to make a file too deep to read: %v", err)
 	}
 
-	return []string{
+	return []string{filepath.Join(deepest, name), filepath.Join(deepest, fileName)}
-		filepath.Join(deepest.Name(), dirName),
-		filepath.Join(deepest.Name(), fileName),
-	}
-}
-
-// makeTempFileTooLongToRead makes a temp file in the deepest directory of
-// the chain that openDirectoryNearPathLimit makes inside dir. Even the
-// root user then cannot read the temp file's info. It returns the temp
-// file's path.
-func makeTempFileTooLongToRead(t *testing.T, dir string) string {
-	t.Helper()
-
-	deepest := openDirectoryNearPathLimit(t, dir)
-	name := tempFilePrefix + strings.Repeat("1", 255-len(tempFilePrefix))
-
-	err := deepest.WriteFile(name, []byte("partial"), StorageFilePerm)
-	if err != nil {
-		t.Fatalf("failed to make a temp file too deep to read: %v", err)
-	}
-
-	return filepath.Join(deepest.Name(), name)
-}
-
-// assertSkippedEntryWarning checks that logs hold the warning "skipped
-// unreadable entry in cache directory" naming path, with the error
-// ENAMETOOLONG for that path.
-func assertSkippedEntryWarning(t *testing.T, logs, path string) {
-	t.Helper()
-
-	for line := range strings.Lines(logs) {
-		if strings.Contains(line, `"level":"WARN"`) &&
-			strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) &&
-			strings.Contains(line, `"path":"`+path+`"`) &&
-			strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) {
-			return
-		}
-	}
-
-	t.Errorf("no warning %q names %s with the error %q",
-		"skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG)
 }
 
 // TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
@@ -936,7 +884,11 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
 	logs := reconcileAsAtStartup(t, cache)
 
 	for _, path := range unreadable {
-		assertSkippedEntryWarning(t, logs, path)
+		assertWarningNames(t, logs, path)
+
+		if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
+			t.Errorf("no log line gives the error reading %s", path)
+		}
 	}
 
 	rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
@@ -965,27 +917,6 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
 	}
 }
 
-// TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead puts, in each
-// storage directory, a temp file whose info the pass cannot read. The
-// pass must log a warning naming each one's path and the error, as it
-// does for any other file whose info it cannot read.
-func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
-	t.Parallel()
-
-	cache, _ := newEvictionTestCache(t, 1<<30)
-
-	unreadable := []string{
-		makeTempFileTooLongToRead(t, cache.variants.baseDir),
-		makeTempFileTooLongToRead(t, cache.srcContent.baseDir),
-	}
-
-	logs := reconcileAsAtStartup(t, cache)
-
-	for _, path := range unreadable {
-		assertSkippedEntryWarning(t, logs, path)
-	}
-}
-
 // TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves
 // reconciliation is not a one-shot startup-only pass: it must also run
 // on the periodic ticker, so a variant file that lands on disk with no