2 Commits
Author SHA1 Message Date
clawbot 2c90144e7b Go on past cache entries the reconciliation pass cannot read (closes #343)
check / check (push) Canceled after 0s
The walk of each storage directory returned the error of a directory it
could not list, or of a source or variant file whose info it could not
read. The pass then stopped before its later checks, every time it ran.
Now reconcileFiles skips a directory it cannot list and a file whose
info it cannot read, temp files included. It logs a warning naming the
path and the error, and the walk goes on. It reads each file's info
once and hands it to sweepStaleTempFile, reconcileVariantFile or
reconcileSourceFile. A file renamed or removed after its directory was
listed is skipped with no warning. A cancelled context still stops the
walk with its error. README.md says so under Storage.

Model: opus-5-5
2026-10-27 01:13:55 +00:00
clawbot 26b5160aa8 Test that reconciliation goes on past entries it cannot read
Each storage directory gets a directory the pass cannot list and a file
whose info it cannot read. Their paths are longer than Linux accepts,
even for root. The test wants a warning naming each path and the error.
It also wants the rest of the pass done: a variant file with no row
adopted, the row of a missing variant file dropped, a source file with
no row removed, and a wrong usage total corrected. A second test puts a
temp file whose info cannot be read in each storage directory and wants
the same warning. On next the first test's pass stops at the first such
directory and returns its error. The second test gets no warning.

Model: opus-5-5
2026-10-27 01:12:34 +00:00
2 changed files with 33 additions and 19 deletions
+3 -2
View File
@@ -31,8 +31,9 @@ P2: security: per-origin rate limiting
# Completed Steps
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
whose info it cannot read, and goes on (closes #343). It returned the error,
so one such entry stopped every pass before its later checks.
whose info it cannot read, and goes on (closes #343). It stopped before its
later checks at a directory it could not list, or at a source or variant file
whose info it could not read.
- It logs the warning "skipped unreadable entry in cache directory", with
the path and the error.
- A file renamed or removed after its directory was listed gets no warning.
+30 -17
View File
@@ -819,10 +819,11 @@ func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
return root
}
// makeEntriesTooLongToRead makes a directory and a file in the directory
// that openDirectoryNearPathLimit opens inside dir. Even the root user can
// then neither list that directory nor read the file's info. It returns
// their two paths.
// makeEntriesTooLongToRead makes a new directory and a new file in the
// deepest directory of the chain that openDirectoryNearPathLimit makes
// inside dir. Even the root user can then neither list the new directory
// nor read the new file's info. It returns the paths of the new directory
// and the new file.
func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
t.Helper()
@@ -847,9 +848,10 @@ func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
}
}
// makeTempFileTooLongToRead makes a temp file in the directory that
// openDirectoryNearPathLimit opens inside dir. Even the root user then
// cannot read its info. It returns its path.
// makeTempFileTooLongToRead makes a temp file in the deepest directory of
// the chain that openDirectoryNearPathLimit makes inside dir. Even the
// root user then cannot read the temp file's info. It returns the temp
// file's path.
func makeTempFileTooLongToRead(t *testing.T, dir string) string {
t.Helper()
@@ -864,6 +866,25 @@ func makeTempFileTooLongToRead(t *testing.T, dir string) string {
return filepath.Join(deepest.Name(), name)
}
// assertSkippedEntryWarning checks that logs hold the warning "skipped
// unreadable entry in cache directory" naming path, with the error
// ENAMETOOLONG for that path.
func assertSkippedEntryWarning(t *testing.T, logs, path string) {
t.Helper()
for line := range strings.Lines(logs) {
if strings.Contains(line, `"level":"WARN"`) &&
strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) &&
strings.Contains(line, `"path":"`+path+`"`) &&
strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) {
return
}
}
t.Errorf("no warning %q names %s with the error %q",
"skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG)
}
// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
// directory, a directory the pass cannot list and a file whose info it
// cannot read. The pass walks each directory in name order, so it reaches
@@ -915,11 +936,7 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
logs := reconcileAsAtStartup(t, cache)
for _, path := range unreadable {
assertWarningNames(t, logs, path)
if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
t.Errorf("no log line gives the error reading %s", path)
}
assertSkippedEntryWarning(t, logs, path)
}
rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
@@ -965,11 +982,7 @@ func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
logs := reconcileAsAtStartup(t, cache)
for _, path := range unreadable {
assertWarningNames(t, logs, path)
if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
t.Errorf("no log line gives the error reading %s", path)
}
assertSkippedEntryWarning(t, logs, path)
}
}