Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8669b5cf04 | ||
|
|
2c90144e7b | ||
|
|
26b5160aa8 |
@@ -31,10 +31,12 @@ P2: security: per-origin rate limiting
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
||||||
whose info it cannot read, and goes on (closes #343). It returned the error,
|
whose info it cannot read, and goes on (closes #343). It stopped before its
|
||||||
so one such entry stopped every pass before its later checks.
|
later checks at a directory it could not list, or at a source or variant file
|
||||||
- Each skipped entry gets the warning "skipped unreadable entry in cache
|
whose info it could not read.
|
||||||
directory", with its path and the error.
|
- It logs the warning "skipped unreadable entry in cache directory", with
|
||||||
|
the path and the error.
|
||||||
|
- A file renamed or removed after its directory was listed gets no warning.
|
||||||
- `reconcileFiles` reads each file's info once, before the temp-file check,
|
- `reconcileFiles` reads each file's info once, before the temp-file check,
|
||||||
and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
|
and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
|
||||||
`reconcileSourceFile`.
|
`reconcileSourceFile`.
|
||||||
|
|||||||
@@ -517,9 +517,10 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
|||||||
// reconcileFiles walks baseDir, one of the cache's storage directories,
|
// reconcileFiles walks baseDir, one of the cache's storage directories,
|
||||||
// sweeping stale temp files and passing each other file, with its info and
|
// sweeping stale temp files and passing each other file, with its info and
|
||||||
// name, to reconcileFile. It skips a directory it cannot list and a file
|
// name, to reconcileFile. It skips a directory it cannot list and a file
|
||||||
// whose info it cannot read. Each gets a warning naming its path and the
|
// whose info it cannot read, so one such entry does not stop the pass. It
|
||||||
// error, and the walk goes on, so one such entry does not stop the pass.
|
// logs a warning naming the path and the error. A file renamed or removed
|
||||||
// Once ctx is cancelled it stops at the next file and returns ctx's error.
|
// after its directory was listed is skipped with no warning. Once ctx is
|
||||||
|
// cancelled it stops at the next file and returns ctx's error.
|
||||||
func (c *Cache) reconcileFiles(
|
func (c *Cache) reconcileFiles(
|
||||||
ctx context.Context, baseDir string,
|
ctx context.Context, baseDir string,
|
||||||
reconcileFile func(path string, info fs.FileInfo, name string) error,
|
reconcileFile func(path string, info fs.FileInfo, name string) error,
|
||||||
@@ -543,6 +544,10 @@ func (c *Cache) reconcileFiles(
|
|||||||
}
|
}
|
||||||
|
|
||||||
info, err := entry.Info()
|
info, err := entry.Info()
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.log.Warn("skipped unreadable entry in cache directory",
|
c.log.Warn("skipped unreadable entry in cache directory",
|
||||||
"path", path, "error", err)
|
"path", path, "error", err)
|
||||||
|
|||||||
@@ -819,10 +819,11 @@ func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
|
|||||||
return root
|
return root
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeEntriesTooLongToRead makes a directory and a file in the directory
|
// makeEntriesTooLongToRead makes a new directory and a new file in the
|
||||||
// that openDirectoryNearPathLimit opens inside dir. Even the root user can
|
// deepest directory of the chain that openDirectoryNearPathLimit makes
|
||||||
// then neither list that directory nor read the file's info. It returns
|
// inside dir. Even the root user can then neither list the new directory
|
||||||
// their two paths.
|
// nor read the new file's info. It returns the paths of the new directory
|
||||||
|
// and the new file.
|
||||||
func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -847,9 +848,10 @@ func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeTempFileTooLongToRead makes a temp file in the directory that
|
// makeTempFileTooLongToRead makes a temp file in the deepest directory of
|
||||||
// openDirectoryNearPathLimit opens inside dir. Even the root user then
|
// the chain that openDirectoryNearPathLimit makes inside dir. Even the
|
||||||
// cannot read its info. It returns its path.
|
// root user then cannot read the temp file's info. It returns the temp
|
||||||
|
// file's path.
|
||||||
func makeTempFileTooLongToRead(t *testing.T, dir string) string {
|
func makeTempFileTooLongToRead(t *testing.T, dir string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -864,6 +866,25 @@ func makeTempFileTooLongToRead(t *testing.T, dir string) string {
|
|||||||
return filepath.Join(deepest.Name(), name)
|
return filepath.Join(deepest.Name(), name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// assertSkippedEntryWarning checks that logs hold the warning "skipped
|
||||||
|
// unreadable entry in cache directory" naming path, with the error
|
||||||
|
// ENAMETOOLONG for that path.
|
||||||
|
func assertSkippedEntryWarning(t *testing.T, logs, path string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for line := range strings.Lines(logs) {
|
||||||
|
if strings.Contains(line, `"level":"WARN"`) &&
|
||||||
|
strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) &&
|
||||||
|
strings.Contains(line, `"path":"`+path+`"`) &&
|
||||||
|
strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Errorf("no warning %q names %s with the error %q",
|
||||||
|
"skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG)
|
||||||
|
}
|
||||||
|
|
||||||
// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
|
// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
|
||||||
// directory, a directory the pass cannot list and a file whose info it
|
// directory, a directory the pass cannot list and a file whose info it
|
||||||
// cannot read. The pass walks each directory in name order, so it reaches
|
// cannot read. The pass walks each directory in name order, so it reaches
|
||||||
@@ -915,11 +936,7 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
|
|||||||
logs := reconcileAsAtStartup(t, cache)
|
logs := reconcileAsAtStartup(t, cache)
|
||||||
|
|
||||||
for _, path := range unreadable {
|
for _, path := range unreadable {
|
||||||
assertWarningNames(t, logs, path)
|
assertSkippedEntryWarning(t, logs, path)
|
||||||
|
|
||||||
if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
|
|
||||||
t.Errorf("no log line gives the error reading %s", path)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
|
rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
|
||||||
@@ -948,11 +965,11 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestReconciliationWarnsOfTempFilesItCannotRead puts, in each storage
|
// TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead puts, in each
|
||||||
// directory, a temp file whose info the pass cannot read. The pass must
|
// storage directory, a temp file whose info the pass cannot read. The
|
||||||
// log a warning naming each one's path and the error, as it does for any
|
// pass must log a warning naming each one's path and the error, as it
|
||||||
// other file whose info it cannot read.
|
// does for any other file whose info it cannot read.
|
||||||
func TestReconciliationWarnsOfTempFilesItCannotRead(t *testing.T) {
|
func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
@@ -965,11 +982,7 @@ func TestReconciliationWarnsOfTempFilesItCannotRead(t *testing.T) {
|
|||||||
logs := reconcileAsAtStartup(t, cache)
|
logs := reconcileAsAtStartup(t, cache)
|
||||||
|
|
||||||
for _, path := range unreadable {
|
for _, path := range unreadable {
|
||||||
assertWarningNames(t, logs, path)
|
assertSkippedEntryWarning(t, logs, path)
|
||||||
|
|
||||||
if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
|
|
||||||
t.Errorf("no log line gives the error reading %s", path)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user