Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8669b5cf04 | ||
|
|
2c90144e7b | ||
|
|
26b5160aa8 |
No files matched your search
@@ -31,12 +31,15 @@ P2: security: per-origin rate limiting
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
- 2026-10-27 the reconciliation pass skips a directory it cannot list and a file
|
||||||
whose info it cannot read, and goes on (closes #343). It returned the error,
|
whose info it cannot read, and goes on (closes #343). It stopped before its
|
||||||
so one such entry stopped every pass before its later checks.
|
later checks at a directory it could not list, or at a source or variant file
|
||||||
- Each skipped entry gets the warning "skipped unreadable entry in cache
|
whose info it could not read.
|
||||||
directory", with its path and the error.
|
- It logs the warning "skipped unreadable entry in cache directory", with
|
||||||
- `reconcileFiles` reads each file's info and hands it to
|
the path and the error.
|
||||||
`reconcileVariantFile` and `reconcileSourceFile`.
|
- A file renamed or removed after its directory was listed gets no warning.
|
||||||
|
- `reconcileFiles` reads each file's info once, before the temp-file check,
|
||||||
|
and hands it to `sweepStaleTempFile`, `reconcileVariantFile` or
|
||||||
|
`reconcileSourceFile`.
|
||||||
- A cancelled context still stops the walk with its error.
|
- A cancelled context still stops the walk with its error.
|
||||||
- 2026-10-26 the reconciliation pass drops a variant's accounting row only when
|
- 2026-10-26 the reconciliation pass drops a variant's accounting row only when
|
||||||
its file is missing (closes #340). Any other error reading the file's size
|
its file is missing (closes #340). Any other error reading the file's size
|
||||||
|
|||||||
@@ -517,9 +517,10 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
|||||||
// reconcileFiles walks baseDir, one of the cache's storage directories,
|
// reconcileFiles walks baseDir, one of the cache's storage directories,
|
||||||
// sweeping stale temp files and passing each other file, with its info and
|
// sweeping stale temp files and passing each other file, with its info and
|
||||||
// name, to reconcileFile. It skips a directory it cannot list and a file
|
// name, to reconcileFile. It skips a directory it cannot list and a file
|
||||||
// whose info it cannot read. Each gets a warning naming its path and the
|
// whose info it cannot read, so one such entry does not stop the pass. It
|
||||||
// error, and the walk goes on, so one such entry does not stop the pass.
|
// logs a warning naming the path and the error. A file renamed or removed
|
||||||
// Once ctx is cancelled it stops at the next file and returns ctx's error.
|
// after its directory was listed is skipped with no warning. Once ctx is
|
||||||
|
// cancelled it stops at the next file and returns ctx's error.
|
||||||
func (c *Cache) reconcileFiles(
|
func (c *Cache) reconcileFiles(
|
||||||
ctx context.Context, baseDir string,
|
ctx context.Context, baseDir string,
|
||||||
reconcileFile func(path string, info fs.FileInfo, name string) error,
|
reconcileFile func(path string, info fs.FileInfo, name string) error,
|
||||||
@@ -542,18 +543,22 @@ func (c *Cache) reconcileFiles(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
name := entry.Name()
|
info, err := entry.Info()
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(name, tempFilePrefix) {
|
if err != nil {
|
||||||
c.sweepStaleTempFile(path, entry)
|
c.log.Warn("skipped unreadable entry in cache directory",
|
||||||
|
"path", path, "error", err)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
info, err := entry.Info()
|
name := entry.Name()
|
||||||
if err != nil {
|
|
||||||
c.log.Warn("skipped unreadable entry in cache directory",
|
if strings.HasPrefix(name, tempFilePrefix) {
|
||||||
"path", path, "error", err)
|
c.sweepStaleTempFile(path, info)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -907,17 +912,12 @@ func (c *Cache) correctUsageTotal(
|
|||||||
|
|
||||||
// sweepStaleTempFile removes a temp file left behind by a crashed
|
// sweepStaleTempFile removes a temp file left behind by a crashed
|
||||||
// write once it is old enough that no in-flight store can own it.
|
// write once it is old enough that no in-flight store can own it.
|
||||||
func (c *Cache) sweepStaleTempFile(path string, entry fs.DirEntry) {
|
func (c *Cache) sweepStaleTempFile(path string, info fs.FileInfo) {
|
||||||
info, err := entry.Info()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Since(info.ModTime()) < staleTempFileAge {
|
if time.Since(info.ModTime()) < staleTempFileAge {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.Remove(path)
|
err := os.Remove(path)
|
||||||
if err != nil && !os.IsNotExist(err) {
|
if err != nil && !os.IsNotExist(err) {
|
||||||
c.log.Warn("failed to remove stale temp file", "path", path, "error", err)
|
c.log.Warn("failed to remove stale temp file", "path", path, "error", err)
|
||||||
|
|
||||||
|
|||||||
@@ -787,15 +787,14 @@ func TestReconcileVariantRowsDropsOnlyRowsOfMissingFiles(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// makeEntriesTooLongToRead makes a chain of directories inside dir, each
|
// openDirectoryNearPathLimit makes a chain of directories inside dir, each
|
||||||
// named with 255 zeros, the longest name Linux allows. Linux refuses a path
|
// named with 255 zeros, the longest name Linux allows. Linux refuses a path
|
||||||
// of 4096 bytes or more, and the chain goes as deep as it can below that.
|
// of 4096 bytes or more, and the chain goes as deep as it can below that.
|
||||||
// In its deepest directory it makes a directory and a file whose paths
|
// It returns the chain's deepest directory as an os.Root, closed when the
|
||||||
// reach that length. Even root can then neither list that directory nor
|
// test ends. An entry made through it with a 255-byte name has a path even
|
||||||
// read the file's info. It returns their two paths. It makes them through
|
// the root user cannot read. The os.Root can make that entry because it
|
||||||
// an os.Root, which names one directory at a time, as no call may name
|
// names the entry only by its name, not by the whole path.
|
||||||
// the whole path.
|
func openDirectoryNearPathLimit(t *testing.T, dir string) *os.Root {
|
||||||
func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
name := strings.Repeat("0", 255)
|
name := strings.Repeat("0", 255)
|
||||||
@@ -805,32 +804,85 @@ func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
|||||||
deepest = filepath.Join(deepest, name)
|
deepest = filepath.Join(deepest, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
chain, err := filepath.Rel(dir, deepest)
|
err := os.MkdirAll(deepest, StorageDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("filepath.Rel() error = %v", err)
|
t.Fatalf("failed to make a chain of directories: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
root, err := os.OpenRoot(dir)
|
root, err := os.OpenRoot(deepest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("os.OpenRoot(%s) error = %v", dir, err)
|
t.Fatalf("os.OpenRoot(%s) error = %v", deepest, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
t.Cleanup(func() { _ = root.Close() })
|
||||||
|
|
||||||
err = root.MkdirAll(filepath.Join(chain, name), StorageDirPerm)
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeEntriesTooLongToRead makes a new directory and a new file in the
|
||||||
|
// deepest directory of the chain that openDirectoryNearPathLimit makes
|
||||||
|
// inside dir. Even the root user can then neither list the new directory
|
||||||
|
// nor read the new file's info. It returns the paths of the new directory
|
||||||
|
// and the new file.
|
||||||
|
func makeEntriesTooLongToRead(t *testing.T, dir string) []string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
deepest := openDirectoryNearPathLimit(t, dir)
|
||||||
|
dirName := strings.Repeat("0", 255)
|
||||||
|
|
||||||
|
err := deepest.Mkdir(dirName, StorageDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to make a directory too deep to list: %v", err)
|
t.Fatalf("failed to make a directory too deep to list: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileName := strings.Repeat("1", 255)
|
fileName := strings.Repeat("1", 255)
|
||||||
|
|
||||||
err = root.WriteFile(filepath.Join(chain, fileName), []byte("unreadable"),
|
err = deepest.WriteFile(fileName, []byte("unreadable"), StorageFilePerm)
|
||||||
StorageFilePerm)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to make a file too deep to read: %v", err)
|
t.Fatalf("failed to make a file too deep to read: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return []string{filepath.Join(deepest, name), filepath.Join(deepest, fileName)}
|
return []string{
|
||||||
|
filepath.Join(deepest.Name(), dirName),
|
||||||
|
filepath.Join(deepest.Name(), fileName),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// makeTempFileTooLongToRead makes a temp file in the deepest directory of
|
||||||
|
// the chain that openDirectoryNearPathLimit makes inside dir. Even the
|
||||||
|
// root user then cannot read the temp file's info. It returns the temp
|
||||||
|
// file's path.
|
||||||
|
func makeTempFileTooLongToRead(t *testing.T, dir string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
deepest := openDirectoryNearPathLimit(t, dir)
|
||||||
|
name := tempFilePrefix + strings.Repeat("1", 255-len(tempFilePrefix))
|
||||||
|
|
||||||
|
err := deepest.WriteFile(name, []byte("partial"), StorageFilePerm)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to make a temp file too deep to read: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Join(deepest.Name(), name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertSkippedEntryWarning checks that logs hold the warning "skipped
|
||||||
|
// unreadable entry in cache directory" naming path, with the error
|
||||||
|
// ENAMETOOLONG for that path.
|
||||||
|
func assertSkippedEntryWarning(t *testing.T, logs, path string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for line := range strings.Lines(logs) {
|
||||||
|
if strings.Contains(line, `"level":"WARN"`) &&
|
||||||
|
strings.Contains(line, `"msg":"skipped unreadable entry in cache directory"`) &&
|
||||||
|
strings.Contains(line, `"path":"`+path+`"`) &&
|
||||||
|
strings.Contains(line, path+": "+syscall.ENAMETOOLONG.Error()) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Errorf("no warning %q names %s with the error %q",
|
||||||
|
"skipped unreadable entry in cache directory", path, syscall.ENAMETOOLONG)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
|
// TestReconciliationGoesOnPastEntriesItCannotRead puts, in each storage
|
||||||
@@ -884,11 +936,7 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
|
|||||||
logs := reconcileAsAtStartup(t, cache)
|
logs := reconcileAsAtStartup(t, cache)
|
||||||
|
|
||||||
for _, path := range unreadable {
|
for _, path := range unreadable {
|
||||||
assertWarningNames(t, logs, path)
|
assertSkippedEntryWarning(t, logs, path)
|
||||||
|
|
||||||
if !strings.Contains(logs, path+": "+syscall.ENAMETOOLONG.Error()) {
|
|
||||||
t.Errorf("no log line gives the error reading %s", path)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
|
rowQuery := `SELECT COUNT(*) FROM variant_content WHERE cache_key = ?`
|
||||||
@@ -917,6 +965,27 @@ func TestReconciliationGoesOnPastEntriesItCannotRead(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead puts, in each
|
||||||
|
// storage directory, a temp file whose info the pass cannot read. The
|
||||||
|
// pass must log a warning naming each one's path and the error, as it
|
||||||
|
// does for any other file whose info it cannot read.
|
||||||
|
func TestReconciliationWarnsOfTempFilesWhoseInfoItCannotRead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||||
|
|
||||||
|
unreadable := []string{
|
||||||
|
makeTempFileTooLongToRead(t, cache.variants.baseDir),
|
||||||
|
makeTempFileTooLongToRead(t, cache.srcContent.baseDir),
|
||||||
|
}
|
||||||
|
|
||||||
|
logs := reconcileAsAtStartup(t, cache)
|
||||||
|
|
||||||
|
for _, path := range unreadable {
|
||||||
|
assertSkippedEntryWarning(t, logs, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves
|
// TestPeriodicReconciliationAdoptsFileThatAppearsAfterStartup proves
|
||||||
// reconciliation is not a one-shot startup-only pass: it must also run
|
// reconciliation is not a one-shot startup-only pass: it must also run
|
||||||
// on the periodic ticker, so a variant file that lands on disk with no
|
// on the periodic ticker, so a variant file that lands on disk with no
|
||||||
|
|||||||
Reference in new issue
Block a user