|
|
|
@@ -515,12 +515,15 @@ func (c *Cache) reconcileAccounting(ctx context.Context) error {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// reconcileFiles walks baseDir, one of the cache's storage directories,
|
|
|
|
|
// sweeping stale temp files and passing each other file, with its name, to
|
|
|
|
|
// reconcileFile. Once ctx is cancelled it stops at the next file and
|
|
|
|
|
// returns ctx's error.
|
|
|
|
|
// sweeping stale temp files and passing each other file, with its info and
|
|
|
|
|
// name, to reconcileFile. It skips a directory it cannot list and a file
|
|
|
|
|
// whose info it cannot read, so one such entry does not stop the pass. It
|
|
|
|
|
// logs a warning naming the path and the error. A file renamed or removed
|
|
|
|
|
// after its directory was listed is skipped with no warning. Once ctx is
|
|
|
|
|
// cancelled it stops at the next file and returns ctx's error.
|
|
|
|
|
func (c *Cache) reconcileFiles(
|
|
|
|
|
ctx context.Context, baseDir string,
|
|
|
|
|
reconcileFile func(path string, entry fs.DirEntry, name string) error,
|
|
|
|
|
reconcileFile func(path string, info fs.FileInfo, name string) error,
|
|
|
|
|
) error {
|
|
|
|
|
return filepath.WalkDir(
|
|
|
|
|
baseDir,
|
|
|
|
@@ -529,19 +532,38 @@ func (c *Cache) reconcileFiles(
|
|
|
|
|
return ctx.Err()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if err != nil || entry.IsDir() {
|
|
|
|
|
return err
|
|
|
|
|
if err != nil {
|
|
|
|
|
c.log.Warn("skipped unreadable entry in cache directory",
|
|
|
|
|
"path", path, "error", err)
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if entry.IsDir() {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
info, err := entry.Info()
|
|
|
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
c.log.Warn("skipped unreadable entry in cache directory",
|
|
|
|
|
"path", path, "error", err)
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
name := entry.Name()
|
|
|
|
|
|
|
|
|
|
if strings.HasPrefix(name, tempFilePrefix) {
|
|
|
|
|
c.sweepStaleTempFile(path, entry)
|
|
|
|
|
c.sweepStaleTempFile(path, info)
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return reconcileFile(path, entry, name)
|
|
|
|
|
return reconcileFile(path, info, name)
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
@@ -551,8 +573,8 @@ func (c *Cache) reconcileFiles(
|
|
|
|
|
// and sweeping stale temp files.
|
|
|
|
|
func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
|
|
|
|
|
return c.reconcileFiles(ctx, c.variants.baseDir,
|
|
|
|
|
func(path string, entry fs.DirEntry, name string) error {
|
|
|
|
|
return c.reconcileVariantFile(ctx, path, entry, imagerequest.VariantKey(name))
|
|
|
|
|
func(path string, info fs.FileInfo, name string) error {
|
|
|
|
|
return c.reconcileVariantFile(ctx, path, info, imagerequest.VariantKey(name))
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -562,16 +584,11 @@ func (c *Cache) reconcileVariantFiles(ctx context.Context) error {
|
|
|
|
|
// crash or a restored snapshot can leave it, is removed with its row and
|
|
|
|
|
// a warning naming it, so the variant is converted again.
|
|
|
|
|
func (c *Cache) reconcileVariantFile(
|
|
|
|
|
ctx context.Context, path string, entry fs.DirEntry, cacheKey imagerequest.VariantKey,
|
|
|
|
|
ctx context.Context, path string, info fs.FileInfo, cacheKey imagerequest.VariantKey,
|
|
|
|
|
) error {
|
|
|
|
|
info, err := entry.Info()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return fmt.Errorf("failed to stat variant file: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var recordedSize int64
|
|
|
|
|
|
|
|
|
|
err = c.db.QueryRowContext(ctx,
|
|
|
|
|
err := c.db.QueryRowContext(ctx,
|
|
|
|
|
`SELECT size_bytes FROM variant_content WHERE cache_key = ?`, string(cacheKey),
|
|
|
|
|
).Scan(&recordedSize)
|
|
|
|
|
|
|
|
|
@@ -714,8 +731,8 @@ func queryStringColumn[T ~string](
|
|
|
|
|
// and sweeping stale temp files.
|
|
|
|
|
func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
|
|
|
|
|
return c.reconcileFiles(ctx, c.srcContent.baseDir,
|
|
|
|
|
func(path string, entry fs.DirEntry, name string) error {
|
|
|
|
|
return c.reconcileSourceFile(ctx, path, entry, imagerequest.SourceKey(name))
|
|
|
|
|
func(path string, info fs.FileInfo, name string) error {
|
|
|
|
|
return c.reconcileSourceFile(ctx, path, info, imagerequest.SourceKey(name))
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -726,16 +743,11 @@ func (c *Cache) reconcileSourceFiles(ctx context.Context) error {
|
|
|
|
|
// is removed with its row and a warning naming it, so the source is
|
|
|
|
|
// fetched again.
|
|
|
|
|
func (c *Cache) reconcileSourceFile(
|
|
|
|
|
ctx context.Context, path string, entry fs.DirEntry, key imagerequest.SourceKey,
|
|
|
|
|
ctx context.Context, path string, info fs.FileInfo, key imagerequest.SourceKey,
|
|
|
|
|
) error {
|
|
|
|
|
info, err := entry.Info()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return fmt.Errorf("failed to stat source content file: %w", err)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
var recordedSize int64
|
|
|
|
|
|
|
|
|
|
err = c.db.QueryRowContext(ctx,
|
|
|
|
|
err := c.db.QueryRowContext(ctx,
|
|
|
|
|
`SELECT size_bytes FROM source_content WHERE source_key = ?`,
|
|
|
|
|
string(key),
|
|
|
|
|
).Scan(&recordedSize)
|
|
|
|
@@ -900,17 +912,12 @@ func (c *Cache) correctUsageTotal(
|
|
|
|
|
|
|
|
|
|
// sweepStaleTempFile removes a temp file left behind by a crashed
|
|
|
|
|
// write once it is old enough that no in-flight store can own it.
|
|
|
|
|
func (c *Cache) sweepStaleTempFile(path string, entry fs.DirEntry) {
|
|
|
|
|
info, err := entry.Info()
|
|
|
|
|
if err != nil {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (c *Cache) sweepStaleTempFile(path string, info fs.FileInfo) {
|
|
|
|
|
if time.Since(info.ModTime()) < staleTempFileAge {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
err = os.Remove(path)
|
|
|
|
|
err := os.Remove(path)
|
|
|
|
|
if err != nil && !os.IsNotExist(err) {
|
|
|
|
|
c.log.Warn("failed to remove stale temp file", "path", path, "error", err)
|
|
|
|
|
|
|
|
|
|