Let prune delete old .partial uploads and remote info count them (closes #466)
check / check (push) Waiting to run

A killed upload leaves a `.partial` object that only remote nuke
removed, and that remote info never showed.

The storage interface's DeletePartialUploads becomes ListPartialUploads,
which returns each such object's key, size and time; Delete removes it.
prune deletes those under metadata/ and blobs/ once older than
prune_grace_period, and reports kept, deleted and failed counts. remote
info counts them on their own line and in --json. remote nuke lists and
deletes them like any other object, and now names any it could not
delete.

Model: opus-5-5
This commit was merged in pull request #474.
This commit is contained in:
clawbot committed 2026-11-19 14:01:04 +01:00
1 parent aaa2219d6b
commit be362080d3
24 files changed
+625 -230

No files matched your search

+5 -3
View File
@@ -200,9 +200,11 @@ func (f *Storer) ListStream(
return f.inner.ListStream(ctx, prefix)
}
// DeletePartialUploads delegates unchanged.
func (f *Storer) DeletePartialUploads(ctx context.Context, prefix string) error {
return f.inner.DeletePartialUploads(ctx, prefix)
// ListPartialUploads delegates unchanged.
func (f *Storer) ListPartialUploads(
ctx context.Context, prefix string,
) ([]storage.ObjectInfo, error) {
return f.inner.ListPartialUploads(ctx, prefix)
}
// Info delegates unchanged.
+26 -13
View File
@@ -56,8 +56,8 @@ const storageDirPerm = 0o755
// truncated object at the key a later run would Stat and trust as a complete
// blob. The rclone backend's upload does the same on remotes with a
// server-side move. List and ListStream skip these files, so a leftover from
// an interrupted write is never listed or trusted as a blob; it is otherwise
// harmless.
// an interrupted write is never listed or trusted as a blob; prune deletes
// it once it is older than the prune grace period.
const tempSuffix = ".partial"
// Put stores data at the specified key.
@@ -248,22 +248,24 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
return ch
}
// DeletePartialUploads removes every file under prefix whose name ends in
// ListPartialUploads returns each file under prefix whose name ends in
// tempSuffix. Like List, it reads through symlinked directories and fails
// on one whose target is missing. A missing prefix has none to remove.
func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
basePath := f.fullPath(prefix)
// on one whose target is missing. A missing prefix has none.
func (f *FileStorer) ListPartialUploads(
ctx context.Context, prefix string,
) ([]ObjectInfo, error) {
exists, err := f.prefixExists(prefix)
if err != nil {
return fmt.Errorf("checking path: %w", err)
return nil, fmt.Errorf("checking path: %w", err)
}
if !exists {
return nil
return nil, nil
}
err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
var partial []ObjectInfo
err = f.walk(f.fullPath(prefix), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -276,13 +278,24 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
return nil
}
return f.fs.Remove(path)
relPath, err := filepath.Rel(f.basePath, path)
if err != nil {
return fmt.Errorf("computing relative path: %w", err)
}
partial = append(partial, ObjectInfo{
Key: strings.ReplaceAll(relPath, string(filepath.Separator), "/"),
Size: info.Size(),
ModTime: info.ModTime(),
})
return nil
})
if err != nil {
return fmt.Errorf("walking directory: %w", err)
return nil, fmt.Errorf("walking directory: %w", err)
}
return nil
return partial, nil
}
// Info returns human-readable storage location information.
+32 -8
View File
@@ -121,9 +121,10 @@ func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
}
}
// TestFileStorer_DeletePartialUploads checks that a leftover temp file is
// removed and the object at the real key is kept.
func TestFileStorer_DeletePartialUploads(t *testing.T) {
// TestFileStorer_ListPartialUploads checks that a leftover temp file is
// listed with its size and time, that the object at the real key is not,
// and that Delete removes the leftover by the listed key.
func TestFileStorer_ListPartialUploads(t *testing.T) {
t.Parallel()
base := t.TempDir()
@@ -133,23 +134,46 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) {
t.Fatalf("NewFileStorer: %v", err)
}
checkListPartialUploads(t, f, base)
}
// checkListPartialUploads stores an object and writes a leftover temp
// file next to it in the directory base that s stores into. It checks
// that ListPartialUploads lists only the leftover, with its size and a
// time, and that Delete removes it by the listed key and keeps the
// object.
func checkListPartialUploads(t *testing.T, s storage.Storer, base string) {
t.Helper()
ctx := context.Background()
err = f.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
err := s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
if err != nil {
t.Fatalf("Put: %v", err)
}
leftover := filepath.Join(base, testBlobKey+"-123456.partial")
leftoverKey := testBlobKey + "-123456.partial"
leftover := filepath.Join(base, leftoverKey)
err = os.WriteFile(leftover, []byte("half"), 0o600)
if err != nil {
t.Fatalf("writing leftover temp file: %v", err)
}
err = f.DeletePartialUploads(ctx, "blobs/")
partial, err := s.ListPartialUploads(ctx, "blobs/")
if err != nil {
t.Fatalf("DeletePartialUploads: %v", err)
t.Fatalf("ListPartialUploads: %v", err)
}
if len(partial) != 1 || partial[0].Key != leftoverKey ||
partial[0].Size != int64(len("half")) || partial[0].ModTime.IsZero() {
t.Fatalf("ListPartialUploads = %+v, want only %s, 4 bytes, "+
"with a time", partial, leftoverKey)
}
err = s.Delete(ctx, partial[0].Key)
if err != nil {
t.Fatalf("Delete: %v", err)
}
_, err = os.Stat(leftover)
@@ -157,7 +181,7 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) {
t.Errorf("leftover temp file was not removed: %v", err)
}
_, err = f.Stat(ctx, testBlobKey)
_, err = s.Stat(ctx, testBlobKey)
if err != nil {
t.Errorf("Stat of the real key: %v", err)
}
+13 -18
View File
@@ -239,11 +239,11 @@ func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
}
}
// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that
// DeletePartialUploads, which remote nuke uses, removes a leftover temp
// file behind a symlinked metadata/ or blobs/ directory and keeps the
// object at the real key.
func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
// TestFileStorerListPartialUploadsThroughSymlinkedDirectory checks that
// ListPartialUploads, which prune and remote nuke use, lists a leftover
// temp file behind a symlinked metadata/ or blobs/ directory under its
// key, and leaves out the object at the real key.
func TestFileStorerListPartialUploadsThroughSymlinkedDirectory(t *testing.T) {
t.Parallel()
ctx := context.Background()
@@ -265,29 +265,24 @@ func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
t.Fatalf("Put: %v", err)
}
leftover := filepath.Join(base, key+"-123456.partial")
leftoverKey := key + "-123456.partial"
err = os.WriteFile(leftover, []byte("half"), 0o600)
err = os.WriteFile(filepath.Join(base, leftoverKey),
[]byte("half"), 0o600)
if err != nil {
t.Fatalf("writing leftover temp file: %v", err)
}
moveBehindSymlink(t, base, dir)
err = s.DeletePartialUploads(ctx, dir+"/")
partial, err := s.ListPartialUploads(ctx, dir+"/")
if err != nil {
t.Fatalf("DeletePartialUploads: %v", err)
t.Fatalf("ListPartialUploads: %v", err)
}
// leftover runs through the link.
_, err = os.Stat(leftover)
if !os.IsNotExist(err) {
t.Errorf("leftover temp file was not removed: %v", err)
}
_, err = s.Stat(ctx, key)
if err != nil {
t.Errorf("Stat of the real key: %v", err)
if len(partial) != 1 || partial[0].Key != leftoverKey {
t.Errorf("ListPartialUploads = %+v, want only %s",
partial, leftoverKey)
}
})
}
+13 -14
View File
@@ -256,29 +256,28 @@ func (r *RcloneStorer) ListStream(
return ch
}
// DeletePartialUploads removes every object under prefix whose name ends
// in tempSuffix.
func (r *RcloneStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
var partial []fs.Object
// ListPartialUploads returns each object under prefix whose name ends in
// tempSuffix.
func (r *RcloneStorer) ListPartialUploads(
ctx context.Context, prefix string,
) ([]ObjectInfo, error) {
var partial []ObjectInfo
err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) {
key := obj.Remote()
if strings.HasPrefix(key, prefix) && strings.HasSuffix(key, tempSuffix) {
partial = append(partial, obj)
partial = append(partial, ObjectInfo{
Key: key,
Size: obj.Size(),
ModTime: obj.ModTime(ctx),
})
}
})
if err != nil {
return fmt.Errorf("listing objects: %w", err)
return nil, fmt.Errorf("listing objects: %w", err)
}
for _, obj := range partial {
err = obj.Remove(ctx)
if err != nil {
return fmt.Errorf("removing object: %w", err)
}
}
return nil
return partial, nil
}
// Info returns human-readable storage location information.
+7 -31
View File
@@ -205,45 +205,21 @@ func TestRcloneStorerListSkipsPartialFiles(t *testing.T) {
}
}
// TestRcloneStorerDeletePartialUploads checks that a temporary file left
// by a killed upload is removed and the object at the real key is kept.
// TestRcloneStorerListPartialUploads checks that a temporary file left by
// a killed upload is listed with its size and time, that the object at
// the real key is not, and that Delete removes the leftover by the listed
// key.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerDeletePartialUploads(t *testing.T) {
func TestRcloneStorerListPartialUploads(t *testing.T) {
dir := t.TempDir()
ctx := context.Background()
s, err := storage.NewRcloneStorer(ctx, ":local", dir)
s, err := storage.NewRcloneStorer(context.Background(), ":local", dir)
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
if err != nil {
t.Fatalf("Put: %v", err)
}
leftover := filepath.Join(dir, testBlobKey+"-123456.partial")
err = os.WriteFile(leftover, []byte("half"), 0o600)
if err != nil {
t.Fatalf("writing leftover temp file: %v", err)
}
err = s.DeletePartialUploads(ctx, "blobs/")
if err != nil {
t.Fatalf("DeletePartialUploads: %v", err)
}
_, err = os.Stat(leftover)
if !os.IsNotExist(err) {
t.Errorf("leftover temp file was not removed: %v", err)
}
_, err = s.Stat(ctx, testBlobKey)
if err != nil {
t.Errorf("Stat of the real key: %v", err)
}
checkListPartialUploads(t, s, dir)
}
// newRcloneStorerOnWrappedLocal registers name as rclone's local backend
+6 -4
View File
@@ -100,10 +100,12 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
return ch
}
// DeletePartialUploads has nothing to remove: S3 shows an object only once
// its upload has completed, so an upload cut off part-way leaves none.
func (s *S3Storer) DeletePartialUploads(_ context.Context, _ string) error {
return nil
// ListPartialUploads finds none: S3 shows an object only once its upload
// has completed, so an upload cut off part-way leaves none.
func (s *S3Storer) ListPartialUploads(
_ context.Context, _ string,
) ([]ObjectInfo, error) {
return nil, nil
}
// Info returns human-readable storage location information.
+6 -5
View File
@@ -77,11 +77,12 @@ type Storer interface {
// If an error occurs during listing, the final item will have Err set.
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
// DeletePartialUploads removes every object under prefix that an
// upload cut off part-way left under a temporary name ending in
// `.partial`. The file and rclone backends' List and ListStream skip
// such an object.
DeletePartialUploads(ctx context.Context, prefix string) error
// ListPartialUploads returns each object under prefix whose name ends
// in `.partial`, as a killed upload leaves it, with its key, size and
// modification time. The file and rclone
// backends' List and ListStream skip such an object; Delete removes
// it.
ListPartialUploads(ctx context.Context, prefix string) ([]ObjectInfo, error)
// Info returns human-readable storage location information.
Info() Info