Let prune delete old .partial uploads and remote info count them (closes #466)
check / check (push) Waiting to run
check / check (push) Waiting to run
A killed upload leaves a `.partial` object that only remote nuke removed, and that remote info never showed. The storage interface's DeletePartialUploads becomes ListPartialUploads, which returns each such object's key, size and time; Delete removes it. prune deletes those under metadata/ and blobs/ once older than prune_grace_period, and reports kept, deleted and failed counts. remote info counts them on their own line and in --json. remote nuke lists and deletes them like any other object, and now names any it could not delete. Model: opus-5-5
This commit was merged in pull request #474.
This commit is contained in:
1 parent
aaa2219d6b
commit
be362080d3
24 files changed
+625
-230
No files matched your search
@@ -200,9 +200,11 @@ func (f *Storer) ListStream(
|
||||
return f.inner.ListStream(ctx, prefix)
|
||||
}
|
||||
|
||||
// DeletePartialUploads delegates unchanged.
|
||||
func (f *Storer) DeletePartialUploads(ctx context.Context, prefix string) error {
|
||||
return f.inner.DeletePartialUploads(ctx, prefix)
|
||||
// ListPartialUploads delegates unchanged.
|
||||
func (f *Storer) ListPartialUploads(
|
||||
ctx context.Context, prefix string,
|
||||
) ([]storage.ObjectInfo, error) {
|
||||
return f.inner.ListPartialUploads(ctx, prefix)
|
||||
}
|
||||
|
||||
// Info delegates unchanged.
|
||||
|
||||
+26
-13
@@ -56,8 +56,8 @@ const storageDirPerm = 0o755
|
||||
// truncated object at the key a later run would Stat and trust as a complete
|
||||
// blob. The rclone backend's upload does the same on remotes with a
|
||||
// server-side move. List and ListStream skip these files, so a leftover from
|
||||
// an interrupted write is never listed or trusted as a blob; it is otherwise
|
||||
// harmless.
|
||||
// an interrupted write is never listed or trusted as a blob; prune deletes
|
||||
// it once it is older than the prune grace period.
|
||||
const tempSuffix = ".partial"
|
||||
|
||||
// Put stores data at the specified key.
|
||||
@@ -248,22 +248,24 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
|
||||
return ch
|
||||
}
|
||||
|
||||
// DeletePartialUploads removes every file under prefix whose name ends in
|
||||
// ListPartialUploads returns each file under prefix whose name ends in
|
||||
// tempSuffix. Like List, it reads through symlinked directories and fails
|
||||
// on one whose target is missing. A missing prefix has none to remove.
|
||||
func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
|
||||
basePath := f.fullPath(prefix)
|
||||
|
||||
// on one whose target is missing. A missing prefix has none.
|
||||
func (f *FileStorer) ListPartialUploads(
|
||||
ctx context.Context, prefix string,
|
||||
) ([]ObjectInfo, error) {
|
||||
exists, err := f.prefixExists(prefix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking path: %w", err)
|
||||
return nil, fmt.Errorf("checking path: %w", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
err = f.walk(basePath, func(path string, info os.FileInfo, err error) error {
|
||||
var partial []ObjectInfo
|
||||
|
||||
err = f.walk(f.fullPath(prefix), func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -276,13 +278,24 @@ func (f *FileStorer) DeletePartialUploads(ctx context.Context, prefix string) er
|
||||
return nil
|
||||
}
|
||||
|
||||
return f.fs.Remove(path)
|
||||
relPath, err := filepath.Rel(f.basePath, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("computing relative path: %w", err)
|
||||
}
|
||||
|
||||
partial = append(partial, ObjectInfo{
|
||||
Key: strings.ReplaceAll(relPath, string(filepath.Separator), "/"),
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("walking directory: %w", err)
|
||||
return nil, fmt.Errorf("walking directory: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return partial, nil
|
||||
}
|
||||
|
||||
// Info returns human-readable storage location information.
|
||||
|
||||
@@ -121,9 +121,10 @@ func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorer_DeletePartialUploads checks that a leftover temp file is
|
||||
// removed and the object at the real key is kept.
|
||||
func TestFileStorer_DeletePartialUploads(t *testing.T) {
|
||||
// TestFileStorer_ListPartialUploads checks that a leftover temp file is
|
||||
// listed with its size and time, that the object at the real key is not,
|
||||
// and that Delete removes the leftover by the listed key.
|
||||
func TestFileStorer_ListPartialUploads(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
base := t.TempDir()
|
||||
@@ -133,23 +134,46 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) {
|
||||
t.Fatalf("NewFileStorer: %v", err)
|
||||
}
|
||||
|
||||
checkListPartialUploads(t, f, base)
|
||||
}
|
||||
|
||||
// checkListPartialUploads stores an object and writes a leftover temp
|
||||
// file next to it in the directory base that s stores into. It checks
|
||||
// that ListPartialUploads lists only the leftover, with its size and a
|
||||
// time, and that Delete removes it by the listed key and keeps the
|
||||
// object.
|
||||
func checkListPartialUploads(t *testing.T, s storage.Storer, base string) {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
err = f.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
|
||||
err := s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
leftover := filepath.Join(base, testBlobKey+"-123456.partial")
|
||||
leftoverKey := testBlobKey + "-123456.partial"
|
||||
leftover := filepath.Join(base, leftoverKey)
|
||||
|
||||
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("writing leftover temp file: %v", err)
|
||||
}
|
||||
|
||||
err = f.DeletePartialUploads(ctx, "blobs/")
|
||||
partial, err := s.ListPartialUploads(ctx, "blobs/")
|
||||
if err != nil {
|
||||
t.Fatalf("DeletePartialUploads: %v", err)
|
||||
t.Fatalf("ListPartialUploads: %v", err)
|
||||
}
|
||||
|
||||
if len(partial) != 1 || partial[0].Key != leftoverKey ||
|
||||
partial[0].Size != int64(len("half")) || partial[0].ModTime.IsZero() {
|
||||
t.Fatalf("ListPartialUploads = %+v, want only %s, 4 bytes, "+
|
||||
"with a time", partial, leftoverKey)
|
||||
}
|
||||
|
||||
err = s.Delete(ctx, partial[0].Key)
|
||||
if err != nil {
|
||||
t.Fatalf("Delete: %v", err)
|
||||
}
|
||||
|
||||
_, err = os.Stat(leftover)
|
||||
@@ -157,7 +181,7 @@ func TestFileStorer_DeletePartialUploads(t *testing.T) {
|
||||
t.Errorf("leftover temp file was not removed: %v", err)
|
||||
}
|
||||
|
||||
_, err = f.Stat(ctx, testBlobKey)
|
||||
_, err = s.Stat(ctx, testBlobKey)
|
||||
if err != nil {
|
||||
t.Errorf("Stat of the real key: %v", err)
|
||||
}
|
||||
|
||||
@@ -239,11 +239,11 @@ func TestFileStorerListFailsOnSymlinkLoop(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory checks that
|
||||
// DeletePartialUploads, which remote nuke uses, removes a leftover temp
|
||||
// file behind a symlinked metadata/ or blobs/ directory and keeps the
|
||||
// object at the real key.
|
||||
func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
|
||||
// TestFileStorerListPartialUploadsThroughSymlinkedDirectory checks that
|
||||
// ListPartialUploads, which prune and remote nuke use, lists a leftover
|
||||
// temp file behind a symlinked metadata/ or blobs/ directory under its
|
||||
// key, and leaves out the object at the real key.
|
||||
func TestFileStorerListPartialUploadsThroughSymlinkedDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -265,29 +265,24 @@ func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
leftover := filepath.Join(base, key+"-123456.partial")
|
||||
leftoverKey := key + "-123456.partial"
|
||||
|
||||
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
||||
err = os.WriteFile(filepath.Join(base, leftoverKey),
|
||||
[]byte("half"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("writing leftover temp file: %v", err)
|
||||
}
|
||||
|
||||
moveBehindSymlink(t, base, dir)
|
||||
|
||||
err = s.DeletePartialUploads(ctx, dir+"/")
|
||||
partial, err := s.ListPartialUploads(ctx, dir+"/")
|
||||
if err != nil {
|
||||
t.Fatalf("DeletePartialUploads: %v", err)
|
||||
t.Fatalf("ListPartialUploads: %v", err)
|
||||
}
|
||||
|
||||
// leftover runs through the link.
|
||||
_, err = os.Stat(leftover)
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("leftover temp file was not removed: %v", err)
|
||||
}
|
||||
|
||||
_, err = s.Stat(ctx, key)
|
||||
if err != nil {
|
||||
t.Errorf("Stat of the real key: %v", err)
|
||||
if len(partial) != 1 || partial[0].Key != leftoverKey {
|
||||
t.Errorf("ListPartialUploads = %+v, want only %s",
|
||||
partial, leftoverKey)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+13
-14
@@ -256,29 +256,28 @@ func (r *RcloneStorer) ListStream(
|
||||
return ch
|
||||
}
|
||||
|
||||
// DeletePartialUploads removes every object under prefix whose name ends
|
||||
// in tempSuffix.
|
||||
func (r *RcloneStorer) DeletePartialUploads(ctx context.Context, prefix string) error {
|
||||
var partial []fs.Object
|
||||
// ListPartialUploads returns each object under prefix whose name ends in
|
||||
// tempSuffix.
|
||||
func (r *RcloneStorer) ListPartialUploads(
|
||||
ctx context.Context, prefix string,
|
||||
) ([]ObjectInfo, error) {
|
||||
var partial []ObjectInfo
|
||||
|
||||
err := operations.ListFn(ctx, r.fsys, func(obj fs.Object) {
|
||||
key := obj.Remote()
|
||||
if strings.HasPrefix(key, prefix) && strings.HasSuffix(key, tempSuffix) {
|
||||
partial = append(partial, obj)
|
||||
partial = append(partial, ObjectInfo{
|
||||
Key: key,
|
||||
Size: obj.Size(),
|
||||
ModTime: obj.ModTime(ctx),
|
||||
})
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing objects: %w", err)
|
||||
return nil, fmt.Errorf("listing objects: %w", err)
|
||||
}
|
||||
|
||||
for _, obj := range partial {
|
||||
err = obj.Remove(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("removing object: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return partial, nil
|
||||
}
|
||||
|
||||
// Info returns human-readable storage location information.
|
||||
|
||||
@@ -205,45 +205,21 @@ func TestRcloneStorerListSkipsPartialFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRcloneStorerDeletePartialUploads checks that a temporary file left
|
||||
// by a killed upload is removed and the object at the real key is kept.
|
||||
// TestRcloneStorerListPartialUploads checks that a temporary file left by
|
||||
// a killed upload is listed with its size and time, that the object at
|
||||
// the real key is not, and that Delete removes the leftover by the listed
|
||||
// key.
|
||||
//
|
||||
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
|
||||
func TestRcloneStorerDeletePartialUploads(t *testing.T) {
|
||||
func TestRcloneStorerListPartialUploads(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ctx := context.Background()
|
||||
|
||||
s, err := storage.NewRcloneStorer(ctx, ":local", dir)
|
||||
s, err := storage.NewRcloneStorer(context.Background(), ":local", dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewRcloneStorer: %v", err)
|
||||
}
|
||||
|
||||
err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
|
||||
if err != nil {
|
||||
t.Fatalf("Put: %v", err)
|
||||
}
|
||||
|
||||
leftover := filepath.Join(dir, testBlobKey+"-123456.partial")
|
||||
|
||||
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
||||
if err != nil {
|
||||
t.Fatalf("writing leftover temp file: %v", err)
|
||||
}
|
||||
|
||||
err = s.DeletePartialUploads(ctx, "blobs/")
|
||||
if err != nil {
|
||||
t.Fatalf("DeletePartialUploads: %v", err)
|
||||
}
|
||||
|
||||
_, err = os.Stat(leftover)
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("leftover temp file was not removed: %v", err)
|
||||
}
|
||||
|
||||
_, err = s.Stat(ctx, testBlobKey)
|
||||
if err != nil {
|
||||
t.Errorf("Stat of the real key: %v", err)
|
||||
}
|
||||
checkListPartialUploads(t, s, dir)
|
||||
}
|
||||
|
||||
// newRcloneStorerOnWrappedLocal registers name as rclone's local backend
|
||||
|
||||
@@ -100,10 +100,12 @@ func (s *S3Storer) ListStream(ctx context.Context, prefix string) <-chan ObjectI
|
||||
return ch
|
||||
}
|
||||
|
||||
// DeletePartialUploads has nothing to remove: S3 shows an object only once
|
||||
// its upload has completed, so an upload cut off part-way leaves none.
|
||||
func (s *S3Storer) DeletePartialUploads(_ context.Context, _ string) error {
|
||||
return nil
|
||||
// ListPartialUploads finds none: S3 shows an object only once its upload
|
||||
// has completed, so an upload cut off part-way leaves none.
|
||||
func (s *S3Storer) ListPartialUploads(
|
||||
_ context.Context, _ string,
|
||||
) ([]ObjectInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Info returns human-readable storage location information.
|
||||
|
||||
@@ -77,11 +77,12 @@ type Storer interface {
|
||||
// If an error occurs during listing, the final item will have Err set.
|
||||
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
||||
|
||||
// DeletePartialUploads removes every object under prefix that an
|
||||
// upload cut off part-way left under a temporary name ending in
|
||||
// `.partial`. The file and rclone backends' List and ListStream skip
|
||||
// such an object.
|
||||
DeletePartialUploads(ctx context.Context, prefix string) error
|
||||
// ListPartialUploads returns each object under prefix whose name ends
|
||||
// in `.partial`, as a killed upload leaves it, with its key, size and
|
||||
// modification time. The file and rclone
|
||||
// backends' List and ListStream skip such an object; Delete removes
|
||||
// it.
|
||||
ListPartialUploads(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||
|
||||
// Info returns human-readable storage location information.
|
||||
Info() Info
|
||||
|
||||
Reference in new issue
Block a user