check / check (push) Waiting to run
A killed upload leaves a `.partial` object that only remote nuke removed, and that remote info never showed. The storage interface's DeletePartialUploads becomes ListPartialUploads, which returns each such object's key, size and time; Delete removes it. prune deletes those under metadata/ and blobs/ once older than prune_grace_period, and reports kept, deleted and failed counts. remote info counts them on their own line and in --json. remote nuke lists and deletes them like any other object, and now names any it could not delete. Model: opus-5-5
189 lines
4.7 KiB
Go
189 lines
4.7 KiB
Go
package storage_test
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"sneak.berlin/go/vaultik/internal/storage"
|
|
)
|
|
|
|
// errStreamInterrupted stands in for an upload cut off mid-stream.
|
|
var errStreamInterrupted = errors.New("connection reset mid-upload")
|
|
|
|
// testBlobKey is a key laid out as a blob's key is.
|
|
const testBlobKey = "blobs/aa/bb/aabbccddeeff"
|
|
|
|
// failingReader yields its data once, then fails.
|
|
type failingReader struct {
|
|
data []byte
|
|
done bool
|
|
}
|
|
|
|
func (r *failingReader) Read(p []byte) (int, error) {
|
|
if r.done {
|
|
return 0, errStreamInterrupted
|
|
}
|
|
|
|
n := copy(p, r.data)
|
|
r.done = true
|
|
|
|
return n, nil
|
|
}
|
|
|
|
// TestFileStorer_InterruptedWriteLeavesNoTrustedObject checks that a write
|
|
// cut off mid-stream leaves nothing at the destination key, so a later run
|
|
// cannot Stat a truncated object and trust it as a complete blob.
|
|
func TestFileStorer_InterruptedWriteLeavesNoTrustedObject(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f, err := storage.NewFileStorer(t.TempDir())
|
|
if err != nil {
|
|
t.Fatalf("NewFileStorer: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
key := testBlobKey
|
|
|
|
err = f.PutWithProgress(ctx, key, &failingReader{data: []byte("partial")}, 4096, nil)
|
|
if err == nil {
|
|
t.Fatal("expected the interrupted write to fail, got nil")
|
|
}
|
|
|
|
_, err = f.Stat(ctx, key)
|
|
if !errors.Is(err, storage.ErrNotFound) {
|
|
t.Fatalf("expected key absent after interrupted write, got Stat err %v", err)
|
|
}
|
|
|
|
keys, err := f.List(ctx, "blobs/")
|
|
if err != nil {
|
|
t.Fatalf("List: %v", err)
|
|
}
|
|
|
|
if len(keys) != 0 {
|
|
t.Fatalf("expected no keys listed after interrupted write, got %v", keys)
|
|
}
|
|
}
|
|
|
|
// TestFileStorer_ListSkipsPartialFiles checks that a leftover temp file (the
|
|
// storage layer names them with a ".partial" suffix) is never surfaced as a
|
|
// key by List or ListStream.
|
|
func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
base := t.TempDir()
|
|
|
|
f, err := storage.NewFileStorer(base)
|
|
if err != nil {
|
|
t.Fatalf("NewFileStorer: %v", err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
realKey := testBlobKey
|
|
|
|
err = f.Put(ctx, realKey, strings.NewReader("blob-bytes"))
|
|
if err != nil {
|
|
t.Fatalf("Put: %v", err)
|
|
}
|
|
|
|
// A stray temp file, as an interrupted write would leave behind.
|
|
leftover := filepath.Join(base, "blobs/aa/bb/aabbccddeeff-123456.partial")
|
|
|
|
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
|
if err != nil {
|
|
t.Fatalf("writing leftover temp file: %v", err)
|
|
}
|
|
|
|
keys, err := f.List(ctx, "blobs/")
|
|
if err != nil {
|
|
t.Fatalf("List: %v", err)
|
|
}
|
|
|
|
if len(keys) != 1 || keys[0] != realKey {
|
|
t.Fatalf("List should return only the real key, got %v", keys)
|
|
}
|
|
|
|
var streamed []string
|
|
|
|
for obj := range f.ListStream(ctx, "blobs/") {
|
|
if obj.Err != nil {
|
|
t.Fatalf("ListStream: %v", obj.Err)
|
|
}
|
|
|
|
streamed = append(streamed, obj.Key)
|
|
}
|
|
|
|
if len(streamed) != 1 || streamed[0] != realKey {
|
|
t.Fatalf("ListStream should return only the real key, got %v", streamed)
|
|
}
|
|
}
|
|
|
|
// TestFileStorer_ListPartialUploads checks that a leftover temp file is
|
|
// listed with its size and time, that the object at the real key is not,
|
|
// and that Delete removes the leftover by the listed key.
|
|
func TestFileStorer_ListPartialUploads(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
base := t.TempDir()
|
|
|
|
f, err := storage.NewFileStorer(base)
|
|
if err != nil {
|
|
t.Fatalf("NewFileStorer: %v", err)
|
|
}
|
|
|
|
checkListPartialUploads(t, f, base)
|
|
}
|
|
|
|
// checkListPartialUploads stores an object and writes a leftover temp
|
|
// file next to it in the directory base that s stores into. It checks
|
|
// that ListPartialUploads lists only the leftover, with its size and a
|
|
// time, and that Delete removes it by the listed key and keeps the
|
|
// object.
|
|
func checkListPartialUploads(t *testing.T, s storage.Storer, base string) {
|
|
t.Helper()
|
|
|
|
ctx := context.Background()
|
|
|
|
err := s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
|
|
if err != nil {
|
|
t.Fatalf("Put: %v", err)
|
|
}
|
|
|
|
leftoverKey := testBlobKey + "-123456.partial"
|
|
leftover := filepath.Join(base, leftoverKey)
|
|
|
|
err = os.WriteFile(leftover, []byte("half"), 0o600)
|
|
if err != nil {
|
|
t.Fatalf("writing leftover temp file: %v", err)
|
|
}
|
|
|
|
partial, err := s.ListPartialUploads(ctx, "blobs/")
|
|
if err != nil {
|
|
t.Fatalf("ListPartialUploads: %v", err)
|
|
}
|
|
|
|
if len(partial) != 1 || partial[0].Key != leftoverKey ||
|
|
partial[0].Size != int64(len("half")) || partial[0].ModTime.IsZero() {
|
|
t.Fatalf("ListPartialUploads = %+v, want only %s, 4 bytes, "+
|
|
"with a time", partial, leftoverKey)
|
|
}
|
|
|
|
err = s.Delete(ctx, partial[0].Key)
|
|
if err != nil {
|
|
t.Fatalf("Delete: %v", err)
|
|
}
|
|
|
|
_, err = os.Stat(leftover)
|
|
if !os.IsNotExist(err) {
|
|
t.Errorf("leftover temp file was not removed: %v", err)
|
|
}
|
|
|
|
_, err = s.Stat(ctx, testBlobKey)
|
|
if err != nil {
|
|
t.Errorf("Stat of the real key: %v", err)
|
|
}
|
|
}
|