package storage_test import ( "bytes" "context" "errors" "io" "os" "path/filepath" "strings" "testing" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configmap" "sneak.berlin/go/vaultik/internal/storage" ) // These tests use rclone's ":local:" on-the-fly backend, which addresses the // local filesystem directly without any configured remote, so they run // entirely in-process. A remote that needs credentials and network access // (drive, s3 via rclone, ...) is out of reach of a unit test. // newRcloneStorer builds an rclone backend on rclone's local backend, // rooted at a fresh temp directory. // //nolint:ireturn // conformance runs against the Storer interface by design func newRcloneStorer(t *testing.T) storage.Storer { t.Helper() s, err := storage.NewRcloneStorer(context.Background(), ":local", t.TempDir()) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } return s } // TestRcloneStorer runs the shared Storer contract against the rclone // backend. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestRcloneStorer(t *testing.T) { runStorerConformance(t, newRcloneStorer) } // TestNewRcloneStorerConstruction checks that a valid remote constructs a // backend and that Info() reports the shaped "remote:path" location. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestNewRcloneStorerConstruction(t *testing.T) { dir := t.TempDir() s, err := storage.NewRcloneStorer(context.Background(), ":local", dir) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } // Info().Location is the "remote:path" string the adapter builds from // its two arguments, so asserting it confirms the argument shaping. want := ":local:" + dir if got := s.Info().Location; got != want { t.Errorf("Info().Location = %q, want %q", got, want) } } // TestRcloneStorerObjectAppearsOnlyWhenComplete checks that on a remote // with a server-side move, such as local, nothing is at the key until the // upload has finished, so a killed upload cannot leave a truncated object // there. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestRcloneStorerObjectAppearsOnlyWhenComplete(t *testing.T) { // Without this, rclone reads ahead of the write and holds a small // upload in memory, so the progress callback would run before anything // is written to the remote. ctx, ci := fs.AddConfig(context.Background()) ci.BufferSize = 0 ci.StreamingUploadCutoff = 0 s, err := storage.NewRcloneStorer(ctx, ":local", t.TempDir()) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } key := testBlobKey data := bytes.Repeat([]byte("blob-bytes"), 1000) seenEarly := false err = s.PutWithProgress(ctx, key, bytes.NewReader(data), int64(len(data)), func(int64) error { _, statErr := s.Stat(ctx, key) if statErr == nil { seenEarly = true } return nil }) if err != nil { t.Fatalf("PutWithProgress: %v", err) } if seenEarly { t.Error("object was at its key before the upload finished") } info, err := s.Stat(ctx, key) if err != nil { t.Fatalf("Stat after upload: %v", err) } if info.Size != int64(len(data)) { t.Errorf("stored size = %d, want %d", info.Size, len(data)) } } // TestRcloneStorerListSkipsPartialFiles checks that a temporary file left // by a killed upload is never listed as a key. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestRcloneStorerListSkipsPartialFiles(t *testing.T) { dir := t.TempDir() ctx := context.Background() s, err := storage.NewRcloneStorer(ctx, ":local", dir) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } realKey := testBlobKey err = s.Put(ctx, realKey, strings.NewReader("blob-bytes")) if err != nil { t.Fatalf("Put: %v", err) } leftover := filepath.Join(dir, realKey+"-123456.partial") err = os.WriteFile(leftover, []byte("half"), 0o600) if err != nil { t.Fatalf("writing leftover temp file: %v", err) } keys, err := s.List(ctx, "blobs/") if err != nil { t.Fatalf("List: %v", err) } if len(keys) != 1 || keys[0] != realKey { t.Fatalf("List should return only the real key, got %v", keys) } var streamed []string for obj := range s.ListStream(ctx, "blobs/") { if obj.Err != nil { t.Fatalf("ListStream: %v", obj.Err) } streamed = append(streamed, obj.Key) } if len(streamed) != 1 || streamed[0] != realKey { t.Fatalf("ListStream should return only the real key, got %v", streamed) } } var errNameConflict = errors.New("an object with this name already exists") // moveRefusesExisting is rclone's local backend with a server-side move // that, like dropbox's or onedrive's, refuses to move onto an existing // object. type moveRefusesExisting struct { fs.Fs } func (f *moveRefusesExisting) Features() *fs.Features { features := *f.Fs.Features() features.Move = f.move return &features } //nolint:ireturn // the signature is rclone's func (f *moveRefusesExisting) move( ctx context.Context, src fs.Object, remote string, ) (fs.Object, error) { _, err := f.NewObject(ctx, remote) if err == nil { return nil, errNameConflict } return f.Fs.Features().Move(ctx, src, remote) } // TestRcloneStorerOverwritesWhereMoveRefusesExisting checks that writing a // key twice replaces the object on a remote whose server-side move will not // replace an existing object. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestRcloneStorerOverwritesWhereMoveRefusesExisting(t *testing.T) { fs.Register(&fs.RegInfo{ Name: "moverefusesexisting", NewFs: func( ctx context.Context, _, root string, _ configmap.Mapper, ) (fs.Fs, error) { local, err := fs.NewFs(ctx, ":local:"+root) if err != nil { return nil, err } return &moveRefusesExisting{Fs: local}, nil }, }) ctx := context.Background() s, err := storage.NewRcloneStorer(ctx, ":moverefusesexisting", t.TempDir()) if err != nil { t.Fatalf("NewRcloneStorer: %v", err) } for _, content := range []string{"first", "second"} { err = s.Put(ctx, testBlobKey, strings.NewReader(content)) if err != nil { t.Fatalf("Put %q: %v", content, err) } } rc, err := s.Get(ctx, testBlobKey) if err != nil { t.Fatalf("Get: %v", err) } defer func() { _ = rc.Close() }() got, err := io.ReadAll(rc) if err != nil { t.Fatalf("reading object: %v", err) } if string(got) != "second" { t.Errorf("object = %q, want %q", got, "second") } } // TestNewRcloneStorerUnknownRemote checks that a remote that is not in the // rclone config fails construction with the ErrRemoteNotFound sentinel, // rather than silently returning a backend pointed nowhere. // //nolint:paralleltest // NewRcloneStorer installs the process-global rclone config func TestNewRcloneStorerUnknownRemote(t *testing.T) { _, err := storage.NewRcloneStorer( context.Background(), "vaultik-no-such-remote", "path") if !errors.Is(err, storage.ErrRemoteNotFound) { t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err) } }