vaultik/internal/storage/rclone_test.go

package storage_test

import (
	"bytes"
	"context"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/config/configmap"
	"sneak.berlin/go/vaultik/internal/storage"
)

// These tests use rclone's ":local:" on-the-fly backend, which addresses the
// local filesystem directly without any configured remote, so they run
// entirely in-process. A remote that needs credentials and network access
// (drive, s3 via rclone, ...) is out of reach of a unit test.

// newRcloneStorer builds an rclone backend on rclone's local backend,
// rooted at a fresh temp directory.
//
//nolint:ireturn // conformance runs against the Storer interface by design
func newRcloneStorer(t *testing.T) storage.Storer {
	t.Helper()

	s, err := storage.NewRcloneStorer(context.Background(), ":local", t.TempDir())
	if err != nil {
		t.Fatalf("NewRcloneStorer: %v", err)
	}

	return s
}

// TestRcloneStorer runs the shared Storer contract against the rclone
// backend.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorer(t *testing.T) {
	runStorerConformance(t, newRcloneStorer)
}

// TestNewRcloneStorerConstruction checks that a valid remote constructs a
// backend and that Info() reports the shaped "remote:path" location.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestNewRcloneStorerConstruction(t *testing.T) {
	dir := t.TempDir()

	s, err := storage.NewRcloneStorer(context.Background(), ":local", dir)
	if err != nil {
		t.Fatalf("NewRcloneStorer: %v", err)
	}

	// Info().Location is the "remote:path" string the adapter builds from
	// its two arguments, so asserting it confirms the argument shaping.
	want := ":local:" + dir
	if got := s.Info().Location; got != want {
		t.Errorf("Info().Location = %q, want %q", got, want)
	}
}

// TestRcloneStorerObjectAppearsOnlyWhenComplete checks that on a remote
// with a server-side move, such as local, nothing is at the key until the
// upload has finished, so a killed upload cannot leave a truncated object
// there.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerObjectAppearsOnlyWhenComplete(t *testing.T) {
	// Without this, rclone reads ahead of the write and holds a small
	// upload in memory, so the progress callback would run before anything
	// is written to the remote.
	ctx, ci := fs.AddConfig(context.Background())
	ci.BufferSize = 0
	ci.StreamingUploadCutoff = 0

	s, err := storage.NewRcloneStorer(ctx, ":local", t.TempDir())
	if err != nil {
		t.Fatalf("NewRcloneStorer: %v", err)
	}

	key := testBlobKey
	data := bytes.Repeat([]byte("blob-bytes"), 1000)
	seenEarly := false

	err = s.PutWithProgress(ctx, key, bytes.NewReader(data), int64(len(data)),
		func(int64) error {
			_, statErr := s.Stat(ctx, key)
			if statErr == nil {
				seenEarly = true
			}

			return nil
		})
	if err != nil {
		t.Fatalf("PutWithProgress: %v", err)
	}

	if seenEarly {
		t.Error("object was at its key before the upload finished")
	}

	info, err := s.Stat(ctx, key)
	if err != nil {
		t.Fatalf("Stat after upload: %v", err)
	}

	if info.Size != int64(len(data)) {
		t.Errorf("stored size = %d, want %d", info.Size, len(data))
	}
}

// TestRcloneStorerListSkipsPartialFiles checks that a temporary file left
// by a killed upload is never listed as a key.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerListSkipsPartialFiles(t *testing.T) {
	dir := t.TempDir()
	ctx := context.Background()

	s, err := storage.NewRcloneStorer(ctx, ":local", dir)
	if err != nil {
		t.Fatalf("NewRcloneStorer: %v", err)
	}

	realKey := testBlobKey

	err = s.Put(ctx, realKey, strings.NewReader("blob-bytes"))
	if err != nil {
		t.Fatalf("Put: %v", err)
	}

	leftover := filepath.Join(dir, realKey+"-123456.partial")

	err = os.WriteFile(leftover, []byte("half"), 0o600)
	if err != nil {
		t.Fatalf("writing leftover temp file: %v", err)
	}

	keys, err := s.List(ctx, "blobs/")
	if err != nil {
		t.Fatalf("List: %v", err)
	}

	if len(keys) != 1 || keys[0] != realKey {
		t.Fatalf("List should return only the real key, got %v", keys)
	}

	var streamed []string

	for obj := range s.ListStream(ctx, "blobs/") {
		if obj.Err != nil {
			t.Fatalf("ListStream: %v", obj.Err)
		}

		streamed = append(streamed, obj.Key)
	}

	if len(streamed) != 1 || streamed[0] != realKey {
		t.Fatalf("ListStream should return only the real key, got %v", streamed)
	}
}

var errNameConflict = errors.New("an object with this name already exists")

// moveRefusesExisting is rclone's local backend with a server-side move
// that, like dropbox's or onedrive's, refuses to move onto an existing
// object.
type moveRefusesExisting struct {
	fs.Fs
}

func (f *moveRefusesExisting) Features() *fs.Features {
	features := *f.Fs.Features()
	features.Move = f.move

	return &features
}

//nolint:ireturn // the signature is rclone's
func (f *moveRefusesExisting) move(
	ctx context.Context, src fs.Object, remote string,
) (fs.Object, error) {
	_, err := f.NewObject(ctx, remote)
	if err == nil {
		return nil, errNameConflict
	}

	return f.Fs.Features().Move(ctx, src, remote)
}

// TestRcloneStorerOverwritesWhereMoveRefusesExisting checks that writing a
// key twice replaces the object on a remote whose server-side move will not
// replace an existing object.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestRcloneStorerOverwritesWhereMoveRefusesExisting(t *testing.T) {
	fs.Register(&fs.RegInfo{
		Name: "moverefusesexisting",
		NewFs: func(
			ctx context.Context, _, root string, _ configmap.Mapper,
		) (fs.Fs, error) {
			local, err := fs.NewFs(ctx, ":local:"+root)
			if err != nil {
				return nil, err
			}

			return &moveRefusesExisting{Fs: local}, nil
		},
	})

	ctx := context.Background()

	s, err := storage.NewRcloneStorer(ctx, ":moverefusesexisting", t.TempDir())
	if err != nil {
		t.Fatalf("NewRcloneStorer: %v", err)
	}

	for _, content := range []string{"first", "second"} {
		err = s.Put(ctx, testBlobKey, strings.NewReader(content))
		if err != nil {
			t.Fatalf("Put %q: %v", content, err)
		}
	}

	rc, err := s.Get(ctx, testBlobKey)
	if err != nil {
		t.Fatalf("Get: %v", err)
	}

	defer func() { _ = rc.Close() }()

	got, err := io.ReadAll(rc)
	if err != nil {
		t.Fatalf("reading object: %v", err)
	}

	if string(got) != "second" {
		t.Errorf("object = %q, want %q", got, "second")
	}
}

// TestNewRcloneStorerUnknownRemote checks that a remote that is not in the
// rclone config fails construction with the ErrRemoteNotFound sentinel,
// rather than silently returning a backend pointed nowhere.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestNewRcloneStorerUnknownRemote(t *testing.T) {
	_, err := storage.NewRcloneStorer(
		context.Background(), "vaultik-no-such-remote", "path")
	if !errors.Is(err, storage.ErrRemoteNotFound) {
		t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
	}
}