Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b496da5223 | ||
|
|
8d5e6f3399 | ||
|
|
abad8161bc |
@@ -578,3 +578,54 @@ func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
|
|||||||
require.Equal(t, 1, result.FilesScanned,
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
"**/ file pattern should match at the top level and nested")
|
"**/ file pattern should match at the top level and nested")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_AnchoredDoubleStarAtEveryDepth(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, /**/ also matches directly under the source dir
|
||||||
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/**/projectname"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// /**/projectname should exclude BOTH:
|
||||||
|
// - /backup/projectname/file.txt
|
||||||
|
// - /backup/otherproject/projectname/file.txt
|
||||||
|
// Total files: 4, excluded: 2
|
||||||
|
require.Equal(t, 2, result.FilesScanned,
|
||||||
|
"/**/ pattern should match at the top level and nested")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExcludePatterns_DoubleStarInMiddleMatchesZeroDirectories(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// As in gitignore, a/**/b matches a/b as well as a/c/b. The last part
|
||||||
|
// is a glob: with a plain name the glob library already matched a/b.
|
||||||
|
fs := &afero.MemMapFs{}
|
||||||
|
files := []string{"/backup/a/x.log", "/backup/a/c/y.log", "/backup/a/c/kept.txt"}
|
||||||
|
|
||||||
|
for _, path := range files {
|
||||||
|
require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0755))
|
||||||
|
require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0644))
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"a/**/*.log"})
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
createSnapshotRecord(ctx, t, repos, "test-snapshot")
|
||||||
|
|
||||||
|
result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// a/**/*.log should exclude /backup/a/x.log and /backup/a/c/y.log
|
||||||
|
// Total files: 3, excluded: 2
|
||||||
|
require.Equal(t, 1, result.FilesScanned,
|
||||||
|
"a/**/*.log should match a/x.log as well as a/c/y.log")
|
||||||
|
}
|
||||||
|
|||||||
@@ -2035,33 +2035,60 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
pattern = strings.TrimSuffix(pattern, "/")
|
pattern = strings.TrimSuffix(pattern, "/")
|
||||||
|
|
||||||
// A leading "**/" demands a "/" before the name and so misses an
|
// A leading "**/" demands a "/" before the name and so misses an
|
||||||
// entry directly under the source path. An unanchored pattern
|
// entry directly under the source path. It means "at every depth",
|
||||||
// already matches at every depth, so drop the prefix: as in
|
// which is what an unanchored pattern already does, so drop the
|
||||||
// gitignore, "**/x" then also matches a top-level "x".
|
// prefix and match unanchored: as in gitignore, "**/x" and
|
||||||
if !anchored {
|
// "/**/x" then also match a top-level "x".
|
||||||
pattern = strings.TrimPrefix(pattern, "**/")
|
if rest, found := strings.CutPrefix(pattern, "**/"); found {
|
||||||
|
pattern = rest
|
||||||
|
anchored = false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compile the glob pattern
|
// Likewise "/**/" inside a pattern demands two slashes, so
|
||||||
// For patterns without path separators, we need to match them as components
|
// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
|
||||||
// e.g., ".git" should match ".git" anywhere in the path
|
// zero or more directories, so also compile the pattern with it
|
||||||
g, err := glob.Compile(pattern, '/')
|
// made a single "/". The glob library's {a,b} is no help: with a
|
||||||
if err != nil {
|
// "**" inside it in mid-pattern, it misses paths it should match.
|
||||||
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
for _, variant := range doubleStarVariants(pattern) {
|
||||||
|
// For patterns without path separators, we need to match them as components
|
||||||
|
// e.g., ".git" should match ".git" anywhere in the path
|
||||||
|
g, err := glob.Compile(variant, '/')
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
||||||
|
|
||||||
continue
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
compiled = append(compiled, compiledPattern{
|
||||||
|
pattern: g,
|
||||||
|
anchored: anchored,
|
||||||
|
original: p,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
compiled = append(compiled, compiledPattern{
|
|
||||||
pattern: g,
|
|
||||||
anchored: anchored,
|
|
||||||
original: p,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return compiled
|
return compiled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// doubleStarVariants returns pattern once for every way of keeping each
|
||||||
|
// "/**/" in it or making it a single "/".
|
||||||
|
func doubleStarVariants(pattern string) []string {
|
||||||
|
const doubleStar = "/**/"
|
||||||
|
|
||||||
|
before, after, found := strings.Cut(pattern, doubleStar)
|
||||||
|
if !found {
|
||||||
|
return []string{pattern}
|
||||||
|
}
|
||||||
|
|
||||||
|
var variants []string
|
||||||
|
|
||||||
|
for _, rest := range doubleStarVariants(after) {
|
||||||
|
variants = append(variants, before+doubleStar+rest, before+"/"+rest)
|
||||||
|
}
|
||||||
|
|
||||||
|
return variants
|
||||||
|
}
|
||||||
|
|
||||||
// shouldExclude checks if a path should be excluded based on exclude patterns
|
// shouldExclude checks if a path should be excluded based on exclude patterns
|
||||||
// filePath is the full path to the file
|
// filePath is the full path to the file
|
||||||
// rootPath is the root of the backup source directory
|
// rootPath is the root of the backup source directory
|
||||||
|
|||||||
@@ -7,10 +7,12 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configfile"
|
"github.com/rclone/rclone/fs/config/configfile"
|
||||||
"github.com/rclone/rclone/fs/operations"
|
"github.com/rclone/rclone/fs/operations"
|
||||||
|
|
||||||
@@ -33,6 +35,15 @@ type RcloneStorer struct {
|
|||||||
// The remote parameter is the rclone remote name (as configured via `rclone config`).
|
// The remote parameter is the rclone remote name (as configured via `rclone config`).
|
||||||
// The path parameter is the path within the remote.
|
// The path parameter is the path within the remote.
|
||||||
func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, error) {
|
func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, error) {
|
||||||
|
// rclone reads RCLONE_CONFIG only as the value of its --config flag,
|
||||||
|
// which vaultik does not have, so it is applied here as that flag would.
|
||||||
|
if configPath, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
|
||||||
|
err := config.SetConfigPath(configPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("using RCLONE_CONFIG %q: %w", configPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Install the default config file handler
|
// Install the default config file handler
|
||||||
configfile.Install()
|
configfile.Install()
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/rclone/rclone/fs"
|
"github.com/rclone/rclone/fs"
|
||||||
|
"github.com/rclone/rclone/fs/config"
|
||||||
"github.com/rclone/rclone/fs/config/configmap"
|
"github.com/rclone/rclone/fs/config/configmap"
|
||||||
"sneak.berlin/go/vaultik/internal/storage"
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
)
|
)
|
||||||
@@ -390,3 +391,41 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
|
|||||||
t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
|
t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
|
||||||
|
// in the file RCLONE_CONFIG names is found and written to, as rclone itself
|
||||||
|
// would.
|
||||||
|
func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
|
||||||
|
// rclone keeps the config file path in a process-global variable, which
|
||||||
|
// NewRcloneStorer leaves pointing at this test's file.
|
||||||
|
previous := config.GetConfigPath()
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
|
||||||
|
|
||||||
|
confFile := filepath.Join(t.TempDir(), "rclone.conf")
|
||||||
|
|
||||||
|
err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("writing rclone config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("RCLONE_CONFIG", confFile)
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
s, err := storage.NewRcloneStorer(ctx, "loc", dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewRcloneStorer: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Put: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = os.Stat(filepath.Join(dir, testBlobKey))
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("object is not in the remote's directory: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user