Author SHA1 Message Date
sneak 0e57ea874a Report a prune count that could not be read as unknown, not 0 (closes #96)
check / check (pull_request) Successful in 3m36s
PruneDatabase read seven table counts with the error discarded, so a
query that could not run silently became 0 and the before/after delta
computed from it looked like real work.

Each read now goes through a helper that logs at warn on failure and
returns nil; nil renders as "unknown", never "0", so an empty table is
distinguishable from one that could not be queried. Deltas built from an
unknown count are themselves unknown. The failure is surfaced, not
propagated, so the command's failure conditions are unchanged. These
counts have no --json output — under --json the summary is suppressed
entirely — so nothing there can show a false 0.

Model: opus-4-8
2026-09-21 18:43:15 +00:00
16 changed files with 494 additions and 765 deletions
+1 -2
View File
@@ -251,8 +251,7 @@ local index alone, and still exits zero.
per-snapshot-name (`--keep-latest` keeps the latest of each name, not the
latest globally).
* `--keep-latest`: Keep only the most recent snapshot of each name
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`,
`4w`, `6mo`, `1y`; `m` is minutes, `mo` is months)
* `--older-than <duration>`: Remove snapshots older than duration (e.g. `30d`, `6m`, `1y`)
* `--snapshot <name>`: Restrict to specific snapshot names (repeat for multiple)
* `--force`: Skip confirmation prompt
-27
View File
@@ -34,15 +34,6 @@ release" is exactly the contradiction
is distinguishable from one that could not be queried. The counts have
no `--json` representation — under `--json` the summary is suppressed
entirely — so nothing there can show a false `0`.
- 2026-09-21: Made the s3 storage backend report a missing object as
`storage.ErrNotFound`, like the `file` and `rclone` backends and as the
`Storer` interface documents. `S3Storer.Get` and `Stat` returned the raw
AWS SDK error, so `errors.Is(err, storage.ErrNotFound)` was false on s3
and callers branched differently per backend. Added a small `s3.IsNotFound`
helper (reused by `HeadObject`) and a test that a missing key maps to
`ErrNotFound`
([issue #129](https://git.eeqj.de/sneak/vaultik/issues/129)).
- 2026-09-21: Fixed `verify --deep` reporting healthy snapshots as
corrupt. Its final blob-integrity check hashed the encrypted
downloaded bytes with a single SHA256 and compared that to the blob
@@ -78,24 +69,6 @@ release" is exactly the contradiction
keeps that exact compiler from auto-switching. Bumping Go now touches
`go.mod`, the checksum, and the `Dockerfile` `golang` digest together.
- 2026-09-21: Collapsed the two duration parsers into one and fixed the
`--older-than` months example
([issue #123](https://git.eeqj.de/sneak/vaultik/issues/123)). Two
functions named `parseDuration` existed with different grammars;
`snapshot purge --older-than` and `--keep-newer-than` both already went
through the one in `internal/vaultik`, while the richer copy in
`internal/cli/duration.go` was reachable only from its own test. Kept
the live-path parser and deleted the unused one, so no flag's accepted
grammar changes. The trap the issue was filed over: `README.md`
documented `6m` as the months example for `--older-than`, but `m` is
minutes, so the documented command deleted every snapshot older than
six minutes on a destructive flag. Corrected the doc to `6mo` and put
both flags' help text on one example list that states `m` is minutes
and `mo` is months. The surviving parser now rejects negatives, which
it previously accepted (`-5h`) or silently made positive (`-5d`).
Table-driven tests cover every unit, `6m` as six minutes, `6mo` as 180
days, and rejection of a bare number, an unknown unit, and a negative.
- 2026-08-10: Moved every lint run into its own container, as a build
step ([issue #113](https://git.eeqj.de/sneak/vaultik/issues/113)).
New root `Dockerfile.lint`, built by `script/lint`, runs
+126
View File
@@ -0,0 +1,126 @@
package cli
import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"time"
)
// Approximate lengths of the extended calendar units accepted by
// parseDuration.
const (
durationDay = 24 * time.Hour
durationWeek = 7 * durationDay
durationMonth = 30 * durationDay
durationYear = 365 * durationDay
)
var (
errNegativeDuration = errors.New("negative durations are not supported")
errInvalidDuration = errors.New("invalid duration format")
errUnknownTimeUnit = errors.New("unknown time unit")
)
// parseDuration parses duration strings. Supports standard Go duration format
// (e.g., "3h30m", "1h45m30s") as well as extended units:
// - d: days (e.g., "30d", "7d")
// - w: weeks (e.g., "2w", "4w")
// - mo: months (30 days) (e.g., "6mo", "1mo")
// - y: years (365 days) (e.g., "1y", "2y")
//
// Can combine units: "1y6mo", "2w3d", "1d12h30m"
func parseDuration(s string) (time.Duration, error) {
// First try standard Go duration parsing
d, err := time.ParseDuration(s)
if err == nil {
return d, nil
}
// Extended duration parsing
// Check for negative values
if strings.HasPrefix(strings.TrimSpace(s), "-") {
return 0, errNegativeDuration
}
// Pattern matches: number + unit, repeated
re := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*([a-zA-Z]+)`)
matches := re.FindAllStringSubmatch(s, -1)
if len(matches) == 0 {
return 0, fmt.Errorf("%w: %q", errInvalidDuration, s)
}
var total time.Duration
for _, match := range matches {
valueStr := match[1]
unit := strings.ToLower(match[2])
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
return 0, fmt.Errorf("invalid number %q: %w", valueStr, err)
}
d, err := durationForUnit(value, unit)
if err != nil {
return 0, err
}
total += d
}
return total, nil
}
// durationForUnit converts a value with a (case-normalized) unit suffix
// into a time.Duration, accepting Go's standard units plus the extended
// calendar units.
func durationForUnit(value float64, unit string) (time.Duration, error) {
switch unit {
// Standard time units
case "ns", "nanosecond", "nanoseconds":
return time.Duration(value), nil
case "us", "µs", "microsecond", "microseconds":
return time.Duration(value * float64(time.Microsecond)), nil
case "ms", "millisecond", "milliseconds":
return time.Duration(value * float64(time.Millisecond)), nil
case "s", "sec", "second", "seconds":
return time.Duration(value * float64(time.Second)), nil
case "m", "min", "minute", "minutes":
return time.Duration(value * float64(time.Minute)), nil
case "h", "hr", "hour", "hours":
return time.Duration(value * float64(time.Hour)), nil
// Extended units
case "d", "day", "days":
return time.Duration(value * float64(durationDay)), nil
case "w", "week", "weeks":
return time.Duration(value * float64(durationWeek)), nil
case "mo", "month", "months":
// Using 30 days as approximation
return time.Duration(value * float64(durationMonth)), nil
case "y", "year", "years":
// Using 365 days as approximation
return time.Duration(value * float64(durationYear)), nil
default:
// Try parsing as standard Go duration unit
testStr := "1" + unit
_, err := time.ParseDuration(testStr)
if err != nil {
return 0, fmt.Errorf("%w: %q", errUnknownTimeUnit, unit)
}
// It's a valid Go duration unit, parse the full value
fullStr := fmt.Sprintf("%g%s", value, unit)
d, err := time.ParseDuration(fullStr)
if err != nil {
return 0, fmt.Errorf("invalid duration %q: %w", fullStr, err)
}
return d, nil
}
}
+299
View File
@@ -0,0 +1,299 @@
package cli //nolint:testpackage // needs access to unexported parseDuration
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type parseDurationCase struct {
name string
input string
expected time.Duration
wantErr bool
}
// runParseDurationCases executes a table of parseDuration cases as
// parallel subtests.
func runParseDurationCases(t *testing.T, tests []parseDurationCase) {
t.Helper()
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := parseDuration(tt.input)
if tt.wantErr {
require.Error(t, err, "expected error for input %q", tt.input)
return
}
require.NoError(t, err, "unexpected error for input %q", tt.input)
assert.Equal(t, tt.expected, got, "duration mismatch for input %q", tt.input)
})
}
}
func TestParseDurationStandard(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
{
name: "standard seconds",
input: "30s",
expected: 30 * time.Second,
},
{
name: "standard minutes",
input: "45m",
expected: 45 * time.Minute,
},
{
name: "standard hours",
input: "2h",
expected: 2 * time.Hour,
},
{
name: "standard combined",
input: "3h30m",
expected: 3*time.Hour + 30*time.Minute,
},
{
name: "standard complex",
input: "1h45m30s",
expected: 1*time.Hour + 45*time.Minute + 30*time.Second,
},
{
name: "standard with milliseconds",
input: "1s500ms",
expected: 1*time.Second + 500*time.Millisecond,
},
})
}
func TestParseDurationExtendedUnits(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
// Extended units - days
{
name: "single day",
input: "1d",
expected: 24 * time.Hour,
},
{
name: "multiple days",
input: "7d",
expected: 7 * 24 * time.Hour,
},
{
name: "fractional days",
input: "1.5d",
expected: 36 * time.Hour,
},
{
name: "days spelled out",
input: "3days",
expected: 3 * 24 * time.Hour,
},
// Extended units - weeks
{
name: "single week",
input: "1w",
expected: 7 * 24 * time.Hour,
},
{
name: "multiple weeks",
input: "4w",
expected: 4 * 7 * 24 * time.Hour,
},
{
name: "weeks spelled out",
input: "2weeks",
expected: 2 * 7 * 24 * time.Hour,
},
// Extended units - months
{
name: "single month",
input: "1mo",
expected: 30 * 24 * time.Hour,
},
{
name: "multiple months",
input: "6mo",
expected: 6 * 30 * 24 * time.Hour,
},
{
name: "months spelled out",
input: "3months",
expected: 3 * 30 * 24 * time.Hour,
},
// Extended units - years
{
name: "single year",
input: "1y",
expected: 365 * 24 * time.Hour,
},
{
name: "multiple years",
input: "2y",
expected: 2 * 365 * 24 * time.Hour,
},
{
name: "years spelled out",
input: "1year",
expected: 365 * 24 * time.Hour,
},
})
}
func TestParseDurationCombinedAndErrors(t *testing.T) {
t.Parallel()
runParseDurationCases(t, []parseDurationCase{
// Combined extended units
{
name: "weeks and days",
input: "2w3d",
expected: 2*7*24*time.Hour + 3*24*time.Hour,
},
{
name: "years and months",
input: "1y6mo",
expected: 365*24*time.Hour + 6*30*24*time.Hour,
},
{
name: "days and hours",
input: "1d12h",
expected: 24*time.Hour + 12*time.Hour,
},
{
name: "complex combination",
input: "1y2mo3w4d5h6m7s",
expected: 365*24*time.Hour + 2*30*24*time.Hour +
3*7*24*time.Hour + 4*24*time.Hour +
5*time.Hour + 6*time.Minute + 7*time.Second,
},
{
name: "with spaces",
input: "1d 12h 30m",
expected: 24*time.Hour + 12*time.Hour + 30*time.Minute,
},
// Edge cases
{
name: "zero duration",
input: "0s",
expected: 0,
},
{
name: "large duration",
input: "10y",
expected: 10 * 365 * 24 * time.Hour,
},
// Error cases
{
name: "empty string",
input: "",
wantErr: true,
},
{
name: "invalid format",
input: "abc",
wantErr: true,
},
{
name: "unknown unit",
input: "5x",
wantErr: true,
},
{
name: "invalid number",
input: "xyzd",
wantErr: true,
},
{
name: "negative not supported",
input: "-5d",
wantErr: true,
},
})
}
func TestParseDurationSpecialCases(t *testing.T) {
t.Parallel()
// Test that standard Go durations work exactly as expected
standardDurations := []string{
"300ms",
"1.5h",
"2h45m",
"72h",
"1us",
"1µs",
"1ns",
}
for _, d := range standardDurations {
expected, err := time.ParseDuration(d)
require.NoError(t, err)
got, err := parseDuration(d)
require.NoError(t, err)
assert.Equal(t, expected, got, "standard duration %q should parse identically", d)
}
}
func TestParseDurationRealWorldExamples(t *testing.T) {
t.Parallel()
// Test real-world snapshot purge scenarios
tests := []struct {
description string
input string
olderThan time.Duration
}{
{
description: "keep snapshots from last 30 days",
input: "30d",
olderThan: 30 * 24 * time.Hour,
},
{
description: "keep snapshots from last 6 months",
input: "6mo",
olderThan: 6 * 30 * 24 * time.Hour,
},
{
description: "keep snapshots from last year",
input: "1y",
olderThan: 365 * 24 * time.Hour,
},
{
description: "keep snapshots from last week and a half",
input: "1w3d",
olderThan: 10 * 24 * time.Hour,
},
{
description: "keep snapshots from last 90 days",
input: "90d",
olderThan: 90 * 24 * time.Hour,
},
}
for _, tt := range tests {
t.Run(tt.description, func(t *testing.T) {
t.Parallel()
got, err := parseDuration(tt.input)
require.NoError(t, err)
assert.Equal(t, tt.olderThan, got)
// Verify the duration makes sense for snapshot purging
assert.Greater(t, got, time.Hour,
"snapshot purge duration should be at least an hour")
})
}
}
+2 -4
View File
@@ -141,8 +141,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
"orphaned blobs")
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
"With --prune: keep snapshots newer than this duration "+
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
"instead of only the latest")
"(e.g. 4w, 30d, 6mo) instead of only the latest")
return cmd
}
@@ -205,8 +204,7 @@ restrict the operation to specific snapshot names.`,
cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false,
"Keep only the latest snapshot of each name")
cmd.Flags().StringVar(&opts.OlderThan, "older-than", "",
"Remove snapshots older than duration "+
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months)")
"Remove snapshots older than duration (e.g., 30d, 6m, 1y)")
cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt")
cmd.Flags().StringArrayVar(&opts.Names, "snapshot", nil,
"Restrict to snapshots with these names (repeat for multiple)")
+5 -13
View File
@@ -219,7 +219,11 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
Key: aws.String(fullKey),
})
if err != nil {
if IsNotFound(err) {
var (
notFound *s3types.NotFound
noSuchKey *s3types.NoSuchKey
)
if errors.As(err, &notFound) || errors.As(err, &noSuchKey) {
return false, nil
}
@@ -229,18 +233,6 @@ func (c *Client) HeadObject(ctx context.Context, key string) (bool, error) {
return true, nil
}
// IsNotFound reports whether err indicates that an object does not exist.
// Head and Get requests surface a missing object as different SDK types,
// so both are checked here.
func IsNotFound(err error) bool {
var (
notFound *s3types.NotFound
noSuchKey *s3types.NoSuchKey
)
return errors.As(err, &notFound) || errors.As(err, &noSuchKey)
}
// ObjectInfo contains information about an S3 object.
// It is used by ListObjectsStream to return object metadata
// along with any errors encountered during listing.
-198
View File
@@ -1,198 +0,0 @@
package storage_test
import (
"bytes"
"context"
"errors"
"io"
"reflect"
"sort"
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// runStorerConformance is the shared Storer contract. Every backend that
// can run in-process is expected to pass it: TestFileStorer runs it against
// file://, TestS3Storer against s3://. A new backend inherits this coverage
// by passing its own constructor, so the contract is defined once.
//
// It exercises the public Storer interface: round-trip, stat, list with
// prefix filtering, overwrite, delete, delete-of-missing, and not-found on
// Get and Stat. Each section takes its own fresh backend instance, so the
// order of sections never matters and no section sees another's objects.
func runStorerConformance(t *testing.T, newStorer func(*testing.T) storage.Storer) {
t.Helper()
conformanceRoundTrip(t, newStorer(t))
conformanceOverwrite(t, newStorer(t))
conformanceList(t, newStorer(t))
conformanceDelete(t, newStorer(t))
conformanceNotFound(t, newStorer(t))
}
// conformanceRoundTrip stores a nested key, then reads it back and stats it.
func conformanceRoundTrip(t *testing.T, s storage.Storer) {
t.Helper()
ctx := context.Background()
key := "blobs/aa/bb/object.bin"
want := []byte("round-trip payload")
err := s.Put(ctx, key, bytes.NewReader(want))
if err != nil {
t.Fatalf("Put: %v", err)
}
got := getBytes(t, s, key)
if !bytes.Equal(got, want) {
t.Errorf("Get returned %q, want %q", got, want)
}
info, err := s.Stat(ctx, key)
if err != nil {
t.Fatalf("Stat: %v", err)
}
if info.Key != key {
t.Errorf("Stat key = %q, want %q", info.Key, key)
}
if info.Size != int64(len(want)) {
t.Errorf("Stat size = %d, want %d", info.Size, len(want))
}
}
// conformanceOverwrite checks that a second Put replaces the first.
func conformanceOverwrite(t *testing.T, s storage.Storer) {
t.Helper()
ctx := context.Background()
key := "meta/snapshot.json"
err := s.Put(ctx, key, bytes.NewReader([]byte("first")))
if err != nil {
t.Fatalf("first Put: %v", err)
}
want := []byte("second and longer payload")
err = s.Put(ctx, key, bytes.NewReader(want))
if err != nil {
t.Fatalf("second Put: %v", err)
}
got := getBytes(t, s, key)
if !bytes.Equal(got, want) {
t.Errorf("after overwrite Get returned %q, want %q", got, want)
}
}
// conformanceList checks prefix filtering and the empty result for a
// prefix that matches nothing.
func conformanceList(t *testing.T, s storage.Storer) {
t.Helper()
ctx := context.Background()
keys := []string{"blobs/aa/one", "blobs/bb/two", "meta/three"}
for _, k := range keys {
err := s.Put(ctx, k, bytes.NewReader([]byte("data")))
if err != nil {
t.Fatalf("Put %q: %v", k, err)
}
}
if got := listSorted(t, s, ""); !reflect.DeepEqual(got, keys) {
t.Errorf("List(\"\") = %v, want %v", got, keys)
}
wantBlobs := []string{"blobs/aa/one", "blobs/bb/two"}
if got := listSorted(t, s, "blobs/"); !reflect.DeepEqual(got, wantBlobs) {
t.Errorf("List(\"blobs/\") = %v, want %v", got, wantBlobs)
}
if got := listSorted(t, s, "absent/"); len(got) != 0 {
t.Errorf("List(\"absent/\") = %v, want empty", got)
}
}
// conformanceDelete checks that Delete removes an object and that deleting
// a missing key is not an error.
func conformanceDelete(t *testing.T, s storage.Storer) {
t.Helper()
ctx := context.Background()
key := "blobs/cc/gone.bin"
err := s.Put(ctx, key, bytes.NewReader([]byte("temporary")))
if err != nil {
t.Fatalf("Put: %v", err)
}
err = s.Delete(ctx, key)
if err != nil {
t.Fatalf("Delete: %v", err)
}
_, err = s.Get(ctx, key)
if !errors.Is(err, storage.ErrNotFound) {
t.Errorf("Get after Delete error = %v, want ErrNotFound", err)
}
err = s.Delete(ctx, key)
if err != nil {
t.Errorf("Delete of missing key = %v, want nil", err)
}
}
// conformanceNotFound checks Get and Stat on an absent key.
func conformanceNotFound(t *testing.T, s storage.Storer) {
t.Helper()
ctx := context.Background()
key := "never/written"
_, err := s.Get(ctx, key)
if !errors.Is(err, storage.ErrNotFound) {
t.Errorf("Get error = %v, want ErrNotFound", err)
}
_, err = s.Stat(ctx, key)
if !errors.Is(err, storage.ErrNotFound) {
t.Errorf("Stat error = %v, want ErrNotFound", err)
}
}
// getBytes reads a key fully and closes the reader.
func getBytes(t *testing.T, s storage.Storer, key string) []byte {
t.Helper()
rc, err := s.Get(context.Background(), key)
if err != nil {
t.Fatalf("Get %q: %v", key, err)
}
defer func() { _ = rc.Close() }()
data, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("read %q: %v", key, err)
}
return data
}
// listSorted returns the keys under a prefix in a stable order.
func listSorted(t *testing.T, s storage.Storer, prefix string) []string {
t.Helper()
keys, err := s.List(context.Background(), prefix)
if err != nil {
t.Fatalf("List %q: %v", prefix, err)
}
sort.Strings(keys)
return keys
}
+54 -79
View File
@@ -46,18 +46,31 @@ func (f *FileStorer) SetFilesystem(fs afero.Fs) {
// storage base path.
const storageDirPerm = 0o755
// tempSuffix marks a partially written object. writeAtomic streams into a
// temp file carrying this suffix and only renames it onto the real key once
// the whole object is on disk, so an interrupted write can never leave a
// truncated object at the key a later run would Stat and trust as a complete
// blob. List and ListStream skip these files, so a leftover from an
// interrupted write is never listed or trusted as a blob; it is otherwise
// harmless and is overwritten when the same key is written again.
const tempSuffix = ".partial"
// Put stores data at the specified key.
func (f *FileStorer) Put(_ context.Context, key string, data io.Reader) error {
return f.writeAtomic(key, data, nil)
path := f.fullPath(key)
// Create parent directories
dir := filepath.Dir(path)
err := f.fs.MkdirAll(dir, storageDirPerm)
if err != nil {
return fmt.Errorf("creating directories: %w", err)
}
file, err := f.fs.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() { _ = file.Close() }()
_, err = io.Copy(file, data)
if err != nil {
return fmt.Errorf("writing file: %w", err)
}
return nil
}
// PutWithProgress stores data with progress reporting.
@@ -65,7 +78,35 @@ func (f *FileStorer) PutWithProgress(
_ context.Context, key string, data io.Reader,
_ int64, progress ProgressCallback,
) error {
return f.writeAtomic(key, data, progress)
path := f.fullPath(key)
// Create parent directories
dir := filepath.Dir(path)
err := f.fs.MkdirAll(dir, storageDirPerm)
if err != nil {
return fmt.Errorf("creating directories: %w", err)
}
file, err := f.fs.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() { _ = file.Close() }()
// Wrap with progress tracking
pw := &progressWriter{
writer: file,
callback: progress,
}
_, err = io.Copy(pw, data)
if err != nil {
return fmt.Errorf("writing file: %w", err)
}
return nil
}
// Get retrieves data from the specified key.
@@ -147,7 +188,7 @@ func (f *FileStorer) List(ctx context.Context, prefix string) ([]string, error)
default:
}
if !info.IsDir() && !strings.HasSuffix(info.Name(), tempSuffix) {
if !info.IsDir() {
// Convert back to key (relative path from basePath)
relPath, err := filepath.Rel(f.basePath, path)
if err != nil {
@@ -204,7 +245,7 @@ func (f *FileStorer) ListStream(ctx context.Context, prefix string) <-chan Objec
return nil //nolint:nilerr // continue walking despite errors
}
if !info.IsDir() && !strings.HasSuffix(info.Name(), tempSuffix) {
if !info.IsDir() {
relPath, err := filepath.Rel(f.basePath, path)
if err != nil {
ch <- ObjectInfo{Err: fmt.Errorf("computing relative path: %w", err)}
@@ -234,72 +275,6 @@ func (f *FileStorer) Info() Info {
}
}
// writeAtomic streams data into a temp file in the destination directory,
// fsyncs it, and renames it onto the final key. The key therefore appears
// only once the whole object has been durably written; a failure part-way
// leaves a temp file (removed here on the failing path) rather than a
// truncated object at the key.
func (f *FileStorer) writeAtomic(
key string, data io.Reader, progress ProgressCallback,
) error {
path := f.fullPath(key)
dir := filepath.Dir(path)
err := f.fs.MkdirAll(dir, storageDirPerm)
if err != nil {
return fmt.Errorf("creating directories: %w", err)
}
tmp, err := afero.TempFile(f.fs, dir, filepath.Base(path)+"-*"+tempSuffix)
if err != nil {
return fmt.Errorf("creating temp file: %w", err)
}
tmpPath := tmp.Name()
// Remove the temp file unless the rename below claims it. On the success
// path renamed is true, so the deferred Close and Remove are harmless
// no-ops on a name that no longer exists.
renamed := false
defer func() {
_ = tmp.Close()
if !renamed {
_ = f.fs.Remove(tmpPath)
}
}()
var w io.Writer = tmp
if progress != nil {
w = &progressWriter{writer: tmp, callback: progress}
}
_, err = io.Copy(w, data)
if err != nil {
return fmt.Errorf("writing file: %w", err)
}
err = tmp.Sync()
if err != nil {
return fmt.Errorf("syncing temp file: %w", err)
}
err = tmp.Close()
if err != nil {
return fmt.Errorf("closing temp file: %w", err)
}
err = f.fs.Rename(tmpPath, path)
if err != nil {
return fmt.Errorf("renaming temp file: %w", err)
}
renamed = true
return nil
}
// fullPath returns the full filesystem path for a key.
func (f *FileStorer) fullPath(key string) string {
return filepath.Join(f.basePath, key)
-119
View File
@@ -1,119 +0,0 @@
package storage_test
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// errStreamInterrupted stands in for an upload cut off mid-stream.
var errStreamInterrupted = errors.New("connection reset mid-upload")
// failingReader yields its data once, then fails.
type failingReader struct {
data []byte
done bool
}
func (r *failingReader) Read(p []byte) (int, error) {
if r.done {
return 0, errStreamInterrupted
}
n := copy(p, r.data)
r.done = true
return n, nil
}
// TestFileStorer_InterruptedWriteLeavesNoTrustedObject checks that a write
// cut off mid-stream leaves nothing at the destination key, so a later run
// cannot Stat a truncated object and trust it as a complete blob.
func TestFileStorer_InterruptedWriteLeavesNoTrustedObject(t *testing.T) {
t.Parallel()
f, err := storage.NewFileStorer(t.TempDir())
if err != nil {
t.Fatalf("NewFileStorer: %v", err)
}
ctx := context.Background()
key := "blobs/aa/bb/aabbccddeeff"
err = f.PutWithProgress(ctx, key, &failingReader{data: []byte("partial")}, 4096, nil)
if err == nil {
t.Fatal("expected the interrupted write to fail, got nil")
}
_, err = f.Stat(ctx, key)
if !errors.Is(err, storage.ErrNotFound) {
t.Fatalf("expected key absent after interrupted write, got Stat err %v", err)
}
keys, err := f.List(ctx, "blobs/")
if err != nil {
t.Fatalf("List: %v", err)
}
if len(keys) != 0 {
t.Fatalf("expected no keys listed after interrupted write, got %v", keys)
}
}
// TestFileStorer_ListSkipsPartialFiles checks that a leftover temp file (the
// storage layer names them with a ".partial" suffix) is never surfaced as a
// key by List or ListStream.
func TestFileStorer_ListSkipsPartialFiles(t *testing.T) {
t.Parallel()
base := t.TempDir()
f, err := storage.NewFileStorer(base)
if err != nil {
t.Fatalf("NewFileStorer: %v", err)
}
ctx := context.Background()
realKey := "blobs/aa/bb/aabbccddeeff"
err = f.Put(ctx, realKey, strings.NewReader("blob-bytes"))
if err != nil {
t.Fatalf("Put: %v", err)
}
// A stray temp file, as an interrupted write would leave behind.
leftover := filepath.Join(base, "blobs/aa/bb/aabbccddeeff-123456.partial")
err = os.WriteFile(leftover, []byte("half"), 0o600)
if err != nil {
t.Fatalf("writing leftover temp file: %v", err)
}
keys, err := f.List(ctx, "blobs/")
if err != nil {
t.Fatalf("List: %v", err)
}
if len(keys) != 1 || keys[0] != realKey {
t.Fatalf("List should return only the real key, got %v", keys)
}
var streamed []string
for obj := range f.ListStream(ctx, "blobs/") {
if obj.Err != nil {
t.Fatalf("ListStream: %v", obj.Err)
}
streamed = append(streamed, obj.Key)
}
if len(streamed) != 1 || streamed[0] != realKey {
t.Fatalf("ListStream should return only the real key, got %v", streamed)
}
}
-27
View File
@@ -1,27 +0,0 @@
package storage_test
import (
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// newFileStorer builds a file:// backend rooted at a fresh temp directory.
//
//nolint:ireturn // conformance runs against the Storer interface by design
func newFileStorer(t *testing.T) storage.Storer {
t.Helper()
s, err := storage.NewFileStorer(t.TempDir())
if err != nil {
t.Fatalf("NewFileStorer: %v", err)
}
return s
}
// TestFileStorer runs the shared Storer contract against the file:// backend.
func TestFileStorer(t *testing.T) {
t.Parallel()
runStorerConformance(t, newFileStorer)
}
-58
View File
@@ -1,58 +0,0 @@
package storage_test
import (
"context"
"errors"
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// The rclone backend is a thin adapter over the rclone library: it turns a
// (remote, path) pair into rclone's "remote:path" string, hands it to
// rclone, and maps rclone's own results back to the Storer interface. What
// can be tested in-process, without a configured remote or network, is that
// adapter layer — how the arguments are shaped and how construction errors
// are reported. The data-plane operations (Put/Get/List/Delete) are rclone's
// own, exercised against a real provider (drive, s3-via-rclone, ...), which
// needs a configured remote with credentials and network access and so is
// out of reach of a unit test. The shared Storer conformance suite therefore
// runs against the in-process file and s3 backends; the rclone backend
// inherits that contract once a remote is configured.
//
// These tests use rclone's ":local:" on-the-fly backend, which addresses the
// local filesystem directly without any configured remote, so construction
// runs entirely in-process.
// TestNewRcloneStorerConstruction checks that a valid remote constructs a
// backend and that Info() reports the shaped "remote:path" location.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestNewRcloneStorerConstruction(t *testing.T) {
dir := t.TempDir()
s, err := storage.NewRcloneStorer(context.Background(), ":local", dir)
if err != nil {
t.Fatalf("NewRcloneStorer: %v", err)
}
// Info().Location is the "remote:path" string the adapter builds from
// its two arguments, so asserting it confirms the argument shaping.
want := ":local:" + dir
if got := s.Info().Location; got != want {
t.Errorf("Info().Location = %q, want %q", got, want)
}
}
// TestNewRcloneStorerUnknownRemote checks that a remote that is not in the
// rclone config fails construction with the ErrRemoteNotFound sentinel,
// rather than silently returning a backend pointed nowhere.
//
//nolint:paralleltest // NewRcloneStorer installs the process-global rclone config
func TestNewRcloneStorerUnknownRemote(t *testing.T) {
_, err := storage.NewRcloneStorer(
context.Background(), "vaultik-no-such-remote", "path")
if !errors.Is(err, storage.ErrRemoteNotFound) {
t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
}
}
+1 -16
View File
@@ -38,29 +38,14 @@ func (s *S3Storer) PutWithProgress(
}
// Get retrieves data from the specified key.
// Returns ErrNotFound if the object does not exist.
func (s *S3Storer) Get(ctx context.Context, key string) (io.ReadCloser, error) {
rc, err := s.client.GetObject(ctx, key)
if err != nil {
if s3.IsNotFound(err) {
return nil, fmt.Errorf("get %q: %w", key, ErrNotFound)
}
return nil, err
}
return rc, nil
return s.client.GetObject(ctx, key)
}
// Stat returns metadata about an object without retrieving its contents.
// Returns ErrNotFound if the object does not exist.
func (s *S3Storer) Stat(ctx context.Context, key string) (*ObjectInfo, error) {
info, err := s.client.StatObject(ctx, key)
if err != nil {
if s3.IsNotFound(err) {
return nil, fmt.Errorf("stat %q: %w", key, ErrNotFound)
}
return nil, err
}
-81
View File
@@ -1,81 +0,0 @@
package storage_test
import (
"context"
"errors"
"net/http/httptest"
"testing"
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
"sneak.berlin/go/vaultik/internal/s3"
"sneak.berlin/go/vaultik/internal/storage"
)
// s3TestBucket is the bucket created for each in-process S3 server.
const s3TestBucket = "test-bucket"
// newS3Storer builds an s3:// backend backed by a fresh in-process
// S3 server. It reuses the same in-memory S3 harness (gofakes3 + s3mem
// over httptest) that internal/s3 and the not-found regression test use,
// so no new mock or dependency is introduced. Each call gets its own
// server, bucket, and client, so the conformance suite's per-section
// instances stay isolated.
//
//nolint:ireturn // conformance runs against the Storer interface by design
func newS3Storer(t *testing.T) storage.Storer {
t.Helper()
backend := s3mem.New()
err := backend.CreateBucket(s3TestBucket)
if err != nil {
t.Fatalf("create bucket: %v", err)
}
srv := httptest.NewServer(gofakes3.New(backend).Server())
t.Cleanup(srv.Close)
client, err := s3.NewClient(context.Background(), s3.Config{
Endpoint: srv.URL,
Bucket: s3TestBucket,
AccessKeyID: "test",
SecretAccessKey: "test",
Region: "us-east-1",
})
if err != nil {
t.Fatalf("new client: %v", err)
}
return storage.NewS3Storer(client)
}
// TestS3Storer runs the shared Storer contract against the s3:// backend,
// so it is held to the same round-trip, list, delete, and not-found
// behaviour as the file:// backend.
func TestS3Storer(t *testing.T) {
t.Parallel()
runStorerConformance(t, newS3Storer)
}
// TestS3StorerMissingKeyMapsToErrNotFound pins the specific contract that a
// missing object surfaces as storage.ErrNotFound rather than the raw AWS SDK
// error. Without the mapping, errors.Is(err, storage.ErrNotFound) is false on
// s3 and callers would branch differently per backend.
func TestS3StorerMissingKeyMapsToErrNotFound(t *testing.T) {
t.Parallel()
storer := newS3Storer(t)
ctx := context.Background()
_, err := storer.Get(ctx, "does-not-exist")
if !errors.Is(err, storage.ErrNotFound) {
t.Errorf("Get on missing key: got %v, want ErrNotFound", err)
}
_, err = storer.Stat(ctx, "does-not-exist")
if !errors.Is(err, storage.ErrNotFound) {
t.Errorf("Stat on missing key: got %v, want ErrNotFound", err)
}
}
-110
View File
@@ -1,110 +0,0 @@
package storage_test
import (
"errors"
"reflect"
"testing"
"sneak.berlin/go/vaultik/internal/storage"
)
// TestParseStorageURLValid checks that each supported scheme parses into
// the expected fields, since those fields decide which backend is built.
func TestParseStorageURLValid(t *testing.T) {
t.Parallel()
const bucket = "mybucket"
cases := []struct {
name string
raw string
want *storage.URL
}{
{
name: "file absolute path",
raw: "file:///var/backups/vaultik",
want: &storage.URL{Scheme: "file", Prefix: "/var/backups/vaultik"},
},
{
name: "s3 bucket and prefix, ssl defaults on",
raw: "s3://mybucket/backups/host",
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Prefix: "backups/host", UseSSL: true,
},
},
{
name: "s3 bucket only",
raw: "s3://mybucket",
want: &storage.URL{Scheme: "s3", Bucket: bucket, UseSSL: true},
},
{
name: "s3 with endpoint, region, ssl off",
raw: "s3://mybucket?endpoint=minio.example.com&region=us-west-2&ssl=false",
want: &storage.URL{
Scheme: "s3", Bucket: bucket,
Endpoint: "minio.example.com", Region: "us-west-2", UseSSL: false,
},
},
{
name: "rclone remote and path",
raw: "rclone://gdrive/backups/host",
want: &storage.URL{
Scheme: "rclone", RcloneRemote: "gdrive", Prefix: "backups/host",
},
},
{
name: "rclone remote only",
raw: "rclone://gdrive",
want: &storage.URL{Scheme: "rclone", RcloneRemote: "gdrive"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := storage.ParseStorageURL(tc.raw)
if err != nil {
t.Fatalf("ParseStorageURL(%q) returned error: %v", tc.raw, err)
}
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("ParseStorageURL(%q) = %+v, want %+v", tc.raw, got, tc.want)
}
})
}
}
// TestParseStorageURLErrors checks that empty, missing, and unknown-scheme
// inputs fail with the documented sentinel errors instead of parsing to a
// wrong destination.
func TestParseStorageURLErrors(t *testing.T) {
t.Parallel()
cases := []struct {
name string
raw string
wantErr error
}{
{"empty url", "", storage.ErrEmptyStorageURL},
{"file empty path", "file://", storage.ErrEmptyFilePath},
{"s3 missing bucket", "s3://", storage.ErrMissingBucket},
{"s3 missing bucket with path", "s3:///justprefix", storage.ErrMissingBucket},
{"rclone missing remote", "rclone://", storage.ErrMissingRemote},
{"unknown scheme", "gs://bucket/x", storage.ErrUnsupportedScheme},
{"no scheme", "/local/path", storage.ErrUnsupportedScheme},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := storage.ParseStorageURL(tc.raw)
if !errors.Is(err, tc.wantErr) {
t.Errorf("ParseStorageURL(%q) error = %v, want %v",
tc.raw, err, tc.wantErr)
}
})
}
}
+3 -9
View File
@@ -33,9 +33,8 @@ func ubytes(n int64) string {
var (
errMalformedSnapshotID = errors.New(
"invalid snapshot ID format: expected hostname_snapshotname_timestamp")
errInvalidDuration = errors.New("invalid duration")
errUnknownTimeUnit = errors.New("unknown time unit")
errNegativeDuration = errors.New("negative durations are not supported")
errInvalidDuration = errors.New("invalid duration")
errUnknownTimeUnit = errors.New("unknown time unit")
)
// Time-unit lengths used by parseDuration.
@@ -139,13 +138,8 @@ func parseSnapshotName(snapshotID string) string {
// parseDuration parses a duration string with support for human-friendly units:
// d/day/days, w/week/weeks, mo/month/months, y/year/years, plus standard Go
// duration units. Following Go, m is minutes and mo is months. A bare number,
// an unknown unit, and a negative value are all rejected.
// duration units (h, m, s).
func parseDuration(s string) (time.Duration, error) {
if strings.HasPrefix(strings.TrimSpace(s), "-") {
return 0, errNegativeDuration
}
d, err := time.ParseDuration(s)
if err == nil {
return d, nil
+3 -22
View File
@@ -51,32 +51,13 @@ func TestParseDuration(t *testing.T) {
want time.Duration
err bool
}{
// Go units, including the m-is-minutes / mo-is-months distinction
// that this parser exists to keep straight.
{"10ns", 10 * time.Nanosecond, false},
{"10us", 10 * time.Microsecond, false},
{"500ms", 500 * time.Millisecond, false},
{"30s", 30 * time.Second, false},
{"6m", 6 * time.Minute, false},
{"1h", time.Hour, false},
// Extended calendar units.
{"30d", 30 * 24 * time.Hour, false},
{"3days", 3 * 24 * time.Hour, false},
{"4w", 4 * 7 * 24 * time.Hour, false},
{"2weeks", 2 * 7 * 24 * time.Hour, false},
{"6mo", 180 * 24 * time.Hour, false},
{"1month", 30 * 24 * time.Hour, false},
{"6mo", 6 * 30 * 24 * time.Hour, false},
{"1y", 365 * 24 * time.Hour, false},
{"2years", 2 * 365 * 24 * time.Hour, false},
// Combined units.
{"2w3d", 2*7*24*time.Hour + 3*24*time.Hour, false},
{"1y6mo", 365*24*time.Hour + 180*24*time.Hour, false},
// Rejected inputs.
{"6", 0, true}, // bare number, no unit
{"5x", 0, true}, // unknown unit
{"-5d", 0, true}, // negative, extended unit
{"-5h", 0, true}, // negative, Go unit
{"", 0, true}, // empty
{"1h", time.Hour, false},
{"30s", 30 * time.Second, false},
{"garbage", 0, true},
}