Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13549a442c |
@@ -10,6 +10,8 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
)
|
)
|
||||||
|
|
||||||
// groupSeparator joins an open group path to an attribute key. This
|
// groupSeparator joins an open group path to an attribute key. This
|
||||||
@@ -304,18 +306,13 @@ func formatDuration(d time.Duration) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// formatBytes formats bytes in a human-readable way
|
// formatBytes formats a byte count with humanize.Bytes, as every command
|
||||||
|
// prints one. humanize.Bytes takes no negative count, so one is printed
|
||||||
|
// as it is.
|
||||||
func formatBytes(b int64) string {
|
func formatBytes(b int64) string {
|
||||||
const unit = 1024
|
if b < 0 {
|
||||||
if b < unit {
|
|
||||||
return fmt.Sprintf("%d B", b)
|
return fmt.Sprintf("%d B", b)
|
||||||
}
|
}
|
||||||
|
|
||||||
div, exp := int64(unit), 0
|
return humanize.Bytes(uint64(b))
|
||||||
for n := b / unit; n >= unit; n /= unit {
|
|
||||||
div *= unit
|
|
||||||
exp++
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -200,7 +200,8 @@ func TestTTYHandlerWithGroupQualifiesKeys(t *testing.T) {
|
|||||||
func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
|
func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const oneAndAHalfKiB = 1536
|
// Printed as "1.0 MB", in powers of 1000 like every other byte count.
|
||||||
|
const oneMegabyte = 1_000_000
|
||||||
|
|
||||||
for name, testCase := range map[string]struct {
|
for name, testCase := range map[string]struct {
|
||||||
derive func(*slog.Logger) *slog.Logger
|
derive func(*slog.Logger) *slog.Logger
|
||||||
@@ -223,13 +224,13 @@ func TestTTYHandlerByteFormattingSurvivesGrouping(t *testing.T) {
|
|||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
|
logger := slog.New(log.NewTTYHandler(&buf, debugHandlerOptions()))
|
||||||
testCase.derive(logger).Info("uploaded", "bytes", oneAndAHalfKiB)
|
testCase.derive(logger).Info("uploaded", "bytes", oneMegabyte)
|
||||||
|
|
||||||
line := ansiEscape.ReplaceAllString(buf.String(), "")
|
line := ansiEscape.ReplaceAllString(buf.String(), "")
|
||||||
|
|
||||||
assert.Contains(t, line, testCase.key+"=1.5 KB",
|
assert.Contains(t, line, testCase.key+"=1.0 MB",
|
||||||
"a byte count must be human-readable however it is qualified")
|
"a byte count must be human-readable however it is qualified")
|
||||||
assert.NotContains(t, line, strconv.Itoa(oneAndAHalfKiB),
|
assert.NotContains(t, line, strconv.Itoa(oneMegabyte),
|
||||||
"the raw number must not survive the formatting")
|
"the raw number must not survive the formatting")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-24
@@ -29,10 +29,7 @@ var (
|
|||||||
ErrURLCredentials = errors.New(
|
ErrURLCredentials = errors.New(
|
||||||
"storage URL must not carry credentials; " +
|
"storage URL must not carry credentials; " +
|
||||||
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
"set s3.access_key_id and s3.secret_access_key in the config instead")
|
||||||
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
ErrURLUnknownParam = errors.New("unknown query parameter in storage URL")
|
||||||
ErrRcloneRemoteParams = errors.New(
|
|
||||||
"rclone URL must not carry parameters in the remote name; " +
|
|
||||||
"set them on the remote with rclone config instead")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// URL represents a parsed storage URL.
|
// URL represents a parsed storage URL.
|
||||||
@@ -93,9 +90,8 @@ func parseFileURL(rawPath string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
// parseS3URL parses an s3://bucket/prefix URL. It rejects credentials in
|
||||||
// the userinfo or in the endpoint parameter's userinfo, and any query
|
// the userinfo and any query parameter other than endpoint, region and
|
||||||
// parameter other than endpoint, region and ssl, so a credential-bearing
|
// ssl, so a credential-bearing URL is never stored or echoed.
|
||||||
// URL is never stored or echoed.
|
|
||||||
func parseS3URL(rawURL string) (*URL, error) {
|
func parseS3URL(rawURL string) (*URL, error) {
|
||||||
u, err := url.Parse(rawURL)
|
u, err := url.Parse(rawURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -118,14 +114,6 @@ func parseS3URL(rawURL string) (*URL, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// A host name never contains "@", so one in an endpoint, with or
|
|
||||||
// without a scheme, can only introduce userinfo.
|
|
||||||
for _, endpoint := range query["endpoint"] {
|
|
||||||
if strings.Contains(endpoint, "@") {
|
|
||||||
return nil, ErrURLCredentials
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &URL{
|
return &URL{
|
||||||
Scheme: schemeS3,
|
Scheme: schemeS3,
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
@@ -137,9 +125,8 @@ func parseS3URL(rawURL string) (*URL, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
|
// parseRcloneURL parses an rclone://remote/path URL. rclone:// takes no
|
||||||
// query parameters, so credentials in the userinfo, parameters in the
|
// query parameters, so credentials in the userinfo and any parameter at
|
||||||
// remote name and any query parameter at all are rejected rather than
|
// all are rejected rather than silently ignored.
|
||||||
// silently ignored.
|
|
||||||
func parseRcloneURL(rawURL string) (*URL, error) {
|
func parseRcloneURL(rawURL string) (*URL, error) {
|
||||||
u, err := url.Parse(rawURL)
|
u, err := url.Parse(rawURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -155,12 +142,6 @@ func parseRcloneURL(rawURL string) (*URL, error) {
|
|||||||
return nil, ErrMissingRemote
|
return nil, ErrMissingRemote
|
||||||
}
|
}
|
||||||
|
|
||||||
// An rclone remote name never contains a comma; one starts rclone's
|
|
||||||
// remote,param=value form, whose values can be credentials.
|
|
||||||
if strings.Contains(remote, ",") {
|
|
||||||
return nil, ErrRcloneRemoteParams
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rejectUnknownParams(u.Query())
|
err = rejectUnknownParams(u.Query())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -150,19 +150,6 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|||||||
wantErr: storage.ErrURLCredentials,
|
wantErr: storage.ErrURLCredentials,
|
||||||
secrets: []string{key, secret, "mybucket"},
|
secrets: []string{key, secret, "mybucket"},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: "s3 userinfo in endpoint",
|
|
||||||
raw: "s3://mybucket?endpoint=https://" +
|
|
||||||
key + ":" + secret + "@minio.example.com",
|
|
||||||
wantErr: storage.ErrURLCredentials,
|
|
||||||
secrets: []string{key, secret},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "s3 userinfo in endpoint without scheme",
|
|
||||||
raw: "s3://mybucket?endpoint=" + key + ":" + secret + "@minio.example.com",
|
|
||||||
wantErr: storage.ErrURLCredentials,
|
|
||||||
secrets: []string{key, secret},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: "s3 unknown query param",
|
name: "s3 unknown query param",
|
||||||
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
raw: "s3://mybucket?access_key=" + key + "&secret=" + secret,
|
||||||
@@ -216,27 +203,6 @@ func TestParseStorageURLRejectsCredentials(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestParseStorageURLRejectsRcloneRemoteParams checks that an rclone
|
|
||||||
// remote name carrying parameters (rclone's "remote,param=value" form,
|
|
||||||
// which can hold a credential) is rejected without echoing the value.
|
|
||||||
func TestParseStorageURLRejectsRcloneRemoteParams(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const secret = "topsecret"
|
|
||||||
|
|
||||||
raw := "rclone://gdrive,client_secret=" + secret + "/backups"
|
|
||||||
|
|
||||||
_, err := storage.ParseStorageURL(raw)
|
|
||||||
if !errors.Is(err, storage.ErrRcloneRemoteParams) {
|
|
||||||
t.Fatalf("ParseStorageURL(%q) error = %v, want %v",
|
|
||||||
raw, err, storage.ErrRcloneRemoteParams)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(err.Error(), secret) {
|
|
||||||
t.Errorf("error message leaked %q: %v", secret, err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
|
// TestParseStorageURLParseFailureHidesURL checks that when url.Parse
|
||||||
// itself fails, the wrapped error carries only the inner cause, not the
|
// itself fails, the wrapped error carries only the inner cause, not the
|
||||||
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
|
// *url.Error whose text embeds the raw (possibly credential-bearing) URL.
|
||||||
|
|||||||
@@ -89,22 +89,6 @@ type SnapshotInfo struct {
|
|||||||
RemotePresent *bool `json:"remote_present"`
|
RemotePresent *bool `json:"remote_present"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// formatBytes formats bytes in a human-readable format
|
|
||||||
func formatBytes(bytes int64) string {
|
|
||||||
const unit = 1024
|
|
||||||
if bytes < unit {
|
|
||||||
return fmt.Sprintf("%d B", bytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
div, exp := int64(unit), 0
|
|
||||||
for n := bytes / unit; n >= unit; n /= unit {
|
|
||||||
div *= unit
|
|
||||||
exp++
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseSnapshotTimestamp extracts the timestamp from a snapshot ID
|
// parseSnapshotTimestamp extracts the timestamp from a snapshot ID
|
||||||
// Format: hostname_snapshotname_2026-01-12T14:41:15Z
|
// Format: hostname_snapshotname_2026-01-12T14:41:15Z
|
||||||
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
func parseSnapshotTimestamp(snapshotID string) (time.Time, error) {
|
||||||
|
|||||||
@@ -633,7 +633,7 @@ func (v *Vaultik) confirmAndExecutePurge(
|
|||||||
v.stdoutf(" %s (%s, %s)\n",
|
v.stdoutf(" %s (%s, %s)\n",
|
||||||
snap.ID,
|
snap.ID,
|
||||||
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
||||||
formatBytes(snap.CompressedSize))
|
ubytes(snap.CompressedSize))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -553,8 +553,8 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
|
|||||||
|
|
||||||
if snap.LocallyTracked {
|
if snap.LocallyTracked {
|
||||||
id = snap.ID.String()
|
id = snap.ID.String()
|
||||||
uncompressed = formatBytes(snap.UncompressedSize)
|
uncompressed = ubytes(snap.UncompressedSize)
|
||||||
newChunks = formatBytes(snap.NewChunkSize)
|
newChunks = ubytes(snap.NewChunkSize)
|
||||||
} else {
|
} else {
|
||||||
id = formatRemoteOnlyID(snap.RemoteKey)
|
id = formatRemoteOnlyID(snap.RemoteKey)
|
||||||
uncompressed = remoteOnlyCell
|
uncompressed = remoteOnlyCell
|
||||||
@@ -564,7 +564,7 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
|
|||||||
_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
|
_, err = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
|
||||||
id,
|
id,
|
||||||
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
||||||
formatBytes(snap.CompressedSize),
|
ubytes(snap.CompressedSize),
|
||||||
uncompressed,
|
uncompressed,
|
||||||
newChunks)
|
newChunks)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -212,8 +212,8 @@ const (
|
|||||||
listConfiguredName = "home"
|
listConfiguredName = "home"
|
||||||
listLocalID = "testhost_home_2026-03-01T10:00:00Z"
|
listLocalID = "testhost_home_2026-03-01T10:00:00Z"
|
||||||
listRemoteID = "otherhost_media_2026-03-02T11:22:33Z"
|
listRemoteID = "otherhost_media_2026-03-02T11:22:33Z"
|
||||||
// fiveMegabytes formats as "5.0 MB" through formatBytes.
|
// fiveMegabytes is printed as "5.0 MB" by every command.
|
||||||
fiveMegabytes = 5 * 1024 * 1024
|
fiveMegabytes = 5_000_000
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
|
// TestListSnapshots_RemoteWithoutSecretKey is the regression guard for
|
||||||
@@ -306,6 +306,48 @@ func TestListSnapshots_RemoteOnlyRowRendering(t *testing.T) {
|
|||||||
"are not in the local index")
|
"are not in the local index")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSnapshotListAndPurgePrintSizesLikeRemoteInfo checks that the
|
||||||
|
// `snapshot list` table and the purge listing print a byte count the way
|
||||||
|
// `remote info` does: in powers of 1000, as the MB label says.
|
||||||
|
func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const wantSize = "5.0 MB" // fiveMegabytes
|
||||||
|
|
||||||
|
env := newListEnv(t)
|
||||||
|
|
||||||
|
// A remote-only snapshot: `snapshot list` and `remote info` both take
|
||||||
|
// its size from the manifest.
|
||||||
|
remoteKey := env.addRemote(t, listRemoteID,
|
||||||
|
time.Date(2026, 3, 2, 11, 22, 33, 0, time.UTC))
|
||||||
|
|
||||||
|
// A locally tracked snapshot of the same size for the purge to list.
|
||||||
|
// The purge prints the snapshot's blob size, the last argument here.
|
||||||
|
start := time.Date(2026, 3, 1, 10, 0, 0, 0, time.UTC)
|
||||||
|
env.addLocal(t, listLocalID, start)
|
||||||
|
env.addRemote(t, listLocalID, start)
|
||||||
|
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
|
||||||
|
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
|
||||||
|
|
||||||
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
|
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
|
||||||
|
wantSize, "remote info")
|
||||||
|
|
||||||
|
env.stdout.Reset()
|
||||||
|
require.NoError(t, env.v.ListSnapshots(false))
|
||||||
|
assert.Contains(t, findTableRow(t, env.stdout.String(),
|
||||||
|
"<remote only:"+remoteKey[:12]+">"), wantSize, "snapshot list")
|
||||||
|
|
||||||
|
// Without --force and with nothing on stdin the purge lists what it
|
||||||
|
// would delete, reads no confirmation, and deletes nothing.
|
||||||
|
env.stdout.Reset()
|
||||||
|
require.NoError(t, env.v.PurgeSnapshotsWithOptions(
|
||||||
|
&vaultik.SnapshotPurgeOptions{OlderThan: "1d"}))
|
||||||
|
assert.Contains(t, findTableRow(t, env.stdout.String(), listLocalID),
|
||||||
|
wantSize, "snapshot purge")
|
||||||
|
}
|
||||||
|
|
||||||
// findTableRow returns the single output line containing needle.
|
// findTableRow returns the single output line containing needle.
|
||||||
func findTableRow(t *testing.T, out, needle string) string {
|
func findTableRow(t *testing.T, out, needle string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|||||||
Reference in New Issue
Block a user