docs/DATAMODEL.md said "each .partial object a killed upload left under metadata/", and the Go doc comments in prune.go, info.go, storer.go and the new test said "the .partial objects uploads cut off part-way left": the verb arrives late and the noun pair reads as a clause. They now say "left by a killed upload" or drop the phrase. Model: opus-5-5
This commit is contained in:
1 parent
59664f219e
commit
5e26e30ba2
5 files changed
+20
-20
No files matched your search
+2
-2
@@ -315,8 +315,8 @@ The restore process doesn't use the local database. Instead:
|
|||||||
and no local snapshot record, once every object in it is older than
|
and no local snapshot record, once every object in it is older than
|
||||||
`prune_grace_period`
|
`prune_grace_period`
|
||||||
3. **Delete Partial Uploads**
|
3. **Delete Partial Uploads**
|
||||||
- Delete each `.partial` object a killed upload left under `metadata/` or
|
- Delete each `.partial` object under `metadata/` or `blobs/` left by a
|
||||||
`blobs/`, once it is older than `prune_grace_period`
|
killed upload, once it is older than `prune_grace_period`
|
||||||
|
|
||||||
### 6. Incomplete Snapshot Cleanup
|
### 6. Incomplete Snapshot Cleanup
|
||||||
|
|
||||||
|
|||||||
@@ -77,9 +77,9 @@ type Storer interface {
|
|||||||
// If an error occurs during listing, the final item will have Err set.
|
// If an error occurs during listing, the final item will have Err set.
|
||||||
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
ListStream(ctx context.Context, prefix string) <-chan ObjectInfo
|
||||||
|
|
||||||
// ListPartialUploads returns each object under prefix that an upload
|
// ListPartialUploads returns each object under prefix whose name ends
|
||||||
// cut off part-way left under a temporary name ending in `.partial`,
|
// in `.partial`, as a killed upload leaves it, with its key, size and
|
||||||
// with its key, size and modification time. The file and rclone
|
// modification time. The file and rclone
|
||||||
// backends' List and ListStream skip such an object; Delete removes
|
// backends' List and ListStream skip such an object; Delete removes
|
||||||
// it.
|
// it.
|
||||||
ListPartialUploads(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
ListPartialUploads(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||||
|
|||||||
@@ -216,8 +216,8 @@ type RemoteInfoResult struct {
|
|||||||
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
||||||
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
||||||
|
|
||||||
// `.partial` objects under metadata/ and blobs/ that uploads cut off
|
// `.partial` objects under metadata/ and blobs/ left by killed
|
||||||
// part-way left. The blob figures above leave them out.
|
// uploads. The blob figures above leave them out.
|
||||||
PartialUploadCount int `json:"partial_upload_count"`
|
PartialUploadCount int `json:"partial_upload_count"`
|
||||||
PartialUploadSize int64 `json:"partial_upload_size"`
|
PartialUploadSize int64 `json:"partial_upload_size"`
|
||||||
|
|
||||||
|
|||||||
+10
-10
@@ -159,8 +159,8 @@ func (v *Vaultik) deleteEverythingUnder(prefix string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listEverythingUnder returns the key of every object under prefix,
|
// listEverythingUnder returns the key of every object under prefix,
|
||||||
// including the `.partial` objects uploads cut off part-way left, which
|
// including the `.partial` objects left by killed uploads, which the file
|
||||||
// the file and rclone backends' List skips.
|
// and rclone backends' List skips.
|
||||||
func (v *Vaultik) listEverythingUnder(prefix string) ([]string, error) {
|
func (v *Vaultik) listEverythingUnder(prefix string) ([]string, error) {
|
||||||
keys, err := v.Storage.List(v.ctx, prefix)
|
keys, err := v.Storage.List(v.ctx, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -181,7 +181,7 @@ func (v *Vaultik) listEverythingUnder(prefix string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listPartialUploads returns the `.partial` objects under metadata/ and
|
// listPartialUploads returns the `.partial` objects under metadata/ and
|
||||||
// blobs/ that uploads cut off part-way left.
|
// blobs/ left by killed uploads.
|
||||||
func (v *Vaultik) listPartialUploads() ([]storage.ObjectInfo, error) {
|
func (v *Vaultik) listPartialUploads() ([]storage.ObjectInfo, error) {
|
||||||
var partialUploads []storage.ObjectInfo
|
var partialUploads []storage.ObjectInfo
|
||||||
|
|
||||||
@@ -215,10 +215,10 @@ type PruneBlobsResult struct {
|
|||||||
// not delete.
|
// not delete.
|
||||||
MetadataDirsDeleted int `json:"metadata_dirs_deleted"`
|
MetadataDirsDeleted int `json:"metadata_dirs_deleted"`
|
||||||
MetadataDirsFailed int `json:"metadata_dirs_failed,omitempty"`
|
MetadataDirsFailed int `json:"metadata_dirs_failed,omitempty"`
|
||||||
// PartialUploadsDeleted counts the `.partial` objects uploads cut off
|
// PartialUploadsDeleted counts the `.partial` objects prune deleted,
|
||||||
// part-way left that prune deleted, and PartialUploadsFailed those it
|
// and PartialUploadsFailed those it could not delete.
|
||||||
// could not delete. PartialUploadsTooNew counts those kept for being
|
// PartialUploadsTooNew counts those kept for being newer than the
|
||||||
// newer than the prune grace period, or of unknown age.
|
// prune grace period, or of unknown age.
|
||||||
PartialUploadsDeleted int `json:"partial_uploads_deleted"`
|
PartialUploadsDeleted int `json:"partial_uploads_deleted"`
|
||||||
PartialUploadsFailed int `json:"partial_uploads_failed,omitempty"`
|
PartialUploadsFailed int `json:"partial_uploads_failed,omitempty"`
|
||||||
PartialUploadsTooNew int `json:"partial_uploads_too_new"`
|
PartialUploadsTooNew int `json:"partial_uploads_too_new"`
|
||||||
@@ -226,8 +226,8 @@ type PruneBlobsResult struct {
|
|||||||
|
|
||||||
// Prune removes orphaned data from the local index database AND, from the
|
// Prune removes orphaned data from the local index database AND, from the
|
||||||
// backup destination store, unreferenced blobs, the directories under
|
// backup destination store, unreferenced blobs, the directories under
|
||||||
// metadata/ with no manifest and the `.partial` objects uploads cut off
|
// metadata/ with no manifest and the `.partial` objects left by killed
|
||||||
// part-way left. This is the single user-facing prune entry
|
// uploads. This is the single user-facing prune entry
|
||||||
// point — the split between local and remote cleanup is an
|
// point — the split between local and remote cleanup is an
|
||||||
// implementation detail. Calling code should prefer this method over
|
// implementation detail. Calling code should prefer this method over
|
||||||
// PruneDatabase or PruneBlobs individually unless it specifically wants
|
// PruneDatabase or PruneBlobs individually unless it specifically wants
|
||||||
@@ -257,7 +257,7 @@ func (v *Vaultik) Prune(opts *PruneOptions) error {
|
|||||||
|
|
||||||
// PruneBlobs removes unreferenced blobs from storage, each directory
|
// PruneBlobs removes unreferenced blobs from storage, each directory
|
||||||
// under metadata/ that a backup left with no manifest, and each
|
// under metadata/ that a backup left with no manifest, and each
|
||||||
// `.partial` object an upload cut off part-way left.
|
// `.partial` object left by a killed upload.
|
||||||
func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
|
||||||
log.Info("Starting prune operation")
|
log.Info("Starting prune operation")
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ import (
|
|||||||
const partialUploadGracePeriod = time.Hour
|
const partialUploadGracePeriod = time.Hour
|
||||||
|
|
||||||
// newPartialUploadStore returns a vaultik on a file:// destination
|
// newPartialUploadStore returns a vaultik on a file:// destination
|
||||||
// holding the `.partial` files killed uploads leave: one under blobs/
|
// holding `.partial` files left by killed uploads: one under blobs/
|
||||||
// and one under metadata/ older than the prune grace period, and one
|
// and one under metadata/ older than the prune grace period, and one
|
||||||
// under blobs/ newer than that. It also returns the buffer the vaultik
|
// under blobs/ newer than that. It also returns the buffer the vaultik
|
||||||
// writes its output to, the paths of the old files and the path of the
|
// writes its output to, the paths of the old files and the path of the
|
||||||
@@ -73,8 +73,8 @@ func newPartialUploadStore(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestPruneDeletesPartialUploadsOlderThanGracePeriod checks that prune
|
// TestPruneDeletesPartialUploadsOlderThanGracePeriod checks that prune
|
||||||
// deletes a `.partial` file a killed upload left under blobs/ or
|
// deletes a `.partial` file under blobs/ or metadata/ left by a killed
|
||||||
// metadata/ once it is older than the grace period, keeps a newer one,
|
// upload once it is older than the grace period, keeps a newer one,
|
||||||
// and counts both in the --json document.
|
// and counts both in the --json document.
|
||||||
func TestPruneDeletesPartialUploadsOlderThanGracePeriod(t *testing.T) {
|
func TestPruneDeletesPartialUploadsOlderThanGracePeriod(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
|
|||||||
Reference in new issue
Block a user