Compare commits
5 Commits
a36b314c79
...
fix/dedup-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea8edd653f | ||
| 60b6746db9 | |||
| f28c8a73b7 | |||
| 1c0f5b8eb2 | |||
| 689109a2b8 |
@@ -103,7 +103,7 @@ CREATE TABLE IF NOT EXISTS snapshot_files (
|
|||||||
file_id TEXT NOT NULL,
|
file_id TEXT NOT NULL,
|
||||||
PRIMARY KEY (snapshot_id, file_id),
|
PRIMARY KEY (snapshot_id, file_id),
|
||||||
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
||||||
FOREIGN KEY (file_id) REFERENCES files(id)
|
FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Index for efficient file lookups (used in orphan detection)
|
-- Index for efficient file lookups (used in orphan detection)
|
||||||
@@ -116,7 +116,7 @@ CREATE TABLE IF NOT EXISTS snapshot_blobs (
|
|||||||
blob_hash TEXT NOT NULL,
|
blob_hash TEXT NOT NULL,
|
||||||
PRIMARY KEY (snapshot_id, blob_id),
|
PRIMARY KEY (snapshot_id, blob_id),
|
||||||
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
|
||||||
FOREIGN KEY (blob_id) REFERENCES blobs(id)
|
FOREIGN KEY (blob_id) REFERENCES blobs(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Index for efficient blob lookups (used in orphan detection)
|
-- Index for efficient blob lookups (used in orphan detection)
|
||||||
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS uploads (
|
|||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
duration_ms INTEGER NOT NULL,
|
duration_ms INTEGER NOT NULL,
|
||||||
FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash),
|
FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash),
|
||||||
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id)
|
FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Index for efficient snapshot lookups
|
-- Index for efficient snapshot lookups
|
||||||
|
|||||||
@@ -419,7 +419,7 @@ func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) {
|
|||||||
return remoteSnapshots, nil
|
return remoteSnapshots, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// reconcileLocalWithRemote removes local snapshots not in remote and returns the surviving local map
|
// reconcileLocalWithRemote builds a map of local snapshots keyed by ID for cross-referencing with remote
|
||||||
func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) {
|
func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) {
|
||||||
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000)
|
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -431,19 +431,6 @@ func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map
|
|||||||
localSnapshotMap[s.ID.String()] = s
|
localSnapshotMap[s.ID.String()] = s
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, snap := range localSnapshots {
|
|
||||||
snapshotIDStr := snap.ID.String()
|
|
||||||
if !remoteSnapshots[snapshotIDStr] {
|
|
||||||
log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID)
|
|
||||||
if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil {
|
|
||||||
log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err)
|
|
||||||
} else {
|
|
||||||
log.Info("Deleted local snapshot not found in remote", "snapshot_id", snap.ID)
|
|
||||||
delete(localSnapshotMap, snapshotIDStr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return localSnapshotMap, nil
|
return localSnapshotMap, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -872,7 +859,7 @@ func (v *Vaultik) syncWithRemote() error {
|
|||||||
snapshotIDStr := snapshot.ID.String()
|
snapshotIDStr := snapshot.ID.String()
|
||||||
if !remoteSnapshots[snapshotIDStr] {
|
if !remoteSnapshots[snapshotIDStr] {
|
||||||
log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID)
|
log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID)
|
||||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil {
|
if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil {
|
||||||
log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err)
|
log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err)
|
||||||
} else {
|
} else {
|
||||||
removedCount++
|
removedCount++
|
||||||
@@ -1001,6 +988,7 @@ func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) {
|
|||||||
log.Info("Listing all snapshots")
|
log.Info("Listing all snapshots")
|
||||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||||
|
|
||||||
|
seen := make(map[string]bool)
|
||||||
var snapshotIDs []string
|
var snapshotIDs []string
|
||||||
for object := range objectCh {
|
for object := range objectCh {
|
||||||
if object.Err != nil {
|
if object.Err != nil {
|
||||||
@@ -1015,14 +1003,8 @@ func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) {
|
|||||||
}
|
}
|
||||||
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
|
if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") {
|
||||||
sid := parts[1]
|
sid := parts[1]
|
||||||
found := false
|
if !seen[sid] {
|
||||||
for _, id := range snapshotIDs {
|
seen[sid] = true
|
||||||
if id == sid {
|
|
||||||
found = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
snapshotIDs = append(snapshotIDs, sid)
|
snapshotIDs = append(snapshotIDs, sid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user