Apply linter autofixes: internal/snapshot (refs #61)
This commit is contained in:
@@ -4,6 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
@@ -30,6 +32,7 @@ func NewMockS3Client() *MockS3Client {
|
|||||||
|
|
||||||
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
|
func (m *MockS3Client) PutBlob(ctx context.Context, hash string, data []byte) error {
|
||||||
m.storage[hash] = data
|
m.storage[hash] = data
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -38,11 +41,13 @@ func (m *MockS3Client) GetBlob(ctx context.Context, hash string) ([]byte, error)
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("blob not found: %s", hash)
|
return nil, fmt.Errorf("blob not found: %s", hash)
|
||||||
}
|
}
|
||||||
|
|
||||||
return data, nil
|
return data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
|
func (m *MockS3Client) BlobExists(ctx context.Context, hash string) (bool, error) {
|
||||||
_, ok := m.storage[hash]
|
_, ok := m.storage[hash]
|
||||||
|
|
||||||
return ok, nil
|
return ok, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,12 +86,15 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
|
|
||||||
// Initialize the database
|
// Initialize the database
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create database: %v", err)
|
t.Fatalf("Failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Logf("Failed to close database: %v", err)
|
t.Logf("Failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -142,12 +150,14 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if !expectedFiles[file.Path.String()] {
|
if !expectedFiles[file.Path.String()] {
|
||||||
t.Errorf("Unexpected file in database: %s", file.Path)
|
t.Errorf("Unexpected file in database: %s", file.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
delete(expectedFiles, file.Path.String())
|
delete(expectedFiles, file.Path.String())
|
||||||
|
|
||||||
// Verify file metadata
|
// Verify file metadata
|
||||||
fsFile := testFS[file.Path.String()]
|
fsFile := testFS[file.Path.String()]
|
||||||
if fsFile == nil {
|
if fsFile == nil {
|
||||||
t.Errorf("File %s not found in test filesystem", file.Path)
|
t.Errorf("File %s not found in test filesystem", file.Path)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,6 +197,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to get blob hashes: %v", err)
|
t.Fatalf("Failed to get blob hashes: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(blobHashes) == 0 {
|
if len(blobHashes) == 0 {
|
||||||
t.Error("Expected at least one blob to be created")
|
t.Error("Expected at least one blob to be created")
|
||||||
}
|
}
|
||||||
@@ -197,6 +208,7 @@ func TestBackupWithInMemoryFS(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("Failed to check blob %s: %v", blobHash, err)
|
t.Errorf("Failed to check blob %s: %v", blobHash, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
t.Errorf("Blob %s not found in S3", blobHash)
|
t.Errorf("Blob %s not found in S3", blobHash)
|
||||||
}
|
}
|
||||||
@@ -229,12 +241,15 @@ func TestBackupDeduplication(t *testing.T) {
|
|||||||
|
|
||||||
// Initialize the database
|
// Initialize the database
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to create database: %v", err)
|
t.Fatalf("Failed to create database: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Logf("Failed to close database: %v", err)
|
t.Logf("Failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -348,6 +363,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
UID: 1000, // Default UID for test
|
UID: 1000, // Default UID for test
|
||||||
GID: 1000, // Default GID for test
|
GID: 1000, // Default GID for test
|
||||||
}
|
}
|
||||||
|
|
||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return b.repos.Files.Create(ctx, tx, file)
|
return b.repos.Files.Create(ctx, tx, file)
|
||||||
})
|
})
|
||||||
@@ -364,7 +380,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := f.Close(); err != nil {
|
err := f.Close()
|
||||||
|
if err != nil {
|
||||||
// Log but don't fail since we're already in an error path potentially
|
// Log but don't fail since we're already in an error path potentially
|
||||||
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
|
fmt.Fprintf(os.Stderr, "Failed to close file: %v\n", err)
|
||||||
}
|
}
|
||||||
@@ -376,9 +393,10 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
|
|
||||||
for {
|
for {
|
||||||
n, err := f.Read(buffer)
|
n, err := f.Read(buffer)
|
||||||
if err != nil && err != io.EOF {
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -395,11 +413,13 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
ChunkHash: types.ChunkHash(chunkHash),
|
ChunkHash: types.ChunkHash(chunkHash),
|
||||||
Size: int64(n),
|
Size: int64(n),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.Chunks.Create(ctx, tx, chunk)
|
return b.repos.Chunks.Create(ctx, tx, chunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
processedChunks[chunkHash] = true
|
processedChunks[chunkHash] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -410,6 +430,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
Idx: chunkIndex,
|
Idx: chunkIndex,
|
||||||
ChunkHash: types.ChunkHash(chunkHash),
|
ChunkHash: types.ChunkHash(chunkHash),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
|
return b.repos.FileChunks.Create(ctx, tx, fileChunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -424,6 +445,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
FileOffset: int64(chunkIndex * defaultChunkSize),
|
FileOffset: int64(chunkIndex * defaultChunkSize),
|
||||||
Length: int64(n),
|
Length: int64(n),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
|
return b.repos.ChunkFiles.Create(ctx, tx, chunkFile)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -435,7 +457,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -464,12 +485,14 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
|
|
||||||
// Create blob entry in a short transaction
|
// Create blob entry in a short transaction
|
||||||
blobID := types.NewBlobID()
|
blobID := types.NewBlobID()
|
||||||
|
|
||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
blob := &database.Blob{
|
blob := &database.Blob{
|
||||||
ID: blobID,
|
ID: blobID,
|
||||||
Hash: types.BlobHash(blobHash),
|
Hash: types.BlobHash(blobHash),
|
||||||
CreatedTS: time.Now(),
|
CreatedTS: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.Blobs.Create(ctx, tx, blob)
|
return b.repos.Blobs.Create(ctx, tx, blob)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -487,6 +510,7 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
Offset: 0,
|
Offset: 0,
|
||||||
Length: chunk.Size,
|
Length: chunk.Size,
|
||||||
}
|
}
|
||||||
|
|
||||||
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
return b.repos.BlobChunks.Create(ctx, tx, blobChunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -506,7 +530,6 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
|
return b.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID, fileCount, chunkCount, blobCount, totalSize, blobSize)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -517,16 +540,18 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str
|
|||||||
func calculateHash(data []byte) string {
|
func calculateHash(data []byte) string {
|
||||||
h := sha256.New()
|
h := sha256.New()
|
||||||
h.Write(data)
|
h.Write(data)
|
||||||
return fmt.Sprintf("%x", h.Sum(nil))
|
|
||||||
|
return hex.EncodeToString(h.Sum(nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func generateLargeFileContent(size int) []byte {
|
func generateLargeFileContent(size int) []byte {
|
||||||
data := make([]byte, size)
|
data := make([]byte, size)
|
||||||
// Fill with pattern that changes every chunk to avoid deduplication
|
// Fill with pattern that changes every chunk to avoid deduplication
|
||||||
for i := 0; i < size; i++ {
|
for i := range size {
|
||||||
chunkNum := i / defaultChunkSize
|
chunkNum := i / defaultChunkSize
|
||||||
data[i] = byte((i + chunkNum) % 256)
|
data[i] = byte((i + chunkNum) % 256)
|
||||||
}
|
}
|
||||||
|
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ func setupExcludeTestFS(t *testing.T) afero.Fs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
err := fs.MkdirAll(dir, 0755)
|
err := fs.MkdirAll(dir, 0755)
|
||||||
@@ -107,6 +108,7 @@ func createTestScanner(t *testing.T, fs afero.Fs, excludePatterns []string) (*sn
|
|||||||
|
|
||||||
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Repositories, snapshotID string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err := repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
snap := &database.Snapshot{
|
snap := &database.Snapshot{
|
||||||
ID: types.SnapshotID(snapshotID),
|
ID: types.SnapshotID(snapshotID),
|
||||||
@@ -121,6 +123,7 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
|||||||
BlobSize: 0,
|
BlobSize: 0,
|
||||||
CompressionRatio: 1.0,
|
CompressionRatio: 1.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snap)
|
return repos.Snapshots.Create(ctx, tx, snap)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -128,8 +131,10 @@ func createSnapshotRecord(t *testing.T, ctx context.Context, repos *database.Rep
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -148,8 +153,10 @@ func TestExcludePatterns_ExcludeGitDirectory(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"*.log"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -165,8 +172,10 @@ func TestExcludePatterns_ExcludeByExtension(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"node_modules"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -182,8 +191,10 @@ func TestExcludePatterns_ExcludeNodeModules(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".git", "node_modules", "*.log", ".DS_Store", "thumbs.db", "cache", "build"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -199,8 +210,10 @@ func TestExcludePatterns_MultiplePatterns(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
func TestExcludePatterns_NoExclusions(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -215,8 +228,10 @@ func TestExcludePatterns_NoExclusions(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{".*"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -232,8 +247,10 @@ func TestExcludePatterns_ExcludeHiddenFiles(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"**/*.pack"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -249,8 +266,10 @@ func TestExcludePatterns_DoubleStarGlob(t *testing.T) {
|
|||||||
|
|
||||||
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
func TestExcludePatterns_ExactFileName(t *testing.T) {
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"thumbs.db", ".DS_Store"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -267,8 +286,10 @@ func TestExcludePatterns_ExactFileName(t *testing.T) {
|
|||||||
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
func TestExcludePatterns_CaseSensitive(t *testing.T) {
|
||||||
// Pattern matching should be case-sensitive
|
// Pattern matching should be case-sensitive
|
||||||
fs := setupExcludeTestFS(t)
|
fs := setupExcludeTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"THUMBS.DB"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -287,6 +308,7 @@ func TestExcludePatterns_DirectoryWithTrailingSlash(t *testing.T) {
|
|||||||
// Some users might add trailing slashes to directory patterns
|
// Some users might add trailing slashes to directory patterns
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"cache/", "build/"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -305,6 +327,7 @@ func TestExcludePatterns_PatternInSubdirectory(t *testing.T) {
|
|||||||
// Exclude .hidden file specifically in src directory
|
// Exclude .hidden file specifically in src directory
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"src/.hidden"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -343,6 +366,7 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
for path, content := range files {
|
for path, content := range files {
|
||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
err := fs.MkdirAll(dir, 0755)
|
err := fs.MkdirAll(dir, 0755)
|
||||||
@@ -359,8 +383,10 @@ func setupAnchoredTestFS(t *testing.T) afero.Fs {
|
|||||||
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
||||||
// Pattern starting with / should only match from root of source dir
|
// Pattern starting with / should only match from root of source dir
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/projectname"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -378,8 +404,10 @@ func TestExcludePatterns_AnchoredPattern(t *testing.T) {
|
|||||||
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
||||||
// Pattern without leading / should match anywhere in path
|
// Pattern without leading / should match anywhere in path
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"projectname"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -398,8 +426,10 @@ func TestExcludePatterns_UnanchoredPattern(t *testing.T) {
|
|||||||
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
||||||
// Anchored pattern with glob
|
// Anchored pattern with glob
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/src/*.go"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -416,8 +446,10 @@ func TestExcludePatterns_AnchoredPatternWithGlob(t *testing.T) {
|
|||||||
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
||||||
// Anchored pattern for exact file at root
|
// Anchored pattern for exact file at root
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"/file.txt"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
@@ -435,8 +467,10 @@ func TestExcludePatterns_AnchoredPatternFile(t *testing.T) {
|
|||||||
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
func TestExcludePatterns_UnanchoredPatternFile(t *testing.T) {
|
||||||
// Unanchored pattern for file should match anywhere
|
// Unanchored pattern for file should match anywhere
|
||||||
fs := setupAnchoredTestFS(t)
|
fs := setupAnchoredTestFS(t)
|
||||||
|
|
||||||
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
scanner, repos, cleanup := createTestScanner(t, fs, []string{"file.txt"})
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
require.NotNil(t, scanner)
|
require.NotNil(t, scanner)
|
||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|||||||
@@ -30,9 +30,11 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
|
|
||||||
// Create test database
|
// Create test database
|
||||||
db, err := database.NewTestDB()
|
db, err := database.NewTestDB()
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -59,6 +61,7 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -81,6 +84,7 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
|
|
||||||
// Modify the file
|
// Modify the file
|
||||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||||
|
|
||||||
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
|
err = afero.WriteFile(fs, "/test.txt", []byte("Modified content with different data"), 0644)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -93,6 +97,7 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -130,6 +135,7 @@ func TestFileContentChange(t *testing.T) {
|
|||||||
// Verify that chunk_files for old chunk no longer references this file
|
// Verify that chunk_files for old chunk no longer references this file
|
||||||
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
|
oldChunkFiles, err := repos.ChunkFiles.GetByChunkHash(ctx, oldChunkHash)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
for _, cf := range oldChunkFiles {
|
for _, cf := range oldChunkFiles {
|
||||||
file, err := repos.Files.GetByID(ctx, cf.FileID)
|
file, err := repos.Files.GetByID(ctx, cf.FileID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -159,9 +165,11 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
|
|
||||||
// Create test database
|
// Create test database
|
||||||
db, err := database.NewTestDB()
|
db, err := database.NewTestDB()
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -188,6 +196,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -200,6 +209,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
|
|
||||||
// Modify two files
|
// Modify two files
|
||||||
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
time.Sleep(10 * time.Millisecond) // Ensure mtime changes
|
||||||
|
|
||||||
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
|
err = afero.WriteFile(fs, "/file1.txt", []byte("Modified content 1"), 0644)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
|
err = afero.WriteFile(fs, "/file3.txt", []byte("Modified content 3"), 0644)
|
||||||
@@ -214,6 +224,7 @@ func TestMultipleFileChanges(t *testing.T) {
|
|||||||
VaultikVersion: "test",
|
VaultikVersion: "test",
|
||||||
StartedAt: time.Now(),
|
StartedAt: time.Now(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
|
|||||||
|
|
||||||
// Compress using zstd
|
// Compress using zstd
|
||||||
var compressedBuf bytes.Buffer
|
var compressedBuf bytes.Buffer
|
||||||
|
|
||||||
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
|
writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
return nil, fmt.Errorf("creating zstd writer: %w", err)
|
||||||
@@ -59,6 +60,7 @@ func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) {
|
|||||||
|
|
||||||
if _, err := writer.Write(jsonData); err != nil {
|
if _, err := writer.Write(jsonData); err != nil {
|
||||||
_ = writer.Close()
|
_ = writer.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("writing compressed data: %w", err)
|
return nil, fmt.Errorf("writing compressed data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ import (
|
|||||||
func TestWrapPermissionError(t *testing.T) {
|
func TestWrapPermissionError(t *testing.T) {
|
||||||
// Non-permission errors pass through unchanged.
|
// Non-permission errors pass through unchanged.
|
||||||
plain := errors.New("disk on fire")
|
plain := errors.New("disk on fire")
|
||||||
if got := wrapPermissionError("/some/path", plain); got != plain {
|
|
||||||
|
got := wrapPermissionError("/some/path", plain)
|
||||||
|
if !errors.Is(got, plain) {
|
||||||
t.Errorf("non-permission error should pass through, got %v", got)
|
t.Errorf("non-permission error should pass through, got %v", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -23,6 +25,7 @@ func TestWrapPermissionError(t *testing.T) {
|
|||||||
if !errors.Is(wrapped, os.ErrPermission) {
|
if !errors.Is(wrapped, os.ErrPermission) {
|
||||||
t.Error("wrapped error should still match os.ErrPermission")
|
t.Error("wrapped error should still match os.ErrPermission")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
|
if !strings.Contains(wrapped.Error(), "/Users/u/Library/Calendars") {
|
||||||
t.Error("wrapped error should name the offending path")
|
t.Error("wrapped error should name the offending path")
|
||||||
}
|
}
|
||||||
@@ -31,6 +34,7 @@ func TestWrapPermissionError(t *testing.T) {
|
|||||||
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
|
if !strings.Contains(wrapped.Error(), "Full Disk Access") {
|
||||||
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
|
t.Errorf("macOS permission error should mention Full Disk Access:\n%s", wrapped.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(wrapped.Error(), "System Settings") {
|
if !strings.Contains(wrapped.Error(), "System Settings") {
|
||||||
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
|
t.Errorf("macOS permission error should point at System Settings:\n%s", wrapped.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -153,6 +153,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
|||||||
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
|
if uploadInfo, ok := pr.stats.CurrentUpload.Load().(*UploadInfo); ok && uploadInfo != nil {
|
||||||
// Show upload progress instead
|
// Show upload progress instead
|
||||||
pr.printUploadProgress(uploadInfo)
|
pr.printUploadProgress(uploadInfo)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -175,16 +176,18 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
|||||||
|
|
||||||
// Calculate ETA if we have total size and are processing
|
// Calculate ETA if we have total size and are processing
|
||||||
etaStr := ""
|
etaStr := ""
|
||||||
|
|
||||||
if totalSize > 0 && bytesProcessed > 0 {
|
if totalSize > 0 && bytesProcessed > 0 {
|
||||||
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
||||||
if ok && !processStart.IsZero() {
|
if ok && !processStart.IsZero() {
|
||||||
processElapsed := time.Since(processStart)
|
processElapsed := time.Since(processStart)
|
||||||
|
|
||||||
rate := float64(bytesProcessed) / processElapsed.Seconds()
|
rate := float64(bytesProcessed) / processElapsed.Seconds()
|
||||||
if rate > 0 {
|
if rate > 0 {
|
||||||
remainingBytes := totalSize - bytesProcessed
|
remainingBytes := totalSize - bytesProcessed
|
||||||
remainingSeconds := float64(remainingBytes) / rate
|
remainingSeconds := float64(remainingBytes) / rate
|
||||||
eta := time.Duration(remainingSeconds * float64(time.Second))
|
eta := time.Duration(remainingSeconds * float64(time.Second))
|
||||||
etaStr = fmt.Sprintf(" | ETA: %s", formatDuration(eta))
|
etaStr = " | ETA: " + formatDuration(eta)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,7 +209,7 @@ func (pr *ProgressReporter) printSummaryStatus() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if currentFile != "" {
|
if currentFile != "" {
|
||||||
status += fmt.Sprintf(" | Current: %s", truncatePath(currentFile, 40))
|
status += " | Current: " + truncatePath(currentFile, 40)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info(status)
|
log.Info(status)
|
||||||
@@ -242,6 +245,7 @@ func (pr *ProgressReporter) printDetailedStatus() {
|
|||||||
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
processStart, ok := pr.stats.ProcessStartTime.Load().(time.Time)
|
||||||
if ok && !processStart.IsZero() {
|
if ok && !processStart.IsZero() {
|
||||||
processElapsed := time.Since(processStart)
|
processElapsed := time.Since(processStart)
|
||||||
|
|
||||||
processRate := float64(bytesProcessed) / processElapsed.Seconds()
|
processRate := float64(bytesProcessed) / processElapsed.Seconds()
|
||||||
if processRate > 0 {
|
if processRate > 0 {
|
||||||
remainingBytes := totalSize - bytesProcessed
|
remainingBytes := totalSize - bytesProcessed
|
||||||
@@ -276,9 +280,11 @@ func (pr *ProgressReporter) printDetailedStatus() {
|
|||||||
log.Info("Total uploaded to remote",
|
log.Info("Total uploaded to remote",
|
||||||
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
|
"uploaded", humanize.Bytes(uint64(bytesUploaded)),
|
||||||
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
|
"compression_ratio", formatRatio(bytesUploaded, bytesScanned))
|
||||||
|
|
||||||
if currentFile != "" {
|
if currentFile != "" {
|
||||||
log.Info("Current file", "path", currentFile)
|
log.Info("Current file", "path", currentFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Notice("=============================")
|
log.Notice("=============================")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,12 +294,15 @@ func formatDuration(d time.Duration) string {
|
|||||||
if d < 0 {
|
if d < 0 {
|
||||||
return "unknown"
|
return "unknown"
|
||||||
}
|
}
|
||||||
|
|
||||||
if d < time.Minute {
|
if d < time.Minute {
|
||||||
return fmt.Sprintf("%ds", int(d.Seconds()))
|
return fmt.Sprintf("%ds", int(d.Seconds()))
|
||||||
}
|
}
|
||||||
|
|
||||||
if d < time.Hour {
|
if d < time.Hour {
|
||||||
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
|
return fmt.Sprintf("%dm%ds", int(d.Minutes()), int(d.Seconds())%60)
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
|
return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,6 +310,7 @@ func formatPercent(numerator, denominator int64) string {
|
|||||||
if denominator == 0 {
|
if denominator == 0 {
|
||||||
return "0.0%"
|
return "0.0%"
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
|
return fmt.Sprintf("%.1f%%", float64(numerator)/float64(denominator)*100)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -308,7 +318,9 @@ func formatRatio(compressed, uncompressed int64) string {
|
|||||||
if uncompressed == 0 {
|
if uncompressed == 0 {
|
||||||
return "1.00"
|
return "1.00"
|
||||||
}
|
}
|
||||||
|
|
||||||
ratio := float64(compressed) / float64(uncompressed)
|
ratio := float64(compressed) / float64(uncompressed)
|
||||||
|
|
||||||
return fmt.Sprintf("%.2f", ratio)
|
return fmt.Sprintf("%.2f", ratio)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,6 +365,7 @@ func (pr *ProgressReporter) ReportUploadComplete(blobHash string, size int64, du
|
|||||||
if duration < time.Millisecond {
|
if duration < time.Millisecond {
|
||||||
duration = time.Millisecond
|
duration = time.Millisecond
|
||||||
}
|
}
|
||||||
|
|
||||||
bytesPerSec := float64(size) / duration.Seconds()
|
bytesPerSec := float64(size) / duration.Seconds()
|
||||||
bitsPerSec := bytesPerSec * 8
|
bitsPerSec := bytesPerSec * 8
|
||||||
|
|
||||||
@@ -398,6 +411,7 @@ func (pr *ProgressReporter) ReportUploadProgress(blobHash string, bytesUploaded,
|
|||||||
|
|
||||||
// Calculate ETA based on current speed
|
// Calculate ETA based on current speed
|
||||||
etaStr := "unknown"
|
etaStr := "unknown"
|
||||||
|
|
||||||
if instantSpeed > 0 && bytesUploaded < totalSize {
|
if instantSpeed > 0 && bytesUploaded < totalSize {
|
||||||
remainingBytes := totalSize - bytesUploaded
|
remainingBytes := totalSize - bytesUploaded
|
||||||
remainingSeconds := float64(remainingBytes) / instantSpeed
|
remainingSeconds := float64(remainingBytes) / instantSpeed
|
||||||
|
|||||||
@@ -36,5 +36,6 @@ const remoteKeyPrefix = "vaultik|"
|
|||||||
func RemoteSnapshotKey(snapshotID string) string {
|
func RemoteSnapshotKey(snapshotID string) string {
|
||||||
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
|
first := sha256.Sum256([]byte(remoteKeyPrefix + snapshotID))
|
||||||
second := sha256.Sum256(first[:])
|
second := sha256.Sum256(first[:])
|
||||||
|
|
||||||
return hex.EncodeToString(second[:])
|
return hex.EncodeToString(second[:])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
|||||||
// Create encryptor (required for blob packing)
|
// Create encryptor (required for blob packing)
|
||||||
if len(cfg.AgeRecipients) == 0 {
|
if len(cfg.AgeRecipients) == 0 {
|
||||||
log.Error("No age recipients configured - encryption is required")
|
log.Error("No age recipients configured - encryption is required")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,9 +131,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
|
|||||||
Repositories: cfg.Repositories,
|
Repositories: cfg.Repositories,
|
||||||
Fs: cfg.FS,
|
Fs: cfg.FS,
|
||||||
}
|
}
|
||||||
|
|
||||||
packer, err := blob.NewPacker(packerCfg)
|
packer, err := blob.NewPacker(packerCfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error("Failed to create packer", "error", err)
|
log.Error("Failed to create packer", "error", err)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,11 +202,14 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
|||||||
// Phase 1: Scan directory, collect files to process, and track existing files
|
// Phase 1: Scan directory, collect files to process, and track existing files
|
||||||
// (builds existingFiles map during walk to avoid double traversal)
|
// (builds existingFiles map during walk to avoid double traversal)
|
||||||
log.Info("Phase 1/3: Scanning directory structure")
|
log.Info("Phase 1/3: Scanning directory structure")
|
||||||
|
|
||||||
existingFiles := make(map[string]struct{})
|
existingFiles := make(map[string]struct{})
|
||||||
|
|
||||||
scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles)
|
scanResult, err := s.scanPhase(ctx, path, result, existingFiles, knownFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scan phase failed: %w", err)
|
return nil, fmt.Errorf("scan phase failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
filesToProcess := scanResult.FilesToProcess
|
filesToProcess := scanResult.FilesToProcess
|
||||||
|
|
||||||
// Phase 1b: Detect deleted files by comparing DB against scanned files
|
// Phase 1b: Detect deleted files by comparing DB against scanned files
|
||||||
@@ -214,7 +220,9 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
|||||||
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
|
// Phase 1c: Associate unchanged files with this snapshot (no new records needed)
|
||||||
if len(scanResult.UnchangedFileIDs) > 0 {
|
if len(scanResult.UnchangedFileIDs) > 0 {
|
||||||
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
|
s.ui.Begin("Associating %s unchanged files with the snapshot.", s.ui.Count(len(scanResult.UnchangedFileIDs)))
|
||||||
if err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs); err != nil {
|
|
||||||
|
err := s.batchAddFilesToSnapshot(ctx, scanResult.UnchangedFileIDs)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("associating unchanged files: %w", err)
|
return nil, fmt.Errorf("associating unchanged files: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -226,7 +234,9 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
|||||||
if len(filesToProcess) > 0 {
|
if len(filesToProcess) > 0 {
|
||||||
s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
|
s.ui.Begin("Backing up %s snapshot source files (chunking, compressing, encrypting, uploading).", s.ui.Count(len(filesToProcess)))
|
||||||
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
|
log.Info("Phase 2/3: Creating snapshot (chunking, compressing, encrypting, and uploading blobs)")
|
||||||
if err := s.processPhase(ctx, filesToProcess, result); err != nil {
|
|
||||||
|
err := s.processPhase(ctx, filesToProcess, result)
|
||||||
|
if err != nil {
|
||||||
return nil, fmt.Errorf("process phase failed: %w", err)
|
return nil, fmt.Errorf("process phase failed: %w", err)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -244,16 +254,20 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc
|
|||||||
// This avoids per-file and per-chunk database queries during the scan and process phases
|
// This avoids per-file and per-chunk database queries during the scan and process phases
|
||||||
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
|
func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) {
|
||||||
s.ui.Begin("Loading known files from local index database.")
|
s.ui.Begin("Loading known files from local index database.")
|
||||||
|
|
||||||
knownFiles, err := s.loadKnownFiles(ctx, path)
|
knownFiles, err := s.loadKnownFiles(ctx, path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("loading known files: %w", err)
|
return nil, fmt.Errorf("loading known files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
|
s.ui.Complete("Loaded %s known files from local index database.", s.ui.Count(len(knownFiles)))
|
||||||
|
|
||||||
s.ui.Begin("Loading known chunks from local index database.")
|
s.ui.Begin("Loading known chunks from local index database.")
|
||||||
|
|
||||||
if err := s.loadKnownChunks(ctx); err != nil {
|
if err := s.loadKnownChunks(ctx); err != nil {
|
||||||
return nil, fmt.Errorf("loading known chunks: %w", err)
|
return nil, fmt.Errorf("loading known chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
|
s.ui.Complete("Loaded %s known chunks from local index database.", s.ui.Count(len(s.knownChunks)))
|
||||||
|
|
||||||
return knownFiles, nil
|
return knownFiles, nil
|
||||||
@@ -288,6 +302,7 @@ func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileT
|
|||||||
s.ui.Count(result.FilesDeleted),
|
s.ui.Count(result.FilesDeleted),
|
||||||
s.ui.Size(result.BytesDeleted))
|
s.ui.Size(result.BytesDeleted))
|
||||||
}
|
}
|
||||||
|
|
||||||
s.ui.Complete("%s.", msg)
|
s.ui.Complete("%s.", msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,6 +352,7 @@ func (s *Scanner) loadKnownChunks(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.knownChunksMu.Lock()
|
s.knownChunksMu.Lock()
|
||||||
|
|
||||||
s.knownChunks = make(map[string]struct{}, len(chunks))
|
s.knownChunks = make(map[string]struct{}, len(chunks))
|
||||||
for _, c := range chunks {
|
for _, c := range chunks {
|
||||||
s.knownChunks[c.ChunkHash.String()] = struct{}{}
|
s.knownChunks[c.ChunkHash.String()] = struct{}{}
|
||||||
@@ -351,6 +367,7 @@ func (s *Scanner) chunkExists(hash string) bool {
|
|||||||
s.knownChunksMu.RLock()
|
s.knownChunksMu.RLock()
|
||||||
_, exists := s.knownChunks[hash]
|
_, exists := s.knownChunks[hash]
|
||||||
s.knownChunksMu.RUnlock()
|
s.knownChunksMu.RUnlock()
|
||||||
|
|
||||||
return exists
|
return exists
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,7 +388,9 @@ func (s *Scanner) addPendingChunkHash(hash string) {
|
|||||||
// removePendingChunkHashes removes committed chunk hashes from the pending set
|
// removePendingChunkHashes removes committed chunk hashes from the pending set
|
||||||
func (s *Scanner) removePendingChunkHashes(hashes []string) {
|
func (s *Scanner) removePendingChunkHashes(hashes []string) {
|
||||||
log.Debug("removePendingChunkHashes: starting", "count", len(hashes))
|
log.Debug("removePendingChunkHashes: starting", "count", len(hashes))
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
s.pendingChunkHashesMu.Lock()
|
s.pendingChunkHashesMu.Lock()
|
||||||
for _, hash := range hashes {
|
for _, hash := range hashes {
|
||||||
delete(s.pendingChunkHashes, hash)
|
delete(s.pendingChunkHashes, hash)
|
||||||
@@ -385,6 +404,7 @@ func (s *Scanner) isChunkPending(hash string) bool {
|
|||||||
s.pendingChunkHashesMu.Lock()
|
s.pendingChunkHashesMu.Lock()
|
||||||
_, pending := s.pendingChunkHashes[hash]
|
_, pending := s.pendingChunkHashes[hash]
|
||||||
s.pendingChunkHashesMu.Unlock()
|
s.pendingChunkHashesMu.Unlock()
|
||||||
|
|
||||||
return pending
|
return pending
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -411,37 +431,45 @@ func (s *Scanner) flushPendingFiles(ctx context.Context) error {
|
|||||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
for _, data := range files {
|
for _, data := range files {
|
||||||
// Create or update the file record
|
// Create or update the file record
|
||||||
if err := s.repos.Files.Create(txCtx, tx, data.file); err != nil {
|
err := s.repos.Files.Create(txCtx, tx, data.file)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("creating file record: %w", err)
|
return fmt.Errorf("creating file record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete any existing file_chunks and chunk_files for this file
|
// Delete any existing file_chunks and chunk_files for this file
|
||||||
if err := s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
|
err = s.repos.FileChunks.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting old file chunks: %w", err)
|
return fmt.Errorf("deleting old file chunks: %w", err)
|
||||||
}
|
}
|
||||||
if err := s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID); err != nil {
|
|
||||||
|
err = s.repos.ChunkFiles.DeleteByFileID(txCtx, tx, data.file.ID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting old chunk files: %w", err)
|
return fmt.Errorf("deleting old chunk files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create file-chunk mappings
|
// Create file-chunk mappings
|
||||||
for i := range data.fileChunks {
|
for i := range data.fileChunks {
|
||||||
if err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i]); err != nil {
|
err := s.repos.FileChunks.Create(txCtx, tx, &data.fileChunks[i])
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("creating file chunk: %w", err)
|
return fmt.Errorf("creating file chunk: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create chunk-file mappings
|
// Create chunk-file mappings
|
||||||
for i := range data.chunkFiles {
|
for i := range data.chunkFiles {
|
||||||
if err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i]); err != nil {
|
err := s.repos.ChunkFiles.Create(txCtx, tx, &data.chunkFiles[i])
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("creating chunk file: %w", err)
|
return fmt.Errorf("creating chunk file: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add file to snapshot
|
// Add file to snapshot
|
||||||
if err := s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID); err != nil {
|
err = s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, data.file.ID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("adding file to snapshot: %w", err)
|
return fmt.Errorf("adding file to snapshot: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -455,6 +483,7 @@ func (s *Scanner) flushAllPending(ctx context.Context) error {
|
|||||||
// Files with pending chunks are kept in the queue for later flushing
|
// Files with pending chunks are kept in the queue for later flushing
|
||||||
func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
||||||
flushStart := time.Now()
|
flushStart := time.Now()
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: starting")
|
log.Debug("flushCompletedPendingFiles: starting")
|
||||||
|
|
||||||
// Partition pending files into those ready to flush and those still waiting
|
// Partition pending files into those ready to flush and those still waiting
|
||||||
@@ -462,6 +491,7 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
|||||||
|
|
||||||
if len(canFlush) == 0 {
|
if len(canFlush) == 0 {
|
||||||
log.Debug("flushCompletedPendingFiles: nothing to flush")
|
log.Debug("flushCompletedPendingFiles: nothing to flush")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -474,10 +504,13 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error {
|
|||||||
|
|
||||||
// Execute the batch flush in a single transaction
|
// Execute the batch flush in a single transaction
|
||||||
log.Debug("flushCompletedPendingFiles: starting transaction")
|
log.Debug("flushCompletedPendingFiles: starting transaction")
|
||||||
|
|
||||||
txStart := time.Now()
|
txStart := time.Now()
|
||||||
err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles)
|
err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles)
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart))
|
log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart))
|
||||||
log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart))
|
log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart))
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -492,21 +525,27 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
|
|||||||
var stillPending []pendingFileData
|
var stillPending []pendingFileData
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: checking which files can flush")
|
log.Debug("flushCompletedPendingFiles: checking which files can flush")
|
||||||
|
|
||||||
checkStart := time.Now()
|
checkStart := time.Now()
|
||||||
|
|
||||||
for _, data := range s.pendingFiles {
|
for _, data := range s.pendingFiles {
|
||||||
allChunksCommitted := true
|
allChunksCommitted := true
|
||||||
|
|
||||||
for _, fc := range data.fileChunks {
|
for _, fc := range data.fileChunks {
|
||||||
if s.isChunkPending(fc.ChunkHash.String()) {
|
if s.isChunkPending(fc.ChunkHash.String()) {
|
||||||
allChunksCommitted = false
|
allChunksCommitted = false
|
||||||
|
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if allChunksCommitted {
|
if allChunksCommitted {
|
||||||
canFlush = append(canFlush, data)
|
canFlush = append(canFlush, data)
|
||||||
} else {
|
} else {
|
||||||
stillPending = append(stillPending, data)
|
stillPending = append(stillPending, data)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending))
|
log.Debug("flushCompletedPendingFiles: check done", "duration", time.Since(checkStart), "can_flush", len(canFlush), "still_pending", len(stillPending))
|
||||||
|
|
||||||
s.pendingFiles = stillPending
|
s.pendingFiles = stillPending
|
||||||
@@ -520,12 +559,15 @@ func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, s
|
|||||||
// mappings from the given pending file data for efficient batch database operations
|
// mappings from the given pending file data for efficient batch database operations
|
||||||
func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) {
|
func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) {
|
||||||
log.Debug("flushCompletedPendingFiles: collecting data for batch ops")
|
log.Debug("flushCompletedPendingFiles: collecting data for batch ops")
|
||||||
|
|
||||||
collectStart := time.Now()
|
collectStart := time.Now()
|
||||||
|
|
||||||
var allFileChunks []database.FileChunk
|
var (
|
||||||
var allChunkFiles []database.ChunkFile
|
allFileChunks []database.FileChunk
|
||||||
var allFileIDs []types.FileID
|
allChunkFiles []database.ChunkFile
|
||||||
var allFiles []*database.File
|
allFileIDs []types.FileID
|
||||||
|
allFiles []*database.File
|
||||||
|
)
|
||||||
|
|
||||||
for _, data := range canFlush {
|
for _, data := range canFlush {
|
||||||
allFileChunks = append(allFileChunks, data.fileChunks...)
|
allFileChunks = append(allFileChunks, data.fileChunks...)
|
||||||
@@ -551,52 +593,77 @@ func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*databas
|
|||||||
|
|
||||||
// Batch delete old file_chunks and chunk_files
|
// Batch delete old file_chunks and chunk_files
|
||||||
log.Debug("flushCompletedPendingFiles: deleting old file_chunks")
|
log.Debug("flushCompletedPendingFiles: deleting old file_chunks")
|
||||||
|
|
||||||
opStart := time.Now()
|
opStart := time.Now()
|
||||||
if err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
|
|
||||||
|
err := s.repos.FileChunks.DeleteByFileIDs(txCtx, tx, allFileIDs)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch deleting old file chunks: %w", err)
|
return fmt.Errorf("batch deleting old file chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: deleted file_chunks", "duration", time.Since(opStart))
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: deleting old chunk_files")
|
log.Debug("flushCompletedPendingFiles: deleting old chunk_files")
|
||||||
|
|
||||||
opStart = time.Now()
|
opStart = time.Now()
|
||||||
if err := s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs); err != nil {
|
|
||||||
|
err = s.repos.ChunkFiles.DeleteByFileIDs(txCtx, tx, allFileIDs)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch deleting old chunk files: %w", err)
|
return fmt.Errorf("batch deleting old chunk files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: deleted chunk_files", "duration", time.Since(opStart))
|
||||||
|
|
||||||
// Batch create/update file records
|
// Batch create/update file records
|
||||||
log.Debug("flushCompletedPendingFiles: creating files")
|
log.Debug("flushCompletedPendingFiles: creating files")
|
||||||
|
|
||||||
opStart = time.Now()
|
opStart = time.Now()
|
||||||
if err := s.repos.Files.CreateBatch(txCtx, tx, allFiles); err != nil {
|
|
||||||
|
err = s.repos.Files.CreateBatch(txCtx, tx, allFiles)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch creating file records: %w", err)
|
return fmt.Errorf("batch creating file records: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: created files", "duration", time.Since(opStart))
|
||||||
|
|
||||||
// Batch insert file_chunks
|
// Batch insert file_chunks
|
||||||
log.Debug("flushCompletedPendingFiles: inserting file_chunks")
|
log.Debug("flushCompletedPendingFiles: inserting file_chunks")
|
||||||
|
|
||||||
opStart = time.Now()
|
opStart = time.Now()
|
||||||
if err := s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks); err != nil {
|
|
||||||
|
err = s.repos.FileChunks.CreateBatch(txCtx, tx, allFileChunks)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch creating file chunks: %w", err)
|
return fmt.Errorf("batch creating file chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: inserted file_chunks", "duration", time.Since(opStart))
|
||||||
|
|
||||||
// Batch insert chunk_files
|
// Batch insert chunk_files
|
||||||
log.Debug("flushCompletedPendingFiles: inserting chunk_files")
|
log.Debug("flushCompletedPendingFiles: inserting chunk_files")
|
||||||
|
|
||||||
opStart = time.Now()
|
opStart = time.Now()
|
||||||
if err := s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles); err != nil {
|
|
||||||
|
err = s.repos.ChunkFiles.CreateBatch(txCtx, tx, allChunkFiles)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch creating chunk files: %w", err)
|
return fmt.Errorf("batch creating chunk files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: inserted chunk_files", "duration", time.Since(opStart))
|
||||||
|
|
||||||
// Batch add files to snapshot
|
// Batch add files to snapshot
|
||||||
log.Debug("flushCompletedPendingFiles: adding files to snapshot")
|
log.Debug("flushCompletedPendingFiles: adding files to snapshot")
|
||||||
|
|
||||||
opStart = time.Now()
|
opStart = time.Now()
|
||||||
if err := s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs); err != nil {
|
|
||||||
|
err = s.repos.Snapshots.AddFilesByIDBatch(txCtx, tx, s.snapshotID, allFileIDs)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
return fmt.Errorf("batch adding files to snapshot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart))
|
log.Debug("flushCompletedPendingFiles: added files to snapshot", "duration", time.Since(opStart))
|
||||||
|
|
||||||
log.Debug("flushCompletedPendingFiles: transaction complete")
|
log.Debug("flushCompletedPendingFiles: transaction complete")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -616,24 +683,31 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
estimatedTotal := int64(len(knownFiles))
|
estimatedTotal := int64(len(knownFiles))
|
||||||
|
|
||||||
var filesToProcess []*FileToProcess
|
var filesToProcess []*FileToProcess
|
||||||
|
|
||||||
var unchangedFileIDs []types.FileID // Just IDs - no new records needed
|
var unchangedFileIDs []types.FileID // Just IDs - no new records needed
|
||||||
|
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
|
|
||||||
// Set up periodic status output
|
// Set up periodic status output
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
lastStatusTime := time.Now()
|
lastStatusTime := time.Now()
|
||||||
statusInterval := 15 * time.Second
|
statusInterval := 15 * time.Second
|
||||||
|
|
||||||
var filesScanned int64
|
var filesScanned int64
|
||||||
|
|
||||||
log.Debug("Starting directory walk", "path", path)
|
log.Debug("Starting directory walk", "path", path)
|
||||||
|
|
||||||
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
|
err := afero.Walk(s.fs, path, func(filePath string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if s.skipErrors {
|
if s.skipErrors {
|
||||||
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
|
log.Error("Failed to access file (skipping due to --skip-errors)", "path", filePath, "error", err)
|
||||||
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
|
s.ui.Error("Failed to access %s: %v. Skipping (--skip-errors).", s.ui.Path(filePath), err)
|
||||||
|
|
||||||
return nil // Continue scanning
|
return nil // Continue scanning
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
|
log.Debug("Error accessing filesystem entry", "path", filePath, "error", err)
|
||||||
|
|
||||||
return wrapPermissionError(filePath, err)
|
return wrapPermissionError(filePath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -649,6 +723,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
return filepath.SkipDir
|
return filepath.SkipDir
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -657,7 +732,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
file := s.buildSymlinkEntry(filePath, info)
|
file := s.buildSymlinkEntry(filePath, info)
|
||||||
if file != nil {
|
if file != nil {
|
||||||
existingFiles[filePath] = struct{}{}
|
existingFiles[filePath] = struct{}{}
|
||||||
|
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
|
|
||||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||||
Path: filePath,
|
Path: filePath,
|
||||||
FileInfo: info,
|
FileInfo: info,
|
||||||
@@ -667,6 +744,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
s.updateScanEntryStats(result, true, info)
|
s.updateScanEntryStats(result, true, info)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -674,7 +752,9 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
file := s.buildDirectoryEntry(filePath, info)
|
file := s.buildDirectoryEntry(filePath, info)
|
||||||
existingFiles[filePath] = struct{}{}
|
existingFiles[filePath] = struct{}{}
|
||||||
|
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
|
|
||||||
filesToProcess = append(filesToProcess, &FileToProcess{
|
filesToProcess = append(filesToProcess, &FileToProcess{
|
||||||
Path: filePath,
|
Path: filePath,
|
||||||
FileInfo: info,
|
FileInfo: info,
|
||||||
@@ -682,6 +762,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
})
|
})
|
||||||
filesScanned++
|
filesScanned++
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -708,6 +789,7 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
// Unchanged file with existing ID - just need snapshot association
|
// Unchanged file with existing ID - just need snapshot association
|
||||||
unchangedFileIDs = append(unchangedFileIDs, file.ID)
|
unchangedFileIDs = append(unchangedFileIDs, file.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
filesScanned++
|
filesScanned++
|
||||||
changedCount := len(filesToProcess)
|
changedCount := len(filesToProcess)
|
||||||
mu.Unlock()
|
mu.Unlock()
|
||||||
@@ -718,12 +800,12 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult
|
|||||||
// Output periodic status
|
// Output periodic status
|
||||||
if time.Since(lastStatusTime) >= statusInterval {
|
if time.Since(lastStatusTime) >= statusInterval {
|
||||||
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
|
s.printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime)
|
||||||
|
|
||||||
lastStatusTime = time.Now()
|
lastStatusTime = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -745,11 +827,13 @@ func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool,
|
|||||||
} else {
|
} else {
|
||||||
result.FilesSkipped++
|
result.FilesSkipped++
|
||||||
result.BytesSkipped += info.Size()
|
result.BytesSkipped += info.Size()
|
||||||
|
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
s.progress.GetStats().FilesSkipped.Add(1)
|
s.progress.GetStats().FilesSkipped.Add(1)
|
||||||
s.progress.GetStats().BytesSkipped.Add(info.Size())
|
s.progress.GetStats().BytesSkipped.Add(info.Size())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result.FilesScanned++
|
result.FilesScanned++
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
s.progress.GetStats().FilesScanned.Add(1)
|
s.progress.GetStats().FilesScanned.Add(1)
|
||||||
@@ -768,14 +852,14 @@ func (s *Scanner) printScanProgressLine(filesScanned int64, changedCount int, es
|
|||||||
if pct > 100 {
|
if pct > 100 {
|
||||||
pct = 100 // Cap at 100% for display
|
pct = 100 // Cap at 100% for display
|
||||||
}
|
}
|
||||||
remaining := estimatedTotal - filesScanned
|
|
||||||
if remaining < 0 {
|
remaining := max(estimatedTotal-filesScanned, 0)
|
||||||
remaining = 0
|
|
||||||
}
|
|
||||||
var eta time.Duration
|
var eta time.Duration
|
||||||
if rate > 0 && remaining > 0 {
|
if rate > 0 && remaining > 0 {
|
||||||
eta = time.Duration(float64(remaining)/rate) * time.Second
|
eta = time.Duration(float64(remaining)/rate) * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
if eta > 0 {
|
if eta > 0 {
|
||||||
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
|
s.ui.Progress("Snapshot source files enumeration: %s files (~%s), %s changed or new, %.0f files/sec, enumeration elapsed: %s, enumeration ETA: %s (est remain %s).",
|
||||||
s.ui.Count(int(filesScanned)),
|
s.ui.Count(int(filesScanned)),
|
||||||
@@ -808,6 +892,7 @@ func (s *Scanner) buildSymlinkEntry(path string, info os.FileInfo) *database.Fil
|
|||||||
target, err := os.Readlink(path)
|
target, err := os.Readlink(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug("Cannot read symlink target", "path", path, "error", err)
|
log.Debug("Cannot read symlink target", "path", path, "error", err)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -860,9 +945,11 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
|
|||||||
// and associates it with the current snapshot. No chunking is performed.
|
// and associates it with the current snapshot. No chunking is performed.
|
||||||
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
|
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
|
||||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
if err := s.repos.Files.Create(txCtx, tx, ftp.File); err != nil {
|
err := s.repos.Files.Create(txCtx, tx, ftp.File)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("creating non-regular file record: %w", err)
|
return fmt.Errorf("creating non-regular file record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
|
return s.repos.Snapshots.AddFileByID(txCtx, tx, s.snapshotID, ftp.File.ID)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -941,18 +1028,18 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
end := i + batchSize
|
end := min(i+batchSize, len(fileIDs))
|
||||||
if end > len(fileIDs) {
|
|
||||||
end = len(fileIDs)
|
|
||||||
}
|
|
||||||
batch := fileIDs[i:end]
|
batch := fileIDs[i:end]
|
||||||
|
|
||||||
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err := s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
for _, fileID := range batch {
|
for _, fileID := range batch {
|
||||||
if err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID); err != nil {
|
err := s.repos.Snapshots.AddFileByID(ctx, tx, s.snapshotID, fileID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("adding file to snapshot: %w", err)
|
return fmt.Errorf("adding file to snapshot: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -966,6 +1053,7 @@ func (s *Scanner) batchAddFilesToSnapshot(ctx context.Context, fileIDs []types.F
|
|||||||
pct := float64(end) / float64(len(fileIDs)) * 100
|
pct := float64(end) / float64(len(fileIDs)) * 100
|
||||||
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
|
s.ui.Progress("Snapshot unchanged-file association: %s/%s (%s), %.0f files/sec.",
|
||||||
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
|
s.ui.Count(end), s.ui.Count(len(fileIDs)), s.ui.Percent(pct), rate)
|
||||||
|
|
||||||
lastStatusTime = time.Now()
|
lastStatusTime = time.Now()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -991,7 +1079,9 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
|||||||
statusInterval := 15 * time.Second
|
statusInterval := 15 * time.Second
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
filesProcessed := 0
|
filesProcessed := 0
|
||||||
|
|
||||||
var bytesProcessed int64
|
var bytesProcessed int64
|
||||||
|
|
||||||
totalFiles := len(filesToProcess)
|
totalFiles := len(filesToProcess)
|
||||||
|
|
||||||
// Process each file
|
// Process each file
|
||||||
@@ -1006,6 +1096,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if skipped {
|
if skipped {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -1021,6 +1112,7 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
|||||||
// Output periodic status
|
// Output periodic status
|
||||||
if time.Since(lastStatusTime) >= statusInterval {
|
if time.Since(lastStatusTime) >= statusInterval {
|
||||||
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
|
s.printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime)
|
||||||
|
|
||||||
lastStatusTime = time.Now()
|
lastStatusTime = time.Now()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1032,22 +1124,29 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc
|
|||||||
// processFileWithErrorHandling wraps processFileStreaming with error recovery for
|
// processFileWithErrorHandling wraps processFileStreaming with error recovery for
|
||||||
// deleted files and skip-errors mode. Returns (skipped, error).
|
// deleted files and skip-errors mode. Returns (skipped, error).
|
||||||
func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) {
|
func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) {
|
||||||
if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil {
|
err := s.processFileStreaming(ctx, fileToProcess, result)
|
||||||
|
if err != nil {
|
||||||
// Handle files that were deleted between scan and process phases
|
// Handle files that were deleted between scan and process phases
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path)
|
log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path)
|
||||||
|
|
||||||
result.FilesSkipped++
|
result.FilesSkipped++
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
// Skip file read errors if --skip-errors is enabled
|
// Skip file read errors if --skip-errors is enabled
|
||||||
if s.skipErrors {
|
if s.skipErrors {
|
||||||
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
|
log.Error("Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err)
|
||||||
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
|
s.ui.Error("Failed to process %s: %v. Skipping (--skip-errors).", s.ui.Path(fileToProcess.Path), err)
|
||||||
|
|
||||||
result.FilesSkipped++
|
result.FilesSkipped++
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1061,6 +1160,7 @@ func (s *Scanner) printProcessingProgress(filesProcessed, totalFiles int, bytesP
|
|||||||
|
|
||||||
// Calculate ETA based on bytes (more accurate than files)
|
// Calculate ETA based on bytes (more accurate than files)
|
||||||
remainingBytes := totalBytes - bytesProcessed
|
remainingBytes := totalBytes - bytesProcessed
|
||||||
|
|
||||||
var eta time.Duration
|
var eta time.Duration
|
||||||
if byteRate > 0 {
|
if byteRate > 0 {
|
||||||
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
|
eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second
|
||||||
@@ -1097,15 +1197,19 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
|||||||
// Final packer flush first - this commits remaining chunks to DB
|
// Final packer flush first - this commits remaining chunks to DB
|
||||||
// and handleBlobReady will flush files whose chunks are now committed
|
// and handleBlobReady will flush files whose chunks are now committed
|
||||||
s.packerMu.Lock()
|
s.packerMu.Lock()
|
||||||
if err := s.packer.Flush(); err != nil {
|
|
||||||
|
err := s.packer.Flush()
|
||||||
|
if err != nil {
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
return fmt.Errorf("flushing packer: %w", err)
|
return fmt.Errorf("flushing packer: %w", err)
|
||||||
}
|
}
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
// Flush any remaining pending files (e.g., files with only pre-existing chunks
|
// Flush any remaining pending files (e.g., files with only pre-existing chunks
|
||||||
// that didn't trigger a blob finalize)
|
// that didn't trigger a blob finalize)
|
||||||
if err := s.flushAllPending(ctx); err != nil {
|
err = s.flushAllPending(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("flushing remaining pending files: %w", err)
|
return fmt.Errorf("flushing remaining pending files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1119,6 +1223,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("parsing blob ID: %w", err)
|
return fmt.Errorf("parsing blob ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = s.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash))
|
return s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, blobID, types.BlobHash(b.Hash))
|
||||||
})
|
})
|
||||||
@@ -1126,6 +1231,7 @@ func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult)
|
|||||||
return fmt.Errorf("storing blob metadata: %w", err)
|
return fmt.Errorf("storing blob metadata: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result.BlobsCreated += len(blobs)
|
result.BlobsCreated += len(blobs)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1148,14 +1254,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
|
blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash)
|
||||||
|
|
||||||
blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
|
blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.cleanupBlobTempFile(blobWithReader)
|
s.cleanupBlobTempFile(blobWithReader)
|
||||||
|
|
||||||
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
|
return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil {
|
if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil {
|
||||||
s.cleanupBlobTempFile(blobWithReader)
|
s.cleanupBlobTempFile(blobWithReader)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1183,6 +1292,7 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
|
|||||||
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
|
"hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed)))
|
||||||
s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.",
|
s.ui.Info("Blob %s (%s) already exists at %s. Skipping upload.",
|
||||||
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
|
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Path(destination))
|
||||||
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1191,8 +1301,10 @@ func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobW
|
|||||||
|
|
||||||
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
|
progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
|
||||||
|
|
||||||
if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil {
|
err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback)
|
||||||
|
if err != nil {
|
||||||
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err)
|
||||||
|
|
||||||
return false, fmt.Errorf("uploading blob to storage: %w", err)
|
return false, fmt.Errorf("uploading blob to storage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1228,17 +1340,21 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
|
|||||||
lastProgressTime := time.Now()
|
lastProgressTime := time.Now()
|
||||||
lastProgressBytes := int64(0)
|
lastProgressBytes := int64(0)
|
||||||
lastStdoutTime := time.Now()
|
lastStdoutTime := time.Now()
|
||||||
|
|
||||||
const stdoutInterval = 15 * time.Second
|
const stdoutInterval = 15 * time.Second
|
||||||
|
|
||||||
return func(uploaded int64) error {
|
return func(uploaded int64) error {
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
elapsed := now.Sub(lastProgressTime).Seconds()
|
elapsed := now.Sub(lastProgressTime).Seconds()
|
||||||
if elapsed > 0.5 {
|
if elapsed > 0.5 {
|
||||||
bytesSinceLastUpdate := uploaded - lastProgressBytes
|
bytesSinceLastUpdate := uploaded - lastProgressBytes
|
||||||
|
|
||||||
speed := float64(bytesSinceLastUpdate) / elapsed
|
speed := float64(bytesSinceLastUpdate) / elapsed
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed)
|
s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed)
|
||||||
}
|
}
|
||||||
|
|
||||||
lastProgressTime = now
|
lastProgressTime = now
|
||||||
lastProgressBytes = uploaded
|
lastProgressBytes = uploaded
|
||||||
}
|
}
|
||||||
@@ -1248,10 +1364,12 @@ func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *
|
|||||||
totalElapsed := now.Sub(uploadStart)
|
totalElapsed := now.Sub(uploadStart)
|
||||||
pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100
|
pct := float64(uploaded) / float64(finishedBlob.Compressed) * 100
|
||||||
avgSpeed := float64(uploaded) / totalElapsed.Seconds()
|
avgSpeed := float64(uploaded) / totalElapsed.Seconds()
|
||||||
|
|
||||||
var eta time.Duration
|
var eta time.Duration
|
||||||
if avgSpeed > 0 {
|
if avgSpeed > 0 {
|
||||||
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
|
eta = time.Duration(float64(finishedBlob.Compressed-uploaded)/avgSpeed) * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
|
s.ui.Progress("Blob upload %s: %s / %s (%s) at %s, blob upload elapsed: %s, blob upload ETA: %s (est remain %s).",
|
||||||
s.ui.Hex(finishedBlob.Hash),
|
s.ui.Hex(finishedBlob.Hash),
|
||||||
s.ui.Size(uploaded),
|
s.ui.Size(uploaded),
|
||||||
@@ -1283,11 +1401,13 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
|||||||
uploadDuration := time.Since(startTime)
|
uploadDuration := time.Since(startTime)
|
||||||
|
|
||||||
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
|
||||||
if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil {
|
err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("updating blob upload timestamp: %w", err)
|
return fmt.Errorf("updating blob upload timestamp: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil {
|
err = s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash))
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("adding blob to snapshot: %w", err)
|
return fmt.Errorf("adding blob to snapshot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1299,7 +1419,9 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
|||||||
Size: finishedBlob.Compressed,
|
Size: finishedBlob.Compressed,
|
||||||
DurationMs: uploadDuration.Milliseconds(),
|
DurationMs: uploadDuration.Milliseconds(),
|
||||||
}
|
}
|
||||||
if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil {
|
|
||||||
|
err := s.repos.Uploads.Create(txCtx, tx, upload)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("recording upload metrics: %w", err)
|
return fmt.Errorf("recording upload metrics: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1312,10 +1434,14 @@ func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.Fin
|
|||||||
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) {
|
func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) {
|
||||||
if blobWithReader.TempFile != nil {
|
if blobWithReader.TempFile != nil {
|
||||||
tempName := blobWithReader.TempFile.Name()
|
tempName := blobWithReader.TempFile.Name()
|
||||||
if err := blobWithReader.TempFile.Close(); err != nil {
|
|
||||||
|
err := blobWithReader.TempFile.Close()
|
||||||
|
if err != nil {
|
||||||
log.Fatal("Failed to close temp file", "file", tempName, "error", err)
|
log.Fatal("Failed to close temp file", "file", tempName, "error", err)
|
||||||
}
|
}
|
||||||
if err := s.fs.Remove(tempName); err != nil {
|
|
||||||
|
err = s.fs.Remove(tempName)
|
||||||
|
if err != nil {
|
||||||
log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
|
log.Fatal("Failed to remove temp file", "file", tempName, "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1343,6 +1469,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
|||||||
defer func() { _ = file.Close() }()
|
defer func() { _ = file.Close() }()
|
||||||
|
|
||||||
var chunks []streamingChunkInfo
|
var chunks []streamingChunkInfo
|
||||||
|
|
||||||
chunkIndex := 0
|
chunkIndex := 0
|
||||||
|
|
||||||
fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error {
|
fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error {
|
||||||
@@ -1372,16 +1499,17 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
|||||||
s.updateChunkStats(chunkExists, chunk.Size, result)
|
s.updateChunkStats(chunkExists, chunk.Size, result)
|
||||||
|
|
||||||
if !chunkExists {
|
if !chunkExists {
|
||||||
if err := s.addChunkToPacker(chunk); err != nil {
|
err := s.addChunkToPacker(chunk)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
chunk.Data = nil
|
chunk.Data = nil
|
||||||
chunkIndex++
|
chunkIndex++
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("chunking file: %w", err)
|
return fmt.Errorf("chunking file: %w", err)
|
||||||
}
|
}
|
||||||
@@ -1390,6 +1518,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
|||||||
"path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks))
|
"path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks))
|
||||||
|
|
||||||
s.queueFileForBatchInsert(ctx, fileToProcess, chunks)
|
s.queueFileForBatchInsert(ctx, fileToProcess, chunks)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1397,6 +1526,7 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT
|
|||||||
func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) {
|
func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) {
|
||||||
if chunkExists {
|
if chunkExists {
|
||||||
result.FilesSkipped++
|
result.FilesSkipped++
|
||||||
|
|
||||||
result.BytesSkipped += chunkSize
|
result.BytesSkipped += chunkSize
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
s.progress.GetStats().BytesSkipped.Add(chunkSize)
|
s.progress.GetStats().BytesSkipped.Add(chunkSize)
|
||||||
@@ -1404,6 +1534,7 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
|
|||||||
} else {
|
} else {
|
||||||
result.ChunksCreated++
|
result.ChunksCreated++
|
||||||
result.BytesScanned += chunkSize
|
result.BytesScanned += chunkSize
|
||||||
|
|
||||||
if s.progress != nil {
|
if s.progress != nil {
|
||||||
s.progress.GetStats().ChunksCreated.Add(1)
|
s.progress.GetStats().ChunksCreated.Add(1)
|
||||||
s.progress.GetStats().BytesProcessed.Add(chunkSize)
|
s.progress.GetStats().BytesProcessed.Add(chunkSize)
|
||||||
@@ -1415,27 +1546,36 @@ func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *Sc
|
|||||||
// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed
|
// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed
|
||||||
func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error {
|
func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error {
|
||||||
s.packerMu.Lock()
|
s.packerMu.Lock()
|
||||||
|
|
||||||
err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||||
if err == blob.ErrBlobSizeLimitExceeded {
|
if errors.Is(err, blob.ErrBlobSizeLimitExceeded) {
|
||||||
if err := s.packer.FinalizeBlob(); err != nil {
|
err := s.packer.FinalizeBlob()
|
||||||
|
if err != nil {
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
return fmt.Errorf("finalizing blob: %w", err)
|
return fmt.Errorf("finalizing blob: %w", err)
|
||||||
}
|
}
|
||||||
if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil {
|
|
||||||
|
err = s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data})
|
||||||
|
if err != nil {
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
return fmt.Errorf("adding chunk after finalize: %w", err)
|
return fmt.Errorf("adding chunk after finalize: %w", err)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
return fmt.Errorf("adding chunk to packer: %w", err)
|
return fmt.Errorf("adding chunk to packer: %w", err)
|
||||||
}
|
}
|
||||||
s.packerMu.Unlock()
|
s.packerMu.Unlock()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert
|
// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert
|
||||||
func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) {
|
func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) {
|
||||||
fileChunks := make([]database.FileChunk, len(chunks))
|
fileChunks := make([]database.FileChunk, len(chunks))
|
||||||
|
|
||||||
chunkFiles := make([]database.ChunkFile, len(chunks))
|
chunkFiles := make([]database.ChunkFile, len(chunks))
|
||||||
for i, ci := range chunks {
|
for i, ci := range chunks {
|
||||||
fileChunks[i] = database.FileChunk{
|
fileChunks[i] = database.FileChunk{
|
||||||
@@ -1503,6 +1643,7 @@ func wrapPermissionError(path string, err error) error {
|
|||||||
if !errors.Is(err, os.ErrPermission) {
|
if !errors.Is(err, os.ErrPermission) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if runtime.GOOS == "darwin" {
|
if runtime.GOOS == "darwin" {
|
||||||
return fmt.Errorf("cannot read %s: %w\n\n"+
|
return fmt.Errorf("cannot read %s: %w\n\n"+
|
||||||
"macOS is blocking access to this path. Grant Full Disk Access to your\n"+
|
"macOS is blocking access to this path. Grant Full Disk Access to your\n"+
|
||||||
@@ -1510,12 +1651,14 @@ func wrapPermissionError(path string, err error) error {
|
|||||||
" System Settings → Privacy & Security → Full Disk Access\n\n"+
|
" System Settings → Privacy & Security → Full Disk Access\n\n"+
|
||||||
"then quit and reopen the terminal and re-run the backup", path, err)
|
"then quit and reopen the terminal and re-run the backup", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err)
|
return fmt.Errorf("cannot read %s: %w (check file permissions, or run with --skip-errors to continue past unreadable files)", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// compileExcludePatterns compiles the exclude patterns into glob matchers
|
// compileExcludePatterns compiles the exclude patterns into glob matchers
|
||||||
func compileExcludePatterns(patterns []string) []compiledPattern {
|
func compileExcludePatterns(patterns []string) []compiledPattern {
|
||||||
var compiled []compiledPattern
|
var compiled []compiledPattern
|
||||||
|
|
||||||
for _, p := range patterns {
|
for _, p := range patterns {
|
||||||
if p == "" {
|
if p == "" {
|
||||||
continue
|
continue
|
||||||
@@ -1523,6 +1666,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
|
|
||||||
// Check if pattern is anchored (starts with /)
|
// Check if pattern is anchored (starts with /)
|
||||||
anchored := strings.HasPrefix(p, "/")
|
anchored := strings.HasPrefix(p, "/")
|
||||||
|
|
||||||
pattern := p
|
pattern := p
|
||||||
if anchored {
|
if anchored {
|
||||||
pattern = p[1:] // Remove leading /
|
pattern = p[1:] // Remove leading /
|
||||||
@@ -1537,6 +1681,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
g, err := glob.Compile(pattern, '/')
|
g, err := glob.Compile(pattern, '/')
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1546,6 +1691,7 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
|
|||||||
original: p,
|
original: p,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return compiled
|
return compiled
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -33,16 +33,22 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
|||||||
|
|
||||||
// Create files with specific times
|
// Create files with specific times
|
||||||
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
testTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
for path, content := range testFiles {
|
for path, content := range testFiles {
|
||||||
dir := filepath.Dir(path)
|
dir := filepath.Dir(path)
|
||||||
if err := fs.MkdirAll(dir, 0755); err != nil {
|
|
||||||
|
err := fs.MkdirAll(dir, 0755)
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("failed to create directory %s: %v", dir, err)
|
t.Fatalf("failed to create directory %s: %v", dir, err)
|
||||||
}
|
}
|
||||||
if err := afero.WriteFile(fs, path, []byte(content), 0644); err != nil {
|
|
||||||
|
err = afero.WriteFile(fs, path, []byte(content), 0644)
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("failed to write file %s: %v", path, err)
|
t.Fatalf("failed to write file %s: %v", path, err)
|
||||||
}
|
}
|
||||||
// Set times
|
// Set times
|
||||||
if err := fs.Chtimes(path, testTime, testTime); err != nil {
|
err = fs.Chtimes(path, testTime, testTime)
|
||||||
|
if err != nil {
|
||||||
t.Fatalf("failed to set times for %s: %v", path, err)
|
t.Fatalf("failed to set times for %s: %v", path, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -53,7 +59,8 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test database: %v", err)
|
t.Fatalf("failed to create test database: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -73,6 +80,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
|||||||
// Create a snapshot record for testing
|
// Create a snapshot record for testing
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
snapshotID := "test-snapshot-001"
|
snapshotID := "test-snapshot-001"
|
||||||
|
|
||||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
snapshot := &database.Snapshot{
|
snapshot := &database.Snapshot{
|
||||||
ID: types.SnapshotID(snapshotID),
|
ID: types.SnapshotID(snapshotID),
|
||||||
@@ -87,6 +95,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
|||||||
BlobSize: 0,
|
BlobSize: 0,
|
||||||
CompressionRatio: 1.0,
|
CompressionRatio: 1.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -95,6 +104,7 @@ func TestScannerSimpleDirectory(t *testing.T) {
|
|||||||
|
|
||||||
// Scan the directory
|
// Scan the directory
|
||||||
var result *snapshot.ScanResult
|
var result *snapshot.ScanResult
|
||||||
|
|
||||||
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("scan failed: %v", err)
|
t.Fatalf("scan failed: %v", err)
|
||||||
@@ -170,7 +180,7 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
// Use random content to ensure good chunk boundaries
|
// Use random content to ensure good chunk boundaries
|
||||||
largeContent := make([]byte, 1024*1024) // 1MB
|
largeContent := make([]byte, 1024*1024) // 1MB
|
||||||
// Fill with pseudo-random data to ensure chunk boundaries
|
// Fill with pseudo-random data to ensure chunk boundaries
|
||||||
for i := 0; i < len(largeContent); i++ {
|
for i := range largeContent {
|
||||||
// Simple pseudo-random generator for deterministic tests
|
// Simple pseudo-random generator for deterministic tests
|
||||||
largeContent[i] = byte((i * 7919) ^ (i >> 3))
|
largeContent[i] = byte((i * 7919) ^ (i >> 3))
|
||||||
}
|
}
|
||||||
@@ -178,6 +188,7 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
if err := fs.MkdirAll("/source", 0755); err != nil {
|
if err := fs.MkdirAll("/source", 0755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
|
if err := afero.WriteFile(fs, "/source/large.bin", largeContent, 0644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -188,7 +199,8 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
t.Fatalf("failed to create test database: %v", err)
|
t.Fatalf("failed to create test database: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -208,6 +220,7 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
// Create a snapshot record for testing
|
// Create a snapshot record for testing
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
snapshotID := "test-snapshot-001"
|
snapshotID := "test-snapshot-001"
|
||||||
|
|
||||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
snapshot := &database.Snapshot{
|
snapshot := &database.Snapshot{
|
||||||
ID: types.SnapshotID(snapshotID),
|
ID: types.SnapshotID(snapshotID),
|
||||||
@@ -222,6 +235,7 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
BlobSize: 0,
|
BlobSize: 0,
|
||||||
CompressionRatio: 1.0,
|
CompressionRatio: 1.0,
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Snapshots.Create(ctx, tx, snapshot)
|
return repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -230,6 +244,7 @@ func TestScannerLargeFile(t *testing.T) {
|
|||||||
|
|
||||||
// Scan the directory
|
// Scan the directory
|
||||||
var result *snapshot.ScanResult
|
var result *snapshot.ScanResult
|
||||||
|
|
||||||
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
result, err = scanner.Scan(ctx, "/source", snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("scan failed: %v", err)
|
t.Fatalf("scan failed: %v", err)
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -97,12 +99,13 @@ func (sm *SnapshotManager) CreateSnapshot(ctx context.Context, hostname, version
|
|||||||
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
|
func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname, name, version, gitRevision string) (string, error) {
|
||||||
// Use short hostname (strip domain if present)
|
// Use short hostname (strip domain if present)
|
||||||
shortHostname := hostname
|
shortHostname := hostname
|
||||||
if idx := strings.Index(hostname, "."); idx != -1 {
|
if before, _, ok := strings.Cut(hostname, "."); ok {
|
||||||
shortHostname = hostname[:idx]
|
shortHostname = before
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build snapshot ID with optional name
|
// Build snapshot ID with optional name
|
||||||
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z")
|
||||||
|
|
||||||
var snapshotID string
|
var snapshotID string
|
||||||
if name != "" {
|
if name != "" {
|
||||||
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
|
snapshotID = fmt.Sprintf("%s_%s_%s", shortHostname, name, timestamp)
|
||||||
@@ -128,12 +131,12 @@ func (sm *SnapshotManager) CreateSnapshotWithName(ctx context.Context, hostname,
|
|||||||
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err := sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
return sm.repos.Snapshots.Create(ctx, tx, snapshot)
|
return sm.repos.Snapshots.Create(ctx, tx, snapshot)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("creating snapshot: %w", err)
|
return "", fmt.Errorf("creating snapshot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Created snapshot", "snapshot_id", snapshotID)
|
log.Info("Created snapshot", "snapshot_id", snapshotID)
|
||||||
|
|
||||||
return snapshotID, nil
|
return snapshotID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,7 +151,6 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
|
|||||||
stats.BytesUploaded,
|
stats.BytesUploaded,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("updating snapshot stats: %w", err)
|
return fmt.Errorf("updating snapshot stats: %w", err)
|
||||||
}
|
}
|
||||||
@@ -161,13 +163,14 @@ func (sm *SnapshotManager) UpdateSnapshotStats(ctx context.Context, snapshotID s
|
|||||||
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
|
func (sm *SnapshotManager) UpdateSnapshotStatsExtended(ctx context.Context, snapshotID string, stats ExtendedBackupStats) error {
|
||||||
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
return sm.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
// First update basic stats
|
// First update basic stats
|
||||||
if err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
|
err := sm.repos.Snapshots.UpdateCounts(ctx, tx, snapshotID,
|
||||||
int64(stats.FilesScanned),
|
int64(stats.FilesScanned),
|
||||||
int64(stats.ChunksCreated),
|
int64(stats.ChunksCreated),
|
||||||
int64(stats.BlobsCreated),
|
int64(stats.BlobsCreated),
|
||||||
stats.BytesScanned,
|
stats.BytesScanned,
|
||||||
stats.BytesUploaded,
|
stats.BytesUploaded,
|
||||||
); err != nil {
|
)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -190,18 +193,20 @@ func (sm *SnapshotManager) CompleteSnapshot(ctx context.Context, snapshotID stri
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if added > 0 {
|
if added > 0 {
|
||||||
log.Info("Populated snapshot_blobs with dedup-referenced blobs",
|
log.Info("Populated snapshot_blobs with dedup-referenced blobs",
|
||||||
"snapshot_id", snapshotID, "added", added)
|
"snapshot_id", snapshotID, "added", added)
|
||||||
}
|
}
|
||||||
|
|
||||||
return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID)
|
return sm.repos.Snapshots.MarkComplete(ctx, tx, snapshotID)
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("marking snapshot complete: %w", err)
|
return fmt.Errorf("marking snapshot complete: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Completed snapshot", "snapshot_id", snapshotID)
|
log.Info("Completed snapshot", "snapshot_id", snapshotID)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,10 +234,13 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("creating temp dir: %w", err)
|
return fmt.Errorf("creating temp dir: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Created temporary directory", "path", tempDir)
|
log.Debug("Created temporary directory", "path", tempDir)
|
||||||
defer func() {
|
defer func() {
|
||||||
log.Debug("Cleaning up temporary directory", "path", tempDir)
|
log.Debug("Cleaning up temporary directory", "path", tempDir)
|
||||||
if err := sm.fs.RemoveAll(tempDir); err != nil {
|
|
||||||
|
err := sm.fs.RemoveAll(tempDir)
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to remove temp dir", "path", tempDir, "error", err)
|
log.Debug("Failed to remove temp dir", "path", tempDir, "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -258,6 +266,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st
|
|||||||
"snapshot_id", snapshotID,
|
"snapshot_id", snapshotID,
|
||||||
"db_size", len(finalData),
|
"db_size", len(finalData),
|
||||||
"manifest_size", len(blobManifest))
|
"manifest_size", len(blobManifest))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -268,17 +277,21 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
|||||||
// The main database should be closed at this point
|
// The main database should be closed at this point
|
||||||
tempDBPath := filepath.Join(tempDir, "snapshot.db")
|
tempDBPath := filepath.Join(tempDir, "snapshot.db")
|
||||||
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
|
log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath)
|
||||||
|
|
||||||
if err := sm.copyFile(dbPath, tempDBPath); err != nil {
|
if err := sm.copyFile(dbPath, tempDBPath); err != nil {
|
||||||
return nil, "", fmt.Errorf("copying database: %w", err)
|
return nil, "", fmt.Errorf("copying database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath))
|
log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath))
|
||||||
|
|
||||||
// Step 2: Clean the temp database to only contain current snapshot data
|
// Step 2: Clean the temp database to only contain current snapshot data
|
||||||
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
|
log.Debug("Cleaning temporary database", "snapshot_id", snapshotID)
|
||||||
|
|
||||||
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
|
stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("cleaning snapshot database: %w", err)
|
return nil, "", fmt.Errorf("cleaning snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Temporary database cleanup complete",
|
log.Info("Temporary database cleanup complete",
|
||||||
"db_path", tempDBPath,
|
"db_path", tempDBPath,
|
||||||
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
"size_after_clean", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
||||||
@@ -294,6 +307,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
|||||||
if err := sm.vacuumDatabase(tempDBPath); err != nil {
|
if err := sm.vacuumDatabase(tempDBPath); err != nil {
|
||||||
return nil, "", fmt.Errorf("vacuuming database: %w", err)
|
return nil, "", fmt.Errorf("vacuuming database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
|
log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))))
|
||||||
|
|
||||||
// Step 4: Compress and encrypt the binary database file
|
// Step 4: Compress and encrypt the binary database file
|
||||||
@@ -301,6 +315,7 @@ func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshot
|
|||||||
if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
|
if err := sm.compressFile(tempDBPath, compressedPath); err != nil {
|
||||||
return nil, "", fmt.Errorf("compressing database: %w", err)
|
return nil, "", fmt.Errorf("compressing database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Compression complete",
|
log.Debug("Compression complete",
|
||||||
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
"original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))),
|
||||||
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
|
"compressed_size", humanize.Bytes(uint64(sm.getFileSize(compressedPath))))
|
||||||
@@ -327,9 +342,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
|
|||||||
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
dbKey := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||||
|
|
||||||
dbUploadStart := time.Now()
|
dbUploadStart := time.Now()
|
||||||
if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil {
|
|
||||||
|
err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData))
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("uploading snapshot database: %w", err)
|
return fmt.Errorf("uploading snapshot database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dbUploadDuration := time.Since(dbUploadStart)
|
dbUploadDuration := time.Since(dbUploadStart)
|
||||||
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
|
dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second
|
||||||
log.Info("Uploaded snapshot database",
|
log.Info("Uploaded snapshot database",
|
||||||
@@ -341,9 +359,12 @@ func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshot
|
|||||||
// Upload blob manifest (compressed only, not encrypted)
|
// Upload blob manifest (compressed only, not encrypted)
|
||||||
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
|
manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", remoteKey)
|
||||||
manifestUploadStart := time.Now()
|
manifestUploadStart := time.Now()
|
||||||
if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil {
|
|
||||||
|
err = sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData))
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("uploading blob manifest: %w", err)
|
return fmt.Errorf("uploading blob manifest: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
manifestUploadDuration := time.Since(manifestUploadStart)
|
manifestUploadDuration := time.Since(manifestUploadStart)
|
||||||
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
|
manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second
|
||||||
log.Info("Uploaded blob manifest",
|
log.Info("Uploaded blob manifest",
|
||||||
@@ -383,7 +404,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
|||||||
return nil, fmt.Errorf("opening temp database: %w", err)
|
return nil, fmt.Errorf("opening temp database: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := db.Close(); err != nil {
|
err := db.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close temp database", "error", err)
|
log.Debug("Failed to close temp database", "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -394,7 +416,8 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
|||||||
return nil, fmt.Errorf("beginning transaction: %w", err)
|
return nil, fmt.Errorf("beginning transaction: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if rbErr := tx.Rollback(); rbErr != nil && rbErr != sql.ErrTxDone {
|
rbErr := tx.Rollback()
|
||||||
|
if rbErr != nil && !errors.Is(rbErr, sql.ErrTxDone) {
|
||||||
log.Debug("Failed to rollback transaction", "error", rbErr)
|
log.Debug("Failed to rollback transaction", "error", rbErr)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -430,6 +453,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
|||||||
|
|
||||||
// Commit transaction
|
// Commit transaction
|
||||||
log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
|
log.Debug("[Temp DB Cleanup] Committing cleanup transaction")
|
||||||
|
|
||||||
if err := tx.Commit(); err != nil {
|
if err := tx.Commit(); err != nil {
|
||||||
return nil, fmt.Errorf("committing transaction: %w", err)
|
return nil, fmt.Errorf("committing transaction: %w", err)
|
||||||
}
|
}
|
||||||
@@ -439,23 +463,30 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
|||||||
|
|
||||||
// Count files
|
// Count files
|
||||||
var fileCount int
|
var fileCount int
|
||||||
|
|
||||||
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
|
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM files").Scan(&fileCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("counting files: %w", err)
|
return nil, fmt.Errorf("counting files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stats.FileCount = fileCount
|
stats.FileCount = fileCount
|
||||||
|
|
||||||
// Count chunks
|
// Count chunks
|
||||||
var chunkCount int
|
var chunkCount int
|
||||||
|
|
||||||
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount)
|
err = db.QueryRowWithLog(ctx, "SELECT COUNT(*) FROM chunks").Scan(&chunkCount)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("counting chunks: %w", err)
|
return nil, fmt.Errorf("counting chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stats.ChunkCount = chunkCount
|
stats.ChunkCount = chunkCount
|
||||||
|
|
||||||
// Count blobs and get sizes
|
// Count blobs and get sizes
|
||||||
var blobCount int
|
var (
|
||||||
var compressedSize, uncompressedSize sql.NullInt64
|
blobCount int
|
||||||
|
compressedSize, uncompressedSize sql.NullInt64
|
||||||
|
)
|
||||||
|
|
||||||
err = db.QueryRowWithLog(ctx, `
|
err = db.QueryRowWithLog(ctx, `
|
||||||
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
|
SELECT COUNT(*), COALESCE(SUM(compressed_size), 0), COALESCE(SUM(uncompressed_size), 0)
|
||||||
FROM blobs
|
FROM blobs
|
||||||
@@ -464,6 +495,7 @@ func (sm *SnapshotManager) cleanSnapshotDB(ctx context.Context, dbPath string, s
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
|
return nil, fmt.Errorf("counting blobs and sizes: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stats.BlobCount = blobCount
|
stats.BlobCount = blobCount
|
||||||
stats.CompressedSize = compressedSize.Int64
|
stats.CompressedSize = compressedSize.Int64
|
||||||
stats.UncompressedSize = uncompressedSize.Int64
|
stats.UncompressedSize = uncompressedSize.Int64
|
||||||
@@ -491,7 +523,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
return fmt.Errorf("opening input file: %w", err)
|
return fmt.Errorf("opening input file: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := input.Close(); err != nil {
|
err := input.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close input file", "path", inputPath, "error", err)
|
log.Debug("Failed to close input file", "path", inputPath, "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -501,13 +534,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
return fmt.Errorf("creating output file: %w", err)
|
return fmt.Errorf("creating output file: %w", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := output.Close(); err != nil {
|
err := output.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close output file", "path", outputPath, "error", err)
|
log.Debug("Failed to close output file", "path", outputPath, "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Use blobgen for compression and encryption
|
// Use blobgen for compression and encryption
|
||||||
log.Debug("Compressing and encrypting data")
|
log.Debug("Compressing and encrypting data")
|
||||||
|
|
||||||
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
|
writer, err := blobgen.NewWriter(output, sm.config.CompressionLevel, sm.config.AgeRecipients)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("creating blobgen writer: %w", err)
|
return fmt.Errorf("creating blobgen writer: %w", err)
|
||||||
@@ -517,7 +552,8 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
writerClosed := false
|
writerClosed := false
|
||||||
defer func() {
|
defer func() {
|
||||||
if !writerClosed {
|
if !writerClosed {
|
||||||
if err := writer.Close(); err != nil {
|
err := writer.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close writer", "error", err)
|
log.Debug("Failed to close writer", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -531,9 +567,10 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
if err := writer.Close(); err != nil {
|
if err := writer.Close(); err != nil {
|
||||||
return fmt.Errorf("closing writer: %w", err)
|
return fmt.Errorf("closing writer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
writerClosed = true
|
writerClosed = true
|
||||||
|
|
||||||
log.Debug("Compression complete", "hash", fmt.Sprintf("%x", writer.Sum256()))
|
log.Debug("Compression complete", "hash", hex.EncodeToString(writer.Sum256()))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -541,34 +578,42 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
|||||||
// copyFile copies a file from src to dst
|
// copyFile copies a file from src to dst
|
||||||
func (sm *SnapshotManager) copyFile(src, dst string) error {
|
func (sm *SnapshotManager) copyFile(src, dst string) error {
|
||||||
log.Debug("Opening source file for copy", "path", src)
|
log.Debug("Opening source file for copy", "path", src)
|
||||||
|
|
||||||
sourceFile, err := sm.fs.Open(src)
|
sourceFile, err := sm.fs.Open(src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
log.Debug("Closing source file", "path", src)
|
log.Debug("Closing source file", "path", src)
|
||||||
if err := sourceFile.Close(); err != nil {
|
|
||||||
|
err := sourceFile.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close source file", "path", src, "error", err)
|
log.Debug("Failed to close source file", "path", src, "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
log.Debug("Creating destination file", "path", dst)
|
log.Debug("Creating destination file", "path", dst)
|
||||||
|
|
||||||
destFile, err := sm.fs.Create(dst)
|
destFile, err := sm.fs.Create(dst)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
log.Debug("Closing destination file", "path", dst)
|
log.Debug("Closing destination file", "path", dst)
|
||||||
if err := destFile.Close(); err != nil {
|
|
||||||
|
err := destFile.Close()
|
||||||
|
if err != nil {
|
||||||
log.Debug("Failed to close destination file", "path", dst, "error", err)
|
log.Debug("Failed to close destination file", "path", dst, "error", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
log.Debug("Copying file data")
|
log.Debug("Copying file data")
|
||||||
|
|
||||||
n, err := io.Copy(destFile, sourceFile)
|
n, err := io.Copy(destFile, sourceFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("File copy complete", "bytes_copied", n)
|
log.Debug("File copy complete", "bytes_copied", n)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -576,7 +621,6 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
|
|||||||
|
|
||||||
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
|
// generateBlobManifest creates a compressed JSON list of all blobs in the snapshot
|
||||||
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
|
func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath string, snapshotID string) ([]byte, error) {
|
||||||
|
|
||||||
// Open the cleaned database using the database package
|
// Open the cleaned database using the database package
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -589,10 +633,12 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
|
|||||||
|
|
||||||
// Get all blobs for this snapshot
|
// Get all blobs for this snapshot
|
||||||
log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID)
|
log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID)
|
||||||
|
|
||||||
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
|
blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("getting snapshot blobs: %w", err)
|
return nil, fmt.Errorf("getting snapshot blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug("Found blobs", "count", len(blobHashes))
|
log.Debug("Found blobs", "count", len(blobHashes))
|
||||||
|
|
||||||
// Get blob details including sizes
|
// Get blob details including sizes
|
||||||
@@ -603,8 +649,10 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri
|
|||||||
blob, err := repos.Blobs.GetByHash(ctx, hash)
|
blob, err := repos.Blobs.GetByHash(ctx, hash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("Failed to get blob details", "hash", hash, "error", err)
|
log.Warn("Failed to get blob details", "hash", hash, "error", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if blob != nil {
|
if blob != nil {
|
||||||
blobs = append(blobs, BlobInfo{
|
blobs = append(blobs, BlobInfo{
|
||||||
Hash: hash,
|
Hash: hash,
|
||||||
@@ -648,6 +696,7 @@ func (sm *SnapshotManager) getFileSize(path string) int64 {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return -1
|
return -1
|
||||||
}
|
}
|
||||||
|
|
||||||
return info.Size()
|
return info.Size()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -663,6 +712,7 @@ type BackupStats struct {
|
|||||||
// ExtendedBackupStats contains additional statistics for comprehensive tracking
|
// ExtendedBackupStats contains additional statistics for comprehensive tracking
|
||||||
type ExtendedBackupStats struct {
|
type ExtendedBackupStats struct {
|
||||||
BackupStats
|
BackupStats
|
||||||
|
|
||||||
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
|
BlobUncompressedSize int64 // Total uncompressed size of all referenced blobs
|
||||||
CompressionLevel int // Compression level used for this snapshot
|
CompressionLevel int // Compression level used for this snapshot
|
||||||
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
UploadDurationMs int64 // Total milliseconds spent uploading to S3
|
||||||
@@ -682,6 +732,7 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
|||||||
|
|
||||||
if len(incompleteSnapshots) == 0 {
|
if len(incompleteSnapshots) == 0 {
|
||||||
log.Debug("No incomplete snapshots found")
|
log.Debug("No incomplete snapshots found")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -692,14 +743,15 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
|||||||
// Check if metadata exists in storage (paths use the hashed
|
// Check if metadata exists in storage (paths use the hashed
|
||||||
// remote key so we don't leak host info to the listing).
|
// remote key so we don't leak host info to the listing).
|
||||||
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
|
metadataKey := fmt.Sprintf("metadata/%s/db.zst", RemoteSnapshotKey(snapshot.ID.String()))
|
||||||
_, err := sm.storage.Stat(ctx, metadataKey)
|
|
||||||
|
|
||||||
|
_, err := sm.storage.Stat(ctx, metadataKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
// Metadata doesn't exist in S3 - this is an incomplete snapshot
|
||||||
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
|
log.Info("Cleaning up incomplete snapshot record", "snapshot_id", snapshot.ID, "started_at", snapshot.StartedAt)
|
||||||
|
|
||||||
// Delete the snapshot and all its associations
|
// Delete the snapshot and all its associations
|
||||||
if err := sm.deleteSnapshot(ctx, snapshot.ID.String()); err != nil {
|
err := sm.deleteSnapshot(ctx, snapshot.ID.String())
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
|
return fmt.Errorf("deleting incomplete snapshot %s: %w", snapshot.ID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -708,7 +760,9 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
|||||||
// Metadata exists - this snapshot was completed but database wasn't updated
|
// Metadata exists - this snapshot was completed but database wasn't updated
|
||||||
// This shouldn't happen in normal operation, but mark it complete
|
// This shouldn't happen in normal operation, but mark it complete
|
||||||
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
|
log.Warn("Found snapshot with remote metadata but incomplete in database", "snapshot_id", snapshot.ID)
|
||||||
if err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String()); err != nil {
|
|
||||||
|
err := sm.repos.Snapshots.MarkComplete(ctx, nil, snapshot.ID.String())
|
||||||
|
if err != nil {
|
||||||
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
|
log.Error("Failed to mark snapshot as complete in database", "snapshot_id", snapshot.ID, "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -720,28 +774,34 @@ func (sm *SnapshotManager) CleanupIncompleteSnapshots(ctx context.Context, hostn
|
|||||||
// deleteSnapshot removes a snapshot and all its associations from the database
|
// deleteSnapshot removes a snapshot and all its associations from the database
|
||||||
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
|
func (sm *SnapshotManager) deleteSnapshot(ctx context.Context, snapshotID string) error {
|
||||||
// Delete snapshot_files entries
|
// Delete snapshot_files entries
|
||||||
if err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID); err != nil {
|
err := sm.repos.Snapshots.DeleteSnapshotFiles(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting snapshot files: %w", err)
|
return fmt.Errorf("deleting snapshot files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete snapshot_blobs entries
|
// Delete snapshot_blobs entries
|
||||||
if err := sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID); err != nil {
|
err = sm.repos.Snapshots.DeleteSnapshotBlobs(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting snapshot blobs: %w", err)
|
return fmt.Errorf("deleting snapshot blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete uploads entries (has foreign key to snapshots without CASCADE)
|
// Delete uploads entries (has foreign key to snapshots without CASCADE)
|
||||||
if err := sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID); err != nil {
|
err = sm.repos.Snapshots.DeleteSnapshotUploads(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting snapshot uploads: %w", err)
|
return fmt.Errorf("deleting snapshot uploads: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete the snapshot itself
|
// Delete the snapshot itself
|
||||||
if err := sm.repos.Snapshots.Delete(ctx, snapshotID); err != nil {
|
err = sm.repos.Snapshots.Delete(ctx, snapshotID)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting snapshot: %w", err)
|
return fmt.Errorf("deleting snapshot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Clean up orphaned data
|
// Clean up orphaned data
|
||||||
log.Debug("Cleaning up orphaned records in main database")
|
log.Debug("Cleaning up orphaned records in main database")
|
||||||
if err := sm.CleanupOrphanedData(ctx); err != nil {
|
|
||||||
|
err = sm.CleanupOrphanedData(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("cleaning up orphaned data: %w", err)
|
return fmt.Errorf("cleaning up orphaned data: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -759,28 +819,36 @@ func (sm *SnapshotManager) CleanupOrphanedData(ctx context.Context) error {
|
|||||||
|
|
||||||
// Delete orphaned files (files not in any snapshot)
|
// Delete orphaned files (files not in any snapshot)
|
||||||
log.Debug("Deleting orphaned file records from database")
|
log.Debug("Deleting orphaned file records from database")
|
||||||
if err := sm.repos.Files.DeleteOrphaned(ctx); err != nil {
|
|
||||||
|
err := sm.repos.Files.DeleteOrphaned(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned files: %w", err)
|
return fmt.Errorf("deleting orphaned files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete orphaned blobs (blobs not in any snapshot)
|
// Delete orphaned blobs (blobs not in any snapshot)
|
||||||
// This will cascade delete blob_chunks for deleted blobs
|
// This will cascade delete blob_chunks for deleted blobs
|
||||||
log.Debug("Deleting orphaned blob records from database")
|
log.Debug("Deleting orphaned blob records from database")
|
||||||
if err := sm.repos.Blobs.DeleteOrphaned(ctx); err != nil {
|
|
||||||
|
err = sm.repos.Blobs.DeleteOrphaned(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete orphaned blob_chunks entries
|
// Delete orphaned blob_chunks entries
|
||||||
// This handles cases where the blob still exists but chunks were deleted
|
// This handles cases where the blob still exists but chunks were deleted
|
||||||
log.Debug("Deleting orphaned blob_chunks associations from database")
|
log.Debug("Deleting orphaned blob_chunks associations from database")
|
||||||
if err := sm.repos.BlobChunks.DeleteOrphaned(ctx); err != nil {
|
|
||||||
|
err = sm.repos.BlobChunks.DeleteOrphaned(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete orphaned chunks (chunks not referenced by any file)
|
// Delete orphaned chunks (chunks not referenced by any file)
|
||||||
// This must come after cleaning up blob_chunks to avoid foreign key violations
|
// This must come after cleaning up blob_chunks to avoid foreign key violations
|
||||||
log.Debug("Deleting orphaned chunk records from database")
|
log.Debug("Deleting orphaned chunk records from database")
|
||||||
if err := sm.repos.Chunks.DeleteOrphaned(ctx); err != nil {
|
|
||||||
|
err = sm.repos.Chunks.DeleteOrphaned(ctx)
|
||||||
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -793,21 +861,26 @@ func (sm *SnapshotManager) deleteOtherSnapshots(ctx context.Context, tx *sql.Tx,
|
|||||||
|
|
||||||
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
|
// First delete uploads that reference other snapshots (no CASCADE DELETE on this FK)
|
||||||
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
database.LogSQL("Execute", "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
|
|
||||||
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
uploadResult, err := tx.ExecContext(ctx, "DELETE FROM uploads WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
|
return fmt.Errorf("deleting uploads for other snapshots: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
uploadsDeleted, _ := uploadResult.RowsAffected()
|
uploadsDeleted, _ := uploadResult.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
|
log.Debug("[Temp DB Cleanup] Deleted upload records", "count", uploadsDeleted)
|
||||||
|
|
||||||
// Now we can safely delete the snapshots
|
// Now we can safely delete the snapshots
|
||||||
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
database.LogSQL("Execute", "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
result, err := tx.ExecContext(ctx, "DELETE FROM snapshots WHERE id != ?", currentSnapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting other snapshots: %w", err)
|
return fmt.Errorf("deleting other snapshots: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted snapshot records from database", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -816,22 +889,27 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
|
|||||||
// Delete orphaned snapshot_files
|
// Delete orphaned snapshot_files
|
||||||
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
|
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_files associations")
|
||||||
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
database.LogSQL("Execute", "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
result, err := tx.ExecContext(ctx, "DELETE FROM snapshot_files WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
|
return fmt.Errorf("deleting orphaned snapshot_files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted snapshot_files associations", "count", rowsAffected)
|
||||||
|
|
||||||
// Delete orphaned snapshot_blobs
|
// Delete orphaned snapshot_blobs
|
||||||
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
|
log.Debug("[Temp DB Cleanup] Deleting orphaned snapshot_blobs associations")
|
||||||
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
database.LogSQL("Execute", "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
|
|
||||||
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
result, err = tx.ExecContext(ctx, "DELETE FROM snapshot_blobs WHERE snapshot_id != ?", currentSnapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
|
return fmt.Errorf("deleting orphaned snapshot_blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ = result.RowsAffected()
|
rowsAffected, _ = result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -839,6 +917,7 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(ctx context.Contex
|
|||||||
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
||||||
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
|
log.Debug("[Temp DB Cleanup] Deleting file records not referenced by current snapshot")
|
||||||
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
|
database.LogSQL("Execute", `DELETE FROM files WHERE NOT EXISTS (SELECT 1 FROM snapshot_files WHERE snapshot_files.file_id = files.id AND snapshot_files.snapshot_id = ?)`, currentSnapshotID)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, `
|
result, err := tx.ExecContext(ctx, `
|
||||||
DELETE FROM files
|
DELETE FROM files
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
@@ -849,11 +928,13 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned files: %w", err)
|
return fmt.Errorf("deleting orphaned files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted file records from database", "count", rowsAffected)
|
||||||
|
|
||||||
// Note: file_chunks will be deleted via CASCADE
|
// Note: file_chunks will be deleted via CASCADE
|
||||||
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
|
log.Debug("[Temp DB Cleanup] file_chunks associations deleted via CASCADE")
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -861,6 +942,7 @@ func (sm *SnapshotManager) deleteOrphanedFiles(ctx context.Context, tx *sql.Tx,
|
|||||||
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
|
func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context, tx *sql.Tx) error {
|
||||||
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
|
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk_files associations")
|
||||||
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
|
database.LogSQL("Execute", `DELETE FROM chunk_files WHERE NOT EXISTS (SELECT 1 FROM files WHERE files.id = chunk_files.file_id)`)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, `
|
result, err := tx.ExecContext(ctx, `
|
||||||
DELETE FROM chunk_files
|
DELETE FROM chunk_files
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
@@ -870,8 +952,10 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
|
return fmt.Errorf("deleting orphaned chunk_files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted chunk_files associations", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -879,6 +963,7 @@ func (sm *SnapshotManager) deleteOrphanedChunkToFileMappings(ctx context.Context
|
|||||||
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx, currentSnapshotID string) error {
|
||||||
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
|
log.Debug("[Temp DB Cleanup] Deleting blob records not referenced by current snapshot")
|
||||||
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
|
database.LogSQL("Execute", `DELETE FROM blobs WHERE NOT EXISTS (SELECT 1 FROM snapshot_blobs WHERE snapshot_blobs.blob_hash = blobs.blob_hash AND snapshot_blobs.snapshot_id = ?)`, currentSnapshotID)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, `
|
result, err := tx.ExecContext(ctx, `
|
||||||
DELETE FROM blobs
|
DELETE FROM blobs
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
@@ -889,8 +974,10 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
return fmt.Errorf("deleting orphaned blobs: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted blob records from database", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -898,6 +985,7 @@ func (sm *SnapshotManager) deleteOrphanedBlobs(ctx context.Context, tx *sql.Tx,
|
|||||||
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
|
func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context, tx *sql.Tx) error {
|
||||||
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
|
log.Debug("[Temp DB Cleanup] Deleting orphaned blob_chunks associations")
|
||||||
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
|
database.LogSQL("Execute", `DELETE FROM blob_chunks WHERE NOT EXISTS (SELECT 1 FROM blobs WHERE blobs.id = blob_chunks.blob_id)`)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, `
|
result, err := tx.ExecContext(ctx, `
|
||||||
DELETE FROM blob_chunks
|
DELETE FROM blob_chunks
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
@@ -907,14 +995,17 @@ func (sm *SnapshotManager) deleteOrphanedBlobToChunkMappings(ctx context.Context
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
return fmt.Errorf("deleting orphaned blob_chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted blob_chunks associations", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// deleteOrphanedChunks deletes chunks not referenced by any file or blob
|
// deleteOrphanedChunks deletes chunks not referenced by any file or blob
|
||||||
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
|
func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx) error {
|
||||||
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
|
log.Debug("[Temp DB Cleanup] Deleting orphaned chunk records")
|
||||||
|
|
||||||
query := `
|
query := `
|
||||||
DELETE FROM chunks
|
DELETE FROM chunks
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
@@ -926,11 +1017,14 @@ func (sm *SnapshotManager) deleteOrphanedChunks(ctx context.Context, tx *sql.Tx)
|
|||||||
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
|
WHERE blob_chunks.chunk_hash = chunks.chunk_hash
|
||||||
)`
|
)`
|
||||||
database.LogSQL("Execute", query)
|
database.LogSQL("Execute", query)
|
||||||
|
|
||||||
result, err := tx.ExecContext(ctx, query)
|
result, err := tx.ExecContext(ctx, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
return fmt.Errorf("deleting orphaned chunks: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
rowsAffected, _ := result.RowsAffected()
|
rowsAffected, _ := result.RowsAffected()
|
||||||
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
|
log.Debug("[Temp DB Cleanup] Deleted chunk records from database", "count", rowsAffected)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ func copyFile(fs afero.Fs, src, dst string) error {
|
|||||||
defer func() { _ = destFile.Close() }()
|
defer func() { _ = destFile.Close() }()
|
||||||
|
|
||||||
_, err = io.Copy(destFile, sourceFile)
|
_, err = io.Copy(destFile, sourceFile)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,6 +47,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
// Create a test database
|
// Create a test database
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
dbPath := filepath.Join(tempDir, "test.db")
|
dbPath := filepath.Join(tempDir, "test.db")
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create database: %v", err)
|
t.Fatalf("failed to create database: %v", err)
|
||||||
@@ -71,9 +73,11 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
|
chunk := &database.Chunk{ChunkHash: "orphan-chunk", Size: 500}
|
||||||
|
|
||||||
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error {
|
||||||
if err := repos.Files.Create(ctx, tx, file); err != nil {
|
err := repos.Files.Create(ctx, tx, file)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return repos.Chunks.Create(ctx, tx, chunk)
|
return repos.Chunks.Create(ctx, tx, chunk)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -111,7 +115,8 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
t.Fatalf("failed to open cleaned database: %v", err)
|
t.Fatalf("failed to open cleaned database: %v", err)
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := cleanedDB.Close(); err != nil {
|
err := cleanedDB.Close()
|
||||||
|
if err != nil {
|
||||||
t.Errorf("failed to close database: %v", err)
|
t.Errorf("failed to close database: %v", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
@@ -123,6 +128,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to get snapshot: %v", err)
|
t.Fatalf("failed to get snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if verifySnapshot == nil {
|
if verifySnapshot == nil {
|
||||||
t.Error("snapshot should exist")
|
t.Error("snapshot should exist")
|
||||||
}
|
}
|
||||||
@@ -132,6 +138,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to check file: %v", err)
|
t.Fatalf("failed to check file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if f != nil {
|
if f != nil {
|
||||||
t.Error("orphan file should not exist")
|
t.Error("orphan file should not exist")
|
||||||
}
|
}
|
||||||
@@ -141,6 +148,7 @@ func TestCleanSnapshotDBEmptySnapshot(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to check chunk: %v", err)
|
t.Fatalf("failed to check chunk: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if c != nil {
|
if c != nil {
|
||||||
t.Error("orphan chunk should not exist")
|
t.Error("orphan chunk should not exist")
|
||||||
}
|
}
|
||||||
@@ -156,6 +164,7 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
|
|||||||
// Create a test database
|
// Create a test database
|
||||||
tempDir := t.TempDir()
|
tempDir := t.TempDir()
|
||||||
dbPath := filepath.Join(tempDir, "test.db")
|
dbPath := filepath.Join(tempDir, "test.db")
|
||||||
|
|
||||||
db, err := database.New(ctx, dbPath)
|
db, err := database.New(ctx, dbPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to create database: %v", err)
|
t.Fatalf("failed to create database: %v", err)
|
||||||
|
|||||||
Reference in New Issue
Block a user