optimize file copy performance for large database files

- Increase copy buffer from 4MB to 256MB for NVMe-speed transfers
- Add Linux-specific fadvise hints for sequential reads and prefetching
- Pre-allocate destination file to avoid fragmentation
- Add throughput monitoring to track copy performance
- Platform-specific build tags for Linux optimizations

internal/bsdaily/copy.go

@@ -5,11 +5,16 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"time"
 )
 
-const copyBufferSize = 4 * 1024 * 1024 // 4MB buffer for large file copies
+const (
+	copyBufferSize = 256 * 1024 * 1024 // 256MB buffer for large file copies from fast storage
+	oneGB          = 1024 * 1024 * 1024
+)
 
 func CopyFile(src, dst string) (err error) {
+	startTime := time.Now()
 	slog.Info("copying file", "src", src, "dst", dst)
 
 	srcFile, err := os.Open(src)
@@ -23,6 +28,11 @@ func CopyFile(src, dst string) (err error) {
 		return fmt.Errorf("stat source %s: %w", src, err)
 	}
 
+	// For large files, advise kernel about sequential read pattern
+	if srcInfo.Size() > oneGB {
+		applyFileAdvice(srcFile, srcInfo.Size())
+	}
+
 	dstFile, err := os.Create(dst)
 	if err != nil {
 		return fmt.Errorf("creating destination %s: %w", dst, err)
@@ -33,7 +43,12 @@ func CopyFile(src, dst string) (err error) {
 		}
 	}()
 
-	// Use a larger buffer for better performance with large database files
+	// Pre-allocate space for the destination file to avoid fragmentation
+	if err := dstFile.Truncate(srcInfo.Size()); err != nil {
+		slog.Warn("failed to pre-allocate destination file", "error", err)
+	}
+
+	// Use a much larger buffer for NVMe-speed copies
 	buf := make([]byte, copyBufferSize)
 	written, err := io.CopyBuffer(dstFile, srcFile, buf)
 	if err != nil {
@@ -48,6 +63,10 @@ func CopyFile(src, dst string) (err error) {
 		return fmt.Errorf("syncing destination %s: %w", dst, err)
 	}
 
-	slog.Info("file copied", "dst", dst, "bytes", written)
+	elapsed := time.Since(startTime)
+	throughputMBps := float64(written) / elapsed.Seconds() / (1024 * 1024)
+	slog.Info("file copied", "dst", dst, "bytes", written,
+		"elapsed", elapsed.Round(time.Millisecond),
+		"throughput_mbps", fmt.Sprintf("%.1f", throughputMBps))
 	return nil
 }

internal/bsdaily/copy_linux.go

@@ -0,0 +1,16 @@
+//go:build linux
+
+package bsdaily
+
+import (
+	"os"
+	"syscall"
+)
+
+func applyFileAdvice(file *os.File, size int64) {
+	fd := int(file.Fd())
+	// POSIX_FADV_SEQUENTIAL = 2
+	_ = syscall.Fadvise(fd, 0, size, 2)
+	// POSIX_FADV_WILLNEED = 3 - prefetch file into cache
+	_ = syscall.Fadvise(fd, 0, size, 3)
+}

internal/bsdaily/copy_other.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package bsdaily
+
+import "os"
+
+func applyFileAdvice(file *os.File, size int64) {
+	// Fadvise not available on this platform
+}