clawbot
c16f177575
check / check (push) Waiting to run
Vendor .golangci.yml byte-identical from sneak/prompts at cc440118 and move the Dockerfile lint phase to golangci-lint v2.14.0 by the digest REPO_POLICIES.md names. Fix the code to that config with flags, help text, output files, SQL and the order of steps unchanged; long functions are split into named steps. Judgement call: the extraction transaction is now rolled back on every early return; the old deferred rollback missed most failures and could dereference a nil transaction. Thirteen //nolint directives (gosec, unconvert, mnd, unqueryvet), each with its reason. Model: opus-5-5
123 lines
2.9 KiB
Go
123 lines
2.9 KiB
Go
package bsdaily
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
)
|
|
|
|
var errEmptyOutput = errors.New("compressed output is empty")
|
|
|
|
// DumpAndCompress writes a `sqlite3 .dump` of the database at dbPath,
|
|
// compressed by zstdmt, to outputPath.
|
|
func DumpAndCompress(dbPath, outputPath string) (err error) {
|
|
for _, tool := range []string{"sqlite3", "zstdmt"} {
|
|
_, err = exec.LookPath(tool)
|
|
if err != nil {
|
|
return fmt.Errorf("required tool %q not found in PATH: %w", tool, err)
|
|
}
|
|
}
|
|
|
|
err = CheckFreeSpace(filepath.Dir(outputPath), MinDailiesFreeBytes,
|
|
"dailiesBase (pre-dump)")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
//nolint:gosec // outputPath is a path this package built
|
|
outFile, err := os.Create(outputPath)
|
|
if err != nil {
|
|
return fmt.Errorf("creating output file: %w", err)
|
|
}
|
|
|
|
defer func() {
|
|
cerr := outFile.Close()
|
|
if cerr != nil && err == nil {
|
|
err = fmt.Errorf("closing output: %w", cerr)
|
|
}
|
|
}()
|
|
|
|
err = runDumpPipeline(context.Background(), dbPath, outFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = outFile.Sync()
|
|
if err != nil {
|
|
return fmt.Errorf("syncing output: %w", err)
|
|
}
|
|
|
|
info, err := os.Stat(outputPath)
|
|
if err != nil {
|
|
return fmt.Errorf("stat output: %w", err)
|
|
}
|
|
|
|
const bytesPerMB = 1024 * 1024
|
|
|
|
slog.Info("compressed output written", "path", outputPath,
|
|
"size_bytes", info.Size(), "size_mb", info.Size()/bytesPerMB)
|
|
|
|
if info.Size() == 0 {
|
|
return errEmptyOutput
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// runDumpPipeline runs `sqlite3 dbPath .dump | zstdmt` with the
|
|
// compressed stream going to outFile, and waits for both to finish.
|
|
func runDumpPipeline(ctx context.Context, dbPath string, outFile *os.File) error {
|
|
// The dump holds plain INSERT INTO statements. merge_daily_dumps.sh
|
|
// rewrites them to INSERT OR IGNORE INTO so that several dumps can be
|
|
// merged into one database.
|
|
//nolint:gosec // dbPath is a scratch file this package created
|
|
dumpCmd := exec.CommandContext(ctx, "sqlite3", dbPath, ".dump")
|
|
//nolint:gosec // the argument is built from a constant
|
|
zstdCmd := exec.CommandContext(ctx, "zstdmt",
|
|
fmt.Sprintf("-%d", zstdCompressionLevel))
|
|
|
|
pipe, err := dumpCmd.StdoutPipe()
|
|
if err != nil {
|
|
return fmt.Errorf("creating dump stdout pipe: %w", err)
|
|
}
|
|
|
|
zstdCmd.Stdin = pipe
|
|
zstdCmd.Stdout = outFile
|
|
|
|
var dumpStderr, zstdStderr strings.Builder
|
|
|
|
dumpCmd.Stderr = &dumpStderr
|
|
zstdCmd.Stderr = &zstdStderr
|
|
|
|
slog.Info("starting sqlite3 dump and zstdmt compression")
|
|
|
|
err = zstdCmd.Start()
|
|
if err != nil {
|
|
return fmt.Errorf("starting zstdmt: %w", err)
|
|
}
|
|
|
|
err = dumpCmd.Start()
|
|
if err != nil {
|
|
return fmt.Errorf("starting sqlite3 dump: %w", err)
|
|
}
|
|
|
|
err = dumpCmd.Wait()
|
|
if err != nil {
|
|
return fmt.Errorf("sqlite3 dump failed: %w; stderr: %s",
|
|
err, dumpStderr.String())
|
|
}
|
|
|
|
err = zstdCmd.Wait()
|
|
if err != nil {
|
|
return fmt.Errorf("zstdmt failed: %w; stderr: %s",
|
|
err, zstdStderr.String())
|
|
}
|
|
|
|
return nil
|
|
}
|