remove parallel processing to fix SQLite locking issues

- Remove parallel.go entirely - Process days strictly sequentially to avoid SQLITE_BUSY errors - Complete each day fully (extract, dump, compress, verify, write) before starting next - Clean up temp files immediately after each day to save disk space - SQLite cannot handle multiple concurrent ATTACH operations on same database
2026-02-12 13:26:55 -08:00
parent 56c3fe7804
commit 8e3a868b7a
2 changed files with 63 additions and 222 deletions
@@ -76,88 +76,77 @@ func Run(targetDates []time.Time) error {
 		}
 	}

-	// Process days - use parallel processing for multiple days
-	var processed, skipped int
-	var processErr error
+	// Process each day completely before moving to the next
+	// This ensures we don't have multiple SQLite operations competing for the same source database
+	processed := 0
+	skipped := 0

-	if len(targetDates) > 1 {
-		// Prepare jobs for parallel processing
-		jobs := make([]extractionJob, len(targetDates))
-		for i, targetDay := range targetDates {
-			jobs[i] = extractionJob{
-				targetDay: targetDay,
-				srcDB:     dstDB,
-				tmpDir:    tmpDir,
-			}
+	for _, targetDay := range targetDates {
+		dayStr := targetDay.Format("2006-01-02")
+		slog.Info("processing day", "date", dayStr)
+
+		// Check if output already exists
+		outputDir := filepath.Join(DailiesBase, targetDay.Format("2006-01"))
+		outputFinal := filepath.Join(outputDir, dayStr+".sql.zst")
+		if _, err := os.Stat(outputFinal); err == nil {
+			slog.Info("output already exists, skipping", "path", outputFinal)
+			skipped++
+			continue
 		}
-		processed, skipped, processErr = processParallel(jobs)
-	} else {
-		// Single day - process sequentially (simpler error handling)
-		for _, targetDay := range targetDates {
-			dayStr := targetDay.Format("2006-01-02")
-			slog.Info("processing day", "date", dayStr)

-			// Check if output already exists
-			outputDir := filepath.Join(DailiesBase, targetDay.Format("2006-01"))
-			outputFinal := filepath.Join(outputDir, dayStr+".sql.zst")
-			if _, err := os.Stat(outputFinal); err == nil {
-				slog.Info("output already exists, skipping", "path", outputFinal)
+		// Extract target day into a per-day database
+		extractedDB := filepath.Join(tmpDir, "extracted-"+dayStr+".db")
+		slog.Info("extracting target day", "src", dstDB, "dst", extractedDB)
+		if err := ExtractDay(dstDB, extractedDB, targetDay); err != nil {
+			if errors.Is(err, ErrNoPosts) {
+				slog.Warn("no posts found, skipping day", "date", dayStr)
+				os.Remove(extractedDB)
 				skipped++
 				continue
 			}
-
-			// Extract target day into a per-day database
-			extractedDB := filepath.Join(tmpDir, "extracted-"+dayStr+".db")
-			slog.Info("extracting target day", "src", dstDB, "dst", extractedDB)
-			if err := ExtractDay(dstDB, extractedDB, targetDay); err != nil {
-				if errors.Is(err, ErrNoPosts) {
-					slog.Warn("no posts found, skipping day", "date", dayStr)
-					os.Remove(extractedDB)
-					skipped++
-					continue
-				}
-				return fmt.Errorf("extracting day %s: %w", dayStr, err)
-			}
-
-			// Dump to SQL and compress
-			if err := os.MkdirAll(outputDir, 0755); err != nil {
-				return fmt.Errorf("creating output directory %s: %w", outputDir, err)
-			}
-
-			outputTmp := filepath.Join(outputDir, "."+dayStr+".sql.zst.tmp")
-
-			slog.Info("dumping and compressing", "tmp_output", outputTmp)
-			if err := DumpAndCompress(extractedDB, outputTmp); err != nil {
-				os.Remove(outputTmp)
-				return fmt.Errorf("dump and compress for %s: %w", dayStr, err)
-			}
-
-			slog.Info("verifying compressed output")
-			if err := VerifyOutput(outputTmp); err != nil {
-				os.Remove(outputTmp)
-				return fmt.Errorf("verification failed for %s: %w", dayStr, err)
-			}
-
-			// Atomic rename to final path
-			slog.Info("renaming to final output", "from", outputTmp, "to", outputFinal)
-			if err := os.Rename(outputTmp, outputFinal); err != nil {
-				return fmt.Errorf("atomic rename for %s: %w", dayStr, err)
-			}
-
-			info, err := os.Stat(outputFinal)
-			if err != nil {
-				return fmt.Errorf("stat final output: %w", err)
-			}
-			slog.Info("day completed", "date", dayStr, "path", outputFinal, "size_bytes", info.Size())
-
-			// Remove extracted DB to reclaim space
-			os.Remove(extractedDB)
-			processed++
+			return fmt.Errorf("extracting day %s: %w", dayStr, err)
 		}
-	}

-	if processErr != nil {
-		return processErr
+		// Dump to SQL and compress
+		if err := os.MkdirAll(outputDir, 0755); err != nil {
+			os.Remove(extractedDB)
+			return fmt.Errorf("creating output directory %s: %w", outputDir, err)
+		}
+
+		outputTmp := filepath.Join(outputDir, "."+dayStr+".sql.zst.tmp")
+
+		slog.Info("dumping and compressing", "tmp_output", outputTmp)
+		if err := DumpAndCompress(extractedDB, outputTmp); err != nil {
+			os.Remove(outputTmp)
+			os.Remove(extractedDB)
+			return fmt.Errorf("dump and compress for %s: %w", dayStr, err)
+		}
+
+		slog.Info("verifying compressed output")
+		if err := VerifyOutput(outputTmp); err != nil {
+			os.Remove(outputTmp)
+			os.Remove(extractedDB)
+			return fmt.Errorf("verification failed for %s: %w", dayStr, err)
+		}
+
+		// Atomic rename to final path
+		slog.Info("renaming to final output", "from", outputTmp, "to", outputFinal)
+		if err := os.Rename(outputTmp, outputFinal); err != nil {
+			os.Remove(outputTmp)
+			os.Remove(extractedDB)
+			return fmt.Errorf("atomic rename for %s: %w", dayStr, err)
+		}
+
+		info, err := os.Stat(outputFinal)
+		if err != nil {
+			os.Remove(extractedDB)
+			return fmt.Errorf("stat final output: %w", err)
+		}
+		slog.Info("day completed", "date", dayStr, "path", outputFinal, "size_bytes", info.Size())
+
+		// Remove extracted DB to reclaim space immediately
+		os.Remove(extractedDB)
+		processed++
 	}

 	slog.Info("run summary", "processed", processed, "skipped", skipped, "total", len(targetDates))