speed optimizations

2026-02-12 13:01:52 -08:00
parent 2d46dc70a9
commit cf79e008b5
8 changed files with 301 additions and 79 deletions
@@ -76,70 +76,88 @@ func Run(targetDates []time.Time) error {
 		}
 	}

-	// Process each day
-	processed := 0
-	skipped := 0
+	// Process days - use parallel processing for multiple days
+	var processed, skipped int
+	var processErr error

-	for _, targetDay := range targetDates {
-		dayStr := targetDay.Format("2006-01-02")
-		slog.Info("processing day", "date", dayStr)
-
-		// Check if output already exists
-		outputDir := filepath.Join(DailiesBase, targetDay.Format("2006-01"))
-		outputFinal := filepath.Join(outputDir, dayStr+".sql.zst")
-		if _, err := os.Stat(outputFinal); err == nil {
-			slog.Info("output already exists, skipping", "path", outputFinal)
-			skipped++
-			continue
+	if len(targetDates) > 1 {
+		// Prepare jobs for parallel processing
+		jobs := make([]extractionJob, len(targetDates))
+		for i, targetDay := range targetDates {
+			jobs[i] = extractionJob{
+				targetDay: targetDay,
+				srcDB:     dstDB,
+				tmpDir:    tmpDir,
+			}
 		}
+		processed, skipped, processErr = processParallel(jobs)
+	} else {
+		// Single day - process sequentially (simpler error handling)
+		for _, targetDay := range targetDates {
+			dayStr := targetDay.Format("2006-01-02")
+			slog.Info("processing day", "date", dayStr)

-		// Extract target day into a per-day database
-		extractedDB := filepath.Join(tmpDir, "extracted-"+dayStr+".db")
-		slog.Info("extracting target day", "src", dstDB, "dst", extractedDB)
-		if err := ExtractDay(dstDB, extractedDB, targetDay); err != nil {
-			if errors.Is(err, ErrNoPosts) {
-				slog.Warn("no posts found, skipping day", "date", dayStr)
-				os.Remove(extractedDB)
+			// Check if output already exists
+			outputDir := filepath.Join(DailiesBase, targetDay.Format("2006-01"))
+			outputFinal := filepath.Join(outputDir, dayStr+".sql.zst")
+			if _, err := os.Stat(outputFinal); err == nil {
+				slog.Info("output already exists, skipping", "path", outputFinal)
 				skipped++
 				continue
 			}
-			return fmt.Errorf("extracting day %s: %w", dayStr, err)
+
+			// Extract target day into a per-day database
+			extractedDB := filepath.Join(tmpDir, "extracted-"+dayStr+".db")
+			slog.Info("extracting target day", "src", dstDB, "dst", extractedDB)
+			if err := ExtractDay(dstDB, extractedDB, targetDay); err != nil {
+				if errors.Is(err, ErrNoPosts) {
+					slog.Warn("no posts found, skipping day", "date", dayStr)
+					os.Remove(extractedDB)
+					skipped++
+					continue
+				}
+				return fmt.Errorf("extracting day %s: %w", dayStr, err)
+			}
+
+			// Dump to SQL and compress
+			if err := os.MkdirAll(outputDir, 0755); err != nil {
+				return fmt.Errorf("creating output directory %s: %w", outputDir, err)
+			}
+
+			outputTmp := filepath.Join(outputDir, "."+dayStr+".sql.zst.tmp")
+
+			slog.Info("dumping and compressing", "tmp_output", outputTmp)
+			if err := DumpAndCompress(extractedDB, outputTmp); err != nil {
+				os.Remove(outputTmp)
+				return fmt.Errorf("dump and compress for %s: %w", dayStr, err)
+			}
+
+			slog.Info("verifying compressed output")
+			if err := VerifyOutput(outputTmp); err != nil {
+				os.Remove(outputTmp)
+				return fmt.Errorf("verification failed for %s: %w", dayStr, err)
+			}
+
+			// Atomic rename to final path
+			slog.Info("renaming to final output", "from", outputTmp, "to", outputFinal)
+			if err := os.Rename(outputTmp, outputFinal); err != nil {
+				return fmt.Errorf("atomic rename for %s: %w", dayStr, err)
+			}
+
+			info, err := os.Stat(outputFinal)
+			if err != nil {
+				return fmt.Errorf("stat final output: %w", err)
+			}
+			slog.Info("day completed", "date", dayStr, "path", outputFinal, "size_bytes", info.Size())
+
+			// Remove extracted DB to reclaim space
+			os.Remove(extractedDB)
+			processed++
 		}
+	}

-		// Dump to SQL and compress
-		if err := os.MkdirAll(outputDir, 0755); err != nil {
-			return fmt.Errorf("creating output directory %s: %w", outputDir, err)
-		}
-
-		outputTmp := filepath.Join(outputDir, "."+dayStr+".sql.zst.tmp")
-
-		slog.Info("dumping and compressing", "tmp_output", outputTmp)
-		if err := DumpAndCompress(extractedDB, outputTmp); err != nil {
-			os.Remove(outputTmp)
-			return fmt.Errorf("dump and compress for %s: %w", dayStr, err)
-		}
-
-		slog.Info("verifying compressed output")
-		if err := VerifyOutput(outputTmp); err != nil {
-			os.Remove(outputTmp)
-			return fmt.Errorf("verification failed for %s: %w", dayStr, err)
-		}
-
-		// Atomic rename to final path
-		slog.Info("renaming to final output", "from", outputTmp, "to", outputFinal)
-		if err := os.Rename(outputTmp, outputFinal); err != nil {
-			return fmt.Errorf("atomic rename for %s: %w", dayStr, err)
-		}
-
-		info, err := os.Stat(outputFinal)
-		if err != nil {
-			return fmt.Errorf("stat final output: %w", err)
-		}
-		slog.Info("day completed", "date", dayStr, "path", outputFinal, "size_bytes", info.Size())
-
-		// Remove extracted DB to reclaim space
-		os.Remove(extractedDB)
-		processed++
+	if processErr != nil {
+		return processErr
 	}

 	slog.Info("run summary", "processed", processed, "skipped", skipped, "total", len(targetDates))