latest

merge_daily_dumps.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Script to merge multiple daily Bluesky archive dumps into a single database
+# Usage: ./merge_daily_dumps.sh output.db daily_dumps/*.sql.zst
+
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 output.db input1.sql.zst [input2.sql.zst ...]"
+    exit 1
+fi
+
+OUTPUT_DB="$1"
+shift
+
+if [ -f "$OUTPUT_DB" ]; then
+    echo "Error: Output database $OUTPUT_DB already exists"
+    exit 1
+fi
+
+echo "Creating merged database: $OUTPUT_DB"
+
+# Create the database with schema from the first dump
+FIRST_DUMP="$1"
+echo "Extracting schema from $FIRST_DUMP..."
+zstdcat "$FIRST_DUMP" | grep -E "^(CREATE|PRAGMA)" | sqlite3 "$OUTPUT_DB"
+
+# Import all dumps, converting INSERT INTO to INSERT OR IGNORE
+# This handles duplicates in hashtags, urls, and users tables
+for DUMP in "$@"; do
+    echo "Importing $DUMP..."
+    zstdcat "$DUMP" | \
+        grep -v "^CREATE" | \
+        grep -v "^PRAGMA" | \
+        sed 's/^INSERT INTO/INSERT OR IGNORE INTO/g' | \
+        sqlite3 "$OUTPUT_DB"
+done
+
+echo "Rebuilding aggregate statistics..."
+sqlite3 "$OUTPUT_DB" <<'EOF'
+-- Update use_count for hashtags based on actual usage
+UPDATE hashtags SET use_count = (
+    SELECT COUNT(*) FROM posts_hashtags WHERE hashtag_id = hashtags.id
+);
+
+-- Update use_count for urls based on actual usage
+UPDATE urls SET use_count = (
+    SELECT COUNT(*) FROM posts_urls WHERE url_id = urls.id
+);
+
+-- Update user timestamps to reflect actual data range
+UPDATE users SET
+    resolved_at = (
+        SELECT MIN(timestamp) FROM posts WHERE user_did = users.did
+    ),
+    updated_at = (
+        SELECT MAX(timestamp) FROM posts WHERE user_did = users.did
+    );
+
+-- Analyze database for query optimization
+ANALYZE;
+
+-- Show summary statistics
+SELECT 'Database merge complete!' as status;
+SELECT '---' as separator;
+SELECT 'Total posts: ' || COUNT(*) FROM posts;
+SELECT 'Total users: ' || COUNT(*) FROM users;
+SELECT 'Total hashtags: ' || COUNT(*) FROM hashtags;
+SELECT 'Total URLs: ' || COUNT(*) FROM urls;
+SELECT 'Date range: ' || MIN(timestamp) || ' to ' || MAX(timestamp) FROM posts;
+EOF
+
+echo "Merge complete! Database saved to: $OUTPUT_DB"