72 lines
2.1 KiB
Bash
72 lines
2.1 KiB
Bash
#!/bin/bash
|
|
# Script to merge multiple daily Bluesky archive dumps into a single database
|
|
# Usage: ./merge_daily_dumps.sh output.db daily_dumps/*.sql.zst
|
|
|
|
set -e
|
|
|
|
if [ $# -lt 2 ]; then
|
|
echo "Usage: $0 output.db input1.sql.zst [input2.sql.zst ...]"
|
|
exit 1
|
|
fi
|
|
|
|
OUTPUT_DB="$1"
|
|
shift
|
|
|
|
if [ -f "$OUTPUT_DB" ]; then
|
|
echo "Error: Output database $OUTPUT_DB already exists"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Creating merged database: $OUTPUT_DB"
|
|
|
|
# Create the database with schema from the first dump
|
|
FIRST_DUMP="$1"
|
|
echo "Extracting schema from $FIRST_DUMP..."
|
|
zstdcat "$FIRST_DUMP" | grep -E "^(CREATE|PRAGMA)" | sqlite3 "$OUTPUT_DB"
|
|
|
|
# Import all dumps, converting INSERT INTO to INSERT OR IGNORE
|
|
# This handles duplicates in hashtags, urls, and users tables
|
|
for DUMP in "$@"; do
|
|
echo "Importing $DUMP..."
|
|
zstdcat "$DUMP" | \
|
|
grep -v "^CREATE" | \
|
|
grep -v "^PRAGMA" | \
|
|
sed 's/^INSERT INTO/INSERT OR IGNORE INTO/g' | \
|
|
sqlite3 "$OUTPUT_DB"
|
|
done
|
|
|
|
echo "Rebuilding aggregate statistics..."
|
|
sqlite3 "$OUTPUT_DB" <<'EOF'
|
|
-- Update use_count for hashtags based on actual usage
|
|
UPDATE hashtags SET use_count = (
|
|
SELECT COUNT(*) FROM posts_hashtags WHERE hashtag_id = hashtags.id
|
|
);
|
|
|
|
-- Update use_count for urls based on actual usage
|
|
UPDATE urls SET use_count = (
|
|
SELECT COUNT(*) FROM posts_urls WHERE url_id = urls.id
|
|
);
|
|
|
|
-- Update user timestamps to reflect actual data range
|
|
UPDATE users SET
|
|
resolved_at = (
|
|
SELECT MIN(timestamp) FROM posts WHERE user_did = users.did
|
|
),
|
|
updated_at = (
|
|
SELECT MAX(timestamp) FROM posts WHERE user_did = users.did
|
|
);
|
|
|
|
-- Analyze database for query optimization
|
|
ANALYZE;
|
|
|
|
-- Show summary statistics
|
|
SELECT 'Database merge complete!' as status;
|
|
SELECT '---' as separator;
|
|
SELECT 'Total posts: ' || COUNT(*) FROM posts;
|
|
SELECT 'Total users: ' || COUNT(*) FROM users;
|
|
SELECT 'Total hashtags: ' || COUNT(*) FROM hashtags;
|
|
SELECT 'Total URLs: ' || COUNT(*) FROM urls;
|
|
SELECT 'Date range: ' || MIN(timestamp) || ' to ' || MAX(timestamp) FROM posts;
|
|
EOF
|
|
|
|
echo "Merge complete! Database saved to: $OUTPUT_DB" |