88 lines
2.9 KiB
SQL
88 lines
2.9 KiB
SQL
-- SQL script to fix/regenerate auxiliary table statistics after merging daily dumps
|
|
-- Usage: sqlite3 merged.db < regenerate_auxiliary_tables.sql
|
|
--
|
|
-- This assumes you've already merged dumps using:
|
|
-- zstdcat *.sql.zst | sed 's/INSERT INTO/INSERT OR IGNORE INTO/g' | sqlite3 merged.db
|
|
|
|
BEGIN TRANSACTION;
|
|
|
|
-- Fix hashtags table statistics
|
|
-- Update first_seen to actual first occurrence
|
|
UPDATE hashtags SET first_seen = (
|
|
SELECT MIN(p.timestamp)
|
|
FROM posts p
|
|
JOIN posts_hashtags ph ON p.id = ph.post_id
|
|
WHERE ph.hashtag_id = hashtags.id
|
|
) WHERE EXISTS (
|
|
SELECT 1 FROM posts_hashtags WHERE hashtag_id = hashtags.id
|
|
);
|
|
|
|
-- Update use_count to actual count
|
|
UPDATE hashtags SET use_count = (
|
|
SELECT COUNT(*) FROM posts_hashtags WHERE hashtag_id = hashtags.id
|
|
);
|
|
|
|
-- Fix urls table statistics
|
|
-- Update first_seen to actual first occurrence
|
|
UPDATE urls SET first_seen = (
|
|
SELECT MIN(p.timestamp)
|
|
FROM posts p
|
|
JOIN posts_urls pu ON p.id = pu.post_id
|
|
WHERE pu.url_id = urls.id
|
|
) WHERE EXISTS (
|
|
SELECT 1 FROM posts_urls WHERE url_id = urls.id
|
|
);
|
|
|
|
-- Update use_count to actual count
|
|
UPDATE urls SET use_count = (
|
|
SELECT COUNT(*) FROM posts_urls WHERE url_id = urls.id
|
|
);
|
|
|
|
-- Fix users table timestamps
|
|
-- Set resolved_at to first post timestamp
|
|
UPDATE users SET resolved_at = (
|
|
SELECT MIN(timestamp) FROM posts WHERE user_did = users.did
|
|
) WHERE EXISTS (
|
|
SELECT 1 FROM posts WHERE user_did = users.did
|
|
);
|
|
|
|
-- Set updated_at to last post timestamp
|
|
UPDATE users SET updated_at = (
|
|
SELECT MAX(timestamp) FROM posts WHERE user_did = users.did
|
|
) WHERE EXISTS (
|
|
SELECT 1 FROM posts WHERE user_did = users.did
|
|
);
|
|
|
|
-- Clean up orphaned records (optional)
|
|
-- Delete hashtags with no posts
|
|
DELETE FROM hashtags WHERE id NOT IN (
|
|
SELECT DISTINCT hashtag_id FROM posts_hashtags
|
|
);
|
|
|
|
-- Delete urls with no posts
|
|
DELETE FROM urls WHERE id NOT IN (
|
|
SELECT DISTINCT url_id FROM posts_urls
|
|
);
|
|
|
|
-- Delete users with no posts
|
|
DELETE FROM users WHERE did NOT IN (
|
|
SELECT DISTINCT user_did FROM posts WHERE user_did IS NOT NULL
|
|
);
|
|
|
|
COMMIT;
|
|
|
|
-- Rebuild indexes and analyze for better query performance
|
|
ANALYZE;
|
|
|
|
-- Show summary statistics
|
|
SELECT '=== Merge Statistics ===' as report;
|
|
SELECT 'Total posts: ' || COUNT(*) as stat FROM posts;
|
|
SELECT 'Total users: ' || COUNT(*) as stat FROM users;
|
|
SELECT 'Total hashtags: ' || COUNT(*) as stat FROM hashtags;
|
|
SELECT 'Total URLs: ' || COUNT(*) as stat FROM urls;
|
|
SELECT 'Date range: ' || MIN(timestamp) || ' to ' || MAX(timestamp) as stat FROM posts;
|
|
SELECT 'Avg posts per user: ' || ROUND(CAST(COUNT(*) AS FLOAT) / (SELECT COUNT(*) FROM users), 2) as stat FROM posts;
|
|
SELECT 'Most used hashtags:' as stat;
|
|
SELECT ' #' || tag || ' (' || use_count || ')' as stat FROM hashtags ORDER BY use_count DESC LIMIT 10;
|
|
SELECT 'Most linked domains:' as stat;
|
|
SELECT ' ' || domain || ' (' || COUNT(*) || ')' as stat FROM urls GROUP BY domain ORDER BY COUNT(*) DESC LIMIT 10; |