latest

regenerate_auxiliary_tables.sql

@@ -0,0 +1,88 @@
+-- SQL script to fix/regenerate auxiliary table statistics after merging daily dumps
+-- Usage: sqlite3 merged.db < regenerate_auxiliary_tables.sql
+--
+-- This assumes you've already merged dumps using:
+-- zstdcat *.sql.zst | sed 's/INSERT INTO/INSERT OR IGNORE INTO/g' | sqlite3 merged.db
+
+BEGIN TRANSACTION;
+
+-- Fix hashtags table statistics
+-- Update first_seen to actual first occurrence
+UPDATE hashtags SET first_seen = (
+    SELECT MIN(p.timestamp)
+    FROM posts p
+    JOIN posts_hashtags ph ON p.id = ph.post_id
+    WHERE ph.hashtag_id = hashtags.id
+) WHERE EXISTS (
+    SELECT 1 FROM posts_hashtags WHERE hashtag_id = hashtags.id
+);
+
+-- Update use_count to actual count
+UPDATE hashtags SET use_count = (
+    SELECT COUNT(*) FROM posts_hashtags WHERE hashtag_id = hashtags.id
+);
+
+-- Fix urls table statistics
+-- Update first_seen to actual first occurrence
+UPDATE urls SET first_seen = (
+    SELECT MIN(p.timestamp)
+    FROM posts p
+    JOIN posts_urls pu ON p.id = pu.post_id
+    WHERE pu.url_id = urls.id
+) WHERE EXISTS (
+    SELECT 1 FROM posts_urls WHERE url_id = urls.id
+);
+
+-- Update use_count to actual count
+UPDATE urls SET use_count = (
+    SELECT COUNT(*) FROM posts_urls WHERE url_id = urls.id
+);
+
+-- Fix users table timestamps
+-- Set resolved_at to first post timestamp
+UPDATE users SET resolved_at = (
+    SELECT MIN(timestamp) FROM posts WHERE user_did = users.did
+) WHERE EXISTS (
+    SELECT 1 FROM posts WHERE user_did = users.did
+);
+
+-- Set updated_at to last post timestamp
+UPDATE users SET updated_at = (
+    SELECT MAX(timestamp) FROM posts WHERE user_did = users.did
+) WHERE EXISTS (
+    SELECT 1 FROM posts WHERE user_did = users.did
+);
+
+-- Clean up orphaned records (optional)
+-- Delete hashtags with no posts
+DELETE FROM hashtags WHERE id NOT IN (
+    SELECT DISTINCT hashtag_id FROM posts_hashtags
+);
+
+-- Delete urls with no posts
+DELETE FROM urls WHERE id NOT IN (
+    SELECT DISTINCT url_id FROM posts_urls
+);
+
+-- Delete users with no posts
+DELETE FROM users WHERE did NOT IN (
+    SELECT DISTINCT user_did FROM posts WHERE user_did IS NOT NULL
+);
+
+COMMIT;
+
+-- Rebuild indexes and analyze for better query performance
+ANALYZE;
+
+-- Show summary statistics
+SELECT '=== Merge Statistics ===' as report;
+SELECT 'Total posts: ' || COUNT(*) as stat FROM posts;
+SELECT 'Total users: ' || COUNT(*) as stat FROM users;
+SELECT 'Total hashtags: ' || COUNT(*) as stat FROM hashtags;
+SELECT 'Total URLs: ' || COUNT(*) as stat FROM urls;
+SELECT 'Date range: ' || MIN(timestamp) || ' to ' || MAX(timestamp) as stat FROM posts;
+SELECT 'Avg posts per user: ' || ROUND(CAST(COUNT(*) AS FLOAT) / (SELECT COUNT(*) FROM users), 2) as stat FROM posts;
+SELECT 'Most used hashtags:' as stat;
+SELECT '  #' || tag || ' (' || use_count || ')' as stat FROM hashtags ORDER BY use_count DESC LIMIT 10;
+SELECT 'Most linked domains:' as stat;
+SELECT '  ' || domain || ' (' || COUNT(*) || ')' as stat FROM urls GROUP BY domain ORDER BY COUNT(*) DESC LIMIT 10;