-- SQL script to fix/regenerate auxiliary table statistics after merging daily dumps
-- Usage: sqlite3 merged.db < regenerate_auxiliary_tables.sql
--
-- This assumes you've already merged dumps using:
-- zstdcat *.sql.zst | sed 's/INSERT INTO/INSERT OR IGNORE INTO/g' | sqlite3 merged.db

BEGIN TRANSACTION;

-- Fix hashtags table statistics
-- Update first_seen to actual first occurrence
UPDATE hashtags SET first_seen = (
    SELECT MIN(p.timestamp)
    FROM posts p
    JOIN posts_hashtags ph ON p.id = ph.post_id
    WHERE ph.hashtag_id = hashtags.id
) WHERE EXISTS (
    SELECT 1 FROM posts_hashtags WHERE hashtag_id = hashtags.id
);

-- Update use_count to actual count
UPDATE hashtags SET use_count = (
    SELECT COUNT(*) FROM posts_hashtags WHERE hashtag_id = hashtags.id
);

-- Fix urls table statistics
-- Update first_seen to actual first occurrence
UPDATE urls SET first_seen = (
    SELECT MIN(p.timestamp)
    FROM posts p
    JOIN posts_urls pu ON p.id = pu.post_id
    WHERE pu.url_id = urls.id
) WHERE EXISTS (
    SELECT 1 FROM posts_urls WHERE url_id = urls.id
);

-- Update use_count to actual count
UPDATE urls SET use_count = (
    SELECT COUNT(*) FROM posts_urls WHERE url_id = urls.id
);

-- Fix users table timestamps
-- Set resolved_at to first post timestamp
UPDATE users SET resolved_at = (
    SELECT MIN(timestamp) FROM posts WHERE user_did = users.did
) WHERE EXISTS (
    SELECT 1 FROM posts WHERE user_did = users.did
);

-- Set updated_at to last post timestamp
UPDATE users SET updated_at = (
    SELECT MAX(timestamp) FROM posts WHERE user_did = users.did
) WHERE EXISTS (
    SELECT 1 FROM posts WHERE user_did = users.did
);

-- Clean up orphaned records (optional)
-- Delete hashtags with no posts
DELETE FROM hashtags WHERE id NOT IN (
    SELECT DISTINCT hashtag_id FROM posts_hashtags
);

-- Delete urls with no posts
DELETE FROM urls WHERE id NOT IN (
    SELECT DISTINCT url_id FROM posts_urls
);

-- Delete users with no posts
DELETE FROM users WHERE did NOT IN (
    SELECT DISTINCT user_did FROM posts WHERE user_did IS NOT NULL
);

COMMIT;

-- Rebuild indexes and analyze for better query performance
ANALYZE;

-- Show summary statistics
SELECT '=== Merge Statistics ===' as report;
SELECT 'Total posts: ' || COUNT(*) as stat FROM posts;
SELECT 'Total users: ' || COUNT(*) as stat FROM users;
SELECT 'Total hashtags: ' || COUNT(*) as stat FROM hashtags;
SELECT 'Total URLs: ' || COUNT(*) as stat FROM urls;
SELECT 'Date range: ' || MIN(timestamp) || ' to ' || MAX(timestamp) as stat FROM posts;
SELECT 'Avg posts per user: ' || ROUND(CAST(COUNT(*) AS FLOAT) / (SELECT COUNT(*) FROM users), 2) as stat FROM posts;
SELECT 'Most used hashtags:' as stat;
SELECT '  #' || tag || ' (' || use_count || ')' as stat FROM hashtags ORDER BY use_count DESC LIMIT 10;
SELECT 'Most linked domains:' as stat;
SELECT '  ' || domain || ' (' || COUNT(*) || ')' as stat FROM urls GROUP BY domain ORDER BY COUNT(*) DESC LIMIT 10;