package main import ( "context" "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" "fmt" "io" "log/slog" "os" "sort" "strings" "time" "github.com/oklog/ulid/v2" ) // ColorizedHandler is a custom slog.Handler that outputs colorized logs type ColorizedHandler struct { w io.Writer level slog.Level timeKey string msgKey string } // NewColorizedHandler creates a new ColorizedHandler func NewColorizedHandler(w io.Writer, opts *slog.HandlerOptions) *ColorizedHandler { if opts == nil { opts = &slog.HandlerOptions{} } return &ColorizedHandler{ w: w, level: opts.Level.Level(), timeKey: "time", msgKey: slog.MessageKey, } } // Enabled implements slog.Handler func (h *ColorizedHandler) Enabled(ctx context.Context, level slog.Level) bool { return level >= h.level } // Handle implements slog.Handler func (h *ColorizedHandler) Handle(ctx context.Context, r slog.Record) error { // Skip logs below our level if !h.Enabled(ctx, r.Level) { return nil } // Format time with milliseconds timeStr := r.Time.Format("15:04:05.000") // Get component from attributes var component string var message string // Store other attributes for printing later attributes := make(map[string]interface{}) r.Attrs(func(a slog.Attr) bool { if a.Key == "component" { component = a.Value.String() return true } if a.Key == h.msgKey { message = a.Value.String() return true } // Skip internal or empty values if a.Key == h.timeKey || a.Key == "level" || a.Value.String() == "" { return true } attributes[a.Key] = a.Value.Any() return true }) // Format level with color var levelColor string var levelText string switch r.Level { case slog.LevelDebug: levelColor = colorGray levelText = "DBG" case slog.LevelInfo: levelColor = colorGreen levelText = "INF" case slog.LevelWarn: levelColor = colorYellow levelText = "WRN" case slog.LevelError: levelColor = colorRed levelText = "ERR" default: levelColor = colorReset levelText = "???" } // Build the log line var sb strings.Builder // Timestamp with gray color sb.WriteString(colorGray) sb.WriteString("[") sb.WriteString(timeStr) sb.WriteString("]") sb.WriteString(colorReset) sb.WriteString(" ") // Level with appropriate color sb.WriteString(levelColor) sb.WriteString(levelText) sb.WriteString(colorReset) sb.WriteString(" ") // Component in blue if component != "" { sb.WriteString(colorBlue) sb.WriteString("[") sb.WriteString(component) sb.WriteString("]") sb.WriteString(colorReset) sb.WriteString(" ") } // Message in white+bold if message != "" { sb.WriteString(bold) sb.WriteString(colorWhite) sb.WriteString(message) sb.WriteString(colorReset) sb.WriteString(" ") } // Sort keys for consistent output keys := make([]string, 0, len(attributes)) for k := range attributes { keys = append(keys, k) } sort.Strings(keys) // Add attributes as key=value pairs with colors for _, k := range keys { v := attributes[k] sb.WriteString(colorCyan) // Key in cyan sb.WriteString(k) sb.WriteString(colorReset) sb.WriteString("=") // Value color depends on type switch v := v.(type) { case string: sb.WriteString(colorYellow) // Strings in yellow sb.WriteString(v) case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: sb.WriteString(colorPurple) // Numbers in purple sb.WriteString(fmt.Sprintf("%v", v)) case bool: if v { sb.WriteString(colorGreen) // true in green } else { sb.WriteString(colorRed) // false in red } sb.WriteString(fmt.Sprintf("%v", v)) case error: sb.WriteString(colorRed) // Errors in red sb.WriteString(v.Error()) default: sb.WriteString(colorReset) // Other types with no color sb.WriteString(fmt.Sprintf("%v", v)) } sb.WriteString(colorReset) sb.WriteString(" ") } sb.WriteString("\n") // Write to output _, err := io.WriteString(h.w, sb.String()) return err } // WithAttrs implements slog.Handler func (h *ColorizedHandler) WithAttrs(attrs []slog.Attr) slog.Handler { // This is a simplified implementation that doesn't actually store the attrs // In a real implementation, you would create a new handler with these attrs return h } // WithGroup implements slog.Handler func (h *ColorizedHandler) WithGroup(name string) slog.Handler { // This is a simplified implementation that doesn't handle groups return h } func setupDatabase() error { _, err := db.Exec(` CREATE TABLE IF NOT EXISTS articles ( link TEXT PRIMARY KEY, title TEXT NOT NULL, description TEXT, published TIMESTAMP NOT NULL, originalDate TIMESTAMP, source TEXT NOT NULL, firstseen TIMESTAMP NOT NULL, seen TIMESTAMP, summary TEXT, importance INTEGER, id TEXT, broadcastTime TIMESTAMP ) `) if err != nil { return err } // Create logs table for structured log entries _, err = db.Exec(` CREATE TABLE IF NOT EXISTS logs ( id TEXT PRIMARY KEY, timestamp TIMESTAMP NOT NULL, log JSON NOT NULL ) `) if err != nil { return err } // Create index on timestamp for efficient querying and deletion _, err = db.Exec(` CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs (timestamp) `) if err != nil { return err } // Check if columns exist rows, err := db.Query(`PRAGMA table_info(articles)`) if err != nil { return err } defer rows.Close() hasIDColumn := false hasBroadcastTimeColumn := false hasOriginalDateColumn := false for rows.Next() { var cid, notnull, pk int var name, type_name string var dflt_value interface{} if err := rows.Scan(&cid, &name, &type_name, ¬null, &dflt_value, &pk); err != nil { return err } if name == "id" { hasIDColumn = true } if name == "broadcastTime" { hasBroadcastTimeColumn = true } if name == "originalDate" { hasOriginalDateColumn = true } } // Add missing columns if needed if !hasIDColumn { _, err = db.Exec(`ALTER TABLE articles ADD COLUMN id TEXT`) if err != nil { return err } } if !hasBroadcastTimeColumn { _, err = db.Exec(`ALTER TABLE articles ADD COLUMN broadcastTime TIMESTAMP`) if err != nil { return err } } if !hasOriginalDateColumn { _, err = db.Exec(`ALTER TABLE articles ADD COLUMN originalDate TIMESTAMP`) if err != nil { return err } logInfo("db", "Added originalDate column to articles table", nil) } return nil } // Generate a deterministic ID from a URL func generateID(url string) string { hash := sha256.Sum256([]byte(url)) return hex.EncodeToString(hash[:])[:26] // Return first 26 chars of the hash } func saveArticle(article Article) error { _, err := db.Exec(` INSERT OR IGNORE INTO articles (link, title, description, published, originalDate, source, firstseen, seen, summary, importance, id, broadcastTime) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `, article.Link, article.Title, article.Description, article.Published, article.OriginalDate, article.Source, article.FirstSeen, article.Seen, article.Summary, article.Importance, article.ID, article.BroadcastTime) if err != nil { logEvent("db_insert_error", map[string]interface{}{ "article": article.Link, "id": article.ID, "error": err.Error(), }) } return err } func updateArticle(article Article) error { _, err := db.Exec(` UPDATE articles SET title = ?, description = ?, published = ?, originalDate = ?, source = ?, firstseen = ?, seen = ?, summary = ?, importance = ?, id = ?, broadcastTime = ? WHERE link = ? `, article.Title, article.Description, article.Published, article.OriginalDate, article.Source, article.FirstSeen, article.Seen, article.Summary, article.Importance, article.ID, article.BroadcastTime, article.Link) if err != nil { logEvent("db_update_error", map[string]interface{}{ "article": article.Link, "id": article.ID, "error": err.Error(), }) } return err } func loadArticles() map[string]Article { articles := make(map[string]Article) rows, err := db.Query(` SELECT link, title, description, published, originalDate, source, firstseen, seen, summary, importance, id, broadcastTime FROM articles `) if err != nil { logEvent("db_query_error", map[string]interface{}{ "error": err.Error(), }) return articles } defer rows.Close() for rows.Next() { var a Article var seen sql.NullTime var broadcastTime sql.NullTime var originalDate sql.NullTime err := rows.Scan( &a.Link, &a.Title, &a.Description, &a.Published, &originalDate, &a.Source, &a.FirstSeen, &seen, &a.Summary, &a.Importance, &a.ID, &broadcastTime, ) if err != nil { logEvent("db_scan_error", map[string]interface{}{ "error": err.Error(), }) continue } if seen.Valid { a.Seen = seen.Time } if broadcastTime.Valid { a.BroadcastTime = broadcastTime.Time } if originalDate.Valid { a.OriginalDate = originalDate.Time } articles[a.Link] = a } return articles } // getBroadcastArticles is a common function for retrieving broadcast articles // with consistent filtering criteria func getBroadcastArticles(limit int) ([]Article, error) { rows, err := db.Query(` SELECT link, title, description, published, originalDate, source, firstseen, seen, summary, importance, id, broadcastTime FROM articles WHERE broadcastTime IS NOT NULL AND broadcastTime > 1 AND broadcastTime != 0 AND datetime(broadcastTime) != '1970-01-01 00:00:00' AND datetime(broadcastTime) != '0001-01-01 00:00:00' AND strftime('%Y', broadcastTime) > '2000' -- Ensure year is at least 2000 ORDER BY broadcastTime DESC LIMIT ? `, limit) if err != nil { return nil, err } defer rows.Close() var articles []Article for rows.Next() { var a Article var seen sql.NullTime var broadcastTime sql.NullTime var originalDate sql.NullTime err := rows.Scan( &a.Link, &a.Title, &a.Description, &a.Published, &originalDate, &a.Source, &a.FirstSeen, &seen, &a.Summary, &a.Importance, &a.ID, &broadcastTime, ) if err != nil { continue } if seen.Valid { a.Seen = seen.Time } if broadcastTime.Valid { a.BroadcastTime = broadcastTime.Time } if originalDate.Valid { a.OriginalDate = originalDate.Time } articles = append(articles, a) } return articles, nil } // getBroadcastHistory gets the most recent broadcast articles func getBroadcastHistory(limit int) ([]Article, error) { return getBroadcastArticles(limit) } // getNextUpArticles gets the top 25 articles eligible for broadcast sorted by importance func getNextUpArticles() ([]Article, error) { now := time.Now() cutoff := now.Add(-ARTICLE_FRESHNESS_WINDOW) // Time window for considering articles fresh rows, err := db.Query(` SELECT link, title, description, published, originalDate, source, firstseen, seen, summary, importance, id, broadcastTime FROM articles WHERE broadcastTime IS NULL AND summary IS NOT NULL AND importance > 0 AND firstseen > ? ORDER BY importance DESC LIMIT 25 `, cutoff) if err != nil { return nil, err } defer rows.Close() var articles []Article for rows.Next() { var a Article var seen sql.NullTime var broadcastTime sql.NullTime var originalDate sql.NullTime err := rows.Scan( &a.Link, &a.Title, &a.Description, &a.Published, &originalDate, &a.Source, &a.FirstSeen, &seen, &a.Summary, &a.Importance, &a.ID, &broadcastTime, ) if err != nil { continue } if seen.Valid { a.Seen = seen.Time } if broadcastTime.Valid { a.BroadcastTime = broadcastTime.Time } if originalDate.Valid { a.OriginalDate = originalDate.Time } articles = append(articles, a) } return articles, nil } // getRecentBroadcasts retrieves the n most recently broadcast articles func getRecentBroadcasts(n int) []Article { articles, err := getBroadcastArticles(n) if err != nil { logInfo("db", "Error retrieving recent broadcasts", map[string]interface{}{ "error": err.Error(), }) return []Article{} } return articles } func setupLogging() { var err error logFile, err = os.Create(logPath) if err != nil { slog.Error("Could not create log file", "error", err) os.Exit(1) } // Set up structured logger with custom colorized handler for console // and JSON handler for file logging consoleHandler := NewColorizedHandler(os.Stderr, &slog.HandlerOptions{ Level: slog.LevelDebug, }) // Use the custom handler slog.SetDefault(slog.New(consoleHandler)) } func flushLog() { logMutex.Lock() defer logMutex.Unlock() if logFile == nil { return } enc := json.NewEncoder(logFile) enc.SetIndent("", " ") _ = enc.Encode(logData) logFile.Close() } // logEvent logs a structured message to both console and database func logEvent(event string, details map[string]interface{}) { logMutex.Lock() defer logMutex.Unlock() // Set timestamp if not already provided if _, exists := details["timestamp"]; !exists { details["timestamp"] = time.Now() } // Set event if not already in details if _, exists := details["event"]; !exists { details["event"] = event } timestamp := time.Now() entry := LogEntry{ Timestamp: timestamp, Event: event, Details: details, } // Add to the permanent log data (for file-based logging) logData = append(logData, entry) // Only try to store in database if the database is initialized if db == nil { slog.Debug("Skipping database log storage - database not yet initialized") return } // Store log in database logBytes, err := json.Marshal(entry) if err != nil { slog.Error("Error marshaling log entry", "error", err) return } // Generate ULID for the log entry entropy := ulid.DefaultEntropy() id := ulid.MustNew(ulid.Timestamp(timestamp), entropy).String() // Insert into database _, err = db.Exec("INSERT INTO logs (id, timestamp, log) VALUES (?, ?, ?)", id, timestamp, string(logBytes)) if err != nil { slog.Error("Error storing log in database", "error", err) } } // getRecentLogs retrieves recent log entries from the database func getRecentLogs(limit int) ([]LogEntry, error) { rows, err := db.Query(` SELECT log FROM logs ORDER BY timestamp DESC LIMIT ? `, limit) if err != nil { return nil, err } defer rows.Close() var logs []LogEntry for rows.Next() { var logJSON string if err := rows.Scan(&logJSON); err != nil { return nil, err } var entry LogEntry if err := json.Unmarshal([]byte(logJSON), &entry); err != nil { return nil, err } logs = append(logs, entry) } return logs, nil } // cleanupOldLogs deletes logs older than one month func cleanupOldLogs() error { // Calculate cutoff date (one month ago) cutoff := time.Now().AddDate(0, -1, 0) result, err := db.Exec("DELETE FROM logs WHERE timestamp < ?", cutoff) if err != nil { return err } rowsDeleted, _ := result.RowsAffected() if rowsDeleted > 0 { logInfo("logs", "Deleted old log entries", map[string]interface{}{ "count": rowsDeleted, "olderThan": "1 month", "cutoffDate": cutoff.Format(time.RFC3339), }) } return nil } // logCleanupWorker runs periodically to clean up old logs func logCleanupWorker(shutdown chan struct{}) { logInfo("logs", "Starting log cleanup worker", map[string]interface{}{ "interval": "15 minutes", "retention": "1 month", }) // Run cleanup immediately on startup if err := cleanupOldLogs(); err != nil { logInfo("logs", "Error cleaning up old logs", map[string]interface{}{ "error": err.Error(), }) } // Then run on interval ticker := time.NewTicker(15 * time.Minute) defer ticker.Stop() for { select { case <-ticker.C: if err := cleanupOldLogs(); err != nil { logInfo("logs", "Error cleaning up old logs", map[string]interface{}{ "error": err.Error(), }) } case <-shutdown: logInfo("logs", "Shutting down log cleanup worker", nil) return } } } // logInfo logs a structured message to both console and log file func logInfo(component string, message string, data map[string]interface{}) { // Create a copy of the data map to avoid modifying the original logData := make(map[string]interface{}) for k, v := range data { logData[k] = v } // Add component and message to the log data logData["component"] = component logData["message"] = message // Use slog for structured logging to console attrs := []any{} for k, v := range logData { attrs = append(attrs, k, v) } slog.Info(message, attrs...) // Log to structured log file and database logEvent("info", logData) } // getLastBroadcastTime retrieves the most recent broadcast time from the database func getLastBroadcastTime() time.Time { var lastBroadcastTime time.Time // Query for the most recent valid broadcast time row := db.QueryRow(` SELECT MAX(broadcastTime) FROM articles WHERE broadcastTime IS NOT NULL AND broadcastTime > 1 AND broadcastTime != 0 AND datetime(broadcastTime) != '1970-01-01 00:00:00' AND datetime(broadcastTime) != '0001-01-01 00:00:00' AND strftime('%Y', broadcastTime) > '2000' -- Ensure year is at least 2000 `) // Scan the result var lastTime sql.NullTime if err := row.Scan(&lastTime); err != nil { logInfo("db", "Error retrieving last broadcast time", map[string]interface{}{ "error": err.Error(), }) return time.Time{} // Return zero time on error } if lastTime.Valid { lastBroadcastTime = lastTime.Time logInfo("db", "Retrieved last broadcast time", map[string]interface{}{ "lastBroadcastTime": lastBroadcastTime.Format(time.RFC3339), }) } else { logInfo("db", "No previous broadcasts found in database", nil) } return lastBroadcastTime }