From f21c916eb34c26e566dd84d977acde429236bc61 Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 13 May 2024 21:43:47 -0700 Subject: [PATCH] initial --- Makefile | 14 +++ README.md | 50 +++++++++++ TODO | 5 ++ console_handler.go | 16 ++++ custom_logger.go | 41 +++++++++ event.go | 26 ++++++ go.mod | 14 +++ go.sum | 13 +++ handler.go | 6 ++ json_handler.go | 18 ++++ relp_handler.go | 169 ++++++++++++++++++++++++++++++++++++ simplelog_test.go | 8 ++ tools/relp_log_trial.go | 16 ++++ tools/run_relp_log_trial.sh | 8 ++ webhook_handler.go | 33 +++++++ 15 files changed, 437 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 TODO create mode 100644 console_handler.go create mode 100644 custom_logger.go create mode 100644 event.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 handler.go create mode 100644 json_handler.go create mode 100644 relp_handler.go create mode 100644 simplelog_test.go create mode 100644 tools/relp_log_trial.go create mode 100755 tools/run_relp_log_trial.sh create mode 100644 webhook_handler.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..923eac9 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: test + +default: test + +test: + @go test -v ./... + +fmt: + goimports -l -w . + golangci-lint run --fix + +lint: + golangci-lint run + sh -c 'test -z "$$(gofmt -l .)"' diff --git a/README.md b/README.md new file mode 100644 index 0000000..6e96722 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# simplelog + +simplelog is an opinionated logging package designed to facilitate easy and +structured logging in Go applications with an absolute minimum of +boilerplate. + +The idea is that you can add a single import line which replaces the +stdlib `log/slog` default handler, and solve the 90% case for logging. + +## Features + +* if output is a tty, outputs pretty color logs +* if output is not a tty, outputs json +* supports delivering logs via tcp RELP (e.g. to remote rsyslog using imrelp) +* supports delivering each log message via a webhook + +## Installation + +To use simplelog, first ensure your project is set up with Go modules: + +```bash +go mod init your_project_name +``` + +Then, add SimpleLog to your project: + +```bash +go get git.eeqj.de/sneak/go-simplelog +``` + +## Usage + +Below is an example of how to use SimpleLog in a Go application. This example is provided in the form of a `main.go` file, which demonstrates logging at various levels using structured logging syntax. + +```go +package main + +import ( + "log/slog" + _ "git.eeqj.de/sneak/go-simplelog" +) + +func main() { + + // log structured data with slog as usual: + slog.Info("User login attempt", slog.String("user", "JohnDoe"), slog.Int("attempt", 3)) + slog.Warn("Configuration mismatch", slog.String("expected", "config.json"), slog.String("found", "config.dev.json")) + slog.Error("Failed to save data", slog.String("reason", "permission denied")) +} +``` diff --git a/TODO b/TODO new file mode 100644 index 0000000..4a17802 --- /dev/null +++ b/TODO @@ -0,0 +1,5 @@ +# TODO + +* fix relp output to cache +* add regex for filtering webhook logs +* better console output format diff --git a/console_handler.go b/console_handler.go new file mode 100644 index 0000000..359bd3d --- /dev/null +++ b/console_handler.go @@ -0,0 +1,16 @@ +package simplelog + +import ( + "github.com/fatih/color" +) + +type ConsoleHandler struct{} + +func NewConsoleHandler() *ConsoleHandler { + return &ConsoleHandler{} +} + +func (c *ConsoleHandler) Log(event Event) error { + color.New(color.FgBlue).PrintfFunc()("%s: %s\n", event.Level, event.Message) + return nil +} diff --git a/custom_logger.go b/custom_logger.go new file mode 100644 index 0000000..27467df --- /dev/null +++ b/custom_logger.go @@ -0,0 +1,41 @@ +package simplelog + +import ( + "log" + "os" + + "github.com/mattn/go-isatty" +) + +var ( + relpServerURL = os.Getenv("LOGGER_RELP_URL") + webhookURL = os.Getenv("LOGGER_WEBHOOK_URL") +) + +type CustomLogger struct { + handlers []Handler +} + +func NewCustomLogger() *CustomLogger { + cl := &CustomLogger{} + if isatty.IsTerminal(os.Stdout.Fd()) { + cl.handlers = append(cl.handlers, NewConsoleHandler()) + } else { + cl.handlers = append(cl.handlers, NewJSONHandler()) + } + if relpServerURL != "" { + handler, err := NewRELPHandler(relpServerURL) + if err != nil { + log.Fatalf("Failed to initialize RELP handler: %v", err) + } + cl.handlers = append(cl.handlers, handler) + } + if webhookURL != "" { + handler, err := NewWebhookHandler(webhookURL) + if err != nil { + log.Fatalf("Failed to initialize Webhook handler: %v", err) + } + cl.handlers = append(cl.handlers, handler) + } + return cl +} diff --git a/event.go b/event.go new file mode 100644 index 0000000..271a48c --- /dev/null +++ b/event.go @@ -0,0 +1,26 @@ +package simplelog + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" +) + +type Event struct { + ID uuid.UUID `json:"id"` + Timestamp time.Time `json:"timestamp"` + Level string `json:"level"` + Message string `json:"message"` + Data json.RawMessage `json:"data"` +} + +func NewEvent(level, message string, data json.RawMessage) Event { + return Event{ + ID: uuid.New(), + Timestamp: time.Now().UTC(), + Level: level, + Message: message, + Data: data, + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9b45bf8 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module git.eeqj.de/sneak/go-simplelog + +go 1.22.1 + +require ( + github.com/fatih/color v1.17.0 + github.com/google/uuid v1.6.0 + github.com/mattn/go-isatty v0.0.20 +) + +require ( + github.com/mattn/go-colorable v0.1.13 // indirect + golang.org/x/sys v0.18.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..152870e --- /dev/null +++ b/go.sum @@ -0,0 +1,13 @@ +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/handler.go b/handler.go new file mode 100644 index 0000000..2bf6431 --- /dev/null +++ b/handler.go @@ -0,0 +1,6 @@ +package simplelog + +// Handler defines the interface for different log outputs. +type Handler interface { + Log(event Event) error +} diff --git a/json_handler.go b/json_handler.go new file mode 100644 index 0000000..2d98d1a --- /dev/null +++ b/json_handler.go @@ -0,0 +1,18 @@ +package simplelog + +import ( + "encoding/json" + "log" +) + +type JSONHandler struct{} + +func NewJSONHandler() *JSONHandler { + return &JSONHandler{} +} + +func (j *JSONHandler) Log(event Event) error { + jsonData, _ := json.Marshal(event) + log.Println(string(jsonData)) + return nil +} diff --git a/relp_handler.go b/relp_handler.go new file mode 100644 index 0000000..5a2aa42 --- /dev/null +++ b/relp_handler.go @@ -0,0 +1,169 @@ +package simplelog + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/url" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" +) + +const ( + cacheDir = "/var/cache/relploggercache" + diskBufferLimit = 100 // Write to disk after accumulating 100 failed events + diskWriteInterval = time.Second // Or write every second, whichever comes first +) + +type RELPHandler struct { + relpServerURL string + conn net.Conn + ch chan Event + done chan struct{} + failedCh chan Event + timer *time.Timer +} + +func NewRELPHandler(relpURL string) (*RELPHandler, error) { + parsedURL, err := url.Parse(relpURL) + if err != nil { + return nil, fmt.Errorf("Error parsing RELP URL: %v", err) + } + if parsedURL.Scheme != "tcp" { + return nil, fmt.Errorf("RELP URL must have the tcp scheme, got %s", parsedURL.Scheme) + } + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return nil, fmt.Errorf("Failed to create cache directory: %v", err) + } + return &RELPHandler{ + relpServerURL: parsedURL.Host, + ch: make(chan Event, diskBufferLimit), + done: make(chan struct{}), + failedCh: make(chan Event, diskBufferLimit), + timer: time.NewTimer(diskWriteInterval), + }, nil +} + +func (r *RELPHandler) Startup() error { + var err error + r.conn, err = net.Dial("tcp", r.relpServerURL) + if err != nil { + return fmt.Errorf("Failed to establish TCP connection: %v", err) + } + go r.receiveEventsFromChannel() + go r.processFailedEvents() + go r.watchDirectoryForFailedEventFiles() + return nil +} + +func (r *RELPHandler) Log(event Event) error { + select { + case r.ch <- event: + return nil // Successfully sent event to channel + default: + return fmt.Errorf("failed to log event: channel is full") + } +} + +func (r *RELPHandler) receiveEventsFromChannel() { + for { + select { + case event := <-r.ch: + if err := r.sendEventToRELPServer(event); err != nil { + r.failedCh <- event + } + case <-r.done: + return + } + } +} + +func (r *RELPHandler) processFailedEvents() { + for { + select { + case event := <-r.failedCh: + if err := r.sendEventToRELPServer(event); err != nil { + // If still failing, write to disk + r.writeFailedToDisk(event) + } + case <-r.timer.C: + // Flush all events in failedCh to disk + for len(r.failedCh) > 0 { + event := <-r.failedCh + r.writeFailedToDisk(event) + } + case <-r.done: + return + } + } +} + +func (r *RELPHandler) sendEventToRELPServer(event Event) error { + jsonData, _ := json.Marshal(event) + _, err := r.conn.Write(jsonData) + if err != nil { + return err + } + // Implement TCP read for the acknowledgment with a timeout + r.conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + ack := make([]byte, 256) + n, err := r.conn.Read(ack) + if err != nil || string(ack[:n]) != "ACK" { + return err + } + return nil +} + +func (r *RELPHandler) watchDirectoryForFailedEventFiles() { + for { + select { + case <-r.done: + return + default: + if r.isConnected() { + files, err := ioutil.ReadDir(cacheDir) + if err != nil { + log.Printf("Error reading cache directory: %v", err) + continue + } + for _, file := range files { + filePath := filepath.Join(cacheDir, file.Name()) + data, err := ioutil.ReadFile(filePath) + if err != nil { + log.Printf("Error reading file %s: %v", file.Name(), err) + continue + } + var event Event + if err := json.Unmarshal(data, &event); err != nil { + log.Printf("Error unmarshalling event from file %s: %v", file.Name(), err) + continue + } + r.ch <- event + os.Remove(filePath) // Remove file after processing + } + } + time.Sleep(10 * time.Second) // Check disk every 10 seconds + } + } +} + +func (r *RELPHandler) isConnected() bool { + _, err := r.conn.Write([]byte{}) + return err == nil +} + +func (r *RELPHandler) Shutdown() error { + close(r.done) + return r.conn.Close() +} + +func (r *RELPHandler) writeFailedToDisk(event Event) { + fileName := filepath.Join(cacheDir, uuid.New().String()+".logevent") + data, _ := json.Marshal(event) + ioutil.WriteFile(fileName, data, 0644) +} diff --git a/simplelog_test.go b/simplelog_test.go new file mode 100644 index 0000000..8b97c39 --- /dev/null +++ b/simplelog_test.go @@ -0,0 +1,8 @@ +package simplelog + +import "testing" + +// TestCompile checks if the package compiles successfully. +func TestCompile(t *testing.T) { + // This test ensures that the simplelog package compiles without error. +} diff --git a/tools/relp_log_trial.go b/tools/relp_log_trial.go new file mode 100644 index 0000000..15d2184 --- /dev/null +++ b/tools/relp_log_trial.go @@ -0,0 +1,16 @@ +package main + +import ( + "log/slog" + + _ "git.eeqj.de/sneak/go-simplelog" // Using underscore to only invoke init() +) + +func main() { + // Send some test messages with structured data + slog.Info("Starting the application", slog.String("status", "initialized")) + slog.Info("Attempting to connect to database", slog.String("host", "localhost"), slog.Int("port", 5432)) + slog.Warn("Using default configuration", slog.String("configuration", "default")) + slog.Error("Failed to load module", slog.String("module", "finance"), slog.String("error", "module not found")) + slog.Info("Shutting down the application", slog.String("status", "stopped")) +} diff --git a/tools/run_relp_log_trial.sh b/tools/run_relp_log_trial.sh new file mode 100755 index 0000000..bd28de8 --- /dev/null +++ b/tools/run_relp_log_trial.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Set the environment variables for the RELP server URL and optionally for the webhook URL +export LOGGER_RELP_URL="tcp://10.201.1.18:20514" +export LOGGER_WEBHOOK_URL="https://example.com/webhook" + +# Run the Go program +go run relp_log_trial.go diff --git a/webhook_handler.go b/webhook_handler.go new file mode 100644 index 0000000..107be4f --- /dev/null +++ b/webhook_handler.go @@ -0,0 +1,33 @@ +package simplelog + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" +) + +type WebhookHandler struct { + webhookURL string +} + +func NewWebhookHandler(webhookURL string) (*WebhookHandler, error) { + if _, err := url.ParseRequestURI(webhookURL); err != nil { + return nil, fmt.Errorf("invalid webhook URL: %v", err) + } + return &WebhookHandler{webhookURL: webhookURL}, nil +} + +func (w *WebhookHandler) Log(event Event) error { + jsonData, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("error marshaling event: %v", err) + } + response, err := http.Post(w.webhookURL, "application/json", bytes.NewBuffer(jsonData)) + if err != nil { + return err + } + defer response.Body.Close() + return nil +}