From 035af35d33ee8e68c7563ba253f146c643c80185 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 3 Mar 2020 04:44:52 -0800 Subject: [PATCH] think this might work, time to begin testing --- .drone.yml | 19 ++++++++++++++ Dockerfile | 39 ++++++++++++++++++++++++++++ Makefile | 19 ++++++++++++++ README.md | 16 ++++++++++++ root/etc/rsyslog.d/10-listeners.conf | 9 +++++++ root/etc/rsyslog.d/20-writedisk.conf | 7 +++++ root/etc/service/logspout/run | 3 +++ root/etc/service/rsyslog/run | 6 +++++ run.sh | 33 +++++++++++++++++++++++ 9 files changed, 151 insertions(+) create mode 100644 .drone.yml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 root/etc/rsyslog.d/10-listeners.conf create mode 100644 root/etc/rsyslog.d/20-writedisk.conf create mode 100644 root/etc/service/logspout/run create mode 100644 root/etc/service/rsyslog/run create mode 100644 run.sh diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..fcba5c4 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,19 @@ +kind: pipeline +name: default + +steps: +- name: docker + image: plugins/docker + network_mode: bridge + settings: + repo: sneak/logwriter + build_args: + - UBUNTU_MIRROR=http://ubuntu.datavi.be/ubuntu + username: + from_secret: docker_username + password: + from_secret: docker_password + tags: + - ${DRONE_BRANCH//\//-}-${DRONE_COMMIT_SHA:0:8} + - ${DRONE_COMMIT_SHA} + - ${DRONE_BRANCH/\//-} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..18aaa7e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# focal 2020.01 dev +FROM ubuntu@sha256:d050ed7278c16ff627e4a70d7d353f1a2ec74d8a0b66e5a865356d92f5f6d87b + +ENV DEBIAN_FRONTEND noninteractive + +ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu +RUN echo "deb $UBUNTU_MIRROR focal main universe restricted multiverse" > /etc/apt/sources.list.new && \ +echo "deb $UBUNTU_MIRROR focal-updates main universe restricted multiverse" >> /etc/apt/sources.list.new && \ +echo "deb $UBUNTU_MIRROR focal-security main universe restricted multiverse" >> /etc/apt/sources.list.new && \ +echo "deb $UBUNTU_MIRROR focal-backports main universe restricted multiverse" >> /etc/apt/sources.list.new && \ +mv /etc/apt/sources.list.new /etc/apt/sources.list + +COPY ./root /tmp/rootoverlay +COPY ./run.sh /tmp/run.sh + +# we only install essential system packages here +ARG SYSTEM_PACKAGES="build-essential ca-certificates openssh-server git golang rsync runit" + +RUN \ + apt update && \ + apt -y upgrade && \ + apt -y install $SYSTEM_PACKAGES && \ + rsync -avP /tmp/rootoverlay/ / && \ + rm -rf /tmp/rootoverlay && \ + go get -v -u git.eeqj.de/sneak/runsvinit && \ + cp /root/go/bin/runsvinit /usr/local/sbin/runsvinit && \ + rm -r /root/go && \ + /bin/bash /tmp/run.sh && \ + rm -f /tmp/run.sh + +# /var/log is distro-default stuff, /var/logs is ours. +VOLUME /var/logs + +EXPOSE 514 +EXPOSE 514/udp +# RELP: +EXPOSE 20514 + +CMD ["/usr/local/sbin/runsvinit"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c6608e1 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +default: build + +build: + docker build \ + --build-arg UBUNTU_MIRROR="http://ubuntu.datavi.be/ubuntu" \ + -t sneak/logwriter \ + . + +run: + docker run \ + --cap-add SYSLOG \ + --restart unless-stopped \ + -v /srv/z/logs:/var/logs \ + -p 514:514 \ + -p 514:514/udp \ + -p 20514:20514 \ + --name logwriter \ + sneak/logwriter + diff --git a/README.md b/README.md new file mode 100644 index 0000000..69cbad6 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# sneak/logwriter + +Container to run logspout + rsyslog and ingest docker logs and write them to +disk as json. + +# example + +``` +docker \ + run \ + -d \ + --restart-policy=unless-stopped \ + -v /srv/z/logarchive:/var/logs \ + -v /var/run/docker.sock:/var/run.docker.sock \ + sneak/logwriter +``` diff --git a/root/etc/rsyslog.d/10-listeners.conf b/root/etc/rsyslog.d/10-listeners.conf new file mode 100644 index 0000000..b5edac1 --- /dev/null +++ b/root/etc/rsyslog.d/10-listeners.conf @@ -0,0 +1,9 @@ +$ModLoad imudp.so +$UDPServerRun 514 + +$ModLoad imtcp +$InputTCPMaxSessions 100 +$InputTCPServerRun 514 + +$ModLoad imrelp +$InputRELPServerRun 20514 diff --git a/root/etc/rsyslog.d/20-writedisk.conf b/root/etc/rsyslog.d/20-writedisk.conf new file mode 100644 index 0000000..111c4c5 --- /dev/null +++ b/root/etc/rsyslog.d/20-writedisk.conf @@ -0,0 +1,7 @@ +$PreserveFQDN on + +$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility-text:::jsonf:facility%,%syslogseverity-text:::jsonf:severity%,%app-name:::jsonf:program%,%procid:::jsonf:processid%}}\n" + +$template systemjsonfile,"/var/logs/%$year%%$month%/%$year%%$month%%$day%.%HOSTNAME%.system.json" + +*.* ?systemjsonfile;ls_json diff --git a/root/etc/service/logspout/run b/root/etc/service/logspout/run new file mode 100644 index 0000000..a2093eb --- /dev/null +++ b/root/etc/service/logspout/run @@ -0,0 +1,3 @@ +#!/bin/bash + +/usr/local/bin/logspout syslog://127.0.0.1:514 diff --git a/root/etc/service/rsyslog/run b/root/etc/service/rsyslog/run new file mode 100644 index 0000000..c2cfea7 --- /dev/null +++ b/root/etc/service/rsyslog/run @@ -0,0 +1,6 @@ +#!/bin/bash + +chgrp syslog /var/logs +chmod g+w /var/logs + +exec rsyslogd -n diff --git a/run.sh b/run.sh new file mode 100644 index 0000000..1e46f6a --- /dev/null +++ b/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -x + +PKGS=" + rsyslog + rsyslog-czmq + rsyslog-doc + rsyslog-elasticsearch + rsyslog-gnutls + rsyslog-gssapi + rsyslog-hiredis + rsyslog-kafka + rsyslog-mongodb + rsyslog-mysql + rsyslog-openssl + rsyslog-pgsql + rsyslog-relp + golang + git +" + +export DEBIAN_FRONTEND=noninteractive +apt update +apt install -y $PKGS + +git clone https://github.com/gliderlabs/logspout.git /usr/local/src/logspout +# current master as of 20200303 +LS_VERSION="8617629f8caffe866dece71c60967349f23c8d2d" +cd /usr/local/src/logspout +git checkout $LS_VERSION +go get -v +go build -o /usr/local/bin/logspout