@@ -0,0 +1,19 @@ | |||
kind: pipeline | |||
name: default | |||
steps: | |||
- name: docker | |||
image: plugins/docker | |||
network_mode: bridge | |||
settings: | |||
repo: sneak/logwriter | |||
build_args: | |||
- UBUNTU_MIRROR=http://ubuntu.datavi.be/ubuntu | |||
username: | |||
from_secret: docker_username | |||
password: | |||
from_secret: docker_password | |||
tags: | |||
- ${DRONE_BRANCH//\//-}-${DRONE_COMMIT_SHA:0:8} | |||
- ${DRONE_COMMIT_SHA} | |||
- ${DRONE_BRANCH/\//-} |
@@ -0,0 +1,39 @@ | |||
# focal 2020.01 dev | |||
FROM ubuntu@sha256:d050ed7278c16ff627e4a70d7d353f1a2ec74d8a0b66e5a865356d92f5f6d87b | |||
ENV DEBIAN_FRONTEND noninteractive | |||
ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu | |||
RUN echo "deb $UBUNTU_MIRROR focal main universe restricted multiverse" > /etc/apt/sources.list.new && \ | |||
echo "deb $UBUNTU_MIRROR focal-updates main universe restricted multiverse" >> /etc/apt/sources.list.new && \ | |||
echo "deb $UBUNTU_MIRROR focal-security main universe restricted multiverse" >> /etc/apt/sources.list.new && \ | |||
echo "deb $UBUNTU_MIRROR focal-backports main universe restricted multiverse" >> /etc/apt/sources.list.new && \ | |||
mv /etc/apt/sources.list.new /etc/apt/sources.list | |||
COPY ./root /tmp/rootoverlay | |||
COPY ./run.sh /tmp/run.sh | |||
# we only install essential system packages here | |||
ARG SYSTEM_PACKAGES="build-essential ca-certificates openssh-server git golang rsync runit" | |||
RUN \ | |||
apt update && \ | |||
apt -y upgrade && \ | |||
apt -y install $SYSTEM_PACKAGES && \ | |||
rsync -avP /tmp/rootoverlay/ / && \ | |||
rm -rf /tmp/rootoverlay && \ | |||
go get -v -u git.eeqj.de/sneak/runsvinit && \ | |||
cp /root/go/bin/runsvinit /usr/local/sbin/runsvinit && \ | |||
rm -r /root/go && \ | |||
/bin/bash /tmp/run.sh && \ | |||
rm -f /tmp/run.sh | |||
# /var/log is distro-default stuff, /var/logs is ours. | |||
VOLUME /var/logs | |||
EXPOSE 514 | |||
EXPOSE 514/udp | |||
# RELP: | |||
EXPOSE 20514 | |||
CMD ["/usr/local/sbin/runsvinit"] |
@@ -0,0 +1,19 @@ | |||
default: build | |||
build: | |||
docker build \ | |||
--build-arg UBUNTU_MIRROR="http://ubuntu.datavi.be/ubuntu" \ | |||
-t sneak/logwriter \ | |||
. | |||
run: | |||
docker run \ | |||
--cap-add SYSLOG \ | |||
--restart unless-stopped \ | |||
-v /srv/z/logs:/var/logs \ | |||
-p 514:514 \ | |||
-p 514:514/udp \ | |||
-p 20514:20514 \ | |||
--name logwriter \ | |||
sneak/logwriter | |||
@@ -0,0 +1,16 @@ | |||
# sneak/logwriter | |||
Container to run logspout + rsyslog and ingest docker logs and write them to | |||
disk as json. | |||
# example | |||
``` | |||
docker \ | |||
run \ | |||
-d \ | |||
--restart-policy=unless-stopped \ | |||
-v /srv/z/logarchive:/var/logs \ | |||
-v /var/run/docker.sock:/var/run.docker.sock \ | |||
sneak/logwriter | |||
``` |
@@ -0,0 +1,9 @@ | |||
$ModLoad imudp.so | |||
$UDPServerRun 514 | |||
$ModLoad imtcp | |||
$InputTCPMaxSessions 100 | |||
$InputTCPServerRun 514 | |||
$ModLoad imrelp | |||
$InputRELPServerRun 20514 |
@@ -0,0 +1,7 @@ | |||
$PreserveFQDN on | |||
$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility-text:::jsonf:facility%,%syslogseverity-text:::jsonf:severity%,%app-name:::jsonf:program%,%procid:::jsonf:processid%}}\n" | |||
$template systemjsonfile,"/var/logs/%$year%%$month%/%$year%%$month%%$day%.%HOSTNAME%.system.json" | |||
*.* ?systemjsonfile;ls_json |
@@ -0,0 +1,3 @@ | |||
#!/bin/bash | |||
/usr/local/bin/logspout syslog://127.0.0.1:514 |
@@ -0,0 +1,6 @@ | |||
#!/bin/bash | |||
chgrp syslog /var/logs | |||
chmod g+w /var/logs | |||
exec rsyslogd -n |
@@ -0,0 +1,33 @@ | |||
#!/bin/bash | |||
set -x | |||
PKGS=" | |||
rsyslog | |||
rsyslog-czmq | |||
rsyslog-doc | |||
rsyslog-elasticsearch | |||
rsyslog-gnutls | |||
rsyslog-gssapi | |||
rsyslog-hiredis | |||
rsyslog-kafka | |||
rsyslog-mongodb | |||
rsyslog-mysql | |||
rsyslog-openssl | |||
rsyslog-pgsql | |||
rsyslog-relp | |||
golang | |||
git | |||
" | |||
export DEBIAN_FRONTEND=noninteractive | |||
apt update | |||
apt install -y $PKGS | |||
git clone https://github.com/gliderlabs/logspout.git /usr/local/src/logspout | |||
# current master as of 20200303 | |||
LS_VERSION="8617629f8caffe866dece71c60967349f23c8d2d" | |||
cd /usr/local/src/logspout | |||
git checkout $LS_VERSION | |||
go get -v | |||
go build -o /usr/local/bin/logspout |