think this might work, time to begin testing

This commit is contained in:
Jeffrey Paul 2020-03-03 04:44:52 -08:00
commit 035af35d33
9 changed files with 151 additions and 0 deletions

19
.drone.yml Normal file
View File

@ -0,0 +1,19 @@
kind: pipeline
name: default
steps:
- name: docker
image: plugins/docker
network_mode: bridge
settings:
repo: sneak/logwriter
build_args:
- UBUNTU_MIRROR=http://ubuntu.datavi.be/ubuntu
username:
from_secret: docker_username
password:
from_secret: docker_password
tags:
- ${DRONE_BRANCH//\//-}-${DRONE_COMMIT_SHA:0:8}
- ${DRONE_COMMIT_SHA}
- ${DRONE_BRANCH/\//-}

39
Dockerfile Normal file
View File

@ -0,0 +1,39 @@
# focal 2020.01 dev
FROM ubuntu@sha256:d050ed7278c16ff627e4a70d7d353f1a2ec74d8a0b66e5a865356d92f5f6d87b
ENV DEBIAN_FRONTEND noninteractive
ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu
RUN echo "deb $UBUNTU_MIRROR focal main universe restricted multiverse" > /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-updates main universe restricted multiverse" >> /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-security main universe restricted multiverse" >> /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-backports main universe restricted multiverse" >> /etc/apt/sources.list.new && \
mv /etc/apt/sources.list.new /etc/apt/sources.list
COPY ./root /tmp/rootoverlay
COPY ./run.sh /tmp/run.sh
# we only install essential system packages here
ARG SYSTEM_PACKAGES="build-essential ca-certificates openssh-server git golang rsync runit"
RUN \
apt update && \
apt -y upgrade && \
apt -y install $SYSTEM_PACKAGES && \
rsync -avP /tmp/rootoverlay/ / && \
rm -rf /tmp/rootoverlay && \
go get -v -u git.eeqj.de/sneak/runsvinit && \
cp /root/go/bin/runsvinit /usr/local/sbin/runsvinit && \
rm -r /root/go && \
/bin/bash /tmp/run.sh && \
rm -f /tmp/run.sh
# /var/log is distro-default stuff, /var/logs is ours.
VOLUME /var/logs
EXPOSE 514
EXPOSE 514/udp
# RELP:
EXPOSE 20514
CMD ["/usr/local/sbin/runsvinit"]

19
Makefile Normal file
View File

@ -0,0 +1,19 @@
default: build
build:
docker build \
--build-arg UBUNTU_MIRROR="http://ubuntu.datavi.be/ubuntu" \
-t sneak/logwriter \
.
run:
docker run \
--cap-add SYSLOG \
--restart unless-stopped \
-v /srv/z/logs:/var/logs \
-p 514:514 \
-p 514:514/udp \
-p 20514:20514 \
--name logwriter \
sneak/logwriter

16
README.md Normal file
View File

@ -0,0 +1,16 @@
# sneak/logwriter
Container to run logspout + rsyslog and ingest docker logs and write them to
disk as json.
# example
```
docker \
run \
-d \
--restart-policy=unless-stopped \
-v /srv/z/logarchive:/var/logs \
-v /var/run/docker.sock:/var/run.docker.sock \
sneak/logwriter
```

View File

@ -0,0 +1,9 @@
$ModLoad imudp.so
$UDPServerRun 514
$ModLoad imtcp
$InputTCPMaxSessions 100
$InputTCPServerRun 514
$ModLoad imrelp
$InputRELPServerRun 20514

View File

@ -0,0 +1,7 @@
$PreserveFQDN on
$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility-text:::jsonf:facility%,%syslogseverity-text:::jsonf:severity%,%app-name:::jsonf:program%,%procid:::jsonf:processid%}}\n"
$template systemjsonfile,"/var/logs/%$year%%$month%/%$year%%$month%%$day%.%HOSTNAME%.system.json"
*.* ?systemjsonfile;ls_json

View File

@ -0,0 +1,3 @@
#!/bin/bash
/usr/local/bin/logspout syslog://127.0.0.1:514

View File

@ -0,0 +1,6 @@
#!/bin/bash
chgrp syslog /var/logs
chmod g+w /var/logs
exec rsyslogd -n

33
run.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
set -x
PKGS="
rsyslog
rsyslog-czmq
rsyslog-doc
rsyslog-elasticsearch
rsyslog-gnutls
rsyslog-gssapi
rsyslog-hiredis
rsyslog-kafka
rsyslog-mongodb
rsyslog-mysql
rsyslog-openssl
rsyslog-pgsql
rsyslog-relp
golang
git
"
export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y $PKGS
git clone https://github.com/gliderlabs/logspout.git /usr/local/src/logspout
# current master as of 20200303
LS_VERSION="8617629f8caffe866dece71c60967349f23c8d2d"
cd /usr/local/src/logspout
git checkout $LS_VERSION
go get -v
go build -o /usr/local/bin/logspout