Browse Source

think this might work, time to begin testing

master
Jeffrey Paul 1 year ago
commit
035af35d33
9 changed files with 151 additions and 0 deletions
  1. +19
    -0
      .drone.yml
  2. +39
    -0
      Dockerfile
  3. +19
    -0
      Makefile
  4. +16
    -0
      README.md
  5. +9
    -0
      root/etc/rsyslog.d/10-listeners.conf
  6. +7
    -0
      root/etc/rsyslog.d/20-writedisk.conf
  7. +3
    -0
      root/etc/service/logspout/run
  8. +6
    -0
      root/etc/service/rsyslog/run
  9. +33
    -0
      run.sh

+ 19
- 0
.drone.yml View File

@@ -0,0 +1,19 @@
kind: pipeline
name: default

steps:
- name: docker
image: plugins/docker
network_mode: bridge
settings:
repo: sneak/logwriter
build_args:
- UBUNTU_MIRROR=http://ubuntu.datavi.be/ubuntu
username:
from_secret: docker_username
password:
from_secret: docker_password
tags:
- ${DRONE_BRANCH//\//-}-${DRONE_COMMIT_SHA:0:8}
- ${DRONE_COMMIT_SHA}
- ${DRONE_BRANCH/\//-}

+ 39
- 0
Dockerfile View File

@@ -0,0 +1,39 @@
# focal 2020.01 dev
FROM ubuntu@sha256:d050ed7278c16ff627e4a70d7d353f1a2ec74d8a0b66e5a865356d92f5f6d87b

ENV DEBIAN_FRONTEND noninteractive

ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu
RUN echo "deb $UBUNTU_MIRROR focal main universe restricted multiverse" > /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-updates main universe restricted multiverse" >> /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-security main universe restricted multiverse" >> /etc/apt/sources.list.new && \
echo "deb $UBUNTU_MIRROR focal-backports main universe restricted multiverse" >> /etc/apt/sources.list.new && \
mv /etc/apt/sources.list.new /etc/apt/sources.list

COPY ./root /tmp/rootoverlay
COPY ./run.sh /tmp/run.sh

# we only install essential system packages here
ARG SYSTEM_PACKAGES="build-essential ca-certificates openssh-server git golang rsync runit"

RUN \
apt update && \
apt -y upgrade && \
apt -y install $SYSTEM_PACKAGES && \
rsync -avP /tmp/rootoverlay/ / && \
rm -rf /tmp/rootoverlay && \
go get -v -u git.eeqj.de/sneak/runsvinit && \
cp /root/go/bin/runsvinit /usr/local/sbin/runsvinit && \
rm -r /root/go && \
/bin/bash /tmp/run.sh && \
rm -f /tmp/run.sh

# /var/log is distro-default stuff, /var/logs is ours.
VOLUME /var/logs

EXPOSE 514
EXPOSE 514/udp
# RELP:
EXPOSE 20514

CMD ["/usr/local/sbin/runsvinit"]

+ 19
- 0
Makefile View File

@@ -0,0 +1,19 @@
default: build

build:
docker build \
--build-arg UBUNTU_MIRROR="http://ubuntu.datavi.be/ubuntu" \
-t sneak/logwriter \
.

run:
docker run \
--cap-add SYSLOG \
--restart unless-stopped \
-v /srv/z/logs:/var/logs \
-p 514:514 \
-p 514:514/udp \
-p 20514:20514 \
--name logwriter \
sneak/logwriter


+ 16
- 0
README.md View File

@@ -0,0 +1,16 @@
# sneak/logwriter

Container to run logspout + rsyslog and ingest docker logs and write them to
disk as json.

# example

```
docker \
run \
-d \
--restart-policy=unless-stopped \
-v /srv/z/logarchive:/var/logs \
-v /var/run/docker.sock:/var/run.docker.sock \
sneak/logwriter
```

+ 9
- 0
root/etc/rsyslog.d/10-listeners.conf View File

@@ -0,0 +1,9 @@
$ModLoad imudp.so
$UDPServerRun 514

$ModLoad imtcp
$InputTCPMaxSessions 100
$InputTCPServerRun 514

$ModLoad imrelp
$InputRELPServerRun 20514

+ 7
- 0
root/etc/rsyslog.d/20-writedisk.conf View File

@@ -0,0 +1,7 @@
$PreserveFQDN on

$template ls_json,"{%timestamp:::date-rfc3339,jsonf:@timestamp%,%source:::jsonf:@source_host%,\"@source\":\"syslog://%fromhost-ip:::json%\",\"@message\":\"%timestamp% %app-name%:%msg:::json%\",\"@fields\":{%syslogfacility-text:::jsonf:facility%,%syslogseverity-text:::jsonf:severity%,%app-name:::jsonf:program%,%procid:::jsonf:processid%}}\n"

$template systemjsonfile,"/var/logs/%$year%%$month%/%$year%%$month%%$day%.%HOSTNAME%.system.json"

*.* ?systemjsonfile;ls_json

+ 3
- 0
root/etc/service/logspout/run View File

@@ -0,0 +1,3 @@
#!/bin/bash

/usr/local/bin/logspout syslog://127.0.0.1:514

+ 6
- 0
root/etc/service/rsyslog/run View File

@@ -0,0 +1,6 @@
#!/bin/bash

chgrp syslog /var/logs
chmod g+w /var/logs

exec rsyslogd -n

+ 33
- 0
run.sh View File

@@ -0,0 +1,33 @@
#!/bin/bash

set -x

PKGS="
rsyslog
rsyslog-czmq
rsyslog-doc
rsyslog-elasticsearch
rsyslog-gnutls
rsyslog-gssapi
rsyslog-hiredis
rsyslog-kafka
rsyslog-mongodb
rsyslog-mysql
rsyslog-openssl
rsyslog-pgsql
rsyslog-relp
golang
git
"

export DEBIAN_FRONTEND=noninteractive
apt update
apt install -y $PKGS

git clone https://github.com/gliderlabs/logspout.git /usr/local/src/logspout
# current master as of 20200303
LS_VERSION="8617629f8caffe866dece71c60967349f23c8d2d"
cd /usr/local/src/logspout
git checkout $LS_VERSION
go get -v
go build -o /usr/local/bin/logspout

Loading…
Cancel
Save