Compare commits

..

No commits in common. "master" and "20160411" have entirely different histories.

8 changed files with 69 additions and 145 deletions

View File

@ -1,18 +0,0 @@
kind: pipeline
name: default
steps:
- name: docker
image: plugins/docker
network_mode: bridge
settings:
repo: sneak/ubuntumirror
username:
from_secret: docker_username
password:
from_secret: docker_password
tags:
- ${DRONE_BRANCH//\//-}-${DRONE_COMMIT_SHA:0:8}
- ${DRONE_COMMIT_SHA}
- ${DRONE_BRANCH/\//-}
- latest

View File

@ -1,40 +1,13 @@
FROM phusion/baseimage:0.11
FROM phusion/baseimage:0.9.18
VOLUME /var/mirror
ENV MIRROR_UPDATE_INTERVAL_SECONDS 43200
ENV UBUNTU_MIRROR_ARCHITECTURES amd64
# set to anything but 'false' to mirror source code as well
ENV UBUNTU_ENABLE_SOURCE_MIRRORING false
ENV UBUNTU_MIRROR_CATEGORIES main,universe,restricted,multiverse,main/debian-installer
ENV UBUNTU_MIRROR_UPSTREAM archive.ubuntu.com
ENV UBUNTU_MIRROR_UPSTREAM_PATH /ubuntu
ENV UBUNTU_MIRROR_PROJECTS bionic,bionic-updates,bionic-security,bionic-backports,focal,focal-updates,focal-security,focal-backports
# The following is the URL populated in /mirrors.txt as used by mirror://
# protocol scheme in sources.list
# the idea is that you can DNS rewrite 'mirrors.ubuntu.com' to this mirror
# and it will return a "list" of mirrors that includes only itself.
# then you can safely use something like the following in your sources.list
# and it will work with max speed on your own dns-rewritten lan and in the
# unmodified datacenter talking to the real mirrors.ubuntu.com:
#
# MURM="main universe restricted multiverse"
# C="$(lsb_release -cs)"
# M="mirror://mirrors.ubuntu.com/mirrors.txt"
# echo "deb $M $C $MURM" > /etc/apt/sources.list.new
# echo "deb $M $C-updates $MURM" >> /etc/apt/sources.list.new
# echo "deb $M $C-backports $MURM" >> /etc/apt/sources.list.new
# echo "deb $M $C-security $MURM" >> /etc/apt/sources.list.new
# mv /etc/apt/sources.list.new /etc/apt/sources.list
ENV THIS_MIRROR_URL http://172.17.0.1/ubuntu/
ADD ./sources.list /etc/apt/sources.list
RUN echo "#!/bin/sh\nexit 101" > /usr/sbin/policy-rc.d
# phusion baseimage says i'm not supposed to do this
# and i don't care:
RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y run-one nginx make debmirror xz-utils
RUN apt-get install -y run-one nginx make debmirror
ADD ./mirror /opt/mirror
ADD ./mirror.run /etc/service/mirror/run
@ -49,7 +22,6 @@ RUN \
mkdir -p $GNUPGHOME && \
chmod go-rwx $GNUPGHOME && \
gpg --import /usr/share/keyrings/ubuntu-archive-keyring.gpg && \
cp $GNUPGHOME/pubring.kbx $GNUPGHOME/trustedkeys.kbx
cp $GNUPGHOME/pubring.gpg $GNUPGHOME/trustedkeys.gpg
EXPOSE 80

View File

@ -1,70 +1,45 @@
# sneak/ubuntumirror
Docker image for syncing ubuntu LTS mirrors to reduce network traffic and
speed up build times.
Docker container for syncing trusty and xenial ubuntu LTS mirrors
to reduce network traffic and speed up build times.
Then even your builds (where you can't specify links to other containers by
hostname) will go fast.
I recommend deploying this with a `--dns 172.17.0.1` in your
`/etc/default/docker` and dnsmasq bound to 172.17.0.1:53 to rewrite
`archive.ubuntu.com` to `172.17.0.1` (which this image will be listening and
serving on).
Then even your builds (where you can't specify links) will go fast.
# Levers
## Paths
The mirror stores all the data in a docker volume at path
`/var/mirror/ubuntu`. Map that somewhere else if you like.
## Environment Variables
* `UBUNTU_MIRROR_ARCHITECTURES`
* default `amd64`.
* `MIRROR_ARCHITECTURES`
* default `amd64`.
* You may want `i386,amd64`.
* `UBUNTU_MIRROR_CATEGORIES`
* Default `main,universe,restricted,multiverse`. Will use lots of disk.
* You may want `main,universe`.
* `UBUNTU_MIRROR_PROJECTS`
* Default
`bionic,bionic-updates,bionic-security,bionic-backports,focal,focal-updates,focal-security,focal-backports`.
* You may want only `focal,focal-updates,focal-security,focal-backports`.
* `UBUNTU_MIRROR_UPSTREAM` - upstream server to mirror from
* `MIRROR_CATEGORIES`
* default `main,universe`.
* You may want `main,universe,restricted,multiverse`. Will use lots of
disk.
* `UBUNTU_MIRROR_SOURCE`
* default `archive.ubuntu.com`.
* You may want `xx.archive.ubuntu.com` (where `xx` is your country
code).
* `UBUNTU_ENABLE_SOURCE_MIRRORING` - do we want to mirror all the source code too?
* Default ``
* You may want `1` if you want to use up even more disk for your mirror
* `MIRROR_UPDATE_INTERVAL_SECONDS` - mirror update interval
* `MIRROR_INTERVAL_SECONDS` - mirror sync interval
* default `43200` (12h)
* You may want `10800` (3h)
# Example Startup
# Example dnsmasq setup
```
docker rm -f local-ubuntu-mirror
docker run -d --name local-ubuntu-mirror \
-v /storage/mirrors/ubuntu:/var/mirror/ubuntu \
--env MIRROR_UPDATE_INTERVAL_SECONDS=3600 \
--env UBUNTU_MIRROR_ARCHITECTURES=amd64 \
--env UBUNTU_MIRROR_UPSTREAM=us.archive.ubuntu.com \
--env UBUNTU_MIRROR_PROJECTS=focal,focal-updates,focal-security,focal-backports \
-p 8080:80 \
--restart unless-stopped \
sneak/ubuntumirror@sha256:7de7cc8245106bb574029a2dedca5aa218af08d802b989cdc7857fb57f876a02
apt-get update && apt-get -y install dnsmasq
cat > /etc/dnsmasq.d/docker <<EOF
no-resolv
server=8.8.8.8
server=8.8.4.4
EOF
echo "172.17.0.1 archive.ubuntu.com" >> /etc/hosts
service dnsmasq restart
```
# Example CapRover Definition File
```
{
"schemaVersion": 2,
"imageName": "sneak/ubuntumirror@sha256:7de7cc8245106bb574029a2dedca5aa218af08d802b989cdc7857fb57f876a02"
}
```
# Author
sneak &lt;[sneak@sneak.berlin](mailto:sneak@sneak.berlin)&gt;
# License
This software is released into the public domain.
Then, just add `--dns 172.17.0.1` to your /etc/default/docker opts
and bounce the docker daemon. Future containers will talk to your local
mirror.

15
docker-compose.yml Normal file
View File

@ -0,0 +1,15 @@
version: '2.0'
services:
mirror:
hostname: mirror
image: sneak/ubuntumirror
build: .
network_mode: bridge
volumes:
- "/mnt/scratch/mirror:/var/mirror"
ports:
- "172.17.0.1:80:80"
environment:
- UBUNTU_MIRROR_SOURCE=de.archive.ubuntu.com
- MIRROR_CATEGORIES=main,universe,restricted

View File

@ -6,8 +6,8 @@ export HOME=/root
# cleanup previous runs
rm -rf /var/mirror/ubuntu/.temp
while sleep 1 ; do
while true ; do
cd /opt/mirror && make
sleep ${MIRROR_UPDATE_INTERVAL_SECONDS}
sleep ${MIRROR_INTERVAL_SECONDS:-43200}
done

45
mirror/syncubuntu.sh Normal file → Executable file
View File

@ -4,42 +4,25 @@ set -x
THISDIR="$(cd $(dirname "$BASH_SOURCE") && pwd -P)"
DST="/var/mirror"
echo "$THIS_MIRROR_URL" > $DST/mirrors.txt
echo "http://172.17.0.1/ubuntu/" > $DST/mirrors.txt
chmod a+r $DST/mirrors.txt
mkdir -p "$DST/ubuntu"
export GNUPGHOME="/etc/debmirror/ubuntu"
SOURCEOPT="--nosource"
if [[ "$UBUNTU_ENABLE_SOURCE_MIRRORING" != "false" ]]; then
SOURCEOPT="--source"
fi
debmirror \
-a ${MIRROR_ARCHITECTURES:-amd64} \
--no-source \
-s ${MIRROR_CATEGORIES:-main,universe} \
-h ${UBUNTU_MIRROR_SOURCE:-archive.ubuntu.com} \
-d trusty,trusty-updates,trusty-security,trusty-backports \
-d xenial,xenial-updates,xenial-security,xenial-backports \
--ignore-small-errors \
-r /ubuntu \
--progress \
--method=http \
$DST/ubuntu
DONE=0
DIDIST="$(
echo -n "$UBUNTU_MIRROR_PROJECTS" | tr "," "\n" | grep -v "\-" | tr "\n" ","
)"
while [[ $DONE -eq 0 ]]; do
debmirror \
-a ${UBUNTU_MIRROR_ARCHITECTURES} \
-s ${UBUNTU_MIRROR_CATEGORIES} \
-h ${UBUNTU_MIRROR_UPSTREAM} \
-d ${UBUNTU_MIRROR_PROJECTS} \
$SOURCEOPT \
--di-dist="$DIDIST" \
--di-arch=arches \
--ignore-small-errors \
-r "$UBUNTU_MIRROR_UPSTREAM_PATH" \
--getcontents \
--progress \
--method=http \
$DST/ubuntu
if [[ $? -eq 0 ]]; then
DONE=1
fi
chmod -R a+rX $DST
sleep 1
done
chmod -R a+rX $DST

View File

@ -3,14 +3,11 @@ worker_processes auto;
pid /var/run/nginx.pid;
error_log /dev/stdout info;
events {
worker_connections 1024;
}
http {
access_log /dev/stdout;
include /etc/nginx/mime.types;
default_type application/octet-stream;
@ -33,12 +30,12 @@ http {
index index.html index.htm;
location /ubuntu/ {
#location /ubuntu/ {
# # alias, not root!
# # trailing slash important!
# alias /var/mirror/ubuntu/;
autoindex on;
}
# autoindex on;
#}
}
}

View File

@ -1,4 +1,4 @@
deb mirror://mirrors.ubuntu.com/mirrors.txt bionic main universe restricted multiverse
deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main universe restricted multiverse
deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main universe restricted multiverse
deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-backports main universe restricted multiverse
deb mirror://mirrors.ubuntu.com/mirrors.txt trusty main universe
deb mirror://mirrors.ubuntu.com/mirrors.txt trusty-updates main universe
deb mirror://mirrors.ubuntu.com/mirrors.txt trusty-backports main universe
deb mirror://mirrors.ubuntu.com/mirrors.txt trusty-security main universe