switch to debmirror

Dockerfile

@@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN \
     apt update && \
     apt upgrade -y && \
-    apt install -y golang git build-essential rsync && \
+    apt install -y golang git build-essential rsync debmirror && \
     go get -u github.com/ipfs/ipfs-update && \
     /root/go/bin/ipfs-update install latest && \
     rm -rf /root/go && \

sync.run

@@ -22,6 +22,13 @@ sleep 1 # prevent cpu spike on looping
 export MIRROR_DIR=/var/lib/ubuntumirror
 export IPFS_PATH=/var/lib/ipfs
 
+export GNUPGHOME=/etc/debmirror/ubuntu
+mkdir -p $GNUPGHOME
+chmod go-rwx $GNUPGHOME
+gpg --import /usr/share/keyrings/ubuntu-archive-keyring.gpg
+cp $GNUPGHOME/pubring.gpg $GNUPGHOME/trustedkeys.gpg
+
+chown -R ipfs:ipfs $GNUPGHOME
 
 if [[ ! -d $MIRROR_DIR ]]; then
     mkdir -p $MIRROR_DIR
@@ -30,39 +37,31 @@ fi
 
 chpst -u ipfs /bin/bash << '__EOF__'
 
+export GNUPGHOME=/etc/debmirror/ubuntu
 export IPFS=/usr/local/bin/ipfs
 export HOME=/home/ipfs
 export IPFS_FD_MAX=4096
 
 $IPFS config --json Experimental.FilestoreEnabled true
 
-# Two stage sync of the main Ubuntu mirror.
-# More info and options in https://wiki.ubuntu.com/Mirrors/Scripts.
-echo "Synchronizing with the main Ubuntu mirror, stage 1."
-rsync --recursive --times --links --safe-links --hard-links --stats \
-    --exclude "Packages*" --exclude "Sources*" --exclude "Release*" \
-    --exclude "InRelease" -v \
-    rsync://archive.ubuntu.com/ubuntu "$MIRROR_DIR"
+echo "Synchronizing with the main Ubuntu mirror."
 
-echo "Adding the mirror files to IPFS."
-hash="$(
-    $IPFS add --progress --local --nocopy --fscache --quieter
-        --recursive "$MIRROR_DIR" | tail -n1)"
-echo "Published IPFS hash ${hash}."
-
-echo "Updating IPNS."
-ipns="$($IPFS name publish "${hash}" | cut -d ' ' -f 3 | tr -d ':')"
-
-# FIXME(sneak) use debmirror to improve this in the future
-echo "Synchronizing with the main Ubuntu mirror, stage 2."
-rsync --recursive --times --links --safe-links --hard-links --stats \
-    --delete --delete-after -v \
-    rsync://archive.ubuntu.com/ubuntu \
-    "$MIRROR_DIR"
+debmirror \
+	-a ${MIRROR_ARCHITECTURES:-amd64} \
+	--no-source \
+	-s ${MIRROR_CATEGORIES:-main,universe,multiverse,restricted} \
+	-h ${UBUNTU_MIRROR_SOURCE:-archive.ubuntu.com} \
+	-d ${UBUNTU_MIRROR_PROJECTS:-xenial,xenial-updates,xenial-security,xenial-backports,bionic,bionic-updates,bionic-security,bionic-backports} \
+    --ignore-small-errors \
+	-r ${UBUNTU_MIRROR_PATH:-/ubuntu} \
+	--progress \
+	--method=http \
+	$MIRROR_DIR
 
 # Publish the files on IPFS.
 # This uses the experimental filestore:
 # https://github.com/ipfs/go-ipfs/issues/3397#issuecomment-284337564
+
 echo "Adding the mirror files to IPFS."
 hash="$($IPFS add --progress --local --nocopy --fscache --quieter
 --recursive "$MIRROR_DIR" | tail -n1)"