latest

2022-11-17 01:12:52 -08:00
parent 65d134c4ff
commit b55882ecde
12 changed files with 413 additions and 38 deletions
--- a/archive-to-cloud/tocloud
+++ b/archive-to-cloud/tocloud
@@ -0,0 +1,140 @@
+#!/bin/bash
+
+set -o pipefail
+set -e
+#set -x
+
+# decrypt like so:
+#
+# gpg -d ~/.paths/sneak-sync/secrets/backup-encryption-keys/2022-11-16.sneak-longterm-archive-age-key.gpg 2>/dev/null |
+# age -d -i - priv.age | tail -1 2>/dev/null |
+# age -d -i - archive.age
+
+YYYYMMDD="$(date -u +%Y-%m-%d)"
+YYYY="$(date -u +%Y)"
+MM="$(date -u +%m)"
+THIS="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+MY_PGP="5539AD00DE4C42F3AFE11575052443F4DF2A55C2"
+MY_LONGTERM_AGE_PUBKEY="age1278m9q7dp3chsh2dcy82qk27v047zywyvtxwnj4cvt0z65jw6a7q5dqhfj"
+TD="$(mktemp -d)"
+
+LOGDIR="$HOME/Documents/_SYSADMIN/$YYYY-$MM/$YYYYMMDD"
+if [[ ! -d "$LOGDIR" ]]; then
+    mkdir -p "$LOGDIR"
+fi
+#exec > >(tee -a $LOGDIR/$YYYYMMDD.$(date -u +%s).tocloud-backup.log) 2>&1
+
+function on_exit {
+    rm -rf "$TD"
+}
+
+function on_terminate {
+    echo "### Cleaning up..."
+    rm -rfv "$TD"
+}
+
+trap on_exit ERR EXIT
+trap on_terminate SIGINT SIGTERM
+
+function usage {
+    echo "usage: $0 <backupname> <dir>" > /dev/stderr
+    exit 1
+}
+
+#function getStorageBoxCredentials {
+#    gpg -d $HOME/.paths/sneak-sync/secrets/credentials/storagebox-offsite-backup-subaccount.json.gpg
+#}
+
+function main {
+    if [[ $# -ne 2 ]]; then
+        usage
+    fi
+
+    if [[ -z "$2" ]]; then
+        usage
+    fi
+
+    if [[ -d "$2" ]]; then
+        SRC="$(cd "$2" && pwd -P)"
+    else
+        SRC="$2"
+    fi
+
+    if [[ ! -r "$SRC" ]]; then
+        usage
+    fi
+
+    BACKUPNAME="$YYYYMMDD.$1.$(date +%s)"
+    time do_backup "$BACKUPNAME" "$SRC"
+}
+
+function do_backup {
+
+    BACKUPNAME="$1"
+    SRC="$2"
+
+    cd "$TD"
+    mkdir "$BACKUPNAME"
+    cd "$TD/$BACKUPNAME"
+    echo "### Beginning backup $BACKUPNAME"
+    echo "### Temporary Working Directory: $TD"
+    AGE_PRIV=$(age-keygen 2> ./pub.txt)
+    age -r $MY_LONGTERM_AGE_PUBKEY <<< "$AGE_PRIV" > ./priv.age
+    PUB="$(awk -F' ' '{print $3}' < ./pub.txt)"
+    echo "### Backup Archive Session Pubkey: $PUB"
+    echo "$PUB" > ./pub.txt  # overwrite non-clean one
+    gpg --trust-model always \
+        --compress-algo none \
+        -r $MY_PGP --encrypt \
+        -a <<< "$AGE_PRIV" \
+        > ./priv.sneak-pgp-DF2A55C2.asc
+    echo "### Backup Source Size: $(du -sh "$SRC" | awk '{print $1}')"
+    echo "### Indexing backup..."
+    (find "$SRC" -type f \( -exec sha1sum {} \; \)) |
+        tee /dev/stderr |
+        age -r $PUB > "$TD/$BACKUPNAME/archive-sums.txt.age"
+    echo "### Compressing backup..."
+    tar -P -c "$SRC" |
+        nice -n 20 zstd --compress -T0 -10 |
+        pv --delay-start 3 --progress --eta --size $(du -sb "$SRC" | awk '{print $1}') |
+        age -r $PUB |
+        split -d -b 1G -a 4 - $TD/$BACKUPNAME/archive.tar.zst.age.
+        COUNT="$(cd "$TD/$BACKUPNAME" && ls -1 archive.tar.zst.age.* | wc -l | awk '{print $1}')"
+        if [[ "$COUNT" -eq 1 ]]; then
+            mv "$TD/$BACKUPNAME/archive.tar.zst.age.0000" "$TD/$BACKUPNAME/archive.tar.zst.age"
+        fi
+    cd "$TD/$BACKUPNAME"
+    echo "### Backup Compressed Archive Size: $(du -sh "$TD/$BACKUPNAME" | awk '{print $1}')"
+    echo "### Creating Checksums..."
+    shasum archive.tar.zst.age* archive-sums.txt.age | tee -a SHASUMS.txt
+    echo "### Signing Checksums..."
+    gpg --default-key $MY_PGP --output SHASUMS.txt.gpg --detach-sig SHASUMS.txt
+    #tar -c . | pv --progress --eta --size $(du -sb "$TD/$BACKUPNAME" | awk '{print $1}') |
+    #ssh fsn1-storagebox-10T "mkdir -p $BACKUPNAME ; cd $BACKUPNAME && tar xvf -"
+    #while ! rsync -avvvcP --delete "$TD/$BACKUPNAME/" fsn1-storagebox-10T:"$BACKUPNAME"/
+    #    sleep 1
+    #done
+    echo "### Uploading data..."
+    # i want to use rsync here but rclone gives much better total
+    # progress/ETA display.
+    rclone sync \
+        --retries 99999 \
+        --progress \
+        --progress-terminal-title \
+        --stats-unit bits \
+        "$TD/$BACKUPNAME" \
+        fsn1-storagebox-10T:"$BACKUPNAME"/
+    # belt and suspenders
+    echo "### Verifying uploaded data checksums..."
+    rsync -acP "$TD/$BACKUPNAME/" fsn1-storagebox-10T:"$BACKUPNAME"/
+    RETVAL="$?"
+    if [[ "$RETVAL" -eq 0 ]]; then
+        echo "### Backup successful."
+        exit 0
+    else
+        echo "### Problem detected."
+        exit 1
+    fi
+}
+
+main "$@"