Merge branch 'development'

This commit is contained in:
David Duque 2020-07-22 12:44:04 +01:00
commit 16ae3038b3
No known key found for this signature in database
GPG Key ID: 2F327738A3C0AE3A
15 changed files with 115 additions and 74 deletions

30
Vagrantfile vendored
View File

@ -1,12 +1,11 @@
# -*- mode: ruby -*- # -*- mode: ruby -*-
# vi: set ft=ruby : # vi: set ft=ruby :
Vagrant.configure("2") do |config| Vagrant.configure("2") do |config|
# Recreate our conditions config.vm.box = "ubuntu/focal64"
config.vm.box = "generic/debian10" config.vm.provider :virtualbox do |vb|
config.vm.provider "hyperv" do |v| vb.customize ["modifyvm", :id, "--cpus", 1, "--memory", 1024]
v.memory = 1024
v.cpus = 1
end end
# Network config: Since it's a mail server, the machine must be connected # Network config: Since it's a mail server, the machine must be connected
@ -24,24 +23,9 @@ Vagrant.configure("2") do |config|
export PUBLIC_IP=auto export PUBLIC_IP=auto
export PUBLIC_IPV6=auto export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=auto export PRIMARY_HOSTNAME=auto
#export SKIP_NETWORK_CHECKS=1 export SKIP_NETWORK_CHECKS=1
if [ ! git ]
then
apt update
apt install git
fi
if [ ! -d /mailinabox ];
then
git clone https://github.com/ddavness/power-mailinabox.git /mailinabox
fi
# Start the setup script. # Start the setup script.
cd /mailinabox cd /vagrant
git checkout development
git pull
setup/start.sh setup/start.sh
SH SH
end end

View File

@ -7,6 +7,5 @@
## your own --- please do not ask for help from us. ## your own --- please do not ask for help from us.
upstream php-fpm { upstream php-fpm {
server unix:/var/run/php/php!!___PHPVER___!!-fpm.sock; server unix:/var/run/php/php{{phpver}}-fpm.sock;
} }

View File

@ -10,9 +10,9 @@
import os, os.path, shutil, glob, re, datetime, sys import os, os.path, shutil, glob, re, datetime, sys
import dateutil.parser, dateutil.relativedelta, dateutil.tz import dateutil.parser, dateutil.relativedelta, dateutil.tz
import rtyaml import rtyaml
from exclusiveprocess import Lock from exclusiveprocess import Lock, CannotAcquireLock
from utils import load_environment, shell, wait_for_service, fix_boto from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version
rsync_ssh_options = [ rsync_ssh_options = [
"--ssh-options= -i /root/.ssh/id_rsa_miab", "--ssh-options= -i /root/.ssh/id_rsa_miab",
@ -20,7 +20,7 @@ rsync_ssh_options = [
] ]
def backup_status(env): def backup_status(env):
# If backups are dissbled, return no status. # If backups are disabled, return no status.
config = get_backup_config(env) config = get_backup_config(env)
if config["target"] == "off": if config["target"] == "off":
return { } return { }
@ -210,13 +210,22 @@ def get_target_type(config):
protocol = config["target"].split(":")[0] protocol = config["target"].split(":")[0]
return protocol return protocol
def perform_backup(full_backup): def perform_backup(full_backup, user_initiated=False):
env = load_environment() env = load_environment()
php_fpm = f"php{get_php_version()}-fpm"
# Create an global exclusive lock so that the backup script # Create an global exclusive lock so that the backup script
# cannot be run more than one. # cannot be run more than one.
Lock(die=True).forever() lock = Lock(name="mailinabox_backup_daemon", die=(not user_initiated))
if user_initiated:
# God forgive me for what I'm about to do
try:
lock._acquire()
except CannotAcquireLock:
return "Another backup is already being done!"
else:
lock.forever()
config = get_backup_config(env) config = get_backup_config(env)
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup') backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
backup_cache_dir = os.path.join(backup_root, 'cache') backup_cache_dir = os.path.join(backup_root, 'cache')
@ -247,7 +256,7 @@ def perform_backup(full_backup):
if quit: if quit:
sys.exit(code) sys.exit(code)
service_command("php!!___PHPVER___!!-fpm", "stop", quit=True) service_command(php_fpm, "stop", quit=True)
service_command("postfix", "stop", quit=True) service_command("postfix", "stop", quit=True)
service_command("dovecot", "stop", quit=True) service_command("dovecot", "stop", quit=True)
@ -281,7 +290,7 @@ def perform_backup(full_backup):
# Start services again. # Start services again.
service_command("dovecot", "start", quit=False) service_command("dovecot", "start", quit=False)
service_command("postfix", "start", quit=False) service_command("postfix", "start", quit=False)
service_command("php!!___PHPVER___!!-fpm", "start", quit=False) service_command(php_fpm, "start", quit=False)
# Remove old backups. This deletes all backup data no longer needed # Remove old backups. This deletes all backup data no longer needed
# from more than 3 days ago. # from more than 3 days ago.
@ -329,8 +338,13 @@ def perform_backup(full_backup):
# backup. Since it checks that dovecot and postfix are running, block for a # backup. Since it checks that dovecot and postfix are running, block for a
# bit (maximum of 10 seconds each) to give each a chance to finish restarting # bit (maximum of 10 seconds each) to give each a chance to finish restarting
# before the status checks might catch them down. See #381. # before the status checks might catch them down. See #381.
wait_for_service(25, True, env, 10) if user_initiated:
wait_for_service(993, True, env, 10) # God forgive me for what I'm about to do
lock._release()
# We don't need to wait for the services to be up in this case
else:
wait_for_service(25, True, env, 10)
wait_for_service(993, True, env, 10)
def run_duplicity_verification(): def run_duplicity_verification():
env = load_environment() env = load_environment()

View File

@ -101,9 +101,12 @@ def index():
utils.fix_boto() # must call prior to importing boto utils.fix_boto() # must call prior to importing boto
import boto.s3 import boto.s3
backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()] backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
lsb=utils.shell("check_output", ["/usr/bin/lsb_release", "-d"])
return render_template('index.html', return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'], hostname=env['PRIMARY_HOSTNAME'],
distname=lsb[lsb.find("\t")+1:-1],
storage_root=env['STORAGE_ROOT'], storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist, no_users_exist=no_users_exist,
@ -440,9 +443,8 @@ def system_status():
self.items[-1]["extra"].append({ "text": message, "monospace": monospace }) self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
output = WebOutput() output = WebOutput()
# Create a temporary pool of processes for the status checks # Create a temporary pool of processes for the status checks
pool = multiprocessing.pool.Pool(processes=5) with multiprocessing.pool.Pool(processes=5) as pool:
run_checks(False, env, output, pool) run_checks(False, env, output, pool)
pool.terminate()
return json_response(output.items) return json_response(output.items)
@app.route('/system/updates') @app.route('/system/updates')
@ -509,6 +511,19 @@ def backup_set_custom():
request.form.get('min_age', '') request.form.get('min_age', '')
)) ))
@app.route('/system/backup/new', methods=["POST"])
@authorized_personnel_only
def backup_new():
from backup import perform_backup, get_backup_config
# If backups are disabled, don't perform the backup
config = get_backup_config(env)
if config["target"] == "off":
return "Backups are disabled in this machine. Nothing was done."
msg = perform_backup(request.form.get('full', False) == 'true', True)
return "OK" if msg is None else msg
@app.route('/system/privacy', methods=["GET"]) @app.route('/system/privacy', methods=["GET"])
@authorized_personnel_only @authorized_personnel_only
def privacy_status_get(): def privacy_status_get():

View File

@ -1036,12 +1036,12 @@ if __name__ == "__main__":
env = load_environment() env = load_environment()
if len(sys.argv) == 1: if len(sys.argv) == 1:
pool = multiprocessing.pool.Pool(processes=10) with multiprocessing.pool.Pool(processes=10) as pool:
run_checks(False, env, ConsoleOutput(), pool) run_checks(False, env, ConsoleOutput(), pool)
elif sys.argv[1] == "--show-changes": elif sys.argv[1] == "--show-changes":
pool = multiprocessing.pool.Pool(processes=10) with multiprocessing.pool.Pool(processes=10) as pool:
run_and_output_changes(env, pool) run_and_output_changes(env, pool)
elif sys.argv[1] == "--check-primary-hostname": elif sys.argv[1] == "--check-primary-hostname":
# See if the primary hostname appears resolvable and has a signed certificate. # See if the primary hostname appears resolvable and has a signed certificate.

View File

@ -180,7 +180,7 @@
<hr> <hr>
<footer> <footer>
<p>This is a <a href="https://github.com/ddavness/power-mailinabox">Power Mail-in-a-Box</a>, using !!___DIST_TAG___!! <p>This is a <a href="https://github.com/ddavness/power-mailinabox">Power Mail-in-a-Box</a> - {{distname}}
</p> </p>
</footer> </footer>
</div> <!-- /container --> </div> <!-- /container -->

View File

@ -12,7 +12,7 @@
<form class="form-horizontal" role="form" onsubmit="set_custom_backup(); return false;"> <form class="form-horizontal" role="form" onsubmit="set_custom_backup(); return false;">
<div class="form-group"> <div class="form-group">
<label for="backup-target-type" class="col-sm-2 control-label">Backup to:</label> <label for="backup-target-type" class="col-sm-2 control-label">Backup to:</label>
<div class="col-sm-2"> <div class="col-sm-3">
<select class="form-control" rows="1" id="backup-target-type" onchange="toggle_form()"> <select class="form-control" rows="1" id="backup-target-type" onchange="toggle_form()">
<option value="off">Nowhere (Disable Backups)</option> <option value="off">Nowhere (Disable Backups)</option>
<option value="local">{{hostname}}</option> <option value="local">{{hostname}}</option>
@ -140,6 +140,11 @@
<tbody> <tbody>
</tbody> </tbody>
</table> </table>
<!-- Hide these buttons until we're sure we can use them :) -->
<button id="create-full-backup-button" class="btn btn-primary" onclick="do_backup(true)" style="display: none;">Create Full Backup Now</button>
<button id="create-incremental-backup-button" class="btn btn-primary" onclick="do_backup(false)" style="display: none;">Create Incremental Backup Now</button>
<script> <script>
function toggle_form() { function toggle_form() {
@ -186,12 +191,17 @@ function show_system_backup() {
if (typeof r.backups == "undefined") { if (typeof r.backups == "undefined") {
var tr = $('<tr><td colspan="3">Backups are turned off.</td></tr>'); var tr = $('<tr><td colspan="3">Backups are turned off.</td></tr>');
$('#backup-status tbody').append(tr); $('#backup-status tbody').append(tr);
$('#create-full-backup-button').css("display","none")
$('#create-incremental-backup-button').css("display","none")
return; return;
} else if (r.backups.length == 0) { } else if (r.backups.length == 0) {
var tr = $('<tr><td colspan="3">No backups have been made yet.</td></tr>'); var tr = $('<tr><td colspan="3">No backups have been made yet.</td></tr>');
$('#backup-status tbody').append(tr); $('#backup-status tbody').append(tr);
} }
// Backups ARE enabled.
$('#create-full-backup-button').css("display","unset")
$('#create-incremental-backup-button').css("display","unset")
for (var i = 0; i < r.backups.length; i++) { for (var i = 0; i < r.backups.length; i++) {
var b = r.backups[i]; var b = r.backups[i];
var tr = $('<tr/>'); var tr = $('<tr/>');
@ -303,4 +313,29 @@ function init_inputs(target_type) {
set_host($('#backup-target-s3-host-select').val()); set_host($('#backup-target-s3-host-select').val());
} }
} }
function do_backup(is_full) {
let disclaimer = "The backup process will pause some services (such as PHP, Postfix and Dovecot). Depending on the size of the data this can take a while."
if (!is_full) {
disclaimer += "\nDepending on the amount of incremental backups done after the last full backup, the box may decide to do a full backup instead."
}
show_modal_confirm("Warning!", disclaimer, "Start Backup", () => {
api(
"/system/backup/new",
"POST",
{
full: is_full
},
function(r) {
// use .text() --- it's a text response, not html
show_modal_error("Backup configuration", $("<p/>").text(r), function() { if (r == "OK") show_system_backup(); }); // refresh after modal on success
},
function(r) {
// use .text() --- it's a text response, not html
show_modal_error("Backup configuration", $("<p/>").text(r));
});
return false;
})
}
</script> </script>

View File

@ -182,6 +182,9 @@ def fix_boto():
import os import os
os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg" os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
def get_php_version():
# Gets the version of PHP installed in the system.
return shell("check_output", ["/usr/bin/php", "-v"])[4:7]
if __name__ == "__main__": if __name__ == "__main__":
from web_update import get_web_domains from web_update import get_web_domains

View File

@ -7,7 +7,7 @@ import os.path, re, rtyaml
from mailconfig import get_mail_domains from mailconfig import get_mail_domains
from dns_update import get_custom_dns_config, get_dns_zones from dns_update import get_custom_dns_config, get_dns_zones
from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
from utils import shell, safe_domain_name, sort_domains from utils import shell, safe_domain_name, sort_domains, get_php_version
def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True): def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
# What domains should we serve HTTP(S) for? # What domains should we serve HTTP(S) for?
@ -76,6 +76,7 @@ def do_web_update(env):
# Build an nginx configuration file. # Build an nginx configuration file.
nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read() nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
nginx_conf = re.sub("{{phpver}}", get_php_version(), nginx_conf)
# Load the templates. # Load the templates.
template0 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read() template0 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()

View File

@ -222,14 +222,6 @@ function git_clone {
rm -rf $TMPPATH rm -rf $TMPPATH
} }
OS=`lsb_release -d | sed 's/.*:\s*//' ` function php_version {
php --version | head -n 1 | cut -d " " -f 2 | cut -c 1-3
# Expected php version }
if [ "$OS" == "Debian GNU/Linux 10 (buster)" ]; then
export PHP_VERSION="7.3"
elif [ "$OS" == "Ubuntu 20.04 LTS" ]; then
export PHP_VERSION="7.4"
fi
sed -i "s|!!___PHPVER___!!|${PHP_VERSION}|g" conf/nginx-top.conf
sed -i "s|!!___PHPVER___!!|${PHP_VERSION}|g" management/backup.py

View File

@ -112,7 +112,7 @@ fi
if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
# Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail. # Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail.
service php$PHP_VERSION-fpm stop &> /dev/null || /bin/true service php$(php_version)-fpm stop &> /dev/null || /bin/true
# Backup the existing ownCloud/Nextcloud. # Backup the existing ownCloud/Nextcloud.
# Create a backup directory to store the current installation and database to # Create a backup directory to store the current installation and database to
@ -285,7 +285,7 @@ if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
# Set PHP FPM values to support large file uploads # Set PHP FPM values to support large file uploads
# (semicolon is the comment character in this file, hashes produce deprecation warnings) # (semicolon is the comment character in this file, hashes produce deprecation warnings)
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
upload_max_filesize=16G \ upload_max_filesize=16G \
post_max_size=16G \ post_max_size=16G \
output_buffering=16384 \ output_buffering=16384 \
@ -294,7 +294,7 @@ management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \
short_open_tag=On short_open_tag=On
# Set Nextcloud recommended opcache settings # Set Nextcloud recommended opcache settings
management/editconf.py /etc/php/$PHP_VERSION/cli/conf.d/10-opcache.ini -c ';' \ management/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \
opcache.enable=1 \ opcache.enable=1 \
opcache.enable_cli=1 \ opcache.enable_cli=1 \
opcache.interned_strings_buffer=8 \ opcache.interned_strings_buffer=8 \
@ -304,8 +304,8 @@ management/editconf.py /etc/php/$PHP_VERSION/cli/conf.d/10-opcache.ini -c ';' \
opcache.revalidate_freq=1 opcache.revalidate_freq=1
# If apc is explicitly disabled we need to enable it # If apc is explicitly disabled we need to enable it
if grep -q apc.enabled=0 /etc/php/$PHP_VERSION/mods-available/apcu.ini; then if grep -q apc.enabled=0 /etc/php/$(php_version)/mods-available/apcu.ini; then
management/editconf.py /etc/php/$PHP_VERSION/mods-available/apcu.ini -c ';' \ management/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \
apc.enabled=1 apc.enabled=1
fi fi
@ -330,4 +330,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud
# ``` # ```
# Enable PHP modules and restart PHP. # Enable PHP modules and restart PHP.
restart_service php$PHP_VERSION-fpm restart_service php$(php_version)-fpm

View File

@ -18,8 +18,6 @@ if [ "$OS" != "Debian GNU/Linux 10 (buster)" -a "$OS" != "Ubuntu 20.04 LTS" ]; t
exit 1 exit 1
fi fi
sed -i "s|!!___DIST_TAG___!!|${OS}|g" management/templates/index.html
# Check that we have enough memory. # Check that we have enough memory.
# #
# /proc/meminfo reports free memory in kibibytes. Our baseline will be 512 MB, # /proc/meminfo reports free memory in kibibytes. Our baseline will be 512 MB,

View File

@ -46,15 +46,15 @@ management/editconf.py /etc/nginx/nginx.conf -s \
ssl_protocols="TLSv1.2 TLSv1.3;" ssl_protocols="TLSv1.2 TLSv1.3;"
# Tell PHP not to expose its version number in the X-Powered-By header. # Tell PHP not to expose its version number in the X-Powered-By header.
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
expose_php=Off expose_php=Off
# Set PHPs default charset to UTF-8, since we use it. See #367. # Set PHPs default charset to UTF-8, since we use it. See #367.
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
default_charset="UTF-8" default_charset="UTF-8"
# Configure the path environment for php-fpm # Configure the path environment for php-fpm
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
env[PATH]=/usr/local/bin:/usr/bin:/bin \ env[PATH]=/usr/local/bin:/usr/bin:/bin \
# Configure php-fpm based on the amount of memory the machine has # Configure php-fpm based on the amount of memory the machine has
@ -64,7 +64,7 @@ management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true) TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true)
if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ] if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ]
then then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=ondemand \ pm=ondemand \
pm.max_children=8 \ pm.max_children=8 \
pm.start_servers=2 \ pm.start_servers=2 \
@ -72,7 +72,7 @@ then
pm.max_spare_servers=3 pm.max_spare_servers=3
elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ] elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ]
then then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=ondemand \ pm=ondemand \
pm.max_children=16 \ pm.max_children=16 \
pm.start_servers=4 \ pm.start_servers=4 \
@ -80,14 +80,14 @@ then
pm.max_spare_servers=6 pm.max_spare_servers=6
elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ] elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ]
then then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=dynamic \ pm=dynamic \
pm.max_children=60 \ pm.max_children=60 \
pm.start_servers=6 \ pm.start_servers=6 \
pm.min_spare_servers=3 \ pm.min_spare_servers=3 \
pm.max_spare_servers=9 pm.max_spare_servers=9
else else
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \ management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=dynamic \ pm=dynamic \
pm.max_children=120 \ pm.max_children=120 \
pm.start_servers=12 \ pm.start_servers=12 \
@ -147,7 +147,7 @@ chown -R $STORAGE_USER $STORAGE_ROOT/www
# Start services. # Start services.
restart_service nginx restart_service nginx
restart_service php$PHP_VERSION-fpm restart_service php$(php_version)-fpm
# Open ports. # Open ports.
ufw_allow http ufw_allow http

View File

@ -198,4 +198,4 @@ chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
# Enable PHP modules. # Enable PHP modules.
phpenmod -v php mcrypt imap phpenmod -v php mcrypt imap
restart_service php$PHP_VERSION-fpm restart_service php$(php_version)-fpm

View File

@ -102,7 +102,7 @@ EOF
# Restart service. # Restart service.
restart_service php$PHP_VERSION-fpm restart_service php$(php_version)-fpm
# Fix states after upgrade # Fix states after upgrade