1
0
mirror of https://github.com/mail-in-a-box/mailinabox.git synced 2024-12-24 07:37:04 +00:00

Merge branch 'development'

This commit is contained in:
David Duque 2020-07-22 12:44:04 +01:00
commit 16ae3038b3
No known key found for this signature in database
GPG Key ID: 2F327738A3C0AE3A
15 changed files with 115 additions and 74 deletions

30
Vagrantfile vendored
View File

@ -1,12 +1,11 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
Vagrant.configure("2") do |config|
# Recreate our conditions
config.vm.box = "generic/debian10"
config.vm.provider "hyperv" do |v|
v.memory = 1024
v.cpus = 1
config.vm.box = "ubuntu/focal64"
config.vm.provider :virtualbox do |vb|
vb.customize ["modifyvm", :id, "--cpus", 1, "--memory", 1024]
end
# Network config: Since it's a mail server, the machine must be connected
@ -24,24 +23,9 @@ Vagrant.configure("2") do |config|
export PUBLIC_IP=auto
export PUBLIC_IPV6=auto
export PRIMARY_HOSTNAME=auto
#export SKIP_NETWORK_CHECKS=1
if [ ! git ]
then
apt update
apt install git
fi
if [ ! -d /mailinabox ];
then
git clone https://github.com/ddavness/power-mailinabox.git /mailinabox
fi
export SKIP_NETWORK_CHECKS=1
# Start the setup script.
cd /mailinabox
git checkout development
git pull
cd /vagrant
setup/start.sh
SH
end
end

View File

@ -7,6 +7,5 @@
## your own --- please do not ask for help from us.
upstream php-fpm {
server unix:/var/run/php/php!!___PHPVER___!!-fpm.sock;
server unix:/var/run/php/php{{phpver}}-fpm.sock;
}

View File

@ -10,9 +10,9 @@
import os, os.path, shutil, glob, re, datetime, sys
import dateutil.parser, dateutil.relativedelta, dateutil.tz
import rtyaml
from exclusiveprocess import Lock
from exclusiveprocess import Lock, CannotAcquireLock
from utils import load_environment, shell, wait_for_service, fix_boto
from utils import load_environment, shell, wait_for_service, fix_boto, get_php_version
rsync_ssh_options = [
"--ssh-options= -i /root/.ssh/id_rsa_miab",
@ -20,7 +20,7 @@ rsync_ssh_options = [
]
def backup_status(env):
# If backups are dissbled, return no status.
# If backups are disabled, return no status.
config = get_backup_config(env)
if config["target"] == "off":
return { }
@ -210,13 +210,22 @@ def get_target_type(config):
protocol = config["target"].split(":")[0]
return protocol
def perform_backup(full_backup):
def perform_backup(full_backup, user_initiated=False):
env = load_environment()
php_fpm = f"php{get_php_version()}-fpm"
# Create an global exclusive lock so that the backup script
# cannot be run more than one.
Lock(die=True).forever()
lock = Lock(name="mailinabox_backup_daemon", die=(not user_initiated))
if user_initiated:
# God forgive me for what I'm about to do
try:
lock._acquire()
except CannotAcquireLock:
return "Another backup is already being done!"
else:
lock.forever()
config = get_backup_config(env)
backup_root = os.path.join(env["STORAGE_ROOT"], 'backup')
backup_cache_dir = os.path.join(backup_root, 'cache')
@ -247,7 +256,7 @@ def perform_backup(full_backup):
if quit:
sys.exit(code)
service_command("php!!___PHPVER___!!-fpm", "stop", quit=True)
service_command(php_fpm, "stop", quit=True)
service_command("postfix", "stop", quit=True)
service_command("dovecot", "stop", quit=True)
@ -281,7 +290,7 @@ def perform_backup(full_backup):
# Start services again.
service_command("dovecot", "start", quit=False)
service_command("postfix", "start", quit=False)
service_command("php!!___PHPVER___!!-fpm", "start", quit=False)
service_command(php_fpm, "start", quit=False)
# Remove old backups. This deletes all backup data no longer needed
# from more than 3 days ago.
@ -329,8 +338,13 @@ def perform_backup(full_backup):
# backup. Since it checks that dovecot and postfix are running, block for a
# bit (maximum of 10 seconds each) to give each a chance to finish restarting
# before the status checks might catch them down. See #381.
wait_for_service(25, True, env, 10)
wait_for_service(993, True, env, 10)
if user_initiated:
# God forgive me for what I'm about to do
lock._release()
# We don't need to wait for the services to be up in this case
else:
wait_for_service(25, True, env, 10)
wait_for_service(993, True, env, 10)
def run_duplicity_verification():
env = load_environment()

View File

@ -101,9 +101,12 @@ def index():
utils.fix_boto() # must call prior to importing boto
import boto.s3
backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
lsb=utils.shell("check_output", ["/usr/bin/lsb_release", "-d"])
return render_template('index.html',
hostname=env['PRIMARY_HOSTNAME'],
distname=lsb[lsb.find("\t")+1:-1],
storage_root=env['STORAGE_ROOT'],
no_users_exist=no_users_exist,
@ -440,9 +443,8 @@ def system_status():
self.items[-1]["extra"].append({ "text": message, "monospace": monospace })
output = WebOutput()
# Create a temporary pool of processes for the status checks
pool = multiprocessing.pool.Pool(processes=5)
run_checks(False, env, output, pool)
pool.terminate()
with multiprocessing.pool.Pool(processes=5) as pool:
run_checks(False, env, output, pool)
return json_response(output.items)
@app.route('/system/updates')
@ -509,6 +511,19 @@ def backup_set_custom():
request.form.get('min_age', '')
))
@app.route('/system/backup/new', methods=["POST"])
@authorized_personnel_only
def backup_new():
from backup import perform_backup, get_backup_config
# If backups are disabled, don't perform the backup
config = get_backup_config(env)
if config["target"] == "off":
return "Backups are disabled in this machine. Nothing was done."
msg = perform_backup(request.form.get('full', False) == 'true', True)
return "OK" if msg is None else msg
@app.route('/system/privacy', methods=["GET"])
@authorized_personnel_only
def privacy_status_get():

View File

@ -1036,12 +1036,12 @@ if __name__ == "__main__":
env = load_environment()
if len(sys.argv) == 1:
pool = multiprocessing.pool.Pool(processes=10)
run_checks(False, env, ConsoleOutput(), pool)
with multiprocessing.pool.Pool(processes=10) as pool:
run_checks(False, env, ConsoleOutput(), pool)
elif sys.argv[1] == "--show-changes":
pool = multiprocessing.pool.Pool(processes=10)
run_and_output_changes(env, pool)
with multiprocessing.pool.Pool(processes=10) as pool:
run_and_output_changes(env, pool)
elif sys.argv[1] == "--check-primary-hostname":
# See if the primary hostname appears resolvable and has a signed certificate.

View File

@ -180,7 +180,7 @@
<hr>
<footer>
<p>This is a <a href="https://github.com/ddavness/power-mailinabox">Power Mail-in-a-Box</a>, using !!___DIST_TAG___!!
<p>This is a <a href="https://github.com/ddavness/power-mailinabox">Power Mail-in-a-Box</a> - {{distname}}
</p>
</footer>
</div> <!-- /container -->

View File

@ -12,7 +12,7 @@
<form class="form-horizontal" role="form" onsubmit="set_custom_backup(); return false;">
<div class="form-group">
<label for="backup-target-type" class="col-sm-2 control-label">Backup to:</label>
<div class="col-sm-2">
<div class="col-sm-3">
<select class="form-control" rows="1" id="backup-target-type" onchange="toggle_form()">
<option value="off">Nowhere (Disable Backups)</option>
<option value="local">{{hostname}}</option>
@ -140,6 +140,11 @@
<tbody>
</tbody>
</table>
<!-- Hide these buttons until we're sure we can use them :) -->
<button id="create-full-backup-button" class="btn btn-primary" onclick="do_backup(true)" style="display: none;">Create Full Backup Now</button>
<button id="create-incremental-backup-button" class="btn btn-primary" onclick="do_backup(false)" style="display: none;">Create Incremental Backup Now</button>
<script>
function toggle_form() {
@ -186,12 +191,17 @@ function show_system_backup() {
if (typeof r.backups == "undefined") {
var tr = $('<tr><td colspan="3">Backups are turned off.</td></tr>');
$('#backup-status tbody').append(tr);
$('#create-full-backup-button').css("display","none")
$('#create-incremental-backup-button').css("display","none")
return;
} else if (r.backups.length == 0) {
var tr = $('<tr><td colspan="3">No backups have been made yet.</td></tr>');
$('#backup-status tbody').append(tr);
}
// Backups ARE enabled.
$('#create-full-backup-button').css("display","unset")
$('#create-incremental-backup-button').css("display","unset")
for (var i = 0; i < r.backups.length; i++) {
var b = r.backups[i];
var tr = $('<tr/>');
@ -303,4 +313,29 @@ function init_inputs(target_type) {
set_host($('#backup-target-s3-host-select').val());
}
}
function do_backup(is_full) {
let disclaimer = "The backup process will pause some services (such as PHP, Postfix and Dovecot). Depending on the size of the data this can take a while."
if (!is_full) {
disclaimer += "\nDepending on the amount of incremental backups done after the last full backup, the box may decide to do a full backup instead."
}
show_modal_confirm("Warning!", disclaimer, "Start Backup", () => {
api(
"/system/backup/new",
"POST",
{
full: is_full
},
function(r) {
// use .text() --- it's a text response, not html
show_modal_error("Backup configuration", $("<p/>").text(r), function() { if (r == "OK") show_system_backup(); }); // refresh after modal on success
},
function(r) {
// use .text() --- it's a text response, not html
show_modal_error("Backup configuration", $("<p/>").text(r));
});
return false;
})
}
</script>

View File

@ -182,6 +182,9 @@ def fix_boto():
import os
os.environ["BOTO_CONFIG"] = "/etc/boto3.cfg"
def get_php_version():
# Gets the version of PHP installed in the system.
return shell("check_output", ["/usr/bin/php", "-v"])[4:7]
if __name__ == "__main__":
from web_update import get_web_domains

View File

@ -7,7 +7,7 @@ import os.path, re, rtyaml
from mailconfig import get_mail_domains
from dns_update import get_custom_dns_config, get_dns_zones
from ssl_certificates import get_ssl_certificates, get_domain_ssl_files, check_certificate
from utils import shell, safe_domain_name, sort_domains
from utils import shell, safe_domain_name, sort_domains, get_php_version
def get_web_domains(env, include_www_redirects=True, exclude_dns_elsewhere=True):
# What domains should we serve HTTP(S) for?
@ -76,6 +76,7 @@ def do_web_update(env):
# Build an nginx configuration file.
nginx_conf = open(os.path.join(os.path.dirname(__file__), "../conf/nginx-top.conf")).read()
nginx_conf = re.sub("{{phpver}}", get_php_version(), nginx_conf)
# Load the templates.
template0 = open(os.path.join(os.path.dirname(__file__), "../conf/nginx.conf")).read()

View File

@ -222,14 +222,6 @@ function git_clone {
rm -rf $TMPPATH
}
OS=`lsb_release -d | sed 's/.*:\s*//' `
# Expected php version
if [ "$OS" == "Debian GNU/Linux 10 (buster)" ]; then
export PHP_VERSION="7.3"
elif [ "$OS" == "Ubuntu 20.04 LTS" ]; then
export PHP_VERSION="7.4"
fi
sed -i "s|!!___PHPVER___!!|${PHP_VERSION}|g" conf/nginx-top.conf
sed -i "s|!!___PHPVER___!!|${PHP_VERSION}|g" management/backup.py
function php_version {
php --version | head -n 1 | cut -d " " -f 2 | cut -c 1-3
}

View File

@ -112,7 +112,7 @@ fi
if [ ! -d /usr/local/lib/owncloud/ ] || [[ ! ${CURRENT_NEXTCLOUD_VER} =~ ^$nextcloud_ver ]]; then
# Stop php-fpm if running. If theyre not running (which happens on a previously failed install), dont bail.
service php$PHP_VERSION-fpm stop &> /dev/null || /bin/true
service php$(php_version)-fpm stop &> /dev/null || /bin/true
# Backup the existing ownCloud/Nextcloud.
# Create a backup directory to store the current installation and database to
@ -285,7 +285,7 @@ if [ \( $? -ne 0 \) -a \( $? -ne 3 \) ]; then exit 1; fi
# Set PHP FPM values to support large file uploads
# (semicolon is the comment character in this file, hashes produce deprecation warnings)
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
upload_max_filesize=16G \
post_max_size=16G \
output_buffering=16384 \
@ -294,7 +294,7 @@ management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \
short_open_tag=On
# Set Nextcloud recommended opcache settings
management/editconf.py /etc/php/$PHP_VERSION/cli/conf.d/10-opcache.ini -c ';' \
management/editconf.py /etc/php/$(php_version)/cli/conf.d/10-opcache.ini -c ';' \
opcache.enable=1 \
opcache.enable_cli=1 \
opcache.interned_strings_buffer=8 \
@ -304,8 +304,8 @@ management/editconf.py /etc/php/$PHP_VERSION/cli/conf.d/10-opcache.ini -c ';' \
opcache.revalidate_freq=1
# If apc is explicitly disabled we need to enable it
if grep -q apc.enabled=0 /etc/php/$PHP_VERSION/mods-available/apcu.ini; then
management/editconf.py /etc/php/$PHP_VERSION/mods-available/apcu.ini -c ';' \
if grep -q apc.enabled=0 /etc/php/$(php_version)/mods-available/apcu.ini; then
management/editconf.py /etc/php/$(php_version)/mods-available/apcu.ini -c ';' \
apc.enabled=1
fi
@ -330,4 +330,4 @@ rm -f /etc/cron.hourly/mailinabox-owncloud
# ```
# Enable PHP modules and restart PHP.
restart_service php$PHP_VERSION-fpm
restart_service php$(php_version)-fpm

View File

@ -18,8 +18,6 @@ if [ "$OS" != "Debian GNU/Linux 10 (buster)" -a "$OS" != "Ubuntu 20.04 LTS" ]; t
exit 1
fi
sed -i "s|!!___DIST_TAG___!!|${OS}|g" management/templates/index.html
# Check that we have enough memory.
#
# /proc/meminfo reports free memory in kibibytes. Our baseline will be 512 MB,

View File

@ -46,15 +46,15 @@ management/editconf.py /etc/nginx/nginx.conf -s \
ssl_protocols="TLSv1.2 TLSv1.3;"
# Tell PHP not to expose its version number in the X-Powered-By header.
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
expose_php=Off
# Set PHPs default charset to UTF-8, since we use it. See #367.
management/editconf.py /etc/php/$PHP_VERSION/fpm/php.ini -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
default_charset="UTF-8"
# Configure the path environment for php-fpm
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
env[PATH]=/usr/local/bin:/usr/bin:/bin \
# Configure php-fpm based on the amount of memory the machine has
@ -64,7 +64,7 @@ management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
TOTAL_PHYSICAL_MEM=$(head -n 1 /proc/meminfo | awk '{print $2}' || /bin/true)
if [ $TOTAL_PHYSICAL_MEM -lt 1000000 ]
then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=ondemand \
pm.max_children=8 \
pm.start_servers=2 \
@ -72,7 +72,7 @@ then
pm.max_spare_servers=3
elif [ $TOTAL_PHYSICAL_MEM -lt 2000000 ]
then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=ondemand \
pm.max_children=16 \
pm.start_servers=4 \
@ -80,14 +80,14 @@ then
pm.max_spare_servers=6
elif [ $TOTAL_PHYSICAL_MEM -lt 3000000 ]
then
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=dynamic \
pm.max_children=60 \
pm.start_servers=6 \
pm.min_spare_servers=3 \
pm.max_spare_servers=9
else
management/editconf.py /etc/php/$PHP_VERSION/fpm/pool.d/www.conf -c ';' \
management/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
pm=dynamic \
pm.max_children=120 \
pm.start_servers=12 \
@ -147,7 +147,7 @@ chown -R $STORAGE_USER $STORAGE_ROOT/www
# Start services.
restart_service nginx
restart_service php$PHP_VERSION-fpm
restart_service php$(php_version)-fpm
# Open ports.
ufw_allow http

View File

@ -198,4 +198,4 @@ chmod 664 $STORAGE_ROOT/mail/roundcube/roundcube.sqlite
# Enable PHP modules.
phpenmod -v php mcrypt imap
restart_service php$PHP_VERSION-fpm
restart_service php$(php_version)-fpm

View File

@ -102,7 +102,7 @@ EOF
# Restart service.
restart_service php$PHP_VERSION-fpm
restart_service php$(php_version)-fpm
# Fix states after upgrade