From a3d7e0dfaed596b62b1adef5a9f90427855a8d17 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:21:19 -0800 Subject: [PATCH 001/106] Adapted MIAB Solr install script from https://github.com/jkaberg/ for Ubuntu Bionic --- conf/cronjob/dovecot | 1 + conf/cronjob/solr | 2 ++ setup/solr.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 conf/cronjob/dovecot create mode 100644 conf/cronjob/solr create mode 100644 setup/solr.sh diff --git a/conf/cronjob/dovecot b/conf/cronjob/dovecot new file mode 100644 index 00000000..8de53e63 --- /dev/null +++ b/conf/cronjob/dovecot @@ -0,0 +1 @@ +/usr/bin/doveadm fts rescan -A diff --git a/conf/cronjob/solr b/conf/cronjob/solr new file mode 100644 index 00000000..217f0191 --- /dev/null +++ b/conf/cronjob/solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null +30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..1e9f597f --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its +# HTTP interface, causing mail to be indexed when searches occur, and getting +# results back. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-tomcat dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Bump memory allocation for Solr. +# Not needed? I'll let it sit here for a while. +#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh + +# Install cronjobs to keep FTS up to date +hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service tomcat8 +restart_service dovecot From 4a23a522e1ed2263645c5b73a004b037dc0b0a32 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:29:21 -0800 Subject: [PATCH 002/106] added solr.sh to start.sh --- setup/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/start.sh b/setup/start.sh index 0b145022..14c5700c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -102,6 +102,7 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh +source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh From 2303ac3394b99ef8a6695a27aaa3cd6c29c0b8b2 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 11:32:16 -0800 Subject: [PATCH 003/106] Force kickoff of Solr indexing at install time --- setup/solr.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup/solr.sh b/setup/solr.sh index 1e9f597f..72f8af61 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -61,3 +61,18 @@ chmod -R o-rwx /etc/dovecot # Restart services to reload solr schema & dovecot plugins restart_service tomcat8 restart_service dovecot + + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' From 2fce29d775dfb041454c497dbf2a33adf13fddb6 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 20:46:52 -0800 Subject: [PATCH 004/106] Added Solr (Tomcat) to status_checks.py --- management/status_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..82310d16 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,6 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): From 907c05299c7799d009e37427db9a31855be4ed57 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 09:24:09 +0200 Subject: [PATCH 005/106] Add quota support --- README.md | 12 +++ .../conf.d/15-mailboxes.conf} | 0 conf/dovecot/conf.d/20-imap.conf | 94 +++++++++++++++++++ conf/dovecot/conf.d/90-quota.conf | 84 +++++++++++++++++ conf/dovecot/dovecot-sql.conf.ext | 6 ++ conf/roundcube/config.inc.php | 37 ++++++++ setup/mail-dovecot.sh | 6 +- 7 files changed, 238 insertions(+), 1 deletion(-) rename conf/{dovecot-mailboxes.conf => dovecot/conf.d/15-mailboxes.conf} (100%) create mode 100644 conf/dovecot/conf.d/20-imap.conf create mode 100644 conf/dovecot/conf.d/90-quota.conf create mode 100644 conf/dovecot/dovecot-sql.conf.ext create mode 100644 conf/roundcube/config.inc.php diff --git a/README.md b/README.md index 300fe496..38b3ce86 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +Mail-in-a-Box with Quotas +========================= + +This is an experimental implementation of Main-in-a-box with quotas. Follow the directions below to install except +clone from this repository instead of the official repository. + +There is no support for quotas in control panel at the moment. To set quotas for a user you need to set the `quota` +column for the user in the `users.sqlite` database. The `quota` column is text and allows for the `M` and `G` suffixes +for megabytes and gigabytes respectively. No spaces should be used in the quota value (e.g. `2G` or `100M`). + +\[BEGIN Official README] + Mail-in-a-Box ============= diff --git a/conf/dovecot-mailboxes.conf b/conf/dovecot/conf.d/15-mailboxes.conf similarity index 100% rename from conf/dovecot-mailboxes.conf rename to conf/dovecot/conf.d/15-mailboxes.conf diff --git a/conf/dovecot/conf.d/20-imap.conf b/conf/dovecot/conf.d/20-imap.conf new file mode 100644 index 00000000..b9855b89 --- /dev/null +++ b/conf/dovecot/conf.d/20-imap.conf @@ -0,0 +1,94 @@ +## +## IMAP specific settings +## + +# If nothing happens for this long while client is IDLEing, move the connection +# to imap-hibernate process and close the old imap process. This saves memory, +# because connections use very little memory in imap-hibernate process. The +# downside is that recreating the imap process back uses some resources. +#imap_hibernate_timeout = 0 + +# Maximum IMAP command line length. Some clients generate very long command +# lines with huge mailboxes, so you may need to raise this if you get +# "Too long argument" or "IMAP command line too large" errors often. +#imap_max_line_length = 64k + +# IMAP logout format string: +# %i - total number of bytes read from client +# %o - total number of bytes sent to client +# %{fetch_hdr_count} - Number of mails with mail header data sent to client +# %{fetch_hdr_bytes} - Number of bytes with mail header data sent to client +# %{fetch_body_count} - Number of mails with mail body data sent to client +# %{fetch_body_bytes} - Number of bytes with mail body data sent to client +# %{deleted} - Number of mails where client added \Deleted flag +# %{expunged} - Number of mails that client expunged, which does not +# include automatically expunged mails +# %{autoexpunged} - Number of mails that were automatically expunged after +# client disconnected +# %{trashed} - Number of mails that client copied/moved to the +# special_use=\Trash mailbox. +# %{appended} - Number of mails saved during the session +#imap_logout_format = in=%i out=%o + +# Override the IMAP CAPABILITY response. If the value begins with '+', +# add the given capabilities on top of the defaults (e.g. +XFOO XBAR). +#imap_capability = + +# How long to wait between "OK Still here" notifications when client is +# IDLEing. +#imap_idle_notify_interval = 2 mins +imap_idle_notify_interval=4 mins + +# ID field names and values to send to clients. Using * as the value makes +# Dovecot use the default value. The following fields have default values +# currently: name, version, os, os-version, support-url, support-email. +#imap_id_send = + +# ID fields sent by client to log. * means everything. +#imap_id_log = + +# Workarounds for various client bugs: +# delay-newmail: +# Send EXISTS/RECENT new mail notifications only when replying to NOOP +# and CHECK commands. Some clients ignore them otherwise, for example OSX +# Mail ( + +## +## Quota limits +## + +# Quota limits are set using "quota_rule" parameters. To get per-user quota +# limits, you can set/override them by returning "quota_rule" extra field +# from userdb. It's also possible to give mailbox-specific limits, for example +# to give additional 100 MB when saving to Trash: + +plugin { + #quota_rule = *:storage=1G + #quota_rule2 = Trash:storage=+100M + + # LDA/LMTP allows saving the last mail to bring user from under quota to + # over quota, if the quota doesn't grow too high. Default is to allow as + # long as quota will stay under 10% above the limit. Also allowed e.g. 10M. + #quota_grace = 10%% + + # Quota plugin can also limit the maximum accepted mail size. + #quota_max_mail_size = 100M +} + +## +## Quota warnings +## + +# You can execute a given command when user exceeds a specified quota limit. +# Each quota root has separate limits. Only the command for the first +# exceeded limit is excecuted, so put the highest limit first. +# The commands are executed via script service by connecting to the named +# UNIX socket (quota-warning below). +# Note that % needs to be escaped as %%, otherwise "% " expands to empty. + +plugin { + #quota_warning = storage=95%% quota-warning 95 %u + #quota_warning2 = storage=80%% quota-warning 80 %u +} + +# Example quota-warning service. The unix listener's permissions should be +# set in a way that mail processes can connect to it. Below example assumes +# that mail processes run as vmail user. If you use mode=0666, all system users +# can generate quota warnings to anyone. +#service quota-warning { +# executable = script /usr/local/bin/quota-warning.sh +# user = dovecot +# unix_listener quota-warning { +# user = vmail +# } +#} + +## +## Quota backends +## + +# Multiple backends are supported: +# dirsize: Find and sum all the files found from mail directory. +# Extremely SLOW with Maildir. It'll eat your CPU and disk I/O. +# dict: Keep quota stored in dictionary (eg. SQL) +# maildir: Maildir++ quota +# fs: Read-only support for filesystem quota + +plugin { + quota = maildir + #quota = dirsize:User quota + #quota = maildir:User quota + #quota = dict:User quota::proxy::quota + #quota = fs:User quota +} + +# Multiple quota roots are also possible, for example this gives each user +# their own 100MB quota and one shared 1GB quota within the domain: +plugin { + #quota = dict:user::proxy::quota + #quota2 = dict:domain:%d:proxy::quota_domain + #quota_rule = *:storage=102400 + #quota2_rule = *:storage=1048576 +} diff --git a/conf/dovecot/dovecot-sql.conf.ext b/conf/dovecot/dovecot-sql.conf.ext new file mode 100644 index 00000000..497043c0 --- /dev/null +++ b/conf/dovecot/dovecot-sql.conf.ext @@ -0,0 +1,6 @@ +driver = sqlite +connect = /home/user-data/mail/users.sqlite +default_pass_scheme = SHA512-CRYPT +password_query = SELECT email as user, password FROM users WHERE email='%u'; +user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "/home/user-data/mail/mailboxes/%d/%n" as home, '*:bytes=' || quota AS quota_rule FROM users WHERE email='%u'; +iterate_query = SELECT email AS user FROM users; diff --git a/conf/roundcube/config.inc.php b/conf/roundcube/config.inc.php new file mode 100644 index 00000000..86be2529 --- /dev/null +++ b/conf/roundcube/config.inc.php @@ -0,0 +1,37 @@ + array( + 'verify_peer' => false, + 'verify_peer_name' => false, + ), + ); +$config['imap_timeout'] = 15; +$config['smtp_server'] = 'tls://127.0.0.1'; +$config['smtp_port'] = 587; +$config['smtp_user'] = '%u'; +$config['smtp_pass'] = '%p'; +$config['smtp_conn_options'] = array( + 'ssl' => array( + 'verify_peer' => false, + 'verify_peer_name' => false, + ), + ); +$config['support_url'] = 'https://mailinabox.email/'; +$config['product_name'] = 'box.supplee.com Webmail'; +$config['des_key'] = 'eE4MCgtZQwgVZVBalTwPWMaC'; +$config['plugins'] = array('html5_notifier', 'archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'persistent_login', 'carddav'); +$config['skin'] = 'larry'; +$config['login_autocomplete'] = 2; +$config['password_charset'] = 'UTF-8'; +$config['junk_mbox'] = 'Spam'; +$config['quota_zero_as_unlimited'] = true; +?> diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 4bcc53aa..46f5ae6f 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -66,7 +66,11 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ first_valid_uid=0 # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive. -cp conf/dovecot-mailboxes.conf /etc/dovecot/conf.d/15-mailboxes.conf +cp conf/dovecot/conf.d/15-mailboxes.conf /etc/dovecot/conf.d/ +cp conf/dovecot/conf.d/20-imap.conf /etc/dovecot/conf.d/ +cp conf/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/ +cp conf/dovecot/dovecot-sql.conf.ext /etc/dovecot/ +cp conf/roundcube/config.inc.php /usr/local/lib/roundcubemail/config/ # ### IMAP/POP From 0b68bf87609edf4711b21dab621ae109d229606a Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 09:38:32 +0200 Subject: [PATCH 006/106] put roundcube config change in the right place --- setup/mail-dovecot.sh | 1 - setup/webmail.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 46f5ae6f..2c4378a6 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -70,7 +70,6 @@ cp conf/dovecot/conf.d/15-mailboxes.conf /etc/dovecot/conf.d/ cp conf/dovecot/conf.d/20-imap.conf /etc/dovecot/conf.d/ cp conf/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/ cp conf/dovecot/dovecot-sql.conf.ext /etc/dovecot/ -cp conf/roundcube/config.inc.php /usr/local/lib/roundcubemail/config/ # ### IMAP/POP diff --git a/setup/webmail.sh b/setup/webmail.sh index b0e11c9b..078f4143 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -127,7 +127,7 @@ cat > $RCM_CONFIG < +\$config['quota_zero_as_unlimited'] = true; EOF # Configure CardDav From d443135155cb872b1e446c433bd3e7ccea316c50 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 11:23:49 +0200 Subject: [PATCH 007/106] changes to users.sqlite * add quota column * modify users_query to return quota_rule --- setup/mail-dovecot.sh | 1 - setup/mail-users.sh | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 2c4378a6..ac18adfd 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -69,7 +69,6 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ cp conf/dovecot/conf.d/15-mailboxes.conf /etc/dovecot/conf.d/ cp conf/dovecot/conf.d/20-imap.conf /etc/dovecot/conf.d/ cp conf/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/ -cp conf/dovecot/dovecot-sql.conf.ext /etc/dovecot/ # ### IMAP/POP diff --git a/setup/mail-users.sh b/setup/mail-users.sh index e54485bb..d5f45417 100755 --- a/setup/mail-users.sh +++ b/setup/mail-users.sh @@ -20,7 +20,7 @@ db_path=$STORAGE_ROOT/mail/users.sqlite # Create an empty database if it doesn't yet exist. if [ ! -f $db_path ]; then echo Creating new user database: $db_path; - echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '');" | sqlite3 $db_path; + echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '', quota TEXT NOT NULL DEFAULT '0');" | sqlite3 $db_path; echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 $db_path; fi @@ -49,7 +49,7 @@ driver = sqlite connect = $db_path default_pass_scheme = SHA512-CRYPT password_query = SELECT email as user, password FROM users WHERE email='%u'; -user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "$STORAGE_ROOT/mail/mailboxes/%d/%n" as home FROM users WHERE email='%u'; +user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "$STORAGE_ROOT/mail/mailboxes/%d/%n" as home, '*:bytes=' || quota AS quota_rule FROM users WHERE email='%u'; iterate_query = SELECT email AS user FROM users; EOF chmod 0600 /etc/dovecot/dovecot-sql.conf.ext # per Dovecot instructions From 3cd14bd4b8e1b6573b76800c173687f29be11aa0 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 16:19:36 +0200 Subject: [PATCH 008/106] Add display of quotas --- management/mailconfig.py | 3 ++- management/templates/users.html | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 28e1c623..469a33a3 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -128,13 +128,14 @@ def get_mail_users_ex(env, with_archived=False): users = [] active_accounts = set() c = open_database(env) - c.execute('SELECT email, privileges FROM users') + c.execute('SELECT email, privileges, quota FROM users') for email, privileges in c.fetchall(): active_accounts.add(email) user = { "email": email, "privileges": parse_privs(privileges), + "user": quota, "status": "active", } users.append(user) diff --git a/management/templates/users.html b/management/templates/users.html index dee79d42..09a66bc6 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -42,6 +42,7 @@ Email Address + Quota Actions @@ -54,6 +55,8 @@ + + @@ -152,7 +155,8 @@ function show_users() { n2.addClass("account_" + user.status); n.attr('data-email', user.email); - n.find('.address').text(user.email) + n.find('.address').text(user.email); + n.find('.quota').text(user.quota); n2.find('.restore_info tt').text(user.mailbox); if (user.status == 'inactive') continue; From 587f33b6c03eb8e31dd27100e99a01e104be2be7 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 16:34:34 +0200 Subject: [PATCH 009/106] bug fix for displaying quotas --- management/mailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 469a33a3..d9c89eb2 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -129,7 +129,7 @@ def get_mail_users_ex(env, with_archived=False): active_accounts = set() c = open_database(env) c.execute('SELECT email, privileges, quota FROM users') - for email, privileges in c.fetchall(): + for email, privileges, quota in c.fetchall(): active_accounts.add(email) user = { From dad22f726149a17a8132b67580b1fc7b092a8faa Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 17:17:43 +0200 Subject: [PATCH 010/106] bug fix --- management/mailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index d9c89eb2..5d5f5343 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -135,7 +135,7 @@ def get_mail_users_ex(env, with_archived=False): user = { "email": email, "privileges": parse_privs(privileges), - "user": quota, + "quota": quota, "status": "active", } users.append(user) From b557e69313fdfd5fc18b2f07d735664c0013e0ce Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 17:24:11 +0200 Subject: [PATCH 011/106] display unlimited when quota is 0 --- management/mailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 5d5f5343..783461ec 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -135,7 +135,7 @@ def get_mail_users_ex(env, with_archived=False): user = { "email": email, "privileges": parse_privs(privileges), - "quota": quota, + "quota": 'unlimited' if quota == '0' else quota, "status": "active", } users.append(user) From e2d71d4636f11adadbb5e95f1ab02640b623d088 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 17:30:21 +0200 Subject: [PATCH 012/106] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38b3ce86..7c78198d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ Mail-in-a-Box with Quotas ========================= -This is an experimental implementation of Main-in-a-box with quotas. Follow the directions below to install except +This is an experimental implementation of Mail-in-a-box with quotas. Follow the directions below to install except clone from this repository instead of the official repository. There is no support for quotas in control panel at the moment. To set quotas for a user you need to set the `quota` From 8b95fac8c56b643e55994a2e0df6f5ed3667520e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 19:25:26 +0200 Subject: [PATCH 013/106] remove unnecessary configuration files --- conf/dovecot/dovecot-sql.conf.ext | 6 ----- conf/roundcube/config.inc.php | 37 ------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 conf/dovecot/dovecot-sql.conf.ext delete mode 100644 conf/roundcube/config.inc.php diff --git a/conf/dovecot/dovecot-sql.conf.ext b/conf/dovecot/dovecot-sql.conf.ext deleted file mode 100644 index 497043c0..00000000 --- a/conf/dovecot/dovecot-sql.conf.ext +++ /dev/null @@ -1,6 +0,0 @@ -driver = sqlite -connect = /home/user-data/mail/users.sqlite -default_pass_scheme = SHA512-CRYPT -password_query = SELECT email as user, password FROM users WHERE email='%u'; -user_query = SELECT email AS user, "mail" as uid, "mail" as gid, "/home/user-data/mail/mailboxes/%d/%n" as home, '*:bytes=' || quota AS quota_rule FROM users WHERE email='%u'; -iterate_query = SELECT email AS user FROM users; diff --git a/conf/roundcube/config.inc.php b/conf/roundcube/config.inc.php deleted file mode 100644 index 86be2529..00000000 --- a/conf/roundcube/config.inc.php +++ /dev/null @@ -1,37 +0,0 @@ - array( - 'verify_peer' => false, - 'verify_peer_name' => false, - ), - ); -$config['imap_timeout'] = 15; -$config['smtp_server'] = 'tls://127.0.0.1'; -$config['smtp_port'] = 587; -$config['smtp_user'] = '%u'; -$config['smtp_pass'] = '%p'; -$config['smtp_conn_options'] = array( - 'ssl' => array( - 'verify_peer' => false, - 'verify_peer_name' => false, - ), - ); -$config['support_url'] = 'https://mailinabox.email/'; -$config['product_name'] = 'box.supplee.com Webmail'; -$config['des_key'] = 'eE4MCgtZQwgVZVBalTwPWMaC'; -$config['plugins'] = array('html5_notifier', 'archive', 'zipdownload', 'password', 'managesieve', 'jqueryui', 'persistent_login', 'carddav'); -$config['skin'] = 'larry'; -$config['login_autocomplete'] = 2; -$config['password_charset'] = 'UTF-8'; -$config['junk_mbox'] = 'Spam'; -$config['quota_zero_as_unlimited'] = true; -?> From ccad47937e509d065fbe4bc9173f5ce4a391a0ab Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 23:27:03 +0200 Subject: [PATCH 014/106] Add components to user interface for setting quotas --- management/daemon.py | 10 ++++++++- management/mailconfig.py | 27 ++++++++++++++++++++++- management/templates/users.html | 39 +++++++++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..5b3bd326 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -9,7 +9,7 @@ import auth, utils, multiprocessing.pool from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias - +from mailconfig import set_mail_quota env = utils.load_environment() auth_service = auth.KeyAuthService() @@ -158,6 +158,14 @@ def mail_users_add(): except ValueError as e: return (str(e), 400) +@app.route('/mail/users/quota', methods=['POST']) +@authorized_personnel_only +def mail_users_quota(): + try: + return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env) + except ValueError as e: + return (str(e), 400) + @app.route('/mail/users/password', methods=['POST']) @authorized_personnel_only def mail_users_password(): diff --git a/management/mailconfig.py b/management/mailconfig.py index 783461ec..62ca1d39 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -135,7 +135,7 @@ def get_mail_users_ex(env, with_archived=False): user = { "email": email, "privileges": parse_privs(privileges), - "quota": 'unlimited' if quota == '0' else quota, + "quota": quota, "status": "active", } users.append(user) @@ -333,6 +333,31 @@ def hash_password(pw): # http://wiki2.dovecot.org/Authentication/PasswordSchemes return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip() +def set_mail_quota(email, quota, env): + # validate that password is acceptable + quota = validate_quota(quota) + + # update the database + conn, c = open_database(env, with_connection=True) + c.execute("UPDATE users SET quota=? WHERE email=?", (quota, email)) + if c.rowcount != 1: + return ("That's not a user (%s)." % email, 400) + conn.commit() + return "OK" + +def validate_quota(quota): + # validate quota + quota = quota.strip().upper() + + if quota == "": + raise ValueError("No quota provided.") + if re.search(r"[\s,.]", quota): + raise ValueError("Quotas cannot contain spaces or commas.") + if not re.match(r'^\d+[GM]?$', quota): + raise ValueError("Invalid quota.") + + return quota + def get_mail_password(email, env): # Gets the hashed password for a user. Passwords are stored in Dovecot's # password format, with a prefixed scheme. diff --git a/management/templates/users.html b/management/templates/users.html index 09a66bc6..40f3be08 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -41,7 +41,7 @@ - + @@ -61,6 +61,10 @@ + + set quota + + | set password @@ -155,8 +159,9 @@ function show_users() { n2.addClass("account_" + user.status); n.attr('data-email', user.email); + n.attr('data-quota', user.quota); n.find('.address').text(user.email); - n.find('.quota').text(user.quota); + n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota); n2.find('.restore_info tt').text(user.mailbox); if (user.status == 'inactive') continue; @@ -233,6 +238,36 @@ function users_set_password(elem) { }); } +function users_set_quota(elem) { + var email = $(elem).parents('tr').attr('data-email'); + var quota = $(elem).parents('tr').attr('data-quota'); + + show_modal_confirm( + "Set Quota", + $("

Set quota for " + email + "?

" + + "

" + + "" + + "

" + + "

Quotas may not contain any numbers or commas. Suffixes of G (gigabytes) and M (megabytes) are allowed.

" + + "

For unlimited storage enter 0 (zero)

"), + "Set Quota", + function() { + api( + "/mail/users/quota", + "POST", + { + email: email, + quota: $('#users_set_quota').val() + }, + function(r) { + show_users(); + }, + function(r) { + show_modal_error("Set Quota", r); + }); + }); +} + function users_remove(elem) { var email = $(elem).parents('tr').attr('data-email'); From 832d2ca3756877c907214fcd5bc12acc8a453c72 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 28 Jan 2019 23:38:04 +0200 Subject: [PATCH 015/106] Update README to note support for control panel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c78198d..36f298e9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Mail-in-a-Box with Quotas This is an experimental implementation of Mail-in-a-box with quotas. Follow the directions below to install except clone from this repository instead of the official repository. -There is no support for quotas in control panel at the moment. To set quotas for a user you need to set the `quota` +There is baisc support for quotas in the control panel now. To set quotas from the command line, set the `quota` column for the user in the `users.sqlite` database. The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. No spaces should be used in the quota value (e.g. `2G` or `100M`). From 3840443159987984a08a1f282fd5bdf0e2366913 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 29 Jan 2019 13:51:18 +0200 Subject: [PATCH 016/106] enable setting quotas from the command line --- tools/mail.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/mail.py b/tools/mail.py index 566971e4..7a7b8b8e 100755 --- a/tools/mail.py +++ b/tools/mail.py @@ -57,9 +57,10 @@ def setup_key_auth(mgmt_uri): if len(sys.argv) < 2: print("Usage: ") - print(" tools/mail.py user (lists users)") + print(" tools/mail.py user (lists users with quotas)") print(" tools/mail.py user add user@domain.com [password]") print(" tools/mail.py user password user@domain.com [password]") + print(" tools/mail.py user quota user@domain new-quota") print(" tools/mail.py user remove user@domain.com") print(" tools/mail.py user make-admin user@domain.com") print(" tools/mail.py user remove-admin user@domain.com") @@ -81,6 +82,10 @@ elif sys.argv[1] == "user" and len(sys.argv) == 2: print(user['email'], end='') if "admin" in user['privileges']: print("*", end='') + if user['quota'] == '0': + print(" unlimited", end='') + else: + print(" " + user['quota'], end='') print() elif sys.argv[1] == "user" and sys.argv[2] in ("add", "password"): @@ -116,6 +121,10 @@ elif sys.argv[1] == "user" and sys.argv[2] == "admins": if "admin" in user['privileges']: print(user['email']) +elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 5: + # Set a user's quota + users = mgmt("/mail/users/quota", { "email": sys.argv[3], "quota": sys.argv[4] }) + elif sys.argv[1] == "alias" and len(sys.argv) == 2: print(mgmt("/mail/aliases")) From c5c7de8b2b0681c75bc306ea47389951582f170b Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 29 Jan 2019 17:42:17 +0200 Subject: [PATCH 017/106] allow decimal points in quotas --- management/mailconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 62ca1d39..73b166a1 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -351,9 +351,9 @@ def validate_quota(quota): if quota == "": raise ValueError("No quota provided.") - if re.search(r"[\s,.]", quota): + if re.search(r"[\s,]", quota): raise ValueError("Quotas cannot contain spaces or commas.") - if not re.match(r'^\d+[GM]?$', quota): + if not re.match(r'^[\d\.]+[GM]?$', quota): raise ValueError("Invalid quota.") return quota From f68ef70b949f11877c4dd3d8c60f6a26bbd2dcd1 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 29 Jan 2019 19:18:48 +0200 Subject: [PATCH 018/106] Calculate and display mailbox sizes in user list --- management/mailconfig.py | 17 +++++++++++++++++ management/templates/users.html | 3 +++ 2 files changed, 20 insertions(+) diff --git a/management/mailconfig.py b/management/mailconfig.py index 73b166a1..cdd05079 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -132,10 +132,27 @@ def get_mail_users_ex(env, with_archived=False): for email, privileges, quota in c.fetchall(): active_accounts.add(email) + (user, domain) = email.split('@') + box_size = 0 + box_count = 0 + box_quota = '' + try: + with open('/home/user-data/mail/mailboxes/%s/%s/maildirsize' % (domain, user), 'r') as f: + box_quota = f.readline() + for line in f.readlines(): + (size, count) = line.split(' ') + box_size += int(size) + box_count += int(count) + except: + box_size = '?' + user = { "email": email, "privileges": parse_privs(privileges), "quota": quota, + "box_quota": box_quota, + "box_size": '%iK' % int(box_size / 1024), + "box_count": box_count, "status": "active", } users.append(user) diff --git a/management/templates/users.html b/management/templates/users.html index 40f3be08..0175b929 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -42,6 +42,7 @@ + @@ -55,6 +56,7 @@ + - + + @@ -56,6 +57,7 @@ + @@ -163,6 +165,7 @@ function show_users() { n.attr('data-email', user.email); n.attr('data-quota', user.quota); n.find('.address').text(user.email); + n.find('.box-count').text(user.box_count); n.find('.box-size').text(user.box_size); n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota); n2.find('.restore_info tt').text(user.mailbox); From a23f186c65ee743a98d0b88b162b635fd88ce717 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 08:48:04 +0200 Subject: [PATCH 023/106] initial test config --- conf/dovecot/conf.d/90-quota.conf | 14 ++++++++++++++ setup/mail-postfix.sh | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/conf/dovecot/conf.d/90-quota.conf b/conf/dovecot/conf.d/90-quota.conf index d294c9e5..270fbaf7 100644 --- a/conf/dovecot/conf.d/90-quota.conf +++ b/conf/dovecot/conf.d/90-quota.conf @@ -68,12 +68,26 @@ plugin { plugin { quota = maildir + + quota_grace = 10% + + quota_status_success = DUNNO + quota_status_nouser = DUNNO + quota_status_overquota = "522 5.2.2 Mailbox is full" + #quota = dirsize:User quota #quota = maildir:User quota #quota = dict:User quota::proxy::quota #quota = fs:User quota } +service quota-status { + executable = quota-status -p postfix + inet_listener { + port = 12340 + } +} + # Multiple quota roots are also possible, for example this gives each user # their own 100MB quota and one shared 1GB quota within the domain: plugin { diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..8aa65d76 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -195,7 +195,7 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service inet:127.0.0.1:12340" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). From fbf990092cbd3242a81a2d7afab53f4d2a1a3f02 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 09:44:23 +0200 Subject: [PATCH 024/106] Add progress to README --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cb2cdf86..8224b7ef 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,13 @@ for megabytes and gigabytes respectively. No spaces should be used in the quota Todo ---- -* Get `postfix` to support quotas. See https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html for a start. Right - now the quota message store size is not calculated unless the user accesses the IMAP server (Dovecot). Right now postfix - does not take quotas into account before delivering a message. +* Get `postfix` to support quotas. Right now the quota message store size is not calculated unless the user accesses + the IMAP server (Dovecot). Right now postfix does not take quotas into account before delivering a message. + + * https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html + * https://linuxize.com/post/install-and-configure-postfix-and-dovecot/ + + Initial work on support for this has been added in the `postfix-quota` branch. * Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. From 2bebd727c211b4bd6d590da92690f6d4250fbccb Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 10:47:07 +0200 Subject: [PATCH 025/106] Fix README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8224b7ef..ef659660 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Todo ---- * Get `postfix` to support quotas. Right now the quota message store size is not calculated unless the user accesses - the IMAP server (Dovecot). Right now postfix does not take quotas into account before delivering a message. + the IMAP server (Dovecot). So `postfix` does not take quotas into account before delivering a message. * https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html * https://linuxize.com/post/install-and-configure-postfix-and-dovecot/ From 98e04bb11f7802bafc7b46c420ad10cbdc435a41 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 12:52:02 +0200 Subject: [PATCH 026/106] Update README for first alpha release --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ef659660..5b858004 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,30 @@ There is baisc support for quotas in the control panel now. To set quotas from column for the user in the `users.sqlite` database. The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. No spaces should be used in the quota value (e.g. `2G` or `100M`). +Please report any bugs on github. + Todo ---- -* Get `postfix` to support quotas. Right now the quota message store size is not calculated unless the user accesses - the IMAP server (Dovecot). So `postfix` does not take quotas into account before delivering a message. - - * https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html - * https://linuxize.com/post/install-and-configure-postfix-and-dovecot/ - - Initial work on support for this has been added in the `postfix-quota` branch. - * Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. +Changes +------- + +### .40-quota-0.1-alpha + +* First experimental release of Mail-in-a-Box for quotas. +* Quotas are working and there is basic support in the control panel and `tools/mail.py`. + + +Reference Documents +------------------- + +* https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html +* https://linuxize.com/post/install-and-configure-postfix-and-dovecot/ + + \[BEGIN Official README] Mail-in-a-Box From ede5e09ad07898eafe8eec5198b88315ac28b4cf Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 15:31:15 +0200 Subject: [PATCH 027/106] make version checks from this repository --- management/status_checks.py | 2 +- setup/bootstrap.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..0294c7ba 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -787,7 +787,7 @@ def get_latest_miab_version(): from socket import timeout try: - return re.search(b'TAG=(.*)', urlopen("https://mailinabox.email/setup.sh?ping=1", timeout=5).read()).group(1).decode("utf8") + return re.search(b'TAG=(.*)', urlopen("https://raw.githubusercontent.com/jrsupplee/mailinabox/master/setup/bootstrap.sh", timeout=5).read()).group(1).decode("utf8") except (HTTPError, URLError, timeout): return None diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 3442499d..aadb4147 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40 + TAG=0.40-quota-0.1-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From e40566967e539a61e5f030ab16c391fd3f82f9f4 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 15:35:17 +0200 Subject: [PATCH 028/106] Change up to date message --- management/status_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/status_checks.py b/management/status_checks.py index 0294c7ba..e267a0ed 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -805,7 +805,7 @@ def check_miab_version(env, output): latest_ver = get_latest_miab_version() if this_ver == latest_ver: - output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver) + output.print_ok("Mail-in-a-Box with quota support is up to date. You are running version %s." % this_ver) elif latest_ver is None: output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver) else: From 863e8895a0e863bd55a9e7d2f169c8bea11fe6d9 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 17:14:48 +0200 Subject: [PATCH 029/106] Add the quota column to the users table --- README.md | 12 +++++++----- setup/mail-users.sh | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5b858004..be0fd721 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,17 @@ Mail-in-a-Box with Quotas ========================= -This is an experimental implementation of Mail-in-a-box with quotas. Follow the directions below to install except -clone from this repository instead of the official repository. +This is an experimental implementation of Mail-in-a-box with quotas. Follow the directions below to install except clone from this repository instead of the official repository. -There is baisc support for quotas in the control panel now. To set quotas from the command line, set the `quota` -column for the user in the `users.sqlite` database. The `quota` column is text and allows for the `M` and `G` suffixes -for megabytes and gigabytes respectively. No spaces should be used in the quota value (e.g. `2G` or `100M`). +There is baisc support for quotas in the control panel now. To set quotas from the command line, either use `tools/mail.py` or set the `quota` +column for the user in the `users.sqlite` database. If you modify `users` table using SQL note the following: + +* The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. +* No spaces should be used in the quota value (e.g. `2G` or `100M`). Please report any bugs on github. + Todo ---- diff --git a/setup/mail-users.sh b/setup/mail-users.sh index d5f45417..ef0a99de 100755 --- a/setup/mail-users.sh +++ b/setup/mail-users.sh @@ -22,6 +22,10 @@ if [ ! -f $db_path ]; then echo Creating new user database: $db_path; echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '', quota TEXT NOT NULL DEFAULT '0');" | sqlite3 $db_path; echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 $db_path; +else + set +e + echo "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';" | sqlite3 $db_path; + set -e fi # ### User Authentication From 1dee84949838d90be1aa2d8f85995cd17b602074 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 30 Jan 2019 11:33:43 -0800 Subject: [PATCH 030/106] Removed extra blankline --- setup/web.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index 2e82fe6a..4ba646e4 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -54,7 +54,6 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_socket_timeout=180 - # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand From f58745c04d2e201b76940f0e2591b016beb59c52 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 31 Jan 2019 09:49:07 +0200 Subject: [PATCH 031/106] Update version to v0.40-quota-0.11-alpha --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index aadb4147..9be40167 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=0.40-quota-0.1-alpha + TAG=v0.40-quota-0.11-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 9a9e0116a195224c45576dbe580b9c7f28bd2ab2 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 31 Jan 2019 10:23:42 +0200 Subject: [PATCH 032/106] Update to version v0.40-quota-0.12-alpha --- README.md | 24 +++++++++++++++++++++++- setup/bootstrap.sh | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be0fd721..1c9c2c75 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,20 @@ column for the user in the `users.sqlite` database. If you modify `users` table Please report any bugs on github. +Upgrading v0.40 to v.0.40-quota +------------------------------- + +This is experimental software. You have been warned. + +* Rename your `mailinabox` directory to something like `miab.old` + +* Clone this repository using: + + `git clone https://github.com/jrsupplee/mailinabox.git` + +* cd into `mailinabox` and run `setup/start.sh` with root privileges. + +* On occasion there are lock errors when updating `Munin`. Just re-run `setup/start.sh` until the error does not occur. Todo ---- @@ -21,7 +35,15 @@ Todo Changes ------- -### .40-quota-0.1-alpha +### v0.40-quota-0.12-alpha + +* Update README + +### v0.40-quota-0.11-alpha + +* Read latest version from this repository not the Mail-in-a-Box master repository + +### v0.40-quota-0.1-alpha * First experimental release of Mail-in-a-Box for quotas. * Quotas are working and there is basic support in the control panel and `tools/mail.py`. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 9be40167..cd09e8da 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40-quota-0.11-alpha + TAG=v0.40-quota-0.12-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 6de34b046464c41234cb4c877f77ff31eb780974 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 31 Jan 2019 12:28:32 +0200 Subject: [PATCH 033/106] Update README to have section for installation and upgrading --- README.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1c9c2c75..4991ebb1 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,23 @@ Mail-in-a-Box with Quotas ========================= -This is an experimental implementation of Mail-in-a-box with quotas. Follow the directions below to install except clone from this repository instead of the official repository. +This is an experimental implementation of Mail-in-a-box with quota support. There is baisc support for quotas in the control panel now. To set quotas from the command line, either use `tools/mail.py` or set the `quota` -column for the user in the `users.sqlite` database. If you modify `users` table using SQL note the following: +column for the user in the `users.sqlite` database. If you modify the `users` table using SQL note the following: * The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. * No spaces should be used in the quota value (e.g. `2G` or `100M`). Please report any bugs on github. + +Installing v0.40-quota +---------------------- + +Follow the directions below for installing from a repository except clone from this repository instead of the official one. + + Upgrading v0.40 to v.0.40-quota ------------------------------- @@ -26,11 +33,25 @@ This is experimental software. You have been warned. * On occasion there are lock errors when updating `Munin`. Just re-run `setup/start.sh` until the error does not occur. + +Upgrading v.0.40-quota to a New Version +--------------------------------------- + +* Remember that this is experimental software and review the changes in the repository. + +* `cd` into the `mailinabox` directory. + +* Execute `git pull` to download the latest changes. + +* Execute `setup/start.sh` with root privileges. + + Todo ---- * Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. +* Allow configuration of default quota value on installation. Changes ------- From d1906bd055064276a070a7b350e853396b23bf60 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 31 Jan 2019 22:57:04 +0200 Subject: [PATCH 034/106] Add support for a default quota value and allow setting quota when adding user --- management/daemon.py | 22 ++++++++++++++++++++-- management/mailconfig.py | 15 ++++++++++++--- management/templates/users.html | 8 +++++++- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 3bf95ec5..a007d6c8 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -9,7 +9,7 @@ import auth, utils, multiprocessing.pool from mailconfig import get_mail_users, get_mail_users_ex, get_admins, add_mail_user, set_mail_password, remove_mail_user from mailconfig import get_mail_user_privileges, add_remove_mail_user_privilege from mailconfig import get_mail_aliases, get_mail_aliases_ex, get_mail_domains, add_mail_alias, remove_mail_alias -from mailconfig import set_mail_quota +from mailconfig import set_mail_quota, get_default_quota, validate_quota env = utils.load_environment() auth_service = auth.KeyAuthService() @@ -154,7 +154,7 @@ def mail_users(): @authorized_personnel_only def mail_users_add(): try: - return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), env) + return add_mail_user(request.form.get('email', ''), request.form.get('password', ''), request.form.get('privileges', ''), request.form.get('quota', ''), env) except ValueError as e: return (str(e), 400) @@ -528,6 +528,24 @@ def privacy_status_set(): utils.write_settings(config, env) return "OK" +@app.route('/system/default-quota', methods=["GET"]) +@authorized_personnel_only +def default_quota_get(): + return get_default_quota(env) + +@app.route('/system/default-quota', methods=["POST"]) +@authorized_personnel_only +def default_quota_set(): + config = utils.load_settings(env) + try: + config["default-quota"] = validate_quota(request.form.get('default_quota')) + utils.write_settings(config, env) + + except ValueError as e: + return ("ERROR: %s" % str(e), 400) + + return "OK" + # MUNIN @app.route('/munin/') diff --git a/management/mailconfig.py b/management/mailconfig.py index 9cb79b0d..92f26dfc 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -298,7 +298,7 @@ def get_mail_domains(env, filter_aliases=lambda alias : True): + [get_domain(address, as_unicode=False) for address, *_ in get_mail_aliases(env) if filter_aliases(address) ] ) -def add_mail_user(email, pw, privs, env): +def add_mail_user(email, pw, privs, quota, env): # validate email if email.strip() == "": return ("No email address provided.", 400) @@ -324,6 +324,11 @@ def add_mail_user(email, pw, privs, env): validation = validate_privilege(p) if validation: return validation + try: + quota = validate_quota(quota) + except ValueError as e: + return (str(e), 400) + # get the database conn, c = open_database(env, with_connection=True) @@ -332,8 +337,8 @@ def add_mail_user(email, pw, privs, env): # add the user to the database try: - c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)", - (email, pw, "\n".join(privs))) + c.execute("INSERT INTO users (email, password, privileges, quota) VALUES (?, ?, ?, ?)", + (email, pw, "\n".join(privs)), quota) except sqlite3.IntegrityError: return ("User already exists.", 400) @@ -376,6 +381,10 @@ def set_mail_quota(email, quota, env): conn.commit() return "OK" +def get_default_quota(env): + config = utils.load_settings(env) + return config.get("default-quota", '0') + def validate_quota(quota): # validate quota quota = quota.strip().upper() diff --git a/management/templates/users.html b/management/templates/users.html index ee22b316..ba3985e0 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -28,6 +28,10 @@ +
+ + +
    @@ -196,13 +200,15 @@ function do_add_user() { var email = $("#adduserEmail").val(); var pw = $("#adduserPassword").val(); var privs = $("#adduserPrivs").val(); + var quota = $("#adduserQuota").val(); api( "/mail/users/add", "POST", { email: email, password: pw, - privileges: privs + privileges: privs, + quota: quota }, function(r) { // Responses are multiple lines of pre-formatted text. From 70c607e2566107463e848f15454e7b8fbb04da00 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 31 Jan 2019 23:58:10 +0200 Subject: [PATCH 035/106] more default quota work --- management/daemon.py | 18 ++++++++++-------- management/templates/users.html | 11 +++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index a007d6c8..eb467681 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -531,20 +531,22 @@ def privacy_status_set(): @app.route('/system/default-quota', methods=["GET"]) @authorized_personnel_only def default_quota_get(): - return get_default_quota(env) + return json_response({ + "default-quota": get_default_quota(env) + }) @app.route('/system/default-quota', methods=["POST"]) @authorized_personnel_only def default_quota_set(): - config = utils.load_settings(env) - try: - config["default-quota"] = validate_quota(request.form.get('default_quota')) - utils.write_settings(config, env) + config = utils.load_settings(env) + try: + config["default-quota"] = validate_quota(request.form.get('default_quota')) + utils.write_settings(config, env) - except ValueError as e: - return ("ERROR: %s" % str(e), 400) + except ValueError as e: + return ("ERROR: %s" % str(e), 400) - return "OK" + return "OK" # MUNIN diff --git a/management/templates/users.html b/management/templates/users.html index ba3985e0..a4cbdce6 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -348,4 +348,15 @@ function generate_random_password() { show_modal_error("Random Password", "

    Here, try this:

    " + pw + " From 1a67c94db01dbf3d0760983ec0b26029d20fb361 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 1 Feb 2019 00:56:25 +0200 Subject: [PATCH 036/106] fix problem with jQuery being loaded too late --- management/templates/index.html | 2 +- management/templates/users.html | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/management/templates/index.html b/management/templates/index.html index 2c0d5a9a..77c305b8 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -10,6 +10,7 @@ +

    Add a mail user

    @@ -47,9 +48,10 @@
- - - + + + + @@ -62,9 +64,10 @@ - - - + + + "); + var hdr = $(""); hdr.find('h4').text(r[i].domain); $('#user_table tbody').append(hdr); @@ -180,7 +183,14 @@ function show_users() { n.attr('data-quota', user.quota); n.find('.address').text(user.email); n.find('.box-count').text(user.box_count); + if (user.box_count == '?') { + n.find('.box-count').attr('title', 'Message count is unkown') + } n.find('.box-size').text(user.box_size); + if (user.box_size == '?') { + n.find('.box-size').attr('title', 'Mailbox size is unkown') + } + n.find('.percent').text(user.percent) n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota); n2.find('.restore_info tt').text(user.mailbox); diff --git a/tools/mail.py b/tools/mail.py index 7a7b8b8e..2807d87d 100755 --- a/tools/mail.py +++ b/tools/mail.py @@ -57,10 +57,11 @@ def setup_key_auth(mgmt_uri): if len(sys.argv) < 2: print("Usage: ") + print(" tools/mail.py system default-quota [new default]") print(" tools/mail.py user (lists users with quotas)") print(" tools/mail.py user add user@domain.com [password]") print(" tools/mail.py user password user@domain.com [password]") - print(" tools/mail.py user quota user@domain new-quota") + print(" tools/mail.py user quota user@domain [new-quota]") print(" tools/mail.py user remove user@domain.com") print(" tools/mail.py user make-admin user@domain.com") print(" tools/mail.py user remove-admin user@domain.com") @@ -121,6 +122,10 @@ elif sys.argv[1] == "user" and sys.argv[2] == "admins": if "admin" in user['privileges']: print(user['email']) +elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 4: + # Set a user's quota + print(mgmt("/mail/users/quota?text=1&email=%s" % sys.argv[3])) + elif sys.argv[1] == "user" and sys.argv[2] == "quota" and len(sys.argv) == 5: # Set a user's quota users = mgmt("/mail/users/quota", { "email": sys.argv[3], "quota": sys.argv[4] }) @@ -134,6 +139,12 @@ elif sys.argv[1] == "alias" and sys.argv[2] == "add" and len(sys.argv) == 5: elif sys.argv[1] == "alias" and sys.argv[2] == "remove" and len(sys.argv) == 4: print(mgmt("/mail/aliases/remove", { "address": sys.argv[3] })) +elif sys.argv[1] == "system" and sys.argv[2] == "default-quota" and len(sys.argv) == 3: + print(mgmt("/system/default-quota?text=1")) + +elif sys.argv[1] == "system" and sys.argv[2] == "default-quota" and len(sys.argv) == 4: + print(mgmt("/system/default-quota", { "default_quota": sys.argv[3]})) + else: print("Invalid command-line arguments.") sys.exit(1) From 2c6aea24cb09c8361698fc165c18cef639fb208b Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 1 Feb 2019 18:38:00 +0200 Subject: [PATCH 042/106] Update README to reflect the latest changes --- README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 680ac766..84228cc3 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,19 @@ Mail-in-a-Box with Quotas This is an experimental implementation of Mail-in-a-box with quota support. -There is baisc support for quotas in the control panel now. To set quotas from the command line, either use `tools/mail.py` or set the `quota` -column for the user in the `users.sqlite` database. If you modify the `users` table using SQL note the following: +Quotas can be set and viewed in the control panel -* The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. -* No spaces should be used in the quota value (e.g. `2G` or `100M`). +To set quotas from the command line, use: + + tools/mail.py user quota + +To set the system default quota for new users, use: + + tools/mail.py system default-quota + +Mailbox size recalculation by Dovecot can be forced using the command: + + doveadm quota recalc -A Please report any bugs on github. @@ -29,9 +37,9 @@ This is experimental software. You have been warned. `git clone https://github.com/jrsupplee/mailinabox.git` -* cd into `mailinabox` and run `setup/start.sh` with root privileges. +* cd into `mailinabox` and run `setup/start.sh` with root privileges. On occasion there are lock errors when updating `Munin`. Just re-run `setup/start.sh` until the error does not occur. -* On occasion there are lock errors when updating `Munin`. Just re-run `setup/start.sh` until the error does not occur. +* Optionally execute `sudo doveadm quota recalc -A` to calculate mailbox sizes. If this is not done, a mailbox's size will be recalculated when mail is delivered to it. Upgrading v.0.40-quota to a New Version @@ -53,20 +61,21 @@ Todo * Allow configuration of default quota value on installation. + Changes ------- ### v0.40-quota-0.13-alpha -* Add a `default-quota` setting in `settings.yaml` +* Add a `default-quota` setting in `settings.yaml`. -* Add input for setting quota when entering a new user in control panel +* Add input for setting quota when entering a new user in control panel. -* Modify `tools/mail.py` to allow for setting and getting the default system quota +* Modify `tools/mail.py` to allow for setting and getting the default system quota. -* Modify `tools/mail.py` to allow for getting a user's quota setting +* Modify `tools/mail.py` to allow for getting a user's quota setting. -* Modify the mail users list in control panel to display percentage used +* Modify the mail users list in control panel to display percentage of quota used. ### v0.40-quota-0.12-alpha From d51a32bcfe52d3d6568776f0352627f24b3fd6b8 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 1 Feb 2019 18:39:47 +0200 Subject: [PATCH 043/106] update latest version to v0.40-quota-0.13-alpha --- setup/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index cd09e8da..5c02f1ea 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40-quota-0.12-alpha + TAG=v0.40-quota-0.13-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 6964ed238a9d015f7502124bf615163c6c33b86a Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 1 Feb 2019 19:01:59 +0200 Subject: [PATCH 044/106] Show correct reference for updating versions --- README.md | 2 +- management/status_checks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84228cc3..ef3cf3a7 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Upgrading v.0.40-quota to a New Version * `cd` into the `mailinabox` directory. -* Execute `git pull` to download the latest changes. +* Execute `git pull --tags` to download the latest changes with tags. * Execute `setup/start.sh` with root privileges. diff --git a/management/status_checks.py b/management/status_checks.py index e267a0ed..a172b256 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -809,7 +809,7 @@ def check_miab_version(env, output): elif latest_ver is None: output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver) else: - output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. " + output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://github.com/jrsupplee/mailinabox/blob/master/README.md. " % (this_ver, latest_ver)) def run_and_output_changes(env, pool): From 937ef47c2a63d9be4cce7fe08c4729c2d584fdbd Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 1 Feb 2019 21:51:49 +0200 Subject: [PATCH 045/106] Remove default quota support from the TODOs --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index ef3cf3a7..7bc4382a 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,6 @@ Todo * Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. -* Allow configuration of default quota value on installation. - Changes ------- From 686a074fab462d6a3ef9017c73f5b2eaabaeb10e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 01:47:09 +0200 Subject: [PATCH 046/106] Force recalculation of a user's quota after setting a new value --- management/mailconfig.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 8e4198a6..11283c78 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -361,6 +361,8 @@ def add_mail_user(email, pw, privs, quota, env): # write databasebefore next step conn.commit() + dovecot_quota_recalc(email) + # Update things in case any new domains are added. return kick(env, "mail user added") @@ -385,6 +387,7 @@ def hash_password(pw): # http://wiki2.dovecot.org/Authentication/PasswordSchemes return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip() + def get_mail_quota(email, env): conn, c = open_database(env, with_connection=True) c.execute("SELECT quota FROM users WHERE email=?", (email,)) @@ -394,17 +397,31 @@ def get_mail_quota(email, env): return rows[0][0] -def set_mail_quota(email, quota, env): - # validate that password is acceptable - quota = validate_quota(quota) - # update the database - conn, c = open_database(env, with_connection=True) - c.execute("UPDATE users SET quota=? WHERE email=?", (quota, email)) - if c.rowcount != 1: - return ("That's not a user (%s)." % email, 400) - conn.commit() - return "OK" +def set_mail_quota(email, quota, env): + # validate that password is acceptable + quota = validate_quota(quota) + + # update the database + conn, c = open_database(env, with_connection=True) + c.execute("UPDATE users SET quota=? WHERE email=?", (quota, email)) + if c.rowcount != 1: + return ("That's not a user (%s)." % email, 400) + conn.commit() + + dovecot_quota_recalc(email) + + return "OK" + +def dovecot_quota_recalc(email): + # dovecot processes running for the user will not recognize the new quota setting + # a reload is necessary to reread the quota setting, but it will also shut down + # running dovecot processes. Email clients generally log back in when they lose + # a connection. + subprocess.call(['doveadm', 'reload']) + + # force dovecot to recalculate the quota info for the user. + subprocess.call(["doveadm", "quota", "recalc" "-u", "%s" % email]) def get_default_quota(env): config = utils.load_settings(env) From a60e1c274a84da4c7c964d200d7dca7db3247271 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 01:47:39 +0200 Subject: [PATCH 047/106] Do not allow decimal points in quota sizes --- management/mailconfig.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 11283c78..f8c88a8a 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -428,17 +428,17 @@ def get_default_quota(env): return config.get("default-quota", '0') def validate_quota(quota): - # validate quota - quota = quota.strip().upper() + # validate quota + quota = quota.strip().upper() - if quota == "": - raise ValueError("No quota provided.") - if re.search(r"[\s,]", quota): - raise ValueError("Quotas cannot contain spaces or commas.") - if not re.match(r'^[\d\.]+[GM]?$', quota): - raise ValueError("Invalid quota.") + if quota == "": + raise ValueError("No quota provided.") + if re.search(r"[\s,.]", quota): + raise ValueError("Quotas cannot contain spaces, commas, or decimal points.") + if not re.match(r'^[\d]+[GM]?$', quota): + raise ValueError("Invalid quota.") - return quota + return quota def get_mail_password(email, env): # Gets the hashed password for a user. Passwords are stored in Dovecot's From d1378a55121ffa6a744df91930ea70bc8f6cf19b Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 01:48:38 +0200 Subject: [PATCH 048/106] Add a thousands separator for message count --- management/templates/users.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/management/templates/users.html b/management/templates/users.html index 1d176de8..5ea45da9 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -182,7 +182,7 @@ function show_users() { n.attr('data-email', user.email); n.attr('data-quota', user.quota); n.find('.address').text(user.email); - n.find('.box-count').text(user.box_count); + n.find('.box-count').text((user.box_count).toLocaleString('en')); if (user.box_count == '?') { n.find('.box-count').attr('title', 'Message count is unkown') } @@ -190,7 +190,7 @@ function show_users() { if (user.box_size == '?') { n.find('.box-size').attr('title', 'Mailbox size is unkown') } - n.find('.percent').text(user.percent) + n.find('.percent').text(user.percent); n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota); n2.find('.restore_info tt').text(user.mailbox); From 19f204a9afad6a5189ec9cf0dbdbe4484c08226f Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 13:32:43 +0200 Subject: [PATCH 049/106] Do not execute a dovecot reload on every quota update --- management/mailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index f8c88a8a..9a9f8407 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -418,7 +418,7 @@ def dovecot_quota_recalc(email): # a reload is necessary to reread the quota setting, but it will also shut down # running dovecot processes. Email clients generally log back in when they lose # a connection. - subprocess.call(['doveadm', 'reload']) + # subprocess.call(['doveadm', 'reload']) # force dovecot to recalculate the quota info for the user. subprocess.call(["doveadm", "quota", "recalc" "-u", "%s" % email]) From e44dd937784d8e15b07c6a206ec55b00250965e6 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 13:34:37 +0200 Subject: [PATCH 050/106] mail setup fixes * check for quota column before trying to add it * force recalculation of quotas --- setup/mail-users.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup/mail-users.sh b/setup/mail-users.sh index ef0a99de..8c7be06a 100755 --- a/setup/mail-users.sh +++ b/setup/mail-users.sh @@ -22,10 +22,8 @@ if [ ! -f $db_path ]; then echo Creating new user database: $db_path; echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra, privileges TEXT NOT NULL DEFAULT '', quota TEXT NOT NULL DEFAULT '0');" | sqlite3 $db_path; echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL, permitted_senders TEXT);" | sqlite3 $db_path; -else - set +e +elif sqlite3 $db_path ".schema users" | grep --invert-match quota; then echo "ALTER TABLE users ADD COLUMN quota TEXT NOT NULL DEFAULT '0';" | sqlite3 $db_path; - set -e fi # ### User Authentication @@ -157,4 +155,5 @@ EOF restart_service postfix restart_service dovecot - +# force a recalculation of all user quotas +doveadm quota recalc -A From 44a31733ac4736712a260c7cff42cccc5a33e5b4 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 5 Feb 2019 13:36:53 +0200 Subject: [PATCH 051/106] update latest version and update README --- README.md | 25 ++++++++++++++++--------- setup/bootstrap.sh | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7bc4382a..1c148a63 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,7 @@ This is experimental software. You have been warned. `git clone https://github.com/jrsupplee/mailinabox.git` -* cd into `mailinabox` and run `setup/start.sh` with root privileges. On occasion there are lock errors when updating `Munin`. Just re-run `setup/start.sh` until the error does not occur. - -* Optionally execute `sudo doveadm quota recalc -A` to calculate mailbox sizes. If this is not done, a mailbox's size will be recalculated when mail is delivered to it. +* cd into `mailinabox` and run `sudo setup/start.sh` On occasion there are lock errors when updating `Munin`. Just re-run `sudo setup/start.sh` until the error does not occur. Upgrading v.0.40-quota to a New Version @@ -49,20 +47,30 @@ Upgrading v.0.40-quota to a New Version * `cd` into the `mailinabox` directory. -* Execute `git pull --tags` to download the latest changes with tags. +* Execute `git pull` to download the latest changes. -* Execute `setup/start.sh` with root privileges. +* Execute `sudo bash setup/bootstrap.sh` to checkout the latest version and re-run setup. -Todo ----- +Issues +------ -* Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. +* When a user's quota is changed, any IMAP session running for that user will not recognize the new quota. To solve this a `dovecot reload` could be issued causing all current IMAP sessions to be terminated. On a system with many users, it might not be desirable to reset all users sessions to fix the quota for one user. Also if the administrator is setting the quota for several users it would result in the continual reset of those connections. Changes ------- +### v0.40-quota-0.14-alpha + +* When updating a user's quota, execute `doveadm quota recalc -u ` to forces an immediate recalculation of the user's quota. + +* Add a thousands separator (,) to the messages count in the control panel user list. + +* Execute `doveadm quota recalc -A` to force a recalculation of all user quotas when running `start.sh`. + +* Get rid of the error message complaining that the `quota` column already exists when upgrading from a previous version of `v0.40-quota`. + ### v0.40-quota-0.13-alpha * Add a `default-quota` setting in `settings.yaml`. @@ -75,7 +83,6 @@ Changes * Modify the mail users list in control panel to display percentage of quota used. - ### v0.40-quota-0.12-alpha * Update README diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 5c02f1ea..ea3a5044 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40-quota-0.13-alpha + TAG=v0.40-quota-0.14-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 4bed2221629eea27aacc22db1a5c14c8d0671dc4 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 6 Feb 2019 12:53:46 +0200 Subject: [PATCH 052/106] Fix bug where quotas are not being recalculated --- management/mailconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index 9a9f8407..0bca7b2a 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -421,7 +421,7 @@ def dovecot_quota_recalc(email): # subprocess.call(['doveadm', 'reload']) # force dovecot to recalculate the quota info for the user. - subprocess.call(["doveadm", "quota", "recalc" "-u", "%s" % email]) + subprocess.call(["doveadm", "quota", "recalc", "-u", email]) def get_default_quota(env): config = utils.load_settings(env) From 53d1c1e4e95cf7eb6377ce28ae467f3cc1257596 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 6 Feb 2019 12:58:05 +0200 Subject: [PATCH 053/106] Release new version v0.40-quota-0.15-alpha --- README.md | 4 ++++ setup/bootstrap.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c148a63..e484ff64 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ Issues Changes ------- +### v0.40-quota-0.15-alpha + +* Fix bug where quotas are not recalculated when a user's quota is changed in control panel + ### v0.40-quota-0.14-alpha * When updating a user's quota, execute `doveadm quota recalc -u ` to forces an immediate recalculation of the user's quota. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index ea3a5044..58ddc304 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40-quota-0.14-alpha + TAG=v0.40-quota-0.15-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From ae6394c879649fc8d440b0f250df8f89ed5e761f Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 6 Feb 2019 14:47:30 +0200 Subject: [PATCH 054/106] Fix instructions for quotas on the users page in control panel --- management/templates/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/templates/users.html b/management/templates/users.html index 5ea45da9..4ed33189 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -40,7 +40,7 @@
  • Use aliases to create email addresses that forward to existing accounts.
  • Administrators get access to this control panel.
  • User accounts cannot contain any international (non-ASCII) characters, but aliases can.
    • -
  • Quotas may not contain any spaces or commas. Suffixes of G (gigabytes) and M (megabytes) are allowed. For unlimited storage enter 0 (zero)
    • +
  • Quotas may not contain any spaces, commas or decimal points. Suffixes of G (gigabytes) and M (megabytes) are allowed. For unlimited storage enter 0 (zero)

    • Existing mail users

    From fc1f211af56a3d49a77a933c069d49cd5fbddeaf Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 10 Feb 2019 23:39:38 +0200 Subject: [PATCH 055/106] initial work on extended configuration --- conf/nginx.conf | 6 ++++-- management/auth.py | 2 +- management/daemon.py | 28 ++++++++++++++++++++++++++++ management/web_update.py | 18 ++++++++++++++++++ setup/mail-postfix.sh | 6 +++++- setup/start.sh | 16 ++++++++++++++++ setup/web.sh | 9 ++++++--- tools/dns-auth.sh | 2 ++ 8 files changed, 80 insertions(+), 7 deletions(-) create mode 100755 tools/dns-auth.sh diff --git a/conf/nginx.conf b/conf/nginx.conf index fafd3409..25910764 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,5 +1,6 @@ ## $HOSTNAME +#BEGIN_HTTP # Redirect all HTTP to HTTPS *except* the ACME challenges (Let's Encrypt TLS certificate # domain validation challenges) path, which must be served over HTTP per the ACME spec # (due to some Apache vulnerability). @@ -28,11 +29,12 @@ server { alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/; } } +#END_HTTP # The secure HTTPS server. server { - listen 443 ssl http2; - listen [::]:443 ssl http2; + listen $HTTP_SSL_PORT ssl http2; + listen [::]:$HTTP_SSL_PORT ssl http2; server_name $HOSTNAME; diff --git a/management/auth.py b/management/auth.py index 55f59664..0b082580 100644 --- a/management/auth.py +++ b/management/auth.py @@ -59,7 +59,7 @@ class KeyAuthService: credentials = decode(credentials) if ":" not in credentials: - return None, None + return credentials, None username, password = credentials.split(':', maxsplit=1) return username, password diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..3869cbaa 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -324,6 +324,34 @@ def dns_get_dump(): from dns_update import build_recommended_dns return json_response(build_recommended_dns(env)) +@app.route('/letsencrypt/dns-auth//', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_auth(domain, token): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', token, 'add', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + +@app.route('/letsencrypt/dns-cleanup/', methods=['GET']) +@authorized_personnel_only +def letsencrypt_dns_cleanup(domain): + from dns_update import do_dns_update, set_custom_dns_record + try: + qname = '_acme-challenge.' + domain + if set_custom_dns_record(qname, 'TXT', None, 'remove', env): + if not do_dns_update(env): + return ("Error updating DNS", 400) + return "OK" + + except ValueError as e: + return (str(e), 400) + # SSL @app.route('/ssl/status') diff --git a/management/web_update.py b/management/web_update.py index 61b38a7b..616698ba 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -94,6 +94,20 @@ def do_web_update(env): # Add default 'www.' redirect. nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env) + if str(env['HTTP_SSL_PORT']) != "443": + in_http = False + new_conf = '' + for line in nginx_conf.split('\n'): + if line.strip() == '#BEGIN_HTTP': + in_http = True + elif line.strip() == '#END_HTTP': + in_http = False + + if not in_http: + new_conf += line + '\n' + + nginx_conf = new_conf + # Did the file change? If not, don't bother writing & restarting nginx. nginx_conf_fn = "/etc/nginx/conf.d/local.conf" if os.path.exists(nginx_conf_fn): @@ -178,8 +192,12 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf) # Replace substitution strings in the template & return. + if int(env['HTTP_SSL_PORT']) != 443: + # disable the regular HTTP server + nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE) nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) nginx_conf = nginx_conf.replace("$HOSTNAME", domain) + nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT']) nginx_conf = nginx_conf.replace("$ROOT", root) nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"]) nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"]) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..1ea64a50 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -193,9 +193,13 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC +RECIPIENT_RESTRICTIONS=permit_sasl_authenticated,permit_mynetworks,\"reject_rbl_client zen.spamhaus.org\",reject_unlisted_recipient +if [ $NO_GREYLISTING != "1" ]; then + RECIPIENT_RESTRICTIONS=${RECIPIENT_RESTRICTIONS},\"check_policy_service inet:127.0.0.1:10023\" +fi tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=$RECIPIENT_RESTRICTIONS # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). diff --git a/setup/start.sh b/setup/start.sh index 0b145022..60a0349e 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -42,6 +42,20 @@ else FIRST_TIME_SETUP=1 fi +if [ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]; then + HTTP_SSL_PORT=443 +else + HTTP_SSL_PORT=$DEFAULT_HTTP_SSL_PORT +fi + +if [ -z "${DEFAULT_NO_GREYLISTING:-}" ]; then + NO_GREYLISTING=0 +elif (($DEFAULT_NO_GREYLISTING > 0)); then + NO_GREYLISTING=1 +else + NO_GREYLISTING=0 +fi + # Put a start script in a global location. We tell the user to run 'mailinabox' # in the first dialog prompt, so we should do this before that starts. cat > /usr/local/bin/mailinabox << EOF; @@ -93,6 +107,8 @@ PUBLIC_IP=$PUBLIC_IP PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 +HTTP_SSL_PORT=$HTTP_SSL_PORT +NO_GREYLISTING=$NO_GREYLISTING EOF # Start service configuration. diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..ad7ad310 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -96,6 +96,9 @@ restart_service nginx restart_service php7.2-fpm # Open ports. -ufw_allow http -ufw_allow https - +if [ $HTTP_SSL_PORT == 443 ]; then + ufw_allow http + ufw_allow https +else + ufw_allow $HTTP_SSL_PORT +fi diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh new file mode 100755 index 00000000..20d602bd --- /dev/null +++ b/tools/dns-auth.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash + From 514619b44ab9927f3922e6bfac5096eadd304bce Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 11 Feb 2019 16:05:50 +0200 Subject: [PATCH 056/106] setup dns authentication for letsencrypt --- .gitignore | 1 + management/ssl_certificates.py | 6 +++++- tools/dns-auth.sh | 8 ++++++++ tools/dns-cleanup.sh | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 tools/dns-cleanup.sh diff --git a/.gitignore b/.gitignore index f3cdb1bc..94072693 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ tools/__pycache__/ externals/ .env .vagrant +.idea/ diff --git a/management/ssl_certificates.py b/management/ssl_certificates.py index 76b0f8fa..6d433b54 100755 --- a/management/ssl_certificates.py +++ b/management/ssl_certificates.py @@ -313,6 +313,7 @@ def provision_certificates(env, limit_domains): webroot = os.path.join(account_path, 'webroot') os.makedirs(webroot, exist_ok=True) with tempfile.TemporaryDirectory() as d: + miab_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) cert_file = os.path.join(d, 'cert_and_chain.pem') print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".") certbotret = subprocess.check_output([ @@ -328,7 +329,10 @@ def provision_certificates(env, limit_domains): "--chain-path", os.path.join(d, 'chain'), # we only use the full chain "--fullchain-path", cert_file, - "--webroot", "--webroot-path", webroot, + "--manual", + "--preferred-challenge", "dns", + "--manual-auth-hook", os.path.join(miab_dir, "/tools/dns-auth.sh"), + "--manual-cleanup-hook", os.path.join(miab_dir, "/tools/dns-cleanup.sh"), "--config-dir", account_path, #"--staging", diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh index 20d602bd..e3ee77c8 100755 --- a/tools/dns-auth.sh +++ b/tools/dns-auth.sh @@ -1,2 +1,10 @@ #!/usr/bin/env bash +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT + +sleep 15 diff --git a/tools/dns-cleanup.sh b/tools/dns-cleanup.sh new file mode 100755 index 00000000..f9ac582d --- /dev/null +++ b/tools/dns-cleanup.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# TODO: Make work with port other than 443 + +API_KEY=`cat /var/lib/mailinabox/api.key` +HOSTNAME=`hostname` + +curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT \ No newline at end of file From 7f8336e459060eeba4acf2bfed8f9f1bf21498d0 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Mon, 11 Feb 2019 16:10:09 +0200 Subject: [PATCH 057/106] Fix bug with quota input that prevented adding users --- README.md | 4 ++++ management/templates/users.html | 2 +- setup/bootstrap.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e484ff64..332d6e67 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ Issues Changes ------- +### v0.40-quota-0.16-alpha + +* Fix problem with quota field on control panel that prevented adding users. + ### v0.40-quota-0.15-alpha * Fix bug where quotas are not recalculated when a user's quota is changed in control panel diff --git a/management/templates/users.html b/management/templates/users.html index 4ed33189..dd2fefbe 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -31,7 +31,7 @@
    - +
    diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 58ddc304..418b0557 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.40-quota-0.15-alpha + TAG=v0.40-quota-0.16-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 2b70277a3587d0fddb0bf883fd1f9ccefee6d719 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 15 Feb 2019 16:08:06 +0200 Subject: [PATCH 058/106] Update README instructions for installing and upgrading --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 332d6e67..6c86de63 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,15 @@ Please report any bugs on github. Installing v0.40-quota ---------------------- -Follow the directions below for installing from a repository except clone from this repository instead of the official one. +To install the latest version, log into your box and execute the following commands: + + $ git clone https://github.com/jrsupplee/mailinabox.git + $ cd mailinabox + $ sudo bash setup/bootstrap.sh + +Follow the standard directions for setting up an MiaB installation. There are no special installation steps for installing this version. + +The default quota is set to `0` which means unlimited. If you want to set a different default quota, follow the directions above. Upgrading v0.40 to v.0.40-quota @@ -37,7 +45,9 @@ This is experimental software. You have been warned. `git clone https://github.com/jrsupplee/mailinabox.git` -* cd into `mailinabox` and run `sudo setup/start.sh` On occasion there are lock errors when updating `Munin`. Just re-run `sudo setup/start.sh` until the error does not occur. +* cd into `mailinabox` and run `sudo setup/bootstrap.sh` On occasion there are lock errors when updating `Munin`. Just re-run `sudo setup/start.sh` until the error does not occur. + +* Note: all existing users at the time of the upgrade will have there quota set to `0` (unlimited). Upgrading v.0.40-quota to a New Version From aa234a504e008dff8d71a962ef01289862e89e9e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 12:48:34 +0200 Subject: [PATCH 059/106] changes for SPF on incoming email --- setup/mail-postfix.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index 0c9bc97c..4802adeb 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -42,7 +42,8 @@ source /etc/mailinabox.conf # load global vars # * `ca-certificates`: A trust store used to squelch postfix warnings about # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." -apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates +apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ + postfix-policyd-spf-python # ### Basic Settings @@ -97,7 +98,9 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ -o cleanup_service_name=authclean" \ "authclean=unix n - - - 0 cleanup -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters - -o nested_header_checks=" + -o nested_header_checks=" \ + "policy-spf=unix - n n - - spawn + user=nobody argv=/usr/bin/policyd-spf" # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters @@ -195,7 +198,7 @@ tools/editconf.py /etc/postfix/main.cf virtual_transport=lmtp:[127.0.0.1]:10025 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -214,6 +217,7 @@ tools/editconf.py /etc/postfix/main.cf \ # Allow the two SMTP ports in the firewall. + ufw_allow smtp ufw_allow submission From adf7a31866ed76cb6d8231c97516b020c3a11111 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 15:02:11 +0200 Subject: [PATCH 060/106] Update README and bump the version --- README.md | 16 +++++++++++----- setup/bootstrap.sh | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fedbfd95..79d62f33 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Mailbox size recalculation by Dovecot can be forced using the command: Please report any bugs on github. -Installing v0.40-quota +Installing v0.41-quota ---------------------- To install the latest version, log into your box and execute the following commands: @@ -34,7 +34,7 @@ Follow the standard directions for setting up an MiaB installation. There are n The default quota is set to `0` which means unlimited. If you want to set a different default quota, follow the directions above. -Upgrading v0.40 to v.0.40-quota +Upgrading v0.41 to v.0.41-quota ------------------------------- This is experimental software. You have been warned. @@ -50,11 +50,9 @@ This is experimental software. You have been warned. * Note: all existing users at the time of the upgrade will have there quota set to `0` (unlimited). -Upgrading v.0.40-quota to a New Version +Upgrading MiaB with quotas to a New Version --------------------------------------- -* Remember that this is experimental software and review the changes in the repository. - * `cd` into the `mailinabox` directory. * Execute `git pull` to download the latest changes. @@ -71,6 +69,14 @@ Issues Changes ------- +### v0.41-quota-0.17-alpha + +* Update the README + +### v0.41-quota-0.16-alpha + +* Update to v0.41 of Mail-in-a-Box + ### v0.40-quota-0.16-alpha * Fix problem with quota field on control panel that prevented adding users. diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 7b57ee64..3d30e438 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.41-quota-0.16-alpha + TAG=v0.41-quota-0.17-alpha elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 68f4d1c426ca239b40e1f077305709ed0c6d1feb Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 27 Feb 2019 17:30:59 +0200 Subject: [PATCH 061/106] add SRS support --- setup/mail-postfix.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index e59de474..e974fb91 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -43,7 +43,7 @@ source /etc/mailinabox.conf # load global vars # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ - postfix-policyd-spf-python + postfix-policyd-spf-python postsrsd # ### Basic Settings @@ -201,7 +201,7 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC tools/editconf.py /etc/postfix/main.cf \ smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ - smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service unix:private/policy-spf" + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service unix:private/policy-spf","check_policy_service inet:127.0.0.1:10023" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -218,6 +218,16 @@ tools/editconf.py /etc/default/postgrey \ tools/editconf.py /etc/postfix/main.cf \ message_size_limit=134217728 +# Setup SRS +postconf -e \ + sender_canonical_maps=tcp:localhost:10001 \ + sender_canonical_classes=envelope_sender \ + recipient_canonical_maps=tcp:localhost:10002 \ + recipient_canonical_classes=envelope_recipient,header_recipient + +hide_output systemctl enable postsrsd +hide_output systemctl restart postsrsd + # Allow the two SMTP ports in the firewall. From 887e2927480303014992427ec1bb1a4d023c0785 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:30:08 +0200 Subject: [PATCH 062/106] Add changes for mailgraph webserver --- conf/nginx-primaryonly.conf | 10 ++++++++++ setup/web.sh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index d8d912ca..be6588e2 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -64,4 +64,14 @@ rewrite ^/.well-known/carddav /cloud/remote.php/carddav/ redirect; rewrite ^/.well-known/caldav /cloud/remote.php/caldav/ redirect; + location ~ ^/mailgraph/.*\.cgi { + root /usr/share/mailgraph; + include /etc/nginx/fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.$ + fastcgi_pass unix:/var/run/fcgiwrap.socket; + + auth_basic "box"; + auth_basic_user_file /etc/nginx/htpasswd; + } + # ADDITIONAL DIRECTIVES HERE diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..6cb1a30b 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -19,7 +19,7 @@ fi echo "Installing Nginx (web server)..." -apt_install nginx php-cli php-fpm +apt_install nginx php-cli php-fpm fcgiwrap mailgraph rm -f /etc/nginx/sites-enabled/default From 69a50ed9541712b39939a28d9a2d0336a6fb80ac Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:39:19 +0200 Subject: [PATCH 063/106] fix error in nginx configuration --- conf/nginx-primaryonly.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index be6588e2..2eeacc6f 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -67,7 +67,7 @@ location ~ ^/mailgraph/.*\.cgi { root /usr/share/mailgraph; include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.$ + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi fastcgi_pass unix:/var/run/fcgiwrap.socket; auth_basic "box"; From 8180707cc63ef234f094fc240bbd6f8d2cbcf102 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 15:43:30 +0200 Subject: [PATCH 064/106] add missing semicolon to nginx conf --- conf/nginx-primaryonly.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index 2eeacc6f..552cf28a 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -67,7 +67,7 @@ location ~ ^/mailgraph/.*\.cgi { root /usr/share/mailgraph; include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi + fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi; fastcgi_pass unix:/var/run/fcgiwrap.socket; auth_basic "box"; From 5615031ef8a040a1fc0a90d65da562cfb6fca787 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 17:23:58 +0200 Subject: [PATCH 065/106] initial work to integrate with control panel --- management/templates/index.html | 1 + management/templates/mailgraph.html | 32 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 management/templates/mailgraph.html diff --git a/management/templates/index.html b/management/templates/index.html index 2c0d5a9a..31fe5916 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -102,6 +102,7 @@
  • Instructions
  • Users
  • Aliases
    • +
  • Reports
  • Contacts/Calendar
    • diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html new file mode 100644 index 00000000..9d495b0c --- /dev/null +++ b/management/templates/mailgraph.html @@ -0,0 +1,32 @@ +

    Mail statistics

    + + +

    Last Day

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Week

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Month

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +

    Last Year

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    + +
    + +

    Mailgraph 1.14 by David Schweikert +(built on Tobi Oetiker's RRDtool)

    From fefb5ebc3342cf8a07520091b4fbd7f0da4cdfb6 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:16:39 +0200 Subject: [PATCH 066/106] more work on control panel integration --- management/daemon.py | 14 ++++++++++++ management/templates/index.html | 2 +- management/templates/mailgraph.html | 34 ++++++++++++++--------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 572b6b4a..918370ef 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -520,6 +520,20 @@ def privacy_status_set(): utils.write_settings(config, env) return "OK" +# Mailgraph + +@app.route('/mailgraph/image.cgi') +@authorized_personnel_only +def mailgraph(): + if request.query_string: + return utils.shell( + "check_output", + ["/usr/share/mailgraph/mailgraph.cgi"], + env={"QUERY_STRING": request.query_string} + ) + return '' + + # MUNIN @app.route('/munin/') diff --git a/management/templates/index.html b/management/templates/index.html index 31fe5916..9f8b8652 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -102,7 +102,7 @@
  • Instructions
  • Users
  • Aliases
    • -
  • Reports
    • +
  • Mailgraph
  • Contacts/Calendar
    • diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 9d495b0c..8b1ad893 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -1,4 +1,4 @@ -

    Mail statistics

    +

    Mail statistics

    -

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Day

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Week

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Month

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    -

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    Last Year

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    From fd239db7c1e2ae92bb9bcdfec5c8b65cec373723 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:22:08 +0200 Subject: [PATCH 067/106] integrate template --- management/templates/index.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/management/templates/index.html b/management/templates/index.html index 9f8b8652..26d25e4a 100644 --- a/management/templates/index.html +++ b/management/templates/index.html @@ -152,6 +152,10 @@ {% include "sync-guide.html" %} +
    + {% include "mailgraph.html" %} +
    +
    {% include "web.html" %}
    From d41cdb844cb6965aa2958b5627b8e88d149ff2de Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:31:24 +0200 Subject: [PATCH 068/106] remove nginx config for Mailgraph --- conf/nginx-primaryonly.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/conf/nginx-primaryonly.conf b/conf/nginx-primaryonly.conf index 552cf28a..d8d912ca 100644 --- a/conf/nginx-primaryonly.conf +++ b/conf/nginx-primaryonly.conf @@ -64,14 +64,4 @@ rewrite ^/.well-known/carddav /cloud/remote.php/carddav/ redirect; rewrite ^/.well-known/caldav /cloud/remote.php/caldav/ redirect; - location ~ ^/mailgraph/.*\.cgi { - root /usr/share/mailgraph; - include /etc/nginx/fastcgi_params; - fastcgi_param SCRIPT_FILENAME /usr/share/mailgraph/mailgraph.cgi; - fastcgi_pass unix:/var/run/fcgiwrap.socket; - - auth_basic "box"; - auth_basic_user_file /etc/nginx/htpasswd; - } - # ADDITIONAL DIRECTIVES HERE From 5ffa71999ac42001fcc825a717a53741f54c32ea Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 20:47:54 +0200 Subject: [PATCH 069/106] work on the daemon for mailgraph --- management/daemon.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 918370ef..54b1a565 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -526,12 +526,21 @@ def privacy_status_set(): @authorized_personnel_only def mailgraph(): if request.query_string: - return utils.shell( + print("QUERY_STRING=%s" % request.query_string) + + code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], - env={"QUERY_STRING": request.query_string} + env={"QUERY_STRING": request.query_string}, + return_bytes=True ) - return '' + + if code != 0: + return ('Error generating mailgraph image: %s' % request.query_string, 500) + + return make_response(bin_out) + + return ('Mailgraph: no image requested', 500) # MUNIN From 89677584132d62c0d1152335459887202be599f5 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 23:18:44 +0200 Subject: [PATCH 070/106] Only load mailgraph images when the page is viewed --- management/templates/mailgraph.html | 32 ++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 8b1ad893..fc96f9b1 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,26 +7,34 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Mailgraph 1.14 by David Schweikert (built on Tobi Oetiker's RRDtool)

    + + From 5b5087c9dd1bdf35c35b35c411be4bbf550dfd15 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sat, 2 Mar 2019 23:33:45 +0200 Subject: [PATCH 071/106] fix query params for mailgraph images --- management/templates/mailgraph.html | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index fc96f9b1..086aa9cc 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,24 +7,24 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    From dce4058705714f30c4e8271d26315da75d6a14e3 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 00:34:41 +0200 Subject: [PATCH 072/106] process images returned from mailgraph --- management/daemon.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 54b1a565..b53c3c40 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -522,23 +522,29 @@ def privacy_status_set(): # Mailgraph -@app.route('/mailgraph/image.cgi') +@app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): if request.query_string: - print("QUERY_STRING=%s" % request.query_string) + app.logger.error("QUERY_STRING=%s" % request.query_string) code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], env={"QUERY_STRING": request.query_string}, - return_bytes=True + return_bytes=True, + trap=True ) if code != 0: return ('Error generating mailgraph image: %s' % request.query_string, 500) - return make_response(bin_out) + headers, image_bytes = bin_out.split(b'\n\n', 1) + response = make_response(image_bytes) + for line in headers.splitlines(): + name, value = line.decode("utf8").split(':', 1) + response.headers[name] = value + return response return ('Mailgraph: no image requested', 500) From 6883a60f5d024437a3a8b8a60688a5d73844a91e Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 19:15:59 +0200 Subject: [PATCH 073/106] load images as base64 --- management/daemon.py | 9 +++----- management/templates/mailgraph.html | 33 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index b53c3c40..7e0d4936 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -521,7 +521,7 @@ def privacy_status_set(): return "OK" # Mailgraph - +import base64 @app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): @@ -540,11 +540,8 @@ def mailgraph(): return ('Error generating mailgraph image: %s' % request.query_string, 500) headers, image_bytes = bin_out.split(b'\n\n', 1) - response = make_response(image_bytes) - for line in headers.splitlines(): - name, value = line.decode("utf8").split(':', 1) - response.headers[name] = value - return response + + return base64.b64encode(image_bytes) return ('Mailgraph: no image requested', 500) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 086aa9cc..25428509 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -7,24 +7,24 @@

    Last Day

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Week

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Month

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    Last Year

    -

    mailgraph

    -

    mailgraph

    -

    mailgraph

    +

    mailgraph

    +

    mailgraph

    +

    mailgraph

    @@ -34,7 +34,14 @@ From e29e3a5cba43ac3094dd0b776dc1cfd78b5c26ae Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 19:23:58 +0200 Subject: [PATCH 074/106] fix QUERY_STRING to only have the image request --- management/daemon.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 7e0d4936..b3f5689c 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -526,12 +526,16 @@ import base64 @authorized_personnel_only def mailgraph(): if request.query_string: - app.logger.error("QUERY_STRING=%s" % request.query_string) + query = request.query_string + if '&' in query: + query = query.split('&')[0] + + app.logger.error("QUERY_STRING=%s" % query) code, bin_out = utils.shell( "check_output", ["/usr/share/mailgraph/mailgraph.cgi"], - env={"QUERY_STRING": request.query_string}, + env={"QUERY_STRING": query}, return_bytes=True, trap=True ) From 7600e727c118e92c0338bb32c2908addc2e06445 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Sun, 3 Mar 2019 21:15:16 +0200 Subject: [PATCH 075/106] debugging of image display --- management/daemon.py | 10 ++++++---- management/templates/mailgraph.html | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index b3f5689c..9925245d 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -1,5 +1,7 @@ import os, os.path, re, json, time import subprocess +import base64 +import sys from functools import wraps @@ -521,16 +523,16 @@ def privacy_status_set(): return "OK" # Mailgraph -import base64 + @app.route('/mailgraph/image.cgi', methods=['GET']) @authorized_personnel_only def mailgraph(): if request.query_string: - query = request.query_string + query = request.query_string.decode('utf-8', 'ignore') if '&' in query: query = query.split('&')[0] - app.logger.error("QUERY_STRING=%s" % query) + print("QUERY_STRING=%s" % query, file=sys.stderr) code, bin_out = utils.shell( "check_output", @@ -541,7 +543,7 @@ def mailgraph(): ) if code != 0: - return ('Error generating mailgraph image: %s' % request.query_string, 500) + return ('Error generating mailgraph image: %s' % query, 500) headers, image_bytes = bin_out.split(b'\n\n', 1) diff --git a/management/templates/mailgraph.html b/management/templates/mailgraph.html index 25428509..22d76088 100644 --- a/management/templates/mailgraph.html +++ b/management/templates/mailgraph.html @@ -34,8 +34,9 @@ diff --git a/management/web_update.py b/management/web_update.py index 1d8da03f..12959632 100644 --- a/management/web_update.py +++ b/management/web_update.py @@ -100,20 +100,6 @@ def do_web_update(env): # Add default 'www.' redirect. nginx_conf += make_domain_config(domain, [template0, template3], ssl_certificates, env) - if str(env['HTTP_SSL_PORT']) != "443": - in_http = False - new_conf = '' - for line in nginx_conf.split('\n'): - if line.strip() == '#BEGIN_HTTP': - in_http = True - elif line.strip() == '#END_HTTP': - in_http = False - - if not in_http: - new_conf += line + '\n' - - nginx_conf = new_conf - # Did the file change? If not, don't bother writing & restarting nginx. nginx_conf_fn = "/etc/nginx/conf.d/local.conf" if os.path.exists(nginx_conf_fn): @@ -198,12 +184,8 @@ def make_domain_config(domain, templates, ssl_certificates, env): nginx_conf = re.sub("[ \t]*# ADDITIONAL DIRECTIVES HERE *\n", t, nginx_conf) # Replace substitution strings in the template & return. - if int(env['HTTP_SSL_PORT']) != 443: - # disable the regular HTTP server - nginx_conf = re.sub(r'#BEGIN_HTTP.*?#END_HTTP', repl='', string=nginx_conf, flags=re.MULTILINE) nginx_conf = nginx_conf.replace("$STORAGE_ROOT", env['STORAGE_ROOT']) nginx_conf = nginx_conf.replace("$HOSTNAME", domain) - nginx_conf = nginx_conf.replace("$HTTP_SSL_PORT", env['HTTP_SSL_PORT']) nginx_conf = nginx_conf.replace("$ROOT", root) nginx_conf = nginx_conf.replace("$SSL_KEY", tls_cert["private-key"]) nginx_conf = nginx_conf.replace("$SSL_CERTIFICATE", tls_cert["certificate"]) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1263c703..00000000 --- a/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -rtyaml -email_validator>=1.0.0 -exclusiveprocess -flask -dnspython -python-dateutil -idna>=2.0.0 -cryptography==2.2.2 -boto -psutil -npyscreen diff --git a/setup/mail-postfix.sh b/setup/mail-postfix.sh index e3ba3422..684aafdd 100755 --- a/setup/mail-postfix.sh +++ b/setup/mail-postfix.sh @@ -42,8 +42,7 @@ source /etc/mailinabox.conf # load global vars # * `ca-certificates`: A trust store used to squelch postfix warnings about # untrusted opportunistically-encrypted connections. echo "Installing Postfix (SMTP server)..." -apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates \ - postfix-policyd-spf-python postsrsd +apt_install postfix postfix-sqlite postfix-pcre postgrey ca-certificates # ### Basic Settings @@ -98,9 +97,7 @@ tools/editconf.py /etc/postfix/master.cf -s -w \ -o cleanup_service_name=authclean" \ "authclean=unix n - - - 0 cleanup -o header_checks=pcre:/etc/postfix/outgoing_mail_header_filters - -o nested_header_checks=" \ - "policy-spf=unix - n n - - spawn - user=nobody argv=/usr/bin/policyd-spf" + -o nested_header_checks=" # Install the `outgoing_mail_header_filters` file required by the new 'authclean' service. cp conf/postfix_outgoing_mail_header_filters /etc/postfix/outgoing_mail_header_filters @@ -199,23 +196,9 @@ tools/editconf.py /etc/postfix/main.cf lmtp_destination_recipient_limit=1 # so these IPs get mail delivered quickly. But when an IP is not listed in the permit_dnswl_client list (i.e. it is not #NODOC # whitelisted) then postfix does a DEFER_IF_REJECT, which results in all "unknown user" sorts of messages turning into #NODOC # "450 4.7.1 Client host rejected: Service unavailable". This is a retry code, so the mail doesn't properly bounce. #NODOC - -postconf -e smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" - -RECIPIENT_RESTRICTIONS="permit_sasl_authenticated,permit_mynetworks,reject_rbl_client zen.spamhaus.org,reject_unlisted_recipient" - -if [ $POSTGREY == 1 ]; then - RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:10023" -fi - -if [ $POLICY_SPF == 1 ]; then - RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service unix:private/policy-spf" -fi - -# Add quota check -RECIPIENT_RESTRICTIONS="${RECIPIENT_RESTRICTIONS},check_policy_service inet:127.0.0.1:12340" - -postconf -e smtpd_recipient_restrictions="$RECIPIENT_RESTRICTIONS" +tools/editconf.py /etc/postfix/main.cf \ + smtpd_sender_restrictions="reject_non_fqdn_sender,reject_unknown_sender_domain,reject_authenticated_sender_login_mismatch,reject_rhsbl_sender dbl.spamhaus.org" \ + smtpd_recipient_restrictions=permit_sasl_authenticated,permit_mynetworks,"reject_rbl_client zen.spamhaus.org",reject_unlisted_recipient,"check_policy_service inet:127.0.0.1:10023","check_policy_service inet:127.0.0.1:12340" # Postfix connects to Postgrey on the 127.0.0.1 interface specifically. Ensure that # Postgrey listens on the same interface (and not IPv6, for instance). @@ -259,29 +242,6 @@ chmod +x /etc/cron.daily/mailinabox-postgrey-whitelist tools/editconf.py /etc/postfix/main.cf \ message_size_limit=134217728 -if [ $POSTSRSD == 1 ]; then - # Setup SRS - postconf -e \ - sender_canonical_maps=tcp:localhost:10001 \ - sender_canonical_classes=envelope_sender \ - recipient_canonical_maps=tcp:localhost:10002 \ - recipient_canonical_classes=envelope_recipient,header_recipient - - hide_output systemctl enable postsrsd - hide_output systemctl restart postsrsd - -else - postconf -e \ - sender_canonical_maps= \ - sender_canonical_classes= \ - recipient_canonical_maps= \ - recipient_canonical_classes= - - hide_output systemctl disable postsrsd - hide_output systemctl stop postsrsd -fi - - # Allow the two SMTP ports in the firewall. ufw_allow smtp @@ -290,11 +250,4 @@ ufw_allow submission # Restart services restart_service postfix - -if [ $POSTGREY == 1 ]; then - hide_output systemctl enable postgrey - hide_output systemctl restart postgrey -else - hide_output systemctl disable postgrey - hide_output systemctl stop postgrey -fi +restart_service postgrey diff --git a/setup/options-dialog.py b/setup/options-dialog.py deleted file mode 100644 index 166ee455..00000000 --- a/setup/options-dialog.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import npyscreen -import sys -import os - - -class OptionsApp(npyscreen.NPSApp): - def main(self): - # These lines create the form and populate it with widgets. - # A fairly complex screen in only 8 or so lines of code - a line for each control. - npyscreen.setTheme(npyscreen.Themes.BlackOnWhiteTheme) - - form = npyscreen.Form(name = "Mail-in-a-Box Options",) - form.add( - npyscreen.TitleFixedText, - name="POSTGREY", - value="", - editable=False - ) - form.add( - npyscreen.MultiLineEdit, - value="The Postgrey service greylists incoming messages from unknown senders.\n" - "It can be useful for fighting spam but often causes message delivery\n" - "delays of several minutes.", - max_height=4, - editable=False - ) - - form.add( - npyscreen.TitleFixedText, - name="POSTSRSD", - value="", - editable=False - ) - form.add( - npyscreen.MultiLineEdit, - value="The PostSRSd daemon performs return path rewriting using the SRS protocol.\n" - "Not that all messages, including locally delivered mail will have their return\n" - "paths rewritten", - max_height=4, - editable=False - ) - - form.add( - npyscreen.TitleFixedText, - name="POLICY_SPF", - value="", - editable=False - ) - form.add( - npyscreen.MultiLineEdit, - value="" - "The policy SPF service checks the SPF of incoming mails and rejects those\n" - "that do not qualify. This helps to prevent spoofing, but if valid mail does\n" - "not have SPF configured properly it will be rejected.", - max_height=4, - editable=False - ) - - init_values = [] - if int(os.getenv('POSTGREY', 1)) == 1: - init_values.append(0) - - if int(os.getenv('POSTSRSD', 0)) == 1: - init_values.append(1) - - if int(os.getenv('POLICY_SPF', 0)) == 1: - init_values.append(2) - - options = form.add( - npyscreen.TitleMultiSelect, - max_height=-2, - value=init_values, - name="Options", - values= ["POSTGREY","POSTSRSD","POLICY_SPF"], - scroll_exit=True - ) - - # This lets the user interact with the Form. - form.edit() - - with open('_options.sh', 'w') as output: - print('POSTGREY=%i' % (1 if 0 in options.value else 0), file=output) - print('POSTSRSD=%i' % (1 if 1 in options.value else 0), file=output) - print('POLICY_SPF=%i' % (1 if 2 in options.value else 0), file=output) - # print(npyscreen.ThemeManager.default_colors, file=output) - - -if __name__ == "__main__": - App = OptionsApp() - App.run() diff --git a/setup/questions.sh b/setup/questions.sh index 61ef19bf..bf382f49 100644 --- a/setup/questions.sh +++ b/setup/questions.sh @@ -16,7 +16,6 @@ if [ -z "${NONINTERACTIVE:-}" ]; then # we install it inside a virtualenv. In this script, we don't have the virtualenv yet # so we install the python package globally. hide_output pip3 install "email_validator>=1.0.0" || exit 1 - hide_output pip3 install npyscreen || exit 1 message_box "Mail-in-a-Box Installation" \ "Hello and thanks for deploying a Mail-in-a-Box! @@ -194,16 +193,6 @@ if [ -z "${STORAGE_ROOT:-}" ]; then STORAGE_ROOT=$([[ -z "${DEFAULT_STORAGE_ROOT:-}" ]] && echo "/home/$STORAGE_USER" || echo "$DEFAULT_STORAGE_ROOT") fi -# export options variables so they are visible to the options program -export POSTGREY -export POSTSRSD -export POLICY_SPF - -python3 setup/options-dialog.py -source ./_options.sh -rm _options.sh - - # Show the configuration, since the user may have not entered it manually. echo echo "Primary Hostname: $PRIMARY_HOSTNAME" diff --git a/setup/solr.sh b/setup/solr.sh deleted file mode 100644 index 05e64433..00000000 --- a/setup/solr.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -# -# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) -# with some modifications -# -# IMAP search with lucene via solr -# -------------------------------- -# -# By default dovecot uses its own Squat search index that has awful performance -# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but -# this didn't make it into the Ubuntu packages, so we use Solr instead to run -# Lucene for us. -# -# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its -# HTTP interface, causing mail to be indexed when searches occur, and getting -# results back. - -source setup/functions.sh # load our functions -source /etc/mailinabox.conf # load global vars - -# Install packages and basic configuation -# --------------------------------------- - -echo "Installing Solr..." - -# Install packages -apt_install solr-tomcat dovecot-solr - -# Solr requires a schema to tell it how to index data, this is provided by dovecot -cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml - -# Update the dovecot plugin configuration -# -# Break-imap-search makes search work the way users expect, rather than the way -# the IMAP specification expects -tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ - mail_plugins="fts fts_solr" - -cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; -plugin { - fts = solr - fts_autoindex = yes - fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ -} -EOF - -# Bump memory allocation for Solr. -# Not needed? I'll let it sit here for a while. -#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh - -# Install cronjobs to keep FTS up to date -hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ -hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ - -# PERMISSIONS - -# Ensure configuration files are owned by dovecot and not world readable. -chown -R mail:dovecot /etc/dovecot -chmod -R o-rwx /etc/dovecot - -mkdir -p /etc/systemd/system/tomcat9.service.d -cat > /etc/systemd/system/tomcat9.service.d/solr-permissions.conf << EOF -[Service] -ReadWritePaths=/var/lib/solr/ -ReadWritePaths=/var/lib/solr/data/ -EOF - -# Restart services to reload solr schema & dovecot plugins -restart_service tomcat9 -restart_service dovecot - - -# Kickoff building the index - -# Per doveadm-fts manpage: Scan what mails exist in the full text search index -# and compare those to what actually exist in mailboxes. -# This removes mails from the index that have already been expunged and makes -# sure that the next doveadm index will index all the missing mails (if any). -doveadm fts rescan -A - -# Adds unindexed files to the fts database -# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) -# * `-A`: All users -# * `'*'`: All folders -doveadm index -q -A '*' diff --git a/setup/start.sh b/setup/start.sh index 2bdea975..0b145022 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -42,22 +42,6 @@ else FIRST_TIME_SETUP=1 fi -if [ -z "${HTTP_SSL_PORT:-}" ]; then - HTTP_SSL_PORT=$([[ -z "${DEFAULT_HTTP_SSL_PORT:-}" ]] && echo "443" || echo "$DEFAULT_HTTP_SSL_PORT") -fi - -if [ -z "${POSTGREY:-}" ]; then - POSTGREY=$([[ -z "${DEFAULT_POSTGREY:-}" ]] && echo "1" || echo "$DEFAULT_POSTGREY") -fi - -if [ -z "${POSTSRSD:-}" ]; then - POSTSRSD=$([[ -z "${DEFAULT_POSTSRSD:-}" ]] && echo "0" || echo "$DEFAULT_POSTSRSD") -fi - -if [ -z "${POLICY_SPF:-}" ]; then - POLICY_SPF=$([[ -z "${DEFAULT_POLICY_SPF:-}" ]] && echo "0" || echo "$DEFAULT_POLICY_SPF") -fi - # Put a start script in a global location. We tell the user to run 'mailinabox' # in the first dialog prompt, so we should do this before that starts. cat > /usr/local/bin/mailinabox << EOF; @@ -109,10 +93,6 @@ PUBLIC_IP=$PUBLIC_IP PUBLIC_IPV6=$PUBLIC_IPV6 PRIVATE_IP=$PRIVATE_IP PRIVATE_IPV6=$PRIVATE_IPV6 -HTTP_SSL_PORT=$HTTP_SSL_PORT -POSTGREY=$POSTGREY -POSTSRSD=$POSTSRSD -POLICY_SPF=$POLICY_SPF EOF # Start service configuration. @@ -122,7 +102,6 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh -source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh diff --git a/setup/web.sh b/setup/web.sh index 02c3a45c..ed37e5e3 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -19,7 +19,7 @@ fi echo "Installing Nginx (web server)..." -apt_install nginx php-cli php-fpm fcgiwrap mailgraph +apt_install nginx php-cli php-fpm rm -f /etc/nginx/sites-enabled/default @@ -48,12 +48,6 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" -# Set higher timeout since searches with Roundcube and Solr may take longer -# than the default 60 seconds. We will also match Roundcube's timeout to the -# same value -tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ - default_socket_timeout=180 - # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand @@ -102,9 +96,6 @@ restart_service nginx restart_service php7.2-fpm # Open ports. -if [ $HTTP_SSL_PORT == 443 ]; then - ufw_allow http - ufw_allow https -else - ufw_allow $HTTP_SSL_PORT -fi +ufw_allow http +ufw_allow https + diff --git a/setup/webmail.sh b/setup/webmail.sh index 9182228f..5e1c6233 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 180; +\$config['imap_timeout'] = 15; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u'; diff --git a/tools/dns-auth.sh b/tools/dns-auth.sh deleted file mode 100755 index e3ee77c8..00000000 --- a/tools/dns-auth.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -# TODO: Make work with port other than 443 - -API_KEY=`cat /var/lib/mailinabox/api.key` -HOSTNAME=`hostname` - -curl -s -X PUT -d "$CERTBOT_VALIDATION" --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT - -sleep 15 diff --git a/tools/dns-cleanup.sh b/tools/dns-cleanup.sh deleted file mode 100755 index cefad58f..00000000 --- a/tools/dns-cleanup.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# TODO: Make work with port other than 443 - -API_KEY=`cat /var/lib/mailinabox/api.key` -HOSTNAME=`hostname` - -curl -s -X DELETE --user "$API_KEY:" https://$HOSTNAME/admin/dns/custom/_acme-challenge.$CERTBOT_DOMAIN/TXT From 0860a93e84ad0e8d7b760187df0efce8ee5a5be7 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 11 Oct 2019 12:43:04 +0200 Subject: [PATCH 101/106] New release to remove extra features from the master branch --- README.md | 4 ++++ setup/bootstrap.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ad44c5bd..0ced52c8 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ Issues Changes ------- +### v0.43-quota-0.21-beta + +* Remove extra features from the master branch + ### v0.43-quota-0.20-beta * Hide *set quota* for a mailbox that has been archived diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index c825f55a..d9aad093 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.43-quota-0.20-beta + TAG=v0.43-quota-0.21-beta elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 8e94402282c22471ef1dc56d9a24e105b623f136 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Fri, 22 Nov 2019 17:13:57 +0200 Subject: [PATCH 102/106] Fix bug in displaying users when there is an archived user --- README.md | 4 ++++ management/mailconfig.py | 4 ++++ setup/bootstrap.sh | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ced52c8..3f765c5f 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ Issues Changes ------- +### v0.43-quota-0.22-beta + +* Fix bug that crashed user list when there is an archived user. + ### v0.43-quota-0.21-beta * Remove extra features from the master branch diff --git a/management/mailconfig.py b/management/mailconfig.py index ee4bef2c..383b4cbc 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -198,6 +198,10 @@ def get_mail_users_ex(env, with_archived=False): "privileges": [], "status": "inactive", "mailbox": mbox, + "box_count": '?', + "box_size": '?', + "box_quota": '?', + "percent": '?', } users.append(user) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index d9aad093..9eb65df5 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -20,7 +20,7 @@ if [ -z "$TAG" ]; then # want to display in status checks. if [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/18\.04\.[0-9]/18.04/' `" == "Ubuntu 18.04 LTS" ]; then # This machine is running Ubuntu 18.04. - TAG=v0.43-quota-0.21-beta + TAG=v0.43-quota-0.22-beta elif [ "`lsb_release -d | sed 's/.*:\s*//' | sed 's/14\.04\.[0-9]/14.04/' `" == "Ubuntu 14.04 LTS" ]; then # This machine is running Ubuntu 14.04. From 332d2c3feb67684f4b6d851100df817d978ab1e5 Mon Sep 17 00:00:00 2001 From: "John R. Supplee" Date: Sun, 17 May 2020 18:19:11 +0200 Subject: [PATCH 103/106] Add changes for new version to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index be66271a..c24fb4e7 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,10 @@ Issues Changes ------- +### v0.45-quota-0.22-beta + +* Update to v0.45 of Mail-in-a-Box + ### v0.44-quota-0.22-beta * Update to v0.44 of Mail-in-a-Box From 682550f6c55a6619952e53fee34e62e1b003ddf6 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Thu, 3 Dec 2020 23:25:22 +0200 Subject: [PATCH 104/106] Update README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 0d417d14..2bbebdd9 100644 --- a/README.md +++ b/README.md @@ -65,10 +65,16 @@ Issues * When a user's quota is changed, any IMAP session running for that user will not recognize the new quota. To solve this a `dovecot reload` could be issued causing all current IMAP sessions to be terminated. On a system with many users, it might not be desirable to reset all users sessions to fix the quota for one user. Also if the administrator is setting the quota for several users it would result in the continual reset of those connections. +* API docs do not include the quota endpoints. Quota API endpoints need to be added to `api/mainlinabox.yml`. + Changes ------- +### v0.51-quota-0.22-beta + +* Update to v0.51 of Mail-in-a-Box + ### v0.50-quota-0.22-beta * Update to v0.50 of Mail-in-a-Box From 26913b34a8d364db6aad959bbc397e2c0c05ef13 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 16 Mar 2022 07:44:50 -0400 Subject: [PATCH 105/106] Update to v56 from mail-in-a-box/mailinabox --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5d15ebe5..aee77cce 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,10 @@ Issues Changes ------- +### v56-quota + +* Update to v56 of Mail-in-a-Box + ### v0.55-quota-0.22-beta * Update to v55 of Mail-in-a-Box From 3873b4295f6f13d2accbefc3d4cbe106d531eccc Mon Sep 17 00:00:00 2001 From: Chad Furman Date: Sat, 27 Apr 2024 11:35:35 -0400 Subject: [PATCH 106/106] no longer clobbering config files --- conf/dovecot/conf.d/20-imap.conf | 94 ----------------------------- conf/dovecot/conf.d/90-quota.conf | 98 ------------------------------- setup/mail-dovecot.sh | 24 +++++++- 3 files changed, 22 insertions(+), 194 deletions(-) delete mode 100644 conf/dovecot/conf.d/20-imap.conf delete mode 100644 conf/dovecot/conf.d/90-quota.conf diff --git a/conf/dovecot/conf.d/20-imap.conf b/conf/dovecot/conf.d/20-imap.conf deleted file mode 100644 index b9855b89..00000000 --- a/conf/dovecot/conf.d/20-imap.conf +++ /dev/null @@ -1,94 +0,0 @@ -## -## IMAP specific settings -## - -# If nothing happens for this long while client is IDLEing, move the connection -# to imap-hibernate process and close the old imap process. This saves memory, -# because connections use very little memory in imap-hibernate process. The -# downside is that recreating the imap process back uses some resources. -#imap_hibernate_timeout = 0 - -# Maximum IMAP command line length. Some clients generate very long command -# lines with huge mailboxes, so you may need to raise this if you get -# "Too long argument" or "IMAP command line too large" errors often. -#imap_max_line_length = 64k - -# IMAP logout format string: -# %i - total number of bytes read from client -# %o - total number of bytes sent to client -# %{fetch_hdr_count} - Number of mails with mail header data sent to client -# %{fetch_hdr_bytes} - Number of bytes with mail header data sent to client -# %{fetch_body_count} - Number of mails with mail body data sent to client -# %{fetch_body_bytes} - Number of bytes with mail body data sent to client -# %{deleted} - Number of mails where client added \Deleted flag -# %{expunged} - Number of mails that client expunged, which does not -# include automatically expunged mails -# %{autoexpunged} - Number of mails that were automatically expunged after -# client disconnected -# %{trashed} - Number of mails that client copied/moved to the -# special_use=\Trash mailbox. -# %{appended} - Number of mails saved during the session -#imap_logout_format = in=%i out=%o - -# Override the IMAP CAPABILITY response. If the value begins with '+', -# add the given capabilities on top of the defaults (e.g. +XFOO XBAR). -#imap_capability = - -# How long to wait between "OK Still here" notifications when client is -# IDLEing. -#imap_idle_notify_interval = 2 mins -imap_idle_notify_interval=4 mins - -# ID field names and values to send to clients. Using * as the value makes -# Dovecot use the default value. The following fields have default values -# currently: name, version, os, os-version, support-url, support-email. -#imap_id_send = - -# ID fields sent by client to log. * means everything. -#imap_id_log = - -# Workarounds for various client bugs: -# delay-newmail: -# Send EXISTS/RECENT new mail notifications only when replying to NOOP -# and CHECK commands. Some clients ignore them otherwise, for example OSX -# Mail ( - -## -## Quota limits -## - -# Quota limits are set using "quota_rule" parameters. To get per-user quota -# limits, you can set/override them by returning "quota_rule" extra field -# from userdb. It's also possible to give mailbox-specific limits, for example -# to give additional 100 MB when saving to Trash: - -plugin { - #quota_rule = *:storage=1G - #quota_rule2 = Trash:storage=+100M - - # LDA/LMTP allows saving the last mail to bring user from under quota to - # over quota, if the quota doesn't grow too high. Default is to allow as - # long as quota will stay under 10% above the limit. Also allowed e.g. 10M. - #quota_grace = 10%% - - # Quota plugin can also limit the maximum accepted mail size. - #quota_max_mail_size = 100M -} - -## -## Quota warnings -## - -# You can execute a given command when user exceeds a specified quota limit. -# Each quota root has separate limits. Only the command for the first -# exceeded limit is excecuted, so put the highest limit first. -# The commands are executed via script service by connecting to the named -# UNIX socket (quota-warning below). -# Note that % needs to be escaped as %%, otherwise "% " expands to empty. - -plugin { - #quota_warning = storage=95%% quota-warning 95 %u - #quota_warning2 = storage=80%% quota-warning 80 %u -} - -# Example quota-warning service. The unix listener's permissions should be -# set in a way that mail processes can connect to it. Below example assumes -# that mail processes run as vmail user. If you use mode=0666, all system users -# can generate quota warnings to anyone. -#service quota-warning { -# executable = script /usr/local/bin/quota-warning.sh -# user = dovecot -# unix_listener quota-warning { -# user = vmail -# } -#} - -## -## Quota backends -## - -# Multiple backends are supported: -# dirsize: Find and sum all the files found from mail directory. -# Extremely SLOW with Maildir. It'll eat your CPU and disk I/O. -# dict: Keep quota stored in dictionary (eg. SQL) -# maildir: Maildir++ quota -# fs: Read-only support for filesystem quota - -plugin { - quota = maildir - - quota_grace = 10% - - quota_status_success = DUNNO - quota_status_nouser = DUNNO - quota_status_overquota = "522 5.2.2 Mailbox is full" - - #quota = dirsize:User quota - #quota = maildir:User quota - #quota = dict:User quota::proxy::quota - #quota = fs:User quota -} - -service quota-status { - executable = quota-status -p postfix - inet_listener { - port = 12340 - } -} - -# Multiple quota roots are also possible, for example this gives each user -# their own 100MB quota and one shared 1GB quota within the domain: -plugin { - #quota = dict:user::proxy::quota - #quota2 = dict:domain:%d:proxy::quota_domain - #quota_rule = *:storage=102400 - #quota2_rule = *:storage=1048576 -} diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index 9ddd14ce..2efa29b9 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -67,8 +67,28 @@ tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ # Create, subscribe, and mark as special folders: INBOX, Drafts, Sent, Trash, Spam and Archive. cp conf/dovecot/conf.d/15-mailboxes.conf /etc/dovecot/conf.d/ -cp conf/dovecot/conf.d/20-imap.conf /etc/dovecot/conf.d/ -cp conf/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/ +sed -i "s/mail_plugins = .*/mail_plugins = \$mail_plugins imap-quota/" /etc/dovecot/conf.d/20-imap.conf +sed -i "s/protocol imap/mail_plugins = \$mail_plugins quota\nprotocol imap/" /etc/dovecot/conf.d/20-imap.conf + +# configure stuff for quota support +cat > /etc/dovecot/conf.d/90-quota.conf << EOF; +plugin { + quota = maildir + + quota_grace = 10% + + quota_status_success = DUNNO + quota_status_nouser = DUNNO + quota_status_overquota = "522 5.2.2 Mailbox is full" +} + +service quota-status { + executable = quota-status -p postfix + inet_listener { + port = 12340 + } +} +EOF # ### IMAP/POP
    Email AddressEmail Address Quota Actions
    Email AddressBox Size Quota Actions
    @@ -161,6 +163,7 @@ function show_users() { n.attr('data-email', user.email); n.attr('data-quota', user.quota); n.find('.address').text(user.email); + n.find('.box-size').text(user.box_size); n.find('.quota').text((user.quota == '0') ? 'unlimited' : user.quota); n2.find('.restore_info tt').text(user.mailbox); From b3f9063ae798901c5821031b14e44cda17627f23 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 29 Jan 2019 19:41:54 +0200 Subject: [PATCH 019/106] comment out code generating errors When the `maildirsize` file does not exist it causes the script to fail. The IOError is not caught by the execpt --- management/mailconfig.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/management/mailconfig.py b/management/mailconfig.py index cdd05079..3a471a80 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -136,15 +136,15 @@ def get_mail_users_ex(env, with_archived=False): box_size = 0 box_count = 0 box_quota = '' - try: - with open('/home/user-data/mail/mailboxes/%s/%s/maildirsize' % (domain, user), 'r') as f: - box_quota = f.readline() - for line in f.readlines(): - (size, count) = line.split(' ') - box_size += int(size) - box_count += int(count) - except: - box_size = '?' + # try: + # with open('/home/user-data/mail/mailboxes/%s/%s/maildirsize' % (domain, user), 'r') as f: + # box_quota = f.readline() + # for line in f.readlines(): + # (size, count) = line.split(' ') + # box_size += int(size) + # box_count += int(count) + # except: + # box_size = '?' user = { "email": email, From 8bd9cf38abbecbad60f56df9d9d48e661cfb8cbb Mon Sep 17 00:00:00 2001 From: John Supplee Date: Tue, 29 Jan 2019 23:31:56 +0200 Subject: [PATCH 020/106] Use tabs for indentation --- management/daemon.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/management/daemon.py b/management/daemon.py index 5b3bd326..3bf95ec5 100755 --- a/management/daemon.py +++ b/management/daemon.py @@ -161,10 +161,10 @@ def mail_users_add(): @app.route('/mail/users/quota', methods=['POST']) @authorized_personnel_only def mail_users_quota(): - try: - return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env) - except ValueError as e: - return (str(e), 400) + try: + return set_mail_quota(request.form.get('email', ''), request.form.get('quota'), env) + except ValueError as e: + return (str(e), 400) @app.route('/mail/users/password', methods=['POST']) @authorized_personnel_only From c302606de44a8b4c4c313e5d98f42f8964f482d1 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Tue, 29 Jan 2019 13:46:35 -0800 Subject: [PATCH 021/106] Extended timeout for php/roundcube for text searches that take a long time --- setup/web.sh | 7 +++++++ setup/webmail.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..2e82fe6a 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -48,6 +48,13 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" +# Set higher timeout since searches with Roundcube and Solr may take longer +# than the default 60 seconds. We will also match Roundcube's timeout to the +# same value +tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ + default_socket_timeout=180 + + # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand diff --git a/setup/webmail.sh b/setup/webmail.sh index b0e11c9b..d082382f 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u'; From a8f02c1eb05075383ca9c464c813b795d1e19bb3 Mon Sep 17 00:00:00 2001 From: John Supplee Date: Wed, 30 Jan 2019 00:01:16 +0200 Subject: [PATCH 022/106] Fix problems with users that do not have maildirsize file --- README.md | 10 ++++++++++ management/mailconfig.py | 34 +++++++++++++++++++++++---------- management/templates/users.html | 5 ++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 36f298e9..cb2cdf86 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,16 @@ There is baisc support for quotas in the control panel now. To set quotas from column for the user in the `users.sqlite` database. The `quota` column is text and allows for the `M` and `G` suffixes for megabytes and gigabytes respectively. No spaces should be used in the quota value (e.g. `2G` or `100M`). +Todo +---- + +* Get `postfix` to support quotas. See https://blog.sys4.de/postfix-dovecot-mailbox-quota-en.html for a start. Right + now the quota message store size is not calculated unless the user accesses the IMAP server (Dovecot). Right now postfix + does not take quotas into account before delivering a message. + +* Allow Trash to have a grace percentage to allow users whose quota is full to delete messages. + + \[BEGIN Official README] Mail-in-a-Box diff --git a/management/mailconfig.py b/management/mailconfig.py index 3a471a80..9cb79b0d 100755 --- a/management/mailconfig.py +++ b/management/mailconfig.py @@ -105,6 +105,18 @@ def get_mail_users(env): users = [ row[0] for row in c.fetchall() ] return utils.sort_email_addresses(users, env) +def sizeof_fmt(num): + for unit in ['','K','M','G','T']: + if abs(num) < 1024.0: + if abs(num) > 99: + return "%3.0f%s" % (num, unit) + else: + return "%2.1f%s" % (num, unit) + + num /= 1024.0 + + return str(num) + def get_mail_users_ex(env, with_archived=False): # Returns a complex data structure of all user accounts, optionally # including archived (status="inactive") accounts. @@ -136,22 +148,24 @@ def get_mail_users_ex(env, with_archived=False): box_size = 0 box_count = 0 box_quota = '' - # try: - # with open('/home/user-data/mail/mailboxes/%s/%s/maildirsize' % (domain, user), 'r') as f: - # box_quota = f.readline() - # for line in f.readlines(): - # (size, count) = line.split(' ') - # box_size += int(size) - # box_count += int(count) - # except: - # box_size = '?' + try: + dirsize_file = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes/%s/%s/maildirsize' % (domain, user)) + with open(dirsize_file, 'r') as f: + box_quota = f.readline() + for line in f.readlines(): + (size, count) = line.split(' ') + box_size += int(size) + box_count += int(count) + except: + box_size = '?' + box_count = '?' user = { "email": email, "privileges": parse_privs(privileges), "quota": quota, "box_quota": box_quota, - "box_size": '%iK' % int(box_size / 1024), + "box_size": sizeof_fmt(box_size) if box_size != '?' else box_size, "box_count": box_count, "status": "active", } diff --git a/management/templates/users.html b/management/templates/users.html index 0175b929..ee22b316 100644 --- a/management/templates/users.html +++ b/management/templates/users.html @@ -42,7 +42,8 @@
    Email AddressBox SizeMessagesSize Quota Actions
    Email AddressMessagesSizeQuotaMessagesSizeUsedQuota Actions
    + @@ -159,7 +162,7 @@ function show_users() { function(r) { $('#user_table tbody').html(""); for (var i = 0; i < r.length; i++) { - var hdr = $("