2014-06-30 14:20:58 +00:00
|
|
|
#!/usr/bin/python3
|
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
import subprocess, shutil, os, sqlite3, re
|
2014-06-09 12:09:45 +00:00
|
|
|
import utils
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2014-07-13 12:13:41 +00:00
|
|
|
def validate_email(email, mode=None):
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Checks that an email address is syntactically valid. Returns True/False.
|
|
|
|
# Until Postfix supports SMTPUTF8, an email address may contain ASCII
|
|
|
|
# characters only; IDNs must be IDNA-encoded.
|
|
|
|
#
|
|
|
|
# When mode=="user", we're checking that this can be a user account name.
|
|
|
|
# Dovecot has tighter restrictions - letters, numbers, underscore, and
|
|
|
|
# dash only!
|
|
|
|
#
|
|
|
|
# When mode=="alias", we're allowing anything that can be in a Postfix
|
|
|
|
# alias table, i.e. omitting the local part ("@domain.tld") is OK.
|
2014-06-06 13:58:58 +00:00
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Check that the address isn't absurdly long.
|
2014-06-06 13:58:58 +00:00
|
|
|
if len(email) > 255: return False
|
|
|
|
|
2014-07-13 12:13:41 +00:00
|
|
|
if mode == 'user':
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# There are a lot of characters permitted in email addresses, but
|
|
|
|
# Dovecot's sqlite driver seems to get confused if there are any
|
|
|
|
# unusual characters in the address. Bah. Also note that since
|
|
|
|
# the mailbox path name is based on the email address, the address
|
|
|
|
# shouldn't be absurdly long and must not have a forward slash.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
ATEXT = r'[a-zA-Z0-9_\-]'
|
2014-11-14 13:33:12 +00:00
|
|
|
elif mode in (None, 'alias'):
|
2014-07-13 12:13:41 +00:00
|
|
|
# For aliases, we can allow any valid email address.
|
2014-06-06 13:58:58 +00:00
|
|
|
# Based on RFC 2822 and https://github.com/SyrusAkbary/validate_email/blob/master/validate_email.py,
|
2014-07-13 12:13:41 +00:00
|
|
|
# these characters are permitted in email addresses.
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
ATEXT = r'[a-zA-Z0-9_!#$%&\'\*\+\-/=\?\^`\{\|\}~]' # see 3.2.4
|
2014-07-13 12:13:41 +00:00
|
|
|
else:
|
|
|
|
raise ValueError(mode)
|
|
|
|
|
|
|
|
# per RFC 2822 3.2.4
|
|
|
|
DOT_ATOM_TEXT_LOCAL = ATEXT + r'+(?:\.' + ATEXT + r'+)*'
|
|
|
|
if mode == 'alias':
|
|
|
|
# For aliases, Postfix accepts '@domain.tld' format for
|
2014-11-14 13:33:12 +00:00
|
|
|
# catch-all addresses on the source side and domain aliases
|
|
|
|
# on the destination side. Make the local part optional.
|
2014-07-13 12:13:41 +00:00
|
|
|
DOT_ATOM_TEXT_LOCAL = '(?:' + DOT_ATOM_TEXT_LOCAL + ')?'
|
2014-06-06 13:58:58 +00:00
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# We can require that the host part have at least one period in it,
|
|
|
|
# so use a "+" rather than a "*" at the end.
|
2014-07-13 12:13:41 +00:00
|
|
|
DOT_ATOM_TEXT_HOST = ATEXT + r'+(?:\.' + ATEXT + r'+)+'
|
2014-07-12 11:17:13 +00:00
|
|
|
|
2014-07-13 12:13:41 +00:00
|
|
|
# per RFC 2822 3.4.1
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
ADDR_SPEC = '^(%s)@(%s)$' % (DOT_ATOM_TEXT_LOCAL, DOT_ATOM_TEXT_HOST)
|
2014-06-06 13:58:58 +00:00
|
|
|
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
# Check the regular expression.
|
|
|
|
m = re.match(ADDR_SPEC, email)
|
|
|
|
if not m: return False
|
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Check that the domain part is valid IDNA.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
localpart, domainpart = m.groups()
|
|
|
|
try:
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
domainpart.encode('ascii').decode("idna")
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
except:
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Domain is not valid IDNA.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
return False
|
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Everything looks good.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
return True
|
|
|
|
|
|
|
|
def sanitize_idn_email_address(email):
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# The user may enter Unicode in an email address. Convert the domain part
|
|
|
|
# to IDNA before going into our database. Leave the local part alone ---
|
|
|
|
# although validate_email will reject non-ASCII characters.
|
|
|
|
#
|
|
|
|
# The domain name system only exists in ASCII, so it doesn't make sense
|
|
|
|
# to store domain names in Unicode. We want to store what is meaningful
|
|
|
|
# to the underlying protocols.
|
|
|
|
try:
|
|
|
|
localpart, domainpart = email.split("@")
|
|
|
|
domainpart = domainpart.encode("idna").decode('ascii')
|
|
|
|
return localpart + "@" + domainpart
|
|
|
|
except:
|
|
|
|
# Domain part is not IDNA-valid, so leave unchanged. If there
|
|
|
|
# are non-ASCII characters it will be filtered out by
|
|
|
|
# validate_email.
|
|
|
|
return email
|
|
|
|
|
|
|
|
def prettify_idn_email_address(email):
|
|
|
|
# This is the opposite of sanitize_idn_email_address. We store domain
|
|
|
|
# names in IDNA in the database, but we want to show Unicode to the user.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
try:
|
|
|
|
localpart, domainpart = email.split("@")
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
domainpart = domainpart.encode("ascii").decode('idna')
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
return localpart + "@" + domainpart
|
|
|
|
except:
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Failed to decode IDNA. Should never happen.
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
return email
|
2014-06-06 13:58:58 +00:00
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
def open_database(env, with_connection=False):
|
|
|
|
conn = sqlite3.connect(env["STORAGE_ROOT"] + "/mail/users.sqlite")
|
|
|
|
if not with_connection:
|
|
|
|
return conn.cursor()
|
|
|
|
else:
|
|
|
|
return conn, conn.cursor()
|
|
|
|
|
2014-10-07 19:28:07 +00:00
|
|
|
def get_mail_users(env):
|
|
|
|
# Returns a flat, sorted list of all user accounts.
|
|
|
|
c = open_database(env)
|
|
|
|
c.execute('SELECT email FROM users')
|
|
|
|
users = [ row[0] for row in c.fetchall() ]
|
|
|
|
return utils.sort_email_addresses(users, env)
|
|
|
|
|
2014-10-07 20:24:11 +00:00
|
|
|
def get_mail_users_ex(env, with_archived=False, with_slow_info=False):
|
2014-10-07 19:28:07 +00:00
|
|
|
# Returns a complex data structure of all user accounts, optionally
|
|
|
|
# including archived (status="inactive") accounts.
|
|
|
|
#
|
|
|
|
# [
|
|
|
|
# {
|
|
|
|
# domain: "domain.tld",
|
|
|
|
# users: [
|
|
|
|
# {
|
|
|
|
# email: "name@domain.tld",
|
|
|
|
# privileges: [ "priv1", "priv2", ... ],
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# status: "active" | "inactive",
|
2014-10-07 19:28:07 +00:00
|
|
|
# },
|
|
|
|
# ...
|
|
|
|
# ]
|
|
|
|
# },
|
|
|
|
# ...
|
|
|
|
# ]
|
|
|
|
|
|
|
|
# Get users and their privileges.
|
|
|
|
users = []
|
|
|
|
active_accounts = set()
|
2014-06-03 13:24:48 +00:00
|
|
|
c = open_database(env)
|
2014-08-08 12:31:22 +00:00
|
|
|
c.execute('SELECT email, privileges FROM users')
|
2014-10-07 19:28:07 +00:00
|
|
|
for email, privileges in c.fetchall():
|
|
|
|
active_accounts.add(email)
|
2014-10-07 20:24:11 +00:00
|
|
|
|
|
|
|
user = {
|
2014-10-07 19:28:07 +00:00
|
|
|
"email": email,
|
|
|
|
"privileges": parse_privs(privileges),
|
|
|
|
"status": "active",
|
2014-10-07 20:24:11 +00:00
|
|
|
}
|
|
|
|
users.append(user)
|
|
|
|
|
|
|
|
if with_slow_info:
|
|
|
|
user["mailbox_size"] = utils.du(os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes', *reversed(email.split("@"))))
|
2014-10-07 19:28:07 +00:00
|
|
|
|
|
|
|
# Add in archived accounts.
|
|
|
|
if with_archived:
|
|
|
|
root = os.path.join(env['STORAGE_ROOT'], 'mail/mailboxes')
|
|
|
|
for domain in os.listdir(root):
|
|
|
|
for user in os.listdir(os.path.join(root, domain)):
|
|
|
|
email = user + "@" + domain
|
2014-10-07 20:24:11 +00:00
|
|
|
mbox = os.path.join(root, domain, user)
|
2014-10-07 19:28:07 +00:00
|
|
|
if email in active_accounts: continue
|
2014-10-07 20:24:11 +00:00
|
|
|
user = {
|
2014-10-07 19:28:07 +00:00
|
|
|
"email": email,
|
|
|
|
"privileges": "",
|
|
|
|
"status": "inactive",
|
2014-10-07 20:24:11 +00:00
|
|
|
"mailbox": mbox,
|
|
|
|
}
|
|
|
|
users.append(user)
|
|
|
|
if with_slow_info:
|
|
|
|
user["mailbox_size"] = utils.du(mbox)
|
2014-10-07 19:28:07 +00:00
|
|
|
|
|
|
|
# Group by domain.
|
|
|
|
domains = { }
|
|
|
|
for user in users:
|
|
|
|
domain = get_domain(user["email"])
|
|
|
|
if domain not in domains:
|
|
|
|
domains[domain] = {
|
|
|
|
"domain": domain,
|
|
|
|
"users": []
|
|
|
|
}
|
|
|
|
domains[domain]["users"].append(user)
|
|
|
|
|
|
|
|
# Sort domains.
|
|
|
|
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
|
|
|
|
|
|
|
|
# Sort users within each domain first by status then lexicographically by email address.
|
|
|
|
for domain in domains:
|
|
|
|
domain["users"].sort(key = lambda user : (user["status"] != "active", user["email"]))
|
|
|
|
|
|
|
|
return domains
|
|
|
|
|
|
|
|
def get_admins(env):
|
|
|
|
# Returns a set of users with admin privileges.
|
|
|
|
users = set()
|
|
|
|
for domain in get_mail_users_ex(env):
|
|
|
|
for user in domain["users"]:
|
|
|
|
if "admin" in user["privileges"]:
|
|
|
|
users.add(user["email"])
|
|
|
|
return users
|
2014-08-17 22:43:57 +00:00
|
|
|
|
2014-10-07 19:47:30 +00:00
|
|
|
def get_mail_aliases(env):
|
|
|
|
# Returns a sorted list of tuples of (alias, forward-to string).
|
2014-06-03 13:24:48 +00:00
|
|
|
c = open_database(env)
|
|
|
|
c.execute('SELECT source, destination FROM aliases')
|
2014-08-17 22:43:57 +00:00
|
|
|
aliases = { row[0]: row[1] for row in c.fetchall() } # make dict
|
|
|
|
|
|
|
|
# put in a canonical order: sort by domain, then by email address lexicographically
|
2014-10-07 19:47:30 +00:00
|
|
|
aliases = [ (source, aliases[source]) for source in utils.sort_email_addresses(aliases.keys(), env) ]
|
2014-08-17 22:43:57 +00:00
|
|
|
return aliases
|
|
|
|
|
2014-10-07 19:47:30 +00:00
|
|
|
def get_mail_aliases_ex(env):
|
|
|
|
# Returns a complex data structure of all mail aliases, similar
|
|
|
|
# to get_mail_users_ex.
|
|
|
|
#
|
|
|
|
# [
|
|
|
|
# {
|
|
|
|
# domain: "domain.tld",
|
|
|
|
# alias: [
|
|
|
|
# {
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# source: "name@domain.tld", # IDNA-encoded
|
|
|
|
# source_display: "name@domain.tld", # full Unicode
|
2014-10-07 19:47:30 +00:00
|
|
|
# destination: ["target1@domain.com", "target2@domain.com", ...],
|
|
|
|
# required: True|False
|
|
|
|
# },
|
|
|
|
# ...
|
|
|
|
# ]
|
|
|
|
# },
|
|
|
|
# ...
|
|
|
|
# ]
|
|
|
|
|
|
|
|
required_aliases = get_required_aliases(env)
|
|
|
|
domains = {}
|
|
|
|
for source, destination in get_mail_aliases(env):
|
|
|
|
# get alias info
|
|
|
|
domain = get_domain(source)
|
|
|
|
required = ((source in required_aliases) or (source == get_system_administrator(env)))
|
|
|
|
|
|
|
|
# add to list
|
|
|
|
if not domain in domains:
|
|
|
|
domains[domain] = {
|
|
|
|
"domain": domain,
|
|
|
|
"aliases": [],
|
|
|
|
}
|
|
|
|
domains[domain]["aliases"].append({
|
|
|
|
"source": source,
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
"source_display": prettify_idn_email_address(source),
|
|
|
|
"destination": [prettify_idn_email_address(d.strip()) for d in destination.split(",")],
|
2014-10-07 19:47:30 +00:00
|
|
|
"required": required,
|
|
|
|
})
|
|
|
|
|
|
|
|
# Sort domains.
|
|
|
|
domains = [domains[domain] for domain in utils.sort_domains(domains.keys(), env)]
|
|
|
|
|
|
|
|
# Sort aliases within each domain first by required-ness then lexicographically by source address.
|
|
|
|
for domain in domains:
|
|
|
|
domain["aliases"].sort(key = lambda alias : (alias["required"], alias["source"]))
|
|
|
|
return domains
|
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
def get_domain(emailaddr, as_unicode=True):
|
|
|
|
# Gets the domain part of an email address. Turns IDNA
|
|
|
|
# back to Unicode for display.
|
|
|
|
ret = emailaddr.split('@', 1)[1]
|
|
|
|
if as_unicode: ret = ret.encode('ascii').decode('idna')
|
|
|
|
return ret
|
2014-10-07 19:28:07 +00:00
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
def get_mail_domains(env, filter_aliases=lambda alias : True):
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# Returns the domain names (IDNA-encoded) of all of the email addresses
|
|
|
|
# configured on the system.
|
2014-07-09 19:29:46 +00:00
|
|
|
return set(
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
[get_domain(addr, as_unicode=False) for addr in get_mail_users(env)]
|
|
|
|
+ [get_domain(source, as_unicode=False) for source, target in get_mail_aliases(env) if filter_aliases((source, target)) ]
|
2014-07-09 19:29:46 +00:00
|
|
|
)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
def add_mail_user(email, pw, privs, env):
|
|
|
|
# validate email
|
|
|
|
if email.strip() == "":
|
|
|
|
return ("No email address provided.", 400)
|
2015-03-22 12:33:06 +00:00
|
|
|
elif not validate_email(email):
|
2014-06-03 13:24:48 +00:00
|
|
|
return ("Invalid email address.", 400)
|
2015-03-22 12:33:06 +00:00
|
|
|
elif not validate_email(email, mode='user'):
|
|
|
|
return ("User account email addresses may only use the ASCII letters A-Z, the digits 0-9, underscore (_), hyphen (-), and period (.).", 400)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# validate password
|
2014-09-21 17:24:01 +00:00
|
|
|
validate_password(pw)
|
2014-08-17 22:43:57 +00:00
|
|
|
|
|
|
|
# validate privileges
|
|
|
|
if privs is None or privs.strip() == "":
|
|
|
|
privs = []
|
|
|
|
else:
|
|
|
|
privs = privs.split("\n")
|
|
|
|
for p in privs:
|
|
|
|
validation = validate_privilege(p)
|
|
|
|
if validation: return validation
|
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
# get the database
|
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
|
|
|
|
# hash the password
|
2014-11-30 15:43:07 +00:00
|
|
|
pw = hash_password(pw)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
|
|
|
# add the user to the database
|
|
|
|
try:
|
2014-08-17 22:43:57 +00:00
|
|
|
c.execute("INSERT INTO users (email, password, privileges) VALUES (?, ?, ?)",
|
|
|
|
(email, pw, "\n".join(privs)))
|
2014-06-03 13:24:48 +00:00
|
|
|
except sqlite3.IntegrityError:
|
|
|
|
return ("User already exists.", 400)
|
2014-07-12 11:17:13 +00:00
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
# write databasebefore next step
|
|
|
|
conn.commit()
|
|
|
|
|
2015-03-22 13:24:55 +00:00
|
|
|
# Create & subscribe the user's INBOX, Trash, Spam, and Drafts folders.
|
|
|
|
# * Our sieve rule for spam expects that the Spam folder exists.
|
|
|
|
# * Roundcube will show an error if the user tries to delete a message before the Trash folder exists (#359).
|
|
|
|
# * K-9 mail will poll every 90 seconds if a Drafts folder does not exist, so create it
|
|
|
|
# to avoid unnecessary polling.
|
2014-06-03 13:24:48 +00:00
|
|
|
|
|
|
|
# Check if the mailboxes exist before creating them. When creating a user that had previously
|
|
|
|
# been deleted, the mailboxes will still exist because they are still on disk.
|
2014-06-06 13:58:58 +00:00
|
|
|
try:
|
2014-06-09 12:09:45 +00:00
|
|
|
existing_mboxes = utils.shell('check_output', ["doveadm", "mailbox", "list", "-u", email, "-8"], capture_stderr=True).split("\n")
|
2014-06-06 13:58:58 +00:00
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
c.execute("DELETE FROM users WHERE email=?", (email,))
|
|
|
|
conn.commit()
|
|
|
|
return ("Failed to initialize the user: " + e.output.decode("utf8"), 400)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2015-03-22 13:24:55 +00:00
|
|
|
for folder in ("INBOX", "Trash", "Spam", "Drafts"):
|
2014-08-09 16:49:57 +00:00
|
|
|
if folder not in existing_mboxes:
|
|
|
|
utils.shell('check_call', ["doveadm", "mailbox", "create", "-u", email, "-s", folder])
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
# Update things in case any new domains are added.
|
2014-07-06 12:16:50 +00:00
|
|
|
return kick(env, "mail user added")
|
2014-06-03 13:24:48 +00:00
|
|
|
|
|
|
|
def set_mail_password(email, pw, env):
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
# validate that password is acceptable
|
2014-09-21 17:24:01 +00:00
|
|
|
validate_password(pw)
|
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
# hash the password
|
2014-11-30 15:43:07 +00:00
|
|
|
pw = hash_password(pw)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
|
|
|
# update the database
|
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
c.execute("UPDATE users SET password=? WHERE email=?", (pw, email))
|
|
|
|
if c.rowcount != 1:
|
|
|
|
return ("That's not a user (%s)." % email, 400)
|
|
|
|
conn.commit()
|
|
|
|
return "OK"
|
|
|
|
|
2014-11-30 15:43:07 +00:00
|
|
|
def hash_password(pw):
|
|
|
|
# Turn the plain password into a Dovecot-format hashed password, meaning
|
|
|
|
# something like "{SCHEME}hashedpassworddata".
|
|
|
|
# http://wiki2.dovecot.org/Authentication/PasswordSchemes
|
|
|
|
return utils.shell('check_output', ["/usr/bin/doveadm", "pw", "-s", "SHA512-CRYPT", "-p", pw]).strip()
|
|
|
|
|
|
|
|
def get_mail_password(email, env):
|
|
|
|
# Gets the hashed password for a user. Passwords are stored in Dovecot's
|
|
|
|
# password format, with a prefixed scheme.
|
|
|
|
# http://wiki2.dovecot.org/Authentication/PasswordSchemes
|
|
|
|
# update the database
|
|
|
|
c = open_database(env)
|
|
|
|
c.execute('SELECT password FROM users WHERE email=?', (email,))
|
|
|
|
rows = c.fetchall()
|
|
|
|
if len(rows) != 1:
|
|
|
|
raise ValueError("That's not a user (%s)." % email)
|
|
|
|
return rows[0][0]
|
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
def remove_mail_user(email, env):
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
# remove
|
2014-06-03 13:24:48 +00:00
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
c.execute("DELETE FROM users WHERE email=?", (email,))
|
|
|
|
if c.rowcount != 1:
|
|
|
|
return ("That's not a user (%s)." % email, 400)
|
|
|
|
conn.commit()
|
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
# Update things in case any domains are removed.
|
2014-07-06 12:16:50 +00:00
|
|
|
return kick(env, "mail user removed")
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2014-08-08 12:31:22 +00:00
|
|
|
def parse_privs(value):
|
|
|
|
return [p for p in value.split("\n") if p.strip() != ""]
|
|
|
|
|
|
|
|
def get_mail_user_privileges(email, env):
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
# get privs
|
2014-08-08 12:31:22 +00:00
|
|
|
c = open_database(env)
|
|
|
|
c.execute('SELECT privileges FROM users WHERE email=?', (email,))
|
|
|
|
rows = c.fetchall()
|
|
|
|
if len(rows) != 1:
|
|
|
|
return ("That's not a user (%s)." % email, 400)
|
|
|
|
return parse_privs(rows[0][0])
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
def validate_privilege(priv):
|
2014-08-08 12:31:22 +00:00
|
|
|
if "\n" in priv or priv.strip() == "":
|
|
|
|
return ("That's not a valid privilege (%s)." % priv, 400)
|
2014-08-17 22:43:57 +00:00
|
|
|
return None
|
2014-08-08 12:31:22 +00:00
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
def add_remove_mail_user_privilege(email, priv, action, env):
|
|
|
|
# validate
|
|
|
|
validation = validate_privilege(priv)
|
|
|
|
if validation: return validation
|
|
|
|
|
|
|
|
# get existing privs, but may fail
|
2014-08-08 12:31:22 +00:00
|
|
|
privs = get_mail_user_privileges(email, env)
|
|
|
|
if isinstance(privs, tuple): return privs # error
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
# update privs set
|
2014-08-08 12:31:22 +00:00
|
|
|
if action == "add":
|
|
|
|
if priv not in privs:
|
|
|
|
privs.append(priv)
|
|
|
|
elif action == "remove":
|
|
|
|
privs = [p for p in privs if p != priv]
|
|
|
|
else:
|
|
|
|
return ("Invalid action.", 400)
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
# commit to database
|
2014-08-08 12:31:22 +00:00
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
c.execute("UPDATE users SET privileges=? WHERE email=?", ("\n".join(privs), email))
|
|
|
|
if c.rowcount != 1:
|
|
|
|
return ("Something went wrong.", 400)
|
|
|
|
conn.commit()
|
|
|
|
|
|
|
|
return "OK"
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
def add_mail_alias(source, destination, env, update_if_exists=False, do_kick=True):
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# convert Unicode domain to IDNA
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
source = sanitize_idn_email_address(source)
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
# validate source
|
|
|
|
if source.strip() == "":
|
|
|
|
return ("No incoming email address provided.", 400)
|
2014-07-13 12:13:41 +00:00
|
|
|
if not validate_email(source, mode='alias'):
|
2014-08-17 22:43:57 +00:00
|
|
|
return ("Invalid incoming email address (%s)." % source, 400)
|
|
|
|
|
2014-11-14 13:33:12 +00:00
|
|
|
# validate destination
|
2014-08-17 22:43:57 +00:00
|
|
|
dests = []
|
2014-11-14 13:33:12 +00:00
|
|
|
destination = destination.strip()
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
|
|
|
|
# Postfix allows a single @domain.tld as the destination, which means
|
|
|
|
# the local part on the address is preserved in the rewrite. We must
|
|
|
|
# try to convert Unicode to IDNA first before validating that it's a
|
|
|
|
# legitimate alias address.
|
|
|
|
d1 = sanitize_idn_email_address(destination)
|
|
|
|
if validate_email(d1, mode='alias'):
|
|
|
|
dests.append(d1)
|
|
|
|
|
2014-11-14 13:33:12 +00:00
|
|
|
else:
|
|
|
|
# Parse comma and \n-separated destination emails & validate. In this
|
|
|
|
# case, the recipients must be complete email addresses.
|
|
|
|
for line in destination.split("\n"):
|
|
|
|
for email in line.split(","):
|
|
|
|
email = email.strip()
|
|
|
|
if email == "": continue
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
email = sanitize_idn_email_address(email) # Unicode => IDNA
|
2014-11-14 13:33:12 +00:00
|
|
|
if not validate_email(email):
|
|
|
|
return ("Invalid destination email address (%s)." % email, 400)
|
|
|
|
dests.append(email)
|
2014-08-17 22:43:57 +00:00
|
|
|
if len(destination) == 0:
|
|
|
|
return ("No destination email address(es) provided.", 400)
|
|
|
|
destination = ",".join(dests)
|
2014-06-06 13:58:58 +00:00
|
|
|
|
2014-11-14 13:33:12 +00:00
|
|
|
# save to db
|
2014-06-03 13:24:48 +00:00
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
try:
|
|
|
|
c.execute("INSERT INTO aliases (source, destination) VALUES (?, ?)", (source, destination))
|
2014-08-17 22:43:57 +00:00
|
|
|
return_status = "alias added"
|
2014-06-03 13:24:48 +00:00
|
|
|
except sqlite3.IntegrityError:
|
2014-08-17 22:43:57 +00:00
|
|
|
if not update_if_exists:
|
|
|
|
return ("Alias already exists (%s)." % source, 400)
|
|
|
|
else:
|
|
|
|
c.execute("UPDATE aliases SET destination = ? WHERE source = ?", (destination, source))
|
|
|
|
return_status = "alias updated"
|
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
conn.commit()
|
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
if do_kick:
|
|
|
|
# Update things in case any new domains are added.
|
2014-08-17 22:43:57 +00:00
|
|
|
return kick(env, return_status)
|
2014-06-03 13:24:48 +00:00
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
def remove_mail_alias(source, env, do_kick=True):
|
store IDNs (internationalized domain names) in IDNA (ASCII) in our database, not in Unicode
I changed my mind. In 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431 I allowed Unicode domain names to go into the database. I thought that was nice because it's what the user *means*. But it's not how the web works. Web and DNS were working, but mail wasn't. Postfix (as shipped with Ubuntu 14.04 without support for SMTPUTF8) exists in an ASCII-only world. When it goes to the users/aliases table, it queries in ASCII (IDNA) only and had no hope of delivering mail if the domain was in full Unicode in the database. I was thinking ahead to SMTPUTF8, where we *could* put Unicode in the database (though that would prevent IDNA-encoded addressing from being deliverable) not realizing it isn't well supported yet anyway.
It's IDNA that goes on the wire in most places anyway (SMTP without SMTPUTF8 (and therefore how Postfix queries our users/aliases tables), DNS zone files, nginx config, CSR 'CN' field, X509 Common Name and Subject Alternative Names fields), so we should really be talking in terms of IDNA (i.e. ASCII).
This partially reverts commit 1bf8f1991f6f08e0fb1e3d2572d280d894a5e431, where I added a lot of Unicode=>IDNA conversions when writing configuration files. Instead I'm doing Unicode=>IDNA before email addresses get into the users/aliases table. Now we assume the database uses IDNA-encoded ASCII domain names. When adding/removing aliases, addresses are converted to ASCII (w/ IDNA). User accounts must be ASCII-only anyway because of Dovecot's auth limitations, so we don't do any IDNA conversion (don't want to change the user's login info behind their back!). The aliases control panel page converts domains back to Unicode for display to be nice. The status checks converts the domains to Unicode just for the output headings.
A migration is added to convert existing aliases with Unicode domains into IDNA. Any custom DNS or web settings with Unicode may need to be changed.
Future support for SMTPUTF8 will probably need to add columns in the users/aliases table so that it lists both IDNA and Unicode forms.
2015-03-29 13:33:31 +00:00
|
|
|
# convert Unicode domain to IDNA
|
internationalized domain names (DNS, web, CSRs, normalize to Unicode in database, prohibit non-ASCII characters in user account names)
* For non-ASCII domain names, we will keep the Unicode encoding in our users/aliases table. This is nice for the user and also simplifies things like sorting domain names (using Unicode lexicographic order is good, using ASCII lexicogrpahic order on IDNA is confusing).
* Write nsd config, nsd zone files, nginx config, and SSL CSRs with domains in IDNA-encoded ASCII.
* When checking SSL certificates, treat the CN and SANs as IDNA.
* Since Chrome has an interesting feature of converting Unicode to IDNA in <input type="email"> form fields, we'll also forcibly convert IDNA to Unicode in the domain part of email addresses before saving email addresses in the users/aliases tables so that the table is normalized to Unicode.
* Don't allow non-ASCII characters in user account email addresses. Dovecot gets confused when querying the Sqlite database (which we observed even for non-word ASCII characters too, so it may not be related to the character encoding).
2015-01-17 13:41:53 +00:00
|
|
|
source = sanitize_idn_email_address(source)
|
|
|
|
|
|
|
|
# remove
|
2014-06-03 13:24:48 +00:00
|
|
|
conn, c = open_database(env, with_connection=True)
|
|
|
|
c.execute("DELETE FROM aliases WHERE source=?", (source,))
|
|
|
|
if c.rowcount != 1:
|
|
|
|
return ("That's not an alias (%s)." % source, 400)
|
|
|
|
conn.commit()
|
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
if do_kick:
|
|
|
|
# Update things in case any domains are removed.
|
|
|
|
return kick(env, "alias removed")
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
def get_system_administrator(env):
|
|
|
|
return "administrator@" + env['PRIMARY_HOSTNAME']
|
|
|
|
|
|
|
|
def get_required_aliases(env):
|
|
|
|
# These are the aliases that must exist.
|
|
|
|
aliases = set()
|
|
|
|
|
2014-09-09 11:41:44 +00:00
|
|
|
# The hostmaster alias is exposed in the DNS SOA for each zone.
|
2014-08-17 22:43:57 +00:00
|
|
|
aliases.add("hostmaster@" + env['PRIMARY_HOSTNAME'])
|
|
|
|
|
|
|
|
# Get a list of domains we serve mail for, except ones for which the only
|
2015-01-19 23:18:28 +00:00
|
|
|
# email on that domain is a postmaster/admin alias to the administrator
|
|
|
|
# or a wildcard alias (since it will forward postmaster/admin).
|
2014-08-17 22:43:57 +00:00
|
|
|
real_mail_domains = get_mail_domains(env,
|
2015-01-19 23:18:28 +00:00
|
|
|
filter_aliases = lambda alias :
|
|
|
|
((not alias[0].startswith("postmaster@") and not alias[0].startswith("admin@")) or alias[1] != get_system_administrator(env))
|
|
|
|
and not alias[0].startswith("@")
|
2014-08-17 22:43:57 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
# Create postmaster@ and admin@ for all domains we serve mail on.
|
|
|
|
# postmaster@ is assumed to exist by our Postfix configuration. admin@
|
|
|
|
# isn't anything, but it might save the user some trouble e.g. when
|
|
|
|
# buying an SSL certificate.
|
|
|
|
for domain in real_mail_domains:
|
|
|
|
aliases.add("postmaster@" + domain)
|
|
|
|
aliases.add("admin@" + domain)
|
|
|
|
|
|
|
|
return aliases
|
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
def kick(env, mail_result=None):
|
|
|
|
results = []
|
|
|
|
|
|
|
|
# Inclde the current operation's result in output.
|
|
|
|
|
|
|
|
if mail_result is not None:
|
|
|
|
results.append(mail_result + "\n")
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
# Ensure every required alias exists.
|
2014-07-09 19:29:46 +00:00
|
|
|
|
2014-09-09 11:41:44 +00:00
|
|
|
existing_users = get_mail_users(env)
|
2014-07-09 19:29:46 +00:00
|
|
|
existing_aliases = get_mail_aliases(env)
|
2014-08-17 22:43:57 +00:00
|
|
|
required_aliases = get_required_aliases(env)
|
2014-07-09 19:29:46 +00:00
|
|
|
|
|
|
|
def ensure_admin_alias_exists(source):
|
2014-09-09 11:41:44 +00:00
|
|
|
# If a user account exists with that address, we're good.
|
|
|
|
if source in existing_users:
|
|
|
|
return
|
|
|
|
|
2014-07-09 19:29:46 +00:00
|
|
|
# Does this alias exists?
|
|
|
|
for s, t in existing_aliases:
|
|
|
|
if s == source:
|
|
|
|
return
|
|
|
|
# Doesn't exist.
|
2014-08-17 22:43:57 +00:00
|
|
|
administrator = get_system_administrator(env)
|
2014-07-09 19:29:46 +00:00
|
|
|
add_mail_alias(source, administrator, env, do_kick=False)
|
|
|
|
results.append("added alias %s (=> %s)\n" % (source, administrator))
|
|
|
|
|
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
for alias in required_aliases:
|
|
|
|
ensure_admin_alias_exists(alias)
|
2014-07-09 19:29:46 +00:00
|
|
|
|
2014-08-17 22:43:57 +00:00
|
|
|
# Remove auto-generated postmaster/admin on domains we no
|
2014-07-09 19:29:46 +00:00
|
|
|
# longer have any other email addresses for.
|
|
|
|
for source, target in existing_aliases:
|
|
|
|
user, domain = source.split("@")
|
2014-08-17 22:43:57 +00:00
|
|
|
if user in ("postmaster", "admin") \
|
|
|
|
and source not in required_aliases \
|
|
|
|
and target == get_system_administrator(env):
|
2014-07-09 19:29:46 +00:00
|
|
|
remove_mail_alias(source, env, do_kick=False)
|
|
|
|
results.append("removed alias %s (was to %s; domain no longer used for email)\n" % (source, target))
|
2014-07-06 12:16:50 +00:00
|
|
|
|
|
|
|
# Update DNS and nginx in case any domains are added/removed.
|
2014-07-09 19:29:46 +00:00
|
|
|
|
2014-06-03 13:24:48 +00:00
|
|
|
from dns_update import do_dns_update
|
2014-07-09 19:29:46 +00:00
|
|
|
results.append( do_dns_update(env) )
|
|
|
|
|
2014-07-06 12:16:50 +00:00
|
|
|
from web_update import do_web_update
|
2014-07-09 19:29:46 +00:00
|
|
|
results.append( do_web_update(env) )
|
|
|
|
|
2014-07-06 12:16:50 +00:00
|
|
|
return "".join(s for s in results if s != "")
|
2014-06-30 14:20:58 +00:00
|
|
|
|
2014-09-21 17:24:01 +00:00
|
|
|
def validate_password(pw):
|
|
|
|
# validate password
|
|
|
|
if pw.strip() == "":
|
|
|
|
raise ValueError("No password provided.")
|
|
|
|
if re.search(r"[\s]", pw):
|
|
|
|
raise ValueError("Passwords cannot contain spaces.")
|
|
|
|
if len(pw) < 4:
|
|
|
|
raise ValueError("Passwords must be at least four characters.")
|
|
|
|
|
|
|
|
|
2014-06-30 14:20:58 +00:00
|
|
|
if __name__ == "__main__":
|
|
|
|
import sys
|
|
|
|
if len(sys.argv) > 2 and sys.argv[1] == "validate-email":
|
|
|
|
# Validate that we can create a Dovecot account for a given string.
|
2014-07-13 12:13:41 +00:00
|
|
|
if validate_email(sys.argv[2], mode='user'):
|
2014-06-30 14:20:58 +00:00
|
|
|
sys.exit(0)
|
|
|
|
else:
|
|
|
|
sys.exit(1)
|
2014-07-09 19:29:46 +00:00
|
|
|
|
|
|
|
if len(sys.argv) > 1 and sys.argv[1] == "update":
|
|
|
|
from utils import load_environment
|
|
|
|
print(kick(load_environment()))
|