From eb47a1471b94a6d245388f43af1baea9f07f7a76 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Wed, 21 Aug 2013 09:37:33 -0400 Subject: [PATCH] mail seems to work --- .gitignore | 1 + README.md | 7 ++- ec2/start_instance.sh | 26 +++++++-- scripts/index.sh | 3 + scripts/mail.sh | 115 ++++++++++++++++++++++++++++++++++++--- scripts/mail_testuser.sh | 3 + scripts/new_volume.sh | 5 -- scripts/system.sh | 19 ++++--- tests/imap.py | 13 +++++ tests/smtp.py | 16 ++++++ tools/editconf.py | 23 +++++++- 11 files changed, 200 insertions(+), 31 deletions(-) create mode 100644 .gitignore create mode 100644 scripts/index.sh mode change 100644 => 100755 scripts/mail.sh create mode 100644 scripts/mail_testuser.sh mode change 100644 => 100755 scripts/new_volume.sh mode change 100644 => 100755 scripts/system.sh create mode 100644 tests/imap.py create mode 100644 tests/smtp.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..b25c15b8 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ diff --git a/README.md b/README.md index 169979f2..7ab19178 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,14 @@ Then launch a new instance. We're creating a m1.small instance --- it's the smal It will wait until the instance is available. -Log in: +Configure the server: ssh -i mykey.pem ubuntu@$INSTANCE_IP -Set up: +Somehow download these files. - + sh scripts/index.sh + ... logout Terminate your instance with: diff --git a/ec2/start_instance.sh b/ec2/start_instance.sh index 64c8eb86..93b39ab6 100644 --- a/ec2/start_instance.sh +++ b/ec2/start_instance.sh @@ -1,18 +1,34 @@ -export AMI=`curl http://cloud-images.ubuntu.com/locator/ec2/releasesTable | python3 tools/get_ubunut_ami.py us-east-1 13.04 amd64 instance-store` -ec2run $AMI -k mykey -t m1.small -z $AWS_AZ | tee instance.info +if [ -z "$EC2_KEYPAIR_NAME" ]; then + EC2_KEYPAIR_NAME=mykey +fi + +UBUNTU_CONFIG="us-east-1 13.04 amd64 instance-store" + +export AMI=`curl -s http://cloud-images.ubuntu.com/locator/ec2/releasesTable | python3 tools/get_ubuntu_ami.py $UBUNTU_CONFIG` + +ec2-create-group -d "mailinabox" "mailinabox" +for PORT in 25 587 993; do ec2-authorize mailinabox -P tcp -p $PORT -s 0.0.0.0/0; done + +ec2run $AMI -k $EC2_KEYPAIR_NAME -t m1.small -z $AWS_AZ -g mailinabox > instance.info export INSTANCE=`cat instance.info | grep INSTANCE | awk {'print $2'}` + +echo Started instance $INSTANCE + sleep 5 while [ 1 ] do - export INSTANCE_IP=`ec2-describe-instances $INSTANCE | grep INSTANCE | awk {'print $14'}` + export INSTANCE_IP=`ec2-describe-instances $INSTANCE | grep INSTANCE | awk {'print $14'}` if [ -z "$INSTANCE_IP" ] then echo "Waiting for $INSTANCE to start..." else - exit + break fi sleep 6 done -echo New instance started: $INSTANCE_IP +# Give SSH time to start. +sleep 5 + +echo New instance has IP: $INSTANCE_IP diff --git a/scripts/index.sh b/scripts/index.sh new file mode 100644 index 00000000..3e00c593 --- /dev/null +++ b/scripts/index.sh @@ -0,0 +1,3 @@ +. scripts/system.sh +. scripts/mail.sh + diff --git a/scripts/mail.sh b/scripts/mail.sh old mode 100644 new mode 100755 index 946be7ec..63c61574 --- a/scripts/mail.sh +++ b/scripts/mail.sh @@ -1,13 +1,27 @@ -# Configures a postfix SMTP server. +# Configures a postfix SMTP server and dovecot IMAP server. +# +# We configure these together because postfix delivers mail +# directly to dovecot, so they basically rely on each other. -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y postfix postgrey +# Install packages. +sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y \ + postfix postgrey dovecot-core dovecot-imapd dovecot-lmtpd dovecot-sqlite + +# POSTFIX + +mkdir -p $STORAGE_ROOT/mail + # TLS configuration +sudo sed -i "s/#submission/submission/" /etc/postfix/master.cf # enable submission port (not in Drew Crawford's instructions) sudo tools/editconf.py /etc/postfix/main.cf \ + smtpd_use_tls=yes\ smtpd_tls_auth_only=yes \ smtp_tls_security_level=may \ smtp_tls_loglevel=2 \ smtpd_tls_received_header=yes + + # note: smtpd_use_tls=yes appears to already be the default, but we can never be too sure # authorization via dovecot sudo tools/editconf.py /etc/postfix/main.cf \ @@ -28,11 +42,11 @@ sudo tools/editconf.py /etc/postfix/main.cf \ virtual_alias_maps=sqlite:/etc/postfix/virtual-alias-maps.cf \ local_recipient_maps=\$virtual_mailbox_maps -db_path=/home/ubuntu/storage/mail.sqlite - +db_path=$STORAGE_ROOT/mail/users.sqlite + sudo su root -c "cat > /etc/postfix/virtual-mailbox-domains.cf" << EOF; dbpath=$db_path -query = SELECT 1 FROM users WHERE email LIKE '@%s' +query = SELECT 1 FROM users WHERE email LIKE '%%@%s' EOF sudo su root -c "cat > /etc/postfix/virtual-mailbox-maps.cf" << EOF; @@ -45,11 +59,96 @@ dbpath=$db_path query = SELECT destination FROM aliases WHERE source='%s' EOF -# re-start postfix +# create an empty mail users database if it doesn't yet exist + +if [ ! -f $db_path ]; then + echo Creating new user database: $db_path; + echo "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, email TEXT NOT NULL UNIQUE, password TEXT NOT NULL, extra);" | sqlite3 $db_path; + echo "CREATE TABLE aliases (id INTEGER PRIMARY KEY AUTOINCREMENT, source TEXT NOT NULL UNIQUE, destination TEXT NOT NULL);" | sqlite3 $db_path; +fi + +# DOVECOT + +# The dovecot-imapd dovecot-lmtpd packages automatically enable those protocols. + +# mail storage location +sudo tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_location=maildir:$STORAGE_ROOT/mail/mailboxes/%d/%n \ + mail_privileged_group=mail \ + first_valid_uid=0 + +# authentication mechanisms +sudo tools/editconf.py /etc/dovecot/conf.d/10-auth.conf \ + disable_plaintext_auth=yes \ + "auth_mechanisms=plain login" + +# use SQL-based authentication, not the system users +sudo sed -i "s/\(\!include auth-system.conf.ext\)/#\1/" /etc/dovecot/conf.d/10-auth.conf +sudo sed -i "s/#\(\!include auth-sql.conf.ext\)/\1/" /etc/dovecot/conf.d/10-auth.conf + +# how to access SQL +sudo su root -c "cat > /etc/dovecot/conf.d/auth-sql.conf.ext" << EOF; +passdb { + driver = sql + args = /etc/dovecot/dovecot-sql.conf.ext +} +userdb { + driver = static + args = uid=mail gid=mail home=$STORAGE_ROOT/mail/mailboxes/%d/%n +} +EOF +sudo su root -c "cat > /etc/dovecot/dovecot-sql.conf.ext" << EOF; +driver = sqlite +connect = $db_path +default_pass_scheme = SHA512-CRYPT +password_query = SELECT email as user, password FROM users WHERE email='%u'; +EOF + +# disable in-the-clear IMAP and POP because we're paranoid (we haven't even +# enabled POP). +sudo sed -i "s/#port = 143/port = 0/" /etc/dovecot/conf.d/10-master.conf +sudo sed -i "s/#port = 110/port = 0/" /etc/dovecot/conf.d/10-master.conf + +# Modify the unix socket for LMTP. +sudo sed -i "s/unix_listener lmtp \(.*\)/unix_listener \/var\/spool\/postfix\/private\/dovecot-lmtp \1\n user = postfix\n group = postfix\n/" /etc/dovecot/conf.d/10-master.conf + +# Add an additional auth socket for postfix. Check if it already is +# set to make sure this is idempotent. +if sudo grep -q "mailinabox-postfix-private-auth" /etc/dovecot/conf.d/10-master.conf; then + # already done + true; +else + sudo sed -i "s/\(\s*unix_listener auth-userdb\)/ unix_listener \/var\/spool\/postfix\/private\/auth \{ # mailinabox-postfix-private-auth\n mode = 0666\n user = postfix\n group = postfix\n \}\n\1/" /etc/dovecot/conf.d/10-master.conf +fi + +# Drew Crawford sets the auth-worker process to run as the mail user, but we don't care if it runs as root. + +# Enable SSL. +sudo tools/editconf.py /etc/dovecot/conf.d/10-ssl.conf \ + ssl=required \ + "ssl_cert=<$STORAGE_ROOT/ssl/ssl_certificate.pem" \ + "ssl_key=<$STORAGE_ROOT/ssl/ssl_private_key.pem" \ + +# The Dovecot installation already created a self-signed public/private key pair +# in /etc/dovecot/dovecot.pem and /etc/dovecot/private/dovecot.pem, which we'll +# use unless certificates already exist. +mkdir -p $STORAGE_ROOT/ssl +if [ ! -f $STORAGE_ROOT/ssl/ssl_certificate.pem ]; then sudo cp /etc/dovecot/dovecot.pem $STORAGE_ROOT/ssl/ssl_certificate.pem; fi +if [ ! -f $STORAGE_ROOT/ssl/ssl_private_key.pem ]; then sudo cp /etc/dovecot/private/dovecot.pem $STORAGE_ROOT/ssl/ssl_private_key.pem; fi + +sudo chown -R mail:dovecot /etc/dovecot +sudo chmod -R o-rwx /etc/dovecot + +mkdir -p $STORAGE_ROOT/mail/mailboxes +sudo chown -R mail.mail $STORAGE_ROOT/mail/mailboxes + +# restart services sudo service postfix restart +sudo service dovecot restart -# allow ports in the firewall -sudo ufw allow smtpd +# allow mail-related ports in the firewall +sudo ufw allow smtp sudo ufw allow submission +sudo ufw allow imaps diff --git a/scripts/mail_testuser.sh b/scripts/mail_testuser.sh new file mode 100644 index 00000000..cdc481ed --- /dev/null +++ b/scripts/mail_testuser.sh @@ -0,0 +1,3 @@ +# Create a test user: testuser@testdomain.com with password "testpw" +echo "INSERT INTO users (email, password) VALUES ('testuser@testdomain.com', '`sudo doveadm pw -s SHA512-CRYPT -p testpw`');" | sqlite3 storage/mail/users.sqlite + diff --git a/scripts/new_volume.sh b/scripts/new_volume.sh old mode 100644 new mode 100755 index 8ab1f1f7..8b137891 --- a/scripts/new_volume.sh +++ b/scripts/new_volume.sh @@ -1,6 +1 @@ -mkdir storage - -# mount volume - -echo "CREATE TABLE users (email text, password text);" | sqlite3 /home/ubuntu/storage/mail.sqlite; diff --git a/scripts/system.sh b/scripts/system.sh old mode 100644 new mode 100755 index 9ae5ee0e..25ef93ec --- a/scripts/system.sh +++ b/scripts/system.sh @@ -1,11 +1,11 @@ # Base system configuration. -sudo apt-get update -sudo apt-get -y upgrade +sudo apt-get -q update +sudo apt-get -q -y upgrade # Basic packages. -sudo apt-get -y install sqlite3 +sudo apt-get -q -y install sqlite3 # Turn on basic services: # @@ -15,13 +15,18 @@ sudo apt-get -y install sqlite3 # # These services don't need further configuration and are started immediately after installation. -sudo apt-get install -y ntp fail2ban +sudo apt-get install -q -y ntp fail2ban # Turn on the firewall. First allow incoming SSH, then turn on the firewall. Additional open # ports will be set up in the scripts that set up those services. sudo ufw allow ssh -sudo ufw allow domain -sudo ufw allow http -sudo ufw allow https +#sudo ufw allow domain +#sudo ufw allow http +#sudo ufw allow https sudo ufw --force enable +# Mount the storage volume. +export STORAGE_ROOT=/home/ubuntu/storage +mkdir -p storage + + diff --git a/tests/imap.py b/tests/imap.py new file mode 100644 index 00000000..e37c68d8 --- /dev/null +++ b/tests/imap.py @@ -0,0 +1,13 @@ +import imaplib, os + +M = imaplib.IMAP4_SSL(os.environ["INSTANCE_IP"]) +M.login("testuser@testdomain.com", "testpw") +M.select() +print("Login successful.") +typ, data = M.search(None, 'ALL') +for num in data[0].split(): + typ, data = M.fetch(num, '(RFC822)') + print('Message %s\n%s\n' % (num, data[0][1])) +M.close() +M.logout() + diff --git a/tests/smtp.py b/tests/smtp.py new file mode 100644 index 00000000..8761b899 --- /dev/null +++ b/tests/smtp.py @@ -0,0 +1,16 @@ +import smtplib, sys, os + +fromaddr = "testuser@testdomain.com" + +msg = """From: %s +To: %s + +This is a test message.""" % (fromaddr, sys.argv[1]) + +server = smtplib.SMTP(os.environ["INSTANCE_IP"], 587) +server.set_debuglevel(1) +server.starttls() +server.login("testuser@testdomain.com", "testpw") +server.sendmail(fromaddr, [sys.argv[1]], msg) +server.quit() + diff --git a/tools/editconf.py b/tools/editconf.py index 0dd21a52..c7e3af51 100755 --- a/tools/editconf.py +++ b/tools/editconf.py @@ -17,16 +17,33 @@ buf = "" for line in open(filename): for i in range(len(settings)): name, val = settings[i].split("=", 1) - if re.match("\s*" + re.escape(name) + "\s*=", line): + m = re.match("\s*" + re.escape(name) + "\s*=\s*(.*?)\s*$", line) + if m: + # If this is already the setting, do nothing. + if m.group(1) == val: + buf += line + found.add(i) + break + + # comment-out the existing line buf += "#" + line - if i in found: break # we've already set the directive + + # if this option oddly appears more than once, don't add the settingg again + if i in found: + break + + # add the new setting buf += name + "=" + val + "\n" + + # note that we've applied this option found.add(i) + break else: - # did not match any setting name + # If did not match any setting names, pass this line through. buf += line +# Put any settings we didn't see at the end of the file. for i in range(len(settings)): if i not in found: buf += settings[i] + "\n"