diff --git a/conf/cronjob/dovecot b/conf/cronjob/dovecot new file mode 100644 index 00000000..8de53e63 --- /dev/null +++ b/conf/cronjob/dovecot @@ -0,0 +1 @@ +/usr/bin/doveadm fts rescan -A diff --git a/conf/cronjob/solr b/conf/cronjob/solr new file mode 100644 index 00000000..217f0191 --- /dev/null +++ b/conf/cronjob/solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null +30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null diff --git a/management/status_checks.py b/management/status_checks.py index a172b256..596d4eaa 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,6 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..72f8af61 --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its +# HTTP interface, causing mail to be indexed when searches occur, and getting +# results back. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-tomcat dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Bump memory allocation for Solr. +# Not needed? I'll let it sit here for a while. +#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh + +# Install cronjobs to keep FTS up to date +hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service tomcat8 +restart_service dovecot + + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' diff --git a/setup/start.sh b/setup/start.sh index a3061fe0..2bdea975 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -122,6 +122,7 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh +source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh diff --git a/setup/web.sh b/setup/web.sh index bd839153..02c3a45c 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -48,6 +48,12 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" +# Set higher timeout since searches with Roundcube and Solr may take longer +# than the default 60 seconds. We will also match Roundcube's timeout to the +# same value +tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ + default_socket_timeout=180 + # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand diff --git a/setup/webmail.sh b/setup/webmail.sh index 078f4143..1ad081d1 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u';