From a3d7e0dfaed596b62b1adef5a9f90427855a8d17 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:21:19 -0800 Subject: [PATCH 1/6] Adapted MIAB Solr install script from https://github.com/jkaberg/ for Ubuntu Bionic --- conf/cronjob/dovecot | 1 + conf/cronjob/solr | 2 ++ setup/solr.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 conf/cronjob/dovecot create mode 100644 conf/cronjob/solr create mode 100644 setup/solr.sh diff --git a/conf/cronjob/dovecot b/conf/cronjob/dovecot new file mode 100644 index 00000000..8de53e63 --- /dev/null +++ b/conf/cronjob/dovecot @@ -0,0 +1 @@ +/usr/bin/doveadm fts rescan -A diff --git a/conf/cronjob/solr b/conf/cronjob/solr new file mode 100644 index 00000000..217f0191 --- /dev/null +++ b/conf/cronjob/solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null +30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..1e9f597f --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its +# HTTP interface, causing mail to be indexed when searches occur, and getting +# results back. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-tomcat dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Bump memory allocation for Solr. +# Not needed? I'll let it sit here for a while. +#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh + +# Install cronjobs to keep FTS up to date +hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service tomcat8 +restart_service dovecot From 4a23a522e1ed2263645c5b73a004b037dc0b0a32 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:29:21 -0800 Subject: [PATCH 2/6] added solr.sh to start.sh --- setup/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/start.sh b/setup/start.sh index 0b145022..14c5700c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -102,6 +102,7 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh +source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh From 2303ac3394b99ef8a6695a27aaa3cd6c29c0b8b2 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 11:32:16 -0800 Subject: [PATCH 3/6] Force kickoff of Solr indexing at install time --- setup/solr.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup/solr.sh b/setup/solr.sh index 1e9f597f..72f8af61 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -61,3 +61,18 @@ chmod -R o-rwx /etc/dovecot # Restart services to reload solr schema & dovecot plugins restart_service tomcat8 restart_service dovecot + + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' From 2fce29d775dfb041454c497dbf2a33adf13fddb6 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 20:46:52 -0800 Subject: [PATCH 4/6] Added Solr (Tomcat) to status_checks.py --- management/status_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..82310d16 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,6 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): From c302606de44a8b4c4c313e5d98f42f8964f482d1 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Tue, 29 Jan 2019 13:46:35 -0800 Subject: [PATCH 5/6] Extended timeout for php/roundcube for text searches that take a long time --- setup/web.sh | 7 +++++++ setup/webmail.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..2e82fe6a 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -48,6 +48,13 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" +# Set higher timeout since searches with Roundcube and Solr may take longer +# than the default 60 seconds. We will also match Roundcube's timeout to the +# same value +tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ + default_socket_timeout=180 + + # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand diff --git a/setup/webmail.sh b/setup/webmail.sh index b0e11c9b..d082382f 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u'; From 1dee84949838d90be1aa2d8f85995cd17b602074 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 30 Jan 2019 11:33:43 -0800 Subject: [PATCH 6/6] Removed extra blankline --- setup/web.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index 2e82fe6a..4ba646e4 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -54,7 +54,6 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_socket_timeout=180 - # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand