From a3d7e0dfaed596b62b1adef5a9f90427855a8d17 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:21:19 -0800 Subject: [PATCH 01/11] Adapted MIAB Solr install script from https://github.com/jkaberg/ for Ubuntu Bionic --- conf/cronjob/dovecot | 1 + conf/cronjob/solr | 2 ++ setup/solr.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 conf/cronjob/dovecot create mode 100644 conf/cronjob/solr create mode 100644 setup/solr.sh diff --git a/conf/cronjob/dovecot b/conf/cronjob/dovecot new file mode 100644 index 00000000..8de53e63 --- /dev/null +++ b/conf/cronjob/dovecot @@ -0,0 +1 @@ +/usr/bin/doveadm fts rescan -A diff --git a/conf/cronjob/solr b/conf/cronjob/solr new file mode 100644 index 00000000..217f0191 --- /dev/null +++ b/conf/cronjob/solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?commit=true &>/dev/null +30 3 * * * root /usr/bin/curl http://127.0.0.1:8080/solr/update?optimize=true &>/dev/null diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..1e9f597f --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its +# HTTP interface, causing mail to be indexed when searches occur, and getting +# results back. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-tomcat dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Bump memory allocation for Solr. +# Not needed? I'll let it sit here for a while. +#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh + +# Install cronjobs to keep FTS up to date +hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service tomcat8 +restart_service dovecot From 4a23a522e1ed2263645c5b73a004b037dc0b0a32 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 10:29:21 -0800 Subject: [PATCH 02/11] added solr.sh to start.sh --- setup/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/start.sh b/setup/start.sh index 0b145022..14c5700c 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -102,6 +102,7 @@ source setup/dns.sh source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh +source setup/solr.sh source setup/dkim.sh source setup/spamassassin.sh source setup/web.sh From 2303ac3394b99ef8a6695a27aaa3cd6c29c0b8b2 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 11:32:16 -0800 Subject: [PATCH 03/11] Force kickoff of Solr indexing at install time --- setup/solr.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup/solr.sh b/setup/solr.sh index 1e9f597f..72f8af61 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -61,3 +61,18 @@ chmod -R o-rwx /etc/dovecot # Restart services to reload solr schema & dovecot plugins restart_service tomcat8 restart_service dovecot + + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' From 2fce29d775dfb041454c497dbf2a33adf13fddb6 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 16 Jan 2019 20:46:52 -0800 Subject: [PATCH 04/11] Added Solr (Tomcat) to status_checks.py --- management/status_checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/management/status_checks.py b/management/status_checks.py index 6f9bb1ef..82310d16 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,6 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): From c302606de44a8b4c4c313e5d98f42f8964f482d1 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Tue, 29 Jan 2019 13:46:35 -0800 Subject: [PATCH 05/11] Extended timeout for php/roundcube for text searches that take a long time --- setup/web.sh | 7 +++++++ setup/webmail.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index ed37e5e3..2e82fe6a 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -48,6 +48,13 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_charset="UTF-8" +# Set higher timeout since searches with Roundcube and Solr may take longer +# than the default 60 seconds. We will also match Roundcube's timeout to the +# same value +tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ + default_socket_timeout=180 + + # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand diff --git a/setup/webmail.sh b/setup/webmail.sh index b0e11c9b..d082382f 100755 --- a/setup/webmail.sh +++ b/setup/webmail.sh @@ -108,7 +108,7 @@ cat > $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_port'] = 587; \$config['smtp_user'] = '%u'; From 1dee84949838d90be1aa2d8f85995cd17b602074 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Wed, 30 Jan 2019 11:33:43 -0800 Subject: [PATCH 06/11] Removed extra blankline --- setup/web.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/web.sh b/setup/web.sh index 2e82fe6a..4ba646e4 100755 --- a/setup/web.sh +++ b/setup/web.sh @@ -54,7 +54,6 @@ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ tools/editconf.py /etc/php/7.2/fpm/php.ini -c ';' \ default_socket_timeout=180 - # Switch from the dynamic process manager to the ondemand manager see #1216 tools/editconf.py /etc/php/7.2/fpm/pool.d/www.conf -c ';' \ pm=ondemand From 511d77a03c5087a5e3c74a5903c09cf1d19b80f4 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Thu, 18 Apr 2019 09:27:53 -0700 Subject: [PATCH 07/11] added solr-jetty instead of solr-tomcat --- setup/solr.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/setup/solr.sh b/setup/solr.sh index 72f8af61..377aaf97 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -24,7 +24,7 @@ source /etc/mailinabox.conf # load global vars echo "Installing Solr..." # Install packages -apt_install solr-tomcat dovecot-solr +apt_install solr-jetty dovecot-solr # Solr requires a schema to tell it how to index data, this is provided by dovecot cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml @@ -44,10 +44,6 @@ plugin { } EOF -# Bump memory allocation for Solr. -# Not needed? I'll let it sit here for a while. -#echo 'export JAVA_OPTS=-Xms512M -Xmx1024M' > /usr/share/tomcat7/bin/setenv.sh - # Install cronjobs to keep FTS up to date hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ @@ -58,10 +54,20 @@ hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ chown -R mail:dovecot /etc/dovecot chmod -R o-rwx /etc/dovecot -# Restart services to reload solr schema & dovecot plugins -restart_service tomcat8 -restart_service dovecot +# Newer updates to jetty9 restrict write directories, this allows for +# jetty to write to solr database directories +cat > /etc/systemd/system/jetty9.service.d/solr-permissions.conf << EOF +[Service] +ReadWritePaths=/var/lib/solr/ +ReadWritePaths=/var/lib/solr/data/ +EOF +# Reload systemctl to pickup the above override +systemctl daemon-reload + +# Restart services to reload solr schema & dovecot plugins +restart_service jetty9 +restart_service dovecot # Kickoff building the index From 34b1b8c95633a2a183be72401c388bee96784806 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Thu, 18 Apr 2019 11:37:33 -0700 Subject: [PATCH 08/11] fix logging to writeable directory --- setup/solr.sh | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/setup/solr.sh b/setup/solr.sh index 377aaf97..b835bb9e 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -65,9 +65,46 @@ EOF # Reload systemctl to pickup the above override systemctl daemon-reload -# Restart services to reload solr schema & dovecot plugins +# Fix Logging +# Due to the new systemd security permissions placed when running jetty +# the log file directory at /var/log/jetty9 is reset to jetty:jetty +# at every program start. This causes syslog to fail to add the +# rsyslog filtered output to this folder. We will move this up a +# directory to /var/log/ since solr-jetty is quite noisy. + +# Remove package config file since it points to a folder that +# it does not have permissions to, and is also too far down the +# /etc/rsyslog.d/ order to work anyway +rm -f /etc/rsyslog.d/jetty9.conf + +# Create new rsyslog config for jetty9 for its new location +cat > /etc/rsyslog.d/10-jetty9.conf < /etc/logrotate.d/jetty9.conf < Date: Thu, 23 May 2019 10:35:45 -0700 Subject: [PATCH 09/11] Fixed naming for Service Check --- management/status_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/status_checks.py b/management/status_checks.py index 82310d16..70e9db86 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -38,7 +38,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, - { "name": "Solr Full Text Search (tomcat)", "port": 8080, "public": False, }, + { "name": "Solr Full Text Search (Jetty)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): From 85b64c93f91c76733482a9b130365417a935df62 Mon Sep 17 00:00:00 2001 From: Jeff Volkenant Date: Thu, 23 May 2019 10:54:07 -0700 Subject: [PATCH 10/11] Keep 12 weeks of jetty logs instead of 52 --- setup/solr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/solr.sh b/setup/solr.sh index b835bb9e..e9ae9bda 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -92,7 +92,7 @@ cat > /etc/logrotate.d/jetty9.conf < Date: Thu, 23 May 2019 11:06:34 -0700 Subject: [PATCH 11/11] Cleaning up verbage and spelling. --- setup/solr.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup/solr.sh b/setup/solr.sh index e9ae9bda..ea12774e 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -11,9 +11,8 @@ # this didn't make it into the Ubuntu packages, so we use Solr instead to run # Lucene for us. # -# Solr runs as a tomcat process. The dovecot solr plugin talks to solr via its -# HTTP interface, causing mail to be indexed when searches occur, and getting -# results back. +# Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its +# HTTP interface, searching indexed mail and returning results back to dovecot. source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars @@ -32,7 +31,10 @@ cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml # Update the dovecot plugin configuration # # Break-imap-search makes search work the way users expect, rather than the way -# the IMAP specification expects +# the IMAP specification expects. +# https://wiki.dovecot.org/Plugins/FTS/Solr +# "break-imap-search : Use Solr also for indexing TEXT and BODY searches. +# This makes your server non-IMAP-compliant." tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ mail_plugins="fts fts_solr" @@ -44,7 +46,7 @@ plugin { } EOF -# Install cronjobs to keep FTS up to date +# Install cronjobs to keep FTS up to date. hide_output install -m 755 conf/cronjob/dovecot /etc/cron.daily/ hide_output install -m 644 conf/cronjob/solr /etc/cron.d/ @@ -62,24 +64,24 @@ ReadWritePaths=/var/lib/solr/ ReadWritePaths=/var/lib/solr/data/ EOF -# Reload systemctl to pickup the above override +# Reload systemctl to pickup the above override. systemctl daemon-reload # Fix Logging -# Due to the new systemd security permissions placed when running jetty -# the log file directory at /var/log/jetty9 is reset to jetty:jetty +# Due to the new systemd security permissions placed when running jetty. +# The log file directory at /var/log/jetty9 is reset to jetty:jetty # at every program start. This causes syslog to fail to add the # rsyslog filtered output to this folder. We will move this up a # directory to /var/log/ since solr-jetty is quite noisy. # Remove package config file since it points to a folder that # it does not have permissions to, and is also too far down the -# /etc/rsyslog.d/ order to work anyway +# /etc/rsyslog.d/ order to work anyway. rm -f /etc/rsyslog.d/jetty9.conf # Create new rsyslog config for jetty9 for its new location cat > /etc/rsyslog.d/10-jetty9.conf <