Add solr full text search based on https://github.com/jvolkenant/mailinabox/tree/solr-jetty

conf/cron/miab_dovecot

@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/bin/doveadm fts rescan -A > /dev/null 2>&1

conf/cron/miab_solr

@@ -0,0 +1,2 @@
+*/1 * * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?commit=true >/dev/null 2>&1
+30  3 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?optimize=true >/dev/null 2>&1

conf/solr-jetty.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0"  encoding="ISO-8859-1"?>
+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">
+
+<!-- Context configuration file for the Solr web application in Jetty -->
+
+<Configure class="org.eclipse.jetty.webapp.WebAppContext">
+  <Set name="contextPath">/solr</Set>
+  <Set name="war">/usr/share/solr/web</Set>
+
+  <!-- Set the solr.solr.home system property -->
+  <Call name="setProperty" class="java.lang.System">
+    <Arg type="String">solr.solr.home</Arg>
+    <Arg type="String">/usr/share/solr</Arg>
+  </Call>
+
+  <!-- Enable symlinks -->
+  <!--  <Call name="addAliasCheck">
+    <Arg>
+      <New class="org.eclipse.jetty.server.handler.ContextHandler$ApproveSameSuffixAliases"/>
+    </Arg>
+  </Call>-->
+</Configure>

management/status_checks.py

@@ -40,6 +40,7 @@ def get_services():
 		{ "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, },
 		{ "name": "HTTP Web (nginx)", "port": 80, "public": True, },
 		{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
+		{ "name": "Solr Full Text Search (Jetty)", "port": 8080, "public": False, },
 	]
 
 def run_checks(rounded_values, env, output, pool):

setup/additionals.sh

@@ -2,8 +2,7 @@ source /etc/mailinabox.conf
 source setup/functions.sh
 
 # Cleanup old spam and trash email
-cp -f conf/cron/local_clean_mail /etc/cron.weekly/
-chmod +x /etc/cron.weekly/local_clean_mail
+hide_output install -m 755 conf/cron/miab_clean_mail /etc/cron.weekly/
 
 # Reduce logs by not logging mail output in syslog
 sed -i "s/\*\.\*;auth,authpriv.none.*\-\/var\/log\/syslog/\*\.\*;mail,auth,authpriv.none    \-\/var\/log\/syslog/g" /etc/rsyslog.d/50-default.conf

setup/solr.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+#
+# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo)
+# with some modifications
+#
+# IMAP search with lucene via solr
+# --------------------------------
+#
+# By default dovecot uses its own Squat search index that has awful performance
+# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but
+# this didn't make it into the Ubuntu packages, so we use Solr instead to run
+# Lucene for us.
+#
+# Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its
+# HTTP interface, searching indexed mail and returning results back to dovecot.
+
+source setup/functions.sh # load our functions
+source /etc/mailinabox.conf # load global vars
+
+# Install packages and basic configuation
+# ---------------------------------------
+
+echo "Installing Solr..."
+
+# Install packages
+apt_install solr-jetty dovecot-solr
+
+# Solr requires a schema to tell it how to index data, this is provided by dovecot
+cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml
+
+# Default config has an error with our config, placing our custom version
+cp conf/solr-jetty.xml  /etc/solr/solr-jetty.xml
+
+# Update the dovecot plugin configuration
+#
+# Break-imap-search makes search work the way users expect, rather than the way
+# the IMAP specification expects.
+# https://wiki.dovecot.org/Plugins/FTS/Solr
+# "break-imap-search : Use Solr also for indexing TEXT and BODY searches.
+# This makes your server non-IMAP-compliant."
+tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
+        mail_plugins="fts fts_solr"
+
+cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
+plugin {
+  fts = solr
+  fts_autoindex = yes
+  fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/
+}
+EOF
+
+# Install cronjobs to keep FTS up to date.
+hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/
+hide_output install -m 644 conf/cron/miab_solr /etc/cron.d/
+
+# PERMISSIONS
+
+# Ensure configuration files are owned by dovecot and not world readable.
+chown -R mail:dovecot /etc/dovecot
+chmod -R o-rwx /etc/dovecot
+
+# Newer updates to jetty9 restrict write directories, this allows for
+# jetty to write to solr database directories
+mkdir -p /etc/systemd/system/jetty9.service.d/
+cat > /etc/systemd/system/jetty9.service.d/solr-permissions.conf << EOF
+[Service]
+ReadWritePaths=/var/lib/solr/
+ReadWritePaths=/var/lib/solr/data/
+EOF
+
+# Reload systemctl to pickup the above override.
+systemctl daemon-reload
+
+# Fix Logging
+# Due to the new systemd security permissions placed when running jetty.
+# The log file directory at /var/log/jetty9 is reset to jetty:jetty
+# at every program start.  This causes syslog to fail to add the
+# rsyslog filtered output to this folder.  We will move this up a
+# directory to /var/log/ since solr-jetty is quite noisy.
+
+# Remove package config file since it points to a folder that
+# it does not have permissions to, and is also too far down the
+# /etc/rsyslog.d/ order to work anyway.
+rm -f /etc/rsyslog.d/jetty9.conf 
+
+# Create new rsyslog config for jetty9 for its new location
+cat > /etc/rsyslog.d/10-jetty9.conf <<EOF
+# Send Jetty messages to jetty-console.log when using systemd
+:programname, startswith, "jetty9" {
+ /var/log/jetty-console.log
+ stop
+}
+EOF
+
+# Also adjust logrotated to the new file and correct user
+cat > /etc/logrotate.d/jetty9.conf <<EOF
+/var/log/jetty-console.log {
+    copytruncate
+    weekly
+    rotate 12
+    compress
+    delaycompress
+    missingok
+    create 640 syslog adm
+}
+EOF
+
+
+# Restart services to reload solr schema, dovecot plugins and rsyslog changes
+restart_service jetty9
+restart_service dovecot
+restart_service rsyslog
+
+# Kickoff building the index
+
+# Per doveadm-fts manpage: Scan what mails exist in the full text search index
+# and compare those to what actually exist in mailboxes.
+# This removes mails from the index that have already been expunged  and  makes
+# sure that the next doveadm index will index all the missing mails (if any).
+doveadm fts rescan -A
+
+# Adds unindexed files to the fts database
+# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing)
+# * `-A`: All users
+# * `'*'`: All folders
+doveadm index -q -A '*'

setup/start.sh

@@ -111,6 +111,7 @@ source setup/dns.sh
 source setup/mail-postfix.sh
 source setup/mail-dovecot.sh
 source setup/mail-users.sh
+source setup/solr.sh
 source setup/dkim.sh
 source setup/spamassassin.sh
 source setup/web.sh

setup/web.sh

@@ -53,6 +53,12 @@ tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
 tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
         default_charset="UTF-8"
 
+# Set higher timeout since searches with Roundcube and Solr may take longer
+# than the default 60 seconds. We will also match Roundcube's timeout to the
+# same value
+tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
+        default_socket_timeout=180
+
 # Configure the path environment for php-fpm
 tools/editconf.py /etc/php/$(php_version)/fpm/pool.d/www.conf -c ';' \
 	env[PATH]=/usr/local/bin:/usr/bin:/bin \

setup/webmail.sh

@@ -117,7 +117,7 @@ cat > $RCM_CONFIG <<EOF;
      'verify_peer_name'  => false,
    ),
  );
-\$config['imap_timeout'] = 15;
+\$config['imap_timeout'] = 180;
 \$config['smtp_server'] = 'tls://127.0.0.1';
 \$config['smtp_conn_options'] = array(
   'ssl'         => array(