From 4aaee13c1c5c6bb3e8c801f8331a6caf9d602171 Mon Sep 17 00:00:00 2001 From: "github@kiekerjan.isdronken.nl" Date: Sat, 17 Apr 2021 23:00:14 +0200 Subject: [PATCH] Add solr full text search based on https://github.com/jvolkenant/mailinabox/tree/solr-jetty --- .../{local_clean_mail => miab_clean_mail} | 0 conf/cron/miab_dovecot | 2 + conf/cron/miab_solr | 2 + conf/solr-jetty.xml | 22 +++ management/status_checks.py | 1 + setup/additionals.sh | 3 +- setup/solr.sh | 126 ++++++++++++++++++ setup/start.sh | 1 + setup/web.sh | 6 + setup/webmail.sh | 2 +- 10 files changed, 162 insertions(+), 3 deletions(-) rename conf/cron/{local_clean_mail => miab_clean_mail} (100%) create mode 100644 conf/cron/miab_dovecot create mode 100644 conf/cron/miab_solr create mode 100644 conf/solr-jetty.xml create mode 100644 setup/solr.sh diff --git a/conf/cron/local_clean_mail b/conf/cron/miab_clean_mail similarity index 100% rename from conf/cron/local_clean_mail rename to conf/cron/miab_clean_mail diff --git a/conf/cron/miab_dovecot b/conf/cron/miab_dovecot new file mode 100644 index 00000000..869ca377 --- /dev/null +++ b/conf/cron/miab_dovecot @@ -0,0 +1,2 @@ +#!/bin/bash +/usr/bin/doveadm fts rescan -A > /dev/null 2>&1 diff --git a/conf/cron/miab_solr b/conf/cron/miab_solr new file mode 100644 index 00000000..b37941fc --- /dev/null +++ b/conf/cron/miab_solr @@ -0,0 +1,2 @@ +*/1 * * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?commit=true >/dev/null 2>&1 +30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?optimize=true >/dev/null 2>&1 diff --git a/conf/solr-jetty.xml b/conf/solr-jetty.xml new file mode 100644 index 00000000..27de9994 --- /dev/null +++ b/conf/solr-jetty.xml @@ -0,0 +1,22 @@ + + + + + + + /solr + /usr/share/solr/web + + + + solr.solr.home + /usr/share/solr + + + + + diff --git a/management/status_checks.py b/management/status_checks.py index 6b8ef1fe..9f78a004 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -40,6 +40,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, + { "name": "Solr Full Text Search (Jetty)", "port": 8080, "public": False, }, ] def run_checks(rounded_values, env, output, pool): diff --git a/setup/additionals.sh b/setup/additionals.sh index c0812173..701672af 100644 --- a/setup/additionals.sh +++ b/setup/additionals.sh @@ -2,8 +2,7 @@ source /etc/mailinabox.conf source setup/functions.sh # Cleanup old spam and trash email -cp -f conf/cron/local_clean_mail /etc/cron.weekly/ -chmod +x /etc/cron.weekly/local_clean_mail +hide_output install -m 755 conf/cron/miab_clean_mail /etc/cron.weekly/ # Reduce logs by not logging mail output in syslog sed -i "s/\*\.\*;auth,authpriv.none.*\-\/var\/log\/syslog/\*\.\*;mail,auth,authpriv.none \-\/var\/log\/syslog/g" /etc/rsyslog.d/50-default.conf diff --git a/setup/solr.sh b/setup/solr.sh new file mode 100644 index 00000000..f1ecb6df --- /dev/null +++ b/setup/solr.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# +# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) +# with some modifications +# +# IMAP search with lucene via solr +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we use Solr instead to run +# Lucene for us. +# +# Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its +# HTTP interface, searching indexed mail and returning results back to dovecot. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing Solr..." + +# Install packages +apt_install solr-jetty dovecot-solr + +# Solr requires a schema to tell it how to index data, this is provided by dovecot +cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml + +# Default config has an error with our config, placing our custom version +cp conf/solr-jetty.xml /etc/solr/solr-jetty.xml + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects. +# https://wiki.dovecot.org/Plugins/FTS/Solr +# "break-imap-search : Use Solr also for indexing TEXT and BODY searches. +# This makes your server non-IMAP-compliant." +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_solr" + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = solr + fts_autoindex = yes + fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ +} +EOF + +# Install cronjobs to keep FTS up to date. +hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/ +hide_output install -m 644 conf/cron/miab_solr /etc/cron.d/ + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Newer updates to jetty9 restrict write directories, this allows for +# jetty to write to solr database directories +mkdir -p /etc/systemd/system/jetty9.service.d/ +cat > /etc/systemd/system/jetty9.service.d/solr-permissions.conf << EOF +[Service] +ReadWritePaths=/var/lib/solr/ +ReadWritePaths=/var/lib/solr/data/ +EOF + +# Reload systemctl to pickup the above override. +systemctl daemon-reload + +# Fix Logging +# Due to the new systemd security permissions placed when running jetty. +# The log file directory at /var/log/jetty9 is reset to jetty:jetty +# at every program start. This causes syslog to fail to add the +# rsyslog filtered output to this folder. We will move this up a +# directory to /var/log/ since solr-jetty is quite noisy. + +# Remove package config file since it points to a folder that +# it does not have permissions to, and is also too far down the +# /etc/rsyslog.d/ order to work anyway. +rm -f /etc/rsyslog.d/jetty9.conf + +# Create new rsyslog config for jetty9 for its new location +cat > /etc/rsyslog.d/10-jetty9.conf < /etc/logrotate.d/jetty9.conf < $RCM_CONFIG < false, ), ); -\$config['imap_timeout'] = 15; +\$config['imap_timeout'] = 180; \$config['smtp_server'] = 'tls://127.0.0.1'; \$config['smtp_conn_options'] = array( 'ssl' => array(