From 23c0388bb3607c17dbbb7b75034c79343a4b95d5 Mon Sep 17 00:00:00 2001
From: "github@kiekerjan.isdronken.nl" <github@kiekerjan.isdronken.nl>
Date: Sat, 8 May 2021 22:35:46 +0200
Subject: [PATCH] base for xapian dovecot fts

---
 conf/cron/miab_dovecot      |  1 +
 lib/howto_build_xapian.txt  | 15 +++++++
 setup/dovecot-fts-xapian.sh | 80 +++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)
 create mode 100644 lib/howto_build_xapian.txt
 create mode 100644 setup/dovecot-fts-xapian.sh

diff --git a/conf/cron/miab_dovecot b/conf/cron/miab_dovecot
index 869ca377..343f6e60 100644
--- a/conf/cron/miab_dovecot
+++ b/conf/cron/miab_dovecot
@@ -1,2 +1,3 @@
 #!/bin/bash
 /usr/bin/doveadm fts rescan -A > /dev/null 2>&1
+/usr/bin/doveadm fts optimize -A
diff --git a/lib/howto_build_xapian.txt b/lib/howto_build_xapian.txt
new file mode 100644
index 00000000..85a8dd92
--- /dev/null
+++ b/lib/howto_build_xapian.txt
@@ -0,0 +1,15 @@
+# Install required packages
+apt-get build-dep dovecot-core
+apt-get install dovecot-dev libxapian-dev git libxapian30 libicu-dev
+
+# Clone the project
+git clone https://github.com/grosjo/fts-xapian
+cd fts-xapian
+
+# Compile (don't install)
+autoreconf -vi
+./configure --with-dovecot=/usr/lib/dovecot
+make
+
+# install library found under src/.libs
+cp src/.libs/lib21_fts_xapian_plugin.so /usr/lib/dovecot/modules
\ No newline at end of file
diff --git a/setup/dovecot-fts-xapian.sh b/setup/dovecot-fts-xapian.sh
new file mode 100644
index 00000000..937e6625
--- /dev/null
+++ b/setup/dovecot-fts-xapian.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# IMAP search with xapian
+# --------------------------------
+#
+# By default dovecot uses its own Squat search index that has awful performance
+# on large mailboxes and is obsolete. Dovecot 2.1+ has support for using Lucene 
+# internally but this didn't make it into the Ubuntu packages. Solr uses too 
+# much memory. Same goes for elasticsearch. fts xapian might be a good match 
+# for mail-in-a-box. See https://github.com/grosjo/fts-xapian
+
+source setup/functions.sh # load our functions
+source /etc/mailinabox.conf # load global vars
+
+# Install packages and basic configuation
+# ---------------------------------------
+
+echo "Installing fts-xapian..."
+
+apt_install libxapian30
+
+# Update the dovecot plugin configuration
+#
+# Break-imap-search makes search work the way users expect, rather than the way
+# the IMAP specification expects.
+tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
+        mail_plugins="fts fts_xapian"
+
+# Install cronjobs to keep FTS up to date.
+hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/
+
+if [ ! -f /usr/lib/dovecot/decode2text.sh ]; then
+	cp -f /usr/share/doc/dovecot-core/examples/decode2text.sh /usr/lib/dovecot
+fi
+
+cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
+plugin {
+  plugin = fts fts_xapian
+  
+  fts = xapian
+  fts_xapian = partial=3 full=20 verbose=0
+
+  fts_autoindex = yes
+  fts_enforced = yes
+
+  fts_autoindex_exclude = \Trash
+  fts_autoindex_exclude2 = \Junk
+  
+  fts_decoder = decode2text // To index attachements
+}
+
+service indexer-worker {
+	vsz_limit = 2G // or above (or 0 if you have rather large memory usable on your server, which is preferred for performance)
+}
+
+service decode2text {
+   executable = script /usr/lib/dovecot/decode2text.sh
+   user = dovecot
+   unix_listener decode2text {
+     mode = 0666
+   }
+}
+EOF
+
+restart_service dovecot
+
+# Kickoff building the index
+
+# Per doveadm-fts manpage: Scan what mails exist in the full text search index
+# and compare those to what actually exist in mailboxes.
+# This removes mails from the index that have already been expunged  and  makes
+# sure that the next doveadm index will index all the missing mails (if any).
+doveadm fts rescan -A
+
+# Adds unindexed files to the fts database
+# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing)
+# * `-A`: All users
+# * `'*'`: All folders
+doveadm index -q -A '*'
+