From 23c0388bb3607c17dbbb7b75034c79343a4b95d5 Mon Sep 17 00:00:00 2001 From: "github@kiekerjan.isdronken.nl" Date: Sat, 8 May 2021 22:35:46 +0200 Subject: [PATCH] base for xapian dovecot fts --- conf/cron/miab_dovecot | 1 + lib/howto_build_xapian.txt | 15 +++++++ setup/dovecot-fts-xapian.sh | 80 +++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 lib/howto_build_xapian.txt create mode 100644 setup/dovecot-fts-xapian.sh diff --git a/conf/cron/miab_dovecot b/conf/cron/miab_dovecot index 869ca377..343f6e60 100644 --- a/conf/cron/miab_dovecot +++ b/conf/cron/miab_dovecot @@ -1,2 +1,3 @@ #!/bin/bash /usr/bin/doveadm fts rescan -A > /dev/null 2>&1 +/usr/bin/doveadm fts optimize -A diff --git a/lib/howto_build_xapian.txt b/lib/howto_build_xapian.txt new file mode 100644 index 00000000..85a8dd92 --- /dev/null +++ b/lib/howto_build_xapian.txt @@ -0,0 +1,15 @@ +# Install required packages +apt-get build-dep dovecot-core +apt-get install dovecot-dev libxapian-dev git libxapian30 libicu-dev + +# Clone the project +git clone https://github.com/grosjo/fts-xapian +cd fts-xapian + +# Compile (don't install) +autoreconf -vi +./configure --with-dovecot=/usr/lib/dovecot +make + +# install library found under src/.libs +cp src/.libs/lib21_fts_xapian_plugin.so /usr/lib/dovecot/modules \ No newline at end of file diff --git a/setup/dovecot-fts-xapian.sh b/setup/dovecot-fts-xapian.sh new file mode 100644 index 00000000..937e6625 --- /dev/null +++ b/setup/dovecot-fts-xapian.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# IMAP search with xapian +# -------------------------------- +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes and is obsolete. Dovecot 2.1+ has support for using Lucene +# internally but this didn't make it into the Ubuntu packages. Solr uses too +# much memory. Same goes for elasticsearch. fts xapian might be a good match +# for mail-in-a-box. See https://github.com/grosjo/fts-xapian + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +echo "Installing fts-xapian..." + +apt_install libxapian30 + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects. +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="fts fts_xapian" + +# Install cronjobs to keep FTS up to date. +hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/ + +if [ ! -f /usr/lib/dovecot/decode2text.sh ]; then + cp -f /usr/share/doc/dovecot-core/examples/decode2text.sh /usr/lib/dovecot +fi + +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + plugin = fts fts_xapian + + fts = xapian + fts_xapian = partial=3 full=20 verbose=0 + + fts_autoindex = yes + fts_enforced = yes + + fts_autoindex_exclude = \Trash + fts_autoindex_exclude2 = \Junk + + fts_decoder = decode2text // To index attachements +} + +service indexer-worker { + vsz_limit = 2G // or above (or 0 if you have rather large memory usable on your server, which is preferred for performance) +} + +service decode2text { + executable = script /usr/lib/dovecot/decode2text.sh + user = dovecot + unix_listener decode2text { + mode = 0666 + } +} +EOF + +restart_service dovecot + +# Kickoff building the index + +# Per doveadm-fts manpage: Scan what mails exist in the full text search index +# and compare those to what actually exist in mailboxes. +# This removes mails from the index that have already been expunged and makes +# sure that the next doveadm index will index all the missing mails (if any). +doveadm fts rescan -A + +# Adds unindexed files to the fts database +# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing) +# * `-A`: All users +# * `'*'`: All folders +doveadm index -q -A '*' +