From f289439d1d9681696b55dea235d6e36aa9339f97 Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 17:44:50 -0400 Subject: [PATCH 1/8] Adapted Jonty's original solr.sh script to instead set up lucene full-text search in dovecot. --- setup/lucene.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 setup/lucene.sh diff --git a/setup/lucene.sh b/setup/lucene.sh new file mode 100755 index 00000000..78c7408d --- /dev/null +++ b/setup/lucene.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# IMAP search with lucene +# -------------------------------- +# +# Adapted from https://github.com/Jonty/mailinabox/blob/solr_support/setup/solr.sh +# +# By default dovecot uses its own Squat search index that has awful performance +# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but +# this didn't make it into the Ubuntu packages, so we maintain our own +# dovecot-lucene package in a ppa. + +source setup/functions.sh # load our functions +source /etc/mailinabox.conf # load global vars + +# Install packages and basic configuation +# --------------------------------------- + +# Add official ppa +hide_output add-apt-repository -y ppa:brock/mailinabox-brocktice + +# Install packages +apt_install dovecot-lucene + +# Update the dovecot plugin configuration +# +# Break-imap-search makes search work the way users expect, rather than the way +# the IMAP specification expects +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="$mail_plugins fts fts_lucene" + +cat > /etc/dovecot/conf.d/90-plugin.conf << EOF; +plugin { + fts = lucene + fts_lucene = whitespace_chars=@. +} +EOF + +# PERMISSIONS + +# Ensure configuration files are owned by dovecot and not world readable. +chown -R mail:dovecot /etc/dovecot +chmod -R o-rwx /etc/dovecot + +# Restart services to reload solr schema & dovecot plugins +restart_service dovecot From e4eba49c1be631a10ba98fda3ac7955521badeba Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 17:55:01 -0400 Subject: [PATCH 2/8] Added lucene.sh to start script. --- setup/start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/start.sh b/setup/start.sh index ca461656..d7cc20bf 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -139,6 +139,7 @@ source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh source setup/dkim.sh +source setup/lucene.sh source setup/spamassassin.sh source setup/web.sh source setup/webmail.sh From 6941ca2f63bfa679e8aad5deff8f0a553c2fb85c Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 18:07:02 -0400 Subject: [PATCH 3/8] Added apt-get update before installing dovecot-lucene --- setup/lucene.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup/lucene.sh b/setup/lucene.sh index 78c7408d..ee43ca86 100755 --- a/setup/lucene.sh +++ b/setup/lucene.sh @@ -18,6 +18,9 @@ source /etc/mailinabox.conf # load global vars # Add official ppa hide_output add-apt-repository -y ppa:brock/mailinabox-brocktice +# Update apt +hide_output apt-get update + # Install packages apt_install dovecot-lucene From 6a659fe10dedf6ee27e219e305fb2f56acdfe002 Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 18:47:36 -0400 Subject: [PATCH 4/8] Fixed patches for pull request. Resolved conflict between two patches. --- ppa/dovecot_lucene.diff | 79 +++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/ppa/dovecot_lucene.diff b/ppa/dovecot_lucene.diff index e3582952..f4a2ecab 100644 --- a/ppa/dovecot_lucene.diff +++ b/ppa/dovecot_lucene.diff @@ -1,6 +1,6 @@ --- a/debian/control +++ b/debian/control -@@ -1,210 +1,22 @@ +@@ -1,210 +1,23 @@ Source: dovecot Section: mail Priority: optional @@ -13,11 +13,14 @@ +Build-Depends: debhelper (>= 7.2.3~), dpkg-dev (>= 1.16.1), pkg-config, libssl-dev, libpam0g-dev, libldap2-dev, libpq-dev, libmysqlclient-dev, libsqlite3-dev, libsasl2-dev, zlib1g-dev, libkrb5-dev, drac-dev (>= 1.12-5), libbz2-dev, libdb-dev, libcurl4-gnutls-dev, libexpat-dev, libwrap0-dev, dh-systemd, po-debconf, lsb-release, libclucene-dev (>= 2.3), liblzma-dev, libexttextcat-dev, libstemmer-dev, hardening-wrapper, dh-autoreconf, autotools-dev Standards-Version: 3.9.4 Homepage: http://dovecot.org/ - Vcs-Git: git://git.debian.org/git/collab-maint/dovecot.git - Vcs-Browser: http://git.debian.org/?p=collab-maint/dovecot.git +-Vcs-Git: git://git.debian.org/git/collab-maint/dovecot.git +-Vcs-Browser: http://git.debian.org/?p=collab-maint/dovecot.git ++Vcs-Git: https://github.com/mail-in-a-box/mailinabox ++Vcs-Browser: https://github.com/mail-in-a-box/mailinabox -Package: dovecot-core --Architecture: any ++Package: dovecot-lucene + Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, libpam-runtime (>= 0.76-13.1), openssl, adduser, ucf (>= 2.0020), ssl-cert (>= 1.0-11ubuntu1), lsb-base (>= 3.2-12ubuntu3) -Suggests: ntp, dovecot-gssapi, dovecot-sieve, dovecot-pgsql, dovecot-mysql, dovecot-sqlite, dovecot-ldap, dovecot-imapd, dovecot-pop3d, dovecot-lmtpd, dovecot-managesieved, dovecot-solr, ufw -Recommends: ntpdate @@ -25,12 +28,14 @@ -Replaces: dovecot-common (<< 1:2.0.14-2~), mailavenger (<< 0.8.1-4) -Breaks: dovecot-common (<< 1:2.0.14-2~), mailavenger (<< 0.8.1-4) -Description: secure POP3/IMAP server - core files -- Dovecot is a mail server whose major goals are security and extreme -- reliability. It tries very hard to handle all error conditions and verify -- that all data is valid, making it nearly impossible to crash. It supports -- mbox/Maildir and its own dbox/mdbox formats, and should also be pretty -- fast, extensible, and portable. -- . ++Depends: ${shlibs:Depends}, ${misc:Depends}, dovecot-core (>= 1:2.2.9-1ubuntu2.1) ++Description: secure POP3/IMAP server - Lucene support + Dovecot is a mail server whose major goals are security and extreme + reliability. It tries very hard to handle all error conditions and verify + that all data is valid, making it nearly impossible to crash. It supports + mbox/Maildir and its own dbox/mdbox formats, and should also be pretty + fast, extensible, and portable. + . - This package contains the Dovecot main server and its command line utility. - -Package: dovecot-dev @@ -151,17 +156,15 @@ - This package provides LDAP support for Dovecot. - -Package: dovecot-gssapi -+Package: dovecot-lucene - Architecture: any - Depends: ${shlibs:Depends}, ${misc:Depends}, dovecot-core (= ${binary:Version}) +-Architecture: any +-Depends: ${shlibs:Depends}, ${misc:Depends}, dovecot-core (= ${binary:Version}) -Description: secure POP3/IMAP server - GSSAPI support -+Description: secure POP3/IMAP server - Lucene support - Dovecot is a mail server whose major goals are security and extreme - reliability. It tries very hard to handle all error conditions and verify - that all data is valid, making it nearly impossible to crash. It supports - mbox/Maildir and its own dbox/mdbox formats, and should also be pretty - fast, extensible, and portable. - . +- Dovecot is a mail server whose major goals are security and extreme +- reliability. It tries very hard to handle all error conditions and verify +- that all data is valid, making it nearly impossible to crash. It supports +- mbox/Maildir and its own dbox/mdbox formats, and should also be pretty +- fast, extensible, and portable. +- . - This package provides GSSAPI authentication support for Dovecot. - -Package: dovecot-sieve @@ -216,7 +219,9 @@ - This package contains configuration files for dovecot. - . - This package modifies postfix's configuration to integrate with dovecot -+ This package provides Lucene full text search support for Dovecot. ++ This package provides Lucene full text search support for Dovecot. It has been modified by Mail-in-a-Box ++ to supply a dovecot-lucene package compatible with the official ubuntu trusty dovecot-core. + diff --git a/debian/dovecot-lucene.links b/debian/dovecot-lucene.links new file mode 100644 index 0000000..6ffcbeb @@ -247,11 +252,9 @@ index 0000000..3d933a5 +++ b/debian/dovecot-lucene.triggers @@ -0,0 +1 @@ +activate register-dovecot-plugin -diff --git a/debian/rules b/debian/rules -index dcee2f6..9533a4a 100755 --- a/debian/rules +++ b/debian/rules -@@ -40,6 +40,7 @@ config-stamp: configure +@@ -40,6 +40,7 @@ --with-solr \ --with-ioloop=best \ --with-libwrap \ @@ -259,7 +262,7 @@ index dcee2f6..9533a4a 100755 --host=$(DEB_HOST_GNU_TYPE) \ --build=$(DEB_BUILD_GNU_TYPE) \ --prefix=/usr \ -@@ -95,6 +96,10 @@ install: build +@@ -95,6 +96,10 @@ dh_testroot dh_clean -k dh_installdirs @@ -270,7 +273,7 @@ index dcee2f6..9533a4a 100755 $(MAKE) install DESTDIR=$(CURDIR)/debian/dovecot-core $(MAKE) -C $(PIGEONHOLE_DIR) install DESTDIR=$(CURDIR)/debian/dovecot-core rm `find $(CURDIR)/debian -name '*.la'` -@@ -209,7 +214,7 @@ binary-arch: build install +@@ -209,7 +214,7 @@ dh_installdocs -a dh_installexamples -a dh_installpam -a @@ -279,7 +282,7 @@ index dcee2f6..9533a4a 100755 dh_systemd_enable dh_installinit -pdovecot-core --name=dovecot dh_systemd_start -@@ -220,10 +225,10 @@ binary-arch: build install +@@ -220,10 +225,10 @@ dh_lintian -a dh_installchangelogs -a ChangeLog dh_link -a @@ -292,3 +295,25 @@ index dcee2f6..9533a4a 100755 dh_makeshlibs -a -n dh_installdeb -a dh_shlibdeps -a +--- a/debian/changelog ++++ a/debian/changelog +@@ -1,3 +1,9 @@ ++dovecot (1:2.2.9-1ubuntu2.2~ppa0~trusty0) trusty; urgency=low ++ ++ * Changed to just build dovecot-lucene for Mail-in-a-box PPA ++ ++ -- Joshua Tauberer Sat, 14 May 2015 16:13:00 -0400 ++ + dovecot (1:2.2.9-1ubuntu2.1) trusty-security; urgency=medium + + * SECURITY UPDATE: denial of service via SSL connection exhaustion +--- a/debian/copyright 2014-03-07 07:26:37.000000000 -0500 ++++ b/debian/copyright 2015-05-23 18:17:42.668005535 -0400 +@@ -1,3 +1,7 @@ ++This package is a fork by Mail-in-a-box (https://mailinabox.email). Original ++copyright statement follows: ++---------------------------------------------------------------------------- ++ + This package was debianized by Jaldhar H. Vyas on + Tue, 3 Dec 2002 01:10:07 -0500. + From 005cc08b4038cb208d1daaf1190a814e603305c1 Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 18:53:48 -0400 Subject: [PATCH 5/8] Modified makefile to work with updated debian-lucene patch, use official locations. Fixed conflicting edits to ppa/Makefile due to cherry-pick. --- ppa/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ppa/Makefile b/ppa/Makefile index 23c29371..b744d543 100755 --- a/ppa/Makefile +++ b/ppa/Makefile @@ -41,10 +41,10 @@ build_dovecot_lucene: (cd /tmp/build/dovecot-2.2.9; dpkg-buildpackage -S -us -uc -nc) # Sign the packages. - #debsign /tmp/build/dovecot_2.2.9-1ubuntu2.1_amd64.changes + debsign /tmp/build/dovecot_2.2.9-1ubuntu2.2~ppa0~trusty0_source.changes # Upload it. - #dput ppa:mail-in-a-box/ppa /tmp/build/dovecot_2.2.9-1ubuntu2.1_amd64.changes + dput ppa:mail-in-a-box/ppa /tmp/build/dovecot_2.2.9-1ubuntu2.2~ppa0~trusty0_source.changes # TESTING BINARY PACKAGE # Install build dependencies and build dependencies we've added in our patch, From 32f563262045d052a60f8004b5f406f187894ddd Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Sat, 23 May 2015 18:57:14 -0400 Subject: [PATCH 6/8] Switch to official PPA --- setup/lucene.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/lucene.sh b/setup/lucene.sh index ee43ca86..55b2bf29 100755 --- a/setup/lucene.sh +++ b/setup/lucene.sh @@ -16,7 +16,7 @@ source /etc/mailinabox.conf # load global vars # --------------------------------------- # Add official ppa -hide_output add-apt-repository -y ppa:brock/mailinabox-brocktice +hide_output add-apt-repository -y ppa:mail-in-a-box/ppa # Update apt hide_output apt-get update From f02e0a3ccb0e10b8859cc72f5897723606e781be Mon Sep 17 00:00:00 2001 From: Brock Tice Date: Mon, 25 May 2015 08:50:41 -0600 Subject: [PATCH 7/8] Fixed comment that still referenced solr. --- setup/lucene.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/lucene.sh b/setup/lucene.sh index 55b2bf29..0da7fe02 100755 --- a/setup/lucene.sh +++ b/setup/lucene.sh @@ -44,5 +44,5 @@ EOF chown -R mail:dovecot /etc/dovecot chmod -R o-rwx /etc/dovecot -# Restart services to reload solr schema & dovecot plugins +# Restart services to reload dovecot plugins restart_service dovecot From b23ba6f75ec4d6e7fbd1c436744d42810ab670b6 Mon Sep 17 00:00:00 2001 From: Joshua Tauberer Date: Mon, 1 Jun 2015 21:04:20 -0400 Subject: [PATCH 8/8] simplify build/setup of dovecot-lucene package --- ppa/Makefile | 11 +++++++--- ppa/README.md | 6 ++++-- ppa/dovecot_lucene.diff | 4 ++-- setup/lucene.sh | 48 ----------------------------------------- setup/mail-dovecot.sh | 18 ++++++++++++++-- setup/start.sh | 1 - setup/system.sh | 18 +++++++++++++++- 7 files changed, 47 insertions(+), 59 deletions(-) delete mode 100755 setup/lucene.sh diff --git a/ppa/Makefile b/ppa/Makefile index b744d543..f12e9283 100755 --- a/ppa/Makefile +++ b/ppa/Makefile @@ -1,13 +1,15 @@ +DOVECOT_VERSION=2.2.9-1ubuntu2.1+miab1 + all: clean build_postgrey build_dovecot_lucene clean: # Clean. rm -rf /tmp/build +build_postgrey: # Prepare to build source packages. mkdir -p /tmp/build -build_postgrey: # Download our fork of the Debian postgrey package. git clone https://github.com/mail-in-a-box/postgrey /tmp/build/postgrey @@ -31,6 +33,9 @@ build_postgrey: #(cd /tmp/build/postgrey; dpkg-buildpackage -us -uc -nc) build_dovecot_lucene: + # Prepare to build source packages. + mkdir -p /tmp/build + # Get the upstream source. (cd /tmp/build; apt-get source dovecot) @@ -41,10 +46,10 @@ build_dovecot_lucene: (cd /tmp/build/dovecot-2.2.9; dpkg-buildpackage -S -us -uc -nc) # Sign the packages. - debsign /tmp/build/dovecot_2.2.9-1ubuntu2.2~ppa0~trusty0_source.changes + debsign /tmp/build/dovecot_$(DOVECOT_VERSION)_source.changes # Upload it. - dput ppa:mail-in-a-box/ppa /tmp/build/dovecot_2.2.9-1ubuntu2.2~ppa0~trusty0_source.changes + dput ppa:mail-in-a-box/ppa /tmp/build/dovecot_$(DOVECOT_VERSION)_source.changes # TESTING BINARY PACKAGE # Install build dependencies and build dependencies we've added in our patch, diff --git a/ppa/README.md b/ppa/README.md index 91c2f712..c4e58699 100644 --- a/ppa/README.md +++ b/ppa/README.md @@ -8,6 +8,8 @@ Packages * [postgrey](https://github.com/mail-in-a-box/postgrey), with a modification to whitelist senders that are whitelisted by [dnswl.org](https://www.dnswl.org/) (i.e. don't greylist mail from them). +* dovecot-lucene, [dovecot's lucene full text search plugin](http://wiki2.dovecot.org/Plugins/FTS/Lucene), which isn't built by Ubuntu's dovecot package maintainer unfortunately. + Building -------- @@ -30,9 +32,9 @@ To build: # Build & upload to launchpad. vagrant ssh -- "cd /vagrant && make" -To use on a Mail-in-a-Box box, add the PPA and then upgrade packages: +Mail-in-a-Box adds our PPA during setup, but if you need to do that yourself for testing: apt-add-repository ppa:mail-in-a-box/ppa apt-get update - apt-get upgrade + apt-get install postgrey dovecot-lucene diff --git a/ppa/dovecot_lucene.diff b/ppa/dovecot_lucene.diff index f4a2ecab..4d258351 100644 --- a/ppa/dovecot_lucene.diff +++ b/ppa/dovecot_lucene.diff @@ -267,7 +267,7 @@ index 0000000..3d933a5 dh_clean -k dh_installdirs + mkdir -p $(CURDIR)/debian/dovecot-lucene/usr/lib/dovecot/modules -+ mv $(CURDIR)/src/plugins/fts-lucene/.libs/* $(CURDIR)/debian/dovecot-lucene/usr/lib/dovecot/modules/ ++ mv $(CURDIR)/src/plugins/fts-lucene/.libs/* $(CURDIR)/debian/dovecot-lucene/usr/lib/dovecot/modules/ + +rest_disabled_by_miab: $(MAKE) install DESTDIR=$(CURDIR)/debian/dovecot-core @@ -298,7 +298,7 @@ index 0000000..3d933a5 --- a/debian/changelog +++ a/debian/changelog @@ -1,3 +1,9 @@ -+dovecot (1:2.2.9-1ubuntu2.2~ppa0~trusty0) trusty; urgency=low ++dovecot (1:2.2.9-1ubuntu2.1+miab1) trusty; urgency=low + + * Changed to just build dovecot-lucene for Mail-in-a-box PPA + diff --git a/setup/lucene.sh b/setup/lucene.sh deleted file mode 100755 index 0da7fe02..00000000 --- a/setup/lucene.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# IMAP search with lucene -# -------------------------------- -# -# Adapted from https://github.com/Jonty/mailinabox/blob/solr_support/setup/solr.sh -# -# By default dovecot uses its own Squat search index that has awful performance -# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but -# this didn't make it into the Ubuntu packages, so we maintain our own -# dovecot-lucene package in a ppa. - -source setup/functions.sh # load our functions -source /etc/mailinabox.conf # load global vars - -# Install packages and basic configuation -# --------------------------------------- - -# Add official ppa -hide_output add-apt-repository -y ppa:mail-in-a-box/ppa - -# Update apt -hide_output apt-get update - -# Install packages -apt_install dovecot-lucene - -# Update the dovecot plugin configuration -# -# Break-imap-search makes search work the way users expect, rather than the way -# the IMAP specification expects -tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ - mail_plugins="$mail_plugins fts fts_lucene" - -cat > /etc/dovecot/conf.d/90-plugin.conf << EOF; -plugin { - fts = lucene - fts_lucene = whitespace_chars=@. -} -EOF - -# PERMISSIONS - -# Ensure configuration files are owned by dovecot and not world readable. -chown -R mail:dovecot /etc/dovecot -chmod -R o-rwx /etc/dovecot - -# Restart services to reload dovecot plugins -restart_service dovecot diff --git a/setup/mail-dovecot.sh b/setup/mail-dovecot.sh index ea7a0d1d..4b681765 100755 --- a/setup/mail-dovecot.sh +++ b/setup/mail-dovecot.sh @@ -18,11 +18,14 @@ source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars -# Install packages... + +# Install packages for dovecot. These are all core dovecot plugins, +# but dovecot-lucene is packaged by *us* in the Mail-in-a-Box PPA, +# not by Ubuntu. apt_install \ dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sqlite sqlite3 \ - dovecot-sieve dovecot-managesieved + dovecot-sieve dovecot-managesieved dovecot-lucene # The `dovecot-imapd`, `dovecot-pop3d`, and `dovecot-lmtpd` packages automatically # enable IMAP, POP and LMTP protocols. @@ -92,6 +95,17 @@ tools/editconf.py /etc/dovecot/conf.d/20-imap.conf \ tools/editconf.py /etc/dovecot/conf.d/20-pop3.conf \ pop3_uidl_format="%08Xu%08Xv" +# Full Text Search - Enable full text search of mail using dovecot's lucene plugin, +# which *we* package and distribute (dovecot-lucene package). +tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \ + mail_plugins="\$mail_plugins fts fts_lucene" +cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; +plugin { + fts = lucene + fts_lucene = whitespace_chars=@. +} +EOF + # ### LDA (LMTP) # Enable Dovecot's LDA service with the LMTP protocol. It will listen diff --git a/setup/start.sh b/setup/start.sh index d7cc20bf..ca461656 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -139,7 +139,6 @@ source setup/mail-postfix.sh source setup/mail-dovecot.sh source setup/mail-users.sh source setup/dkim.sh -source setup/lucene.sh source setup/spamassassin.sh source setup/web.sh source setup/webmail.sh diff --git a/setup/system.sh b/setup/system.sh index 259c8768..55612b9d 100755 --- a/setup/system.sh +++ b/setup/system.sh @@ -3,7 +3,21 @@ source setup/functions.sh # load our functions # Basic System Configuration # ------------------------- -# ### Install Packages +# ### Add Mail-in-a-Box's PPA. + +# We've built several .deb packages on our own that we want to include. +# One is a replacement for Ubuntu's stock postgrey package that makes +# some enhancements. The other is dovecot-lucene, a Lucene-based full +# text search plugin for (and by) dovecot, which is not available in +# Ubuntu currently. +# +# Add that to the system's list of repositories: + +hide_output add-apt-repository -y ppa:mail-in-a-box/ppa + +# The apt-get update in the next step will pull in the PPA's index. + +# ### Update Packages # Update system packages to make sure we have the latest upstream versions of things from Ubuntu. @@ -11,6 +25,8 @@ echo Updating system packages... hide_output apt-get update apt_get_quiet upgrade +# ### Install System Packages + # Install basic utilities. # # * haveged: Provides extra entropy to /dev/random so it doesn't stall