mirror of
https://github.com/mail-in-a-box/mailinabox.git
synced 2024-12-24 07:37:04 +00:00
Merge branch 'develop-xapian-fts'
This commit is contained in:
KiekerJan
commit
764a81d335
@ -14,7 +14,7 @@ Functionality changes and additions
|
|||||||
enable postfix filters, lengthen bantime and findtime
|
enable postfix filters, lengthen bantime and findtime
|
||||||
* Add fail2ban jails for both above mentioned geoipblocking filters
|
* Add fail2ban jails for both above mentioned geoipblocking filters
|
||||||
* Add fail2ban filters for web scanners and badbots
|
* Add fail2ban filters for web scanners and badbots
|
||||||
* Add solr full text searching to dovecot
|
* Add xapian full text searching to dovecot
|
||||||
* Add rkhunter and chkrootkit
|
* Add rkhunter and chkrootkit
|
||||||
* Configure domain names for which only www will be hosted. Edit /etc/miabwwwdomains.conf to configure.
|
* Configure domain names for which only www will be hosted. Edit /etc/miabwwwdomains.conf to configure.
|
||||||
* Add some munin plugins
|
* Add some munin plugins
|
||||||
|
|||||||
@ -1,2 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
/usr/bin/doveadm fts rescan -A > /dev/null 2>&1
|
/usr/bin/doveadm fts rescan -A > /dev/null 2>&1
|
||||||
|
/usr/bin/doveadm fts optimize -A > /dev/null 2>&1
|
||||||
|
|||||||
@ -1,2 +0,0 @@
|
|||||||
1 */1 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?commit=true >/dev/null 2>&1
|
|
||||||
30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?optimize=true >/dev/null 2>&1
|
|
||||||
@ -1,289 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8" ?>
|
|
||||||
|
|
||||||
<!-- This is the default config with stuff non-essential to Dovecot removed. -->
|
|
||||||
|
|
||||||
<config>
|
|
||||||
<!-- Controls what version of Lucene various components of Solr
|
|
||||||
adhere to. Generally, you want to use the latest version to
|
|
||||||
get all bug fixes and improvements. It is highly recommended
|
|
||||||
that you fully re-index after changing this setting as it can
|
|
||||||
affect both how text is indexed and queried.
|
|
||||||
-->
|
|
||||||
<luceneMatchVersion>7.7.0</luceneMatchVersion>
|
|
||||||
|
|
||||||
<!-- A 'dir' option by itself adds any files found in the directory
|
|
||||||
to the classpath, this is useful for including all jars in a
|
|
||||||
directory.
|
|
||||||
|
|
||||||
When a 'regex' is specified in addition to a 'dir', only the
|
|
||||||
files in that directory which completely match the regex
|
|
||||||
(anchored on both ends) will be included.
|
|
||||||
|
|
||||||
If a 'dir' option (with or without a regex) is used and nothing
|
|
||||||
is found that matches, a warning will be logged.
|
|
||||||
|
|
||||||
The examples below can be used to load some solr-contribs along
|
|
||||||
with their external dependencies.
|
|
||||||
-->
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
|
|
||||||
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
|
|
||||||
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
|
|
||||||
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
|
|
||||||
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
|
|
||||||
|
|
||||||
<!-- Data Directory
|
|
||||||
|
|
||||||
Used to specify an alternate directory to hold all index data
|
|
||||||
other than the default ./data under the Solr home. If
|
|
||||||
replication is in use, this should match the replication
|
|
||||||
configuration.
|
|
||||||
-->
|
|
||||||
<dataDir>${solr.data.dir:}</dataDir>
|
|
||||||
|
|
||||||
<!-- The default high-performance update handler -->
|
|
||||||
<updateHandler class="solr.DirectUpdateHandler2">
|
|
||||||
|
|
||||||
<!-- Enables a transaction log, used for real-time get, durability, and
|
|
||||||
and solr cloud replica recovery. The log can grow as big as
|
|
||||||
uncommitted changes to the index, so use of a hard autoCommit
|
|
||||||
is recommended (see below).
|
|
||||||
"dir" - the target directory for transaction logs, defaults to the
|
|
||||||
solr data directory.
|
|
||||||
"numVersionBuckets" - sets the number of buckets used to keep
|
|
||||||
track of max version values when checking for re-ordered
|
|
||||||
updates; increase this value to reduce the cost of
|
|
||||||
synchronizing access to version buckets during high-volume
|
|
||||||
indexing, this requires 8 bytes (long) * numVersionBuckets
|
|
||||||
of heap space per Solr core.
|
|
||||||
-->
|
|
||||||
<updateLog>
|
|
||||||
<str name="dir">${solr.ulog.dir:}</str>
|
|
||||||
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
|
|
||||||
</updateLog>
|
|
||||||
|
|
||||||
<!-- AutoCommit
|
|
||||||
|
|
||||||
Perform a hard commit automatically under certain conditions.
|
|
||||||
Instead of enabling autoCommit, consider using "commitWithin"
|
|
||||||
when adding documents.
|
|
||||||
|
|
||||||
http://wiki.apache.org/solr/UpdateXmlMessages
|
|
||||||
|
|
||||||
maxDocs - Maximum number of documents to add since the last
|
|
||||||
commit before automatically triggering a new commit.
|
|
||||||
|
|
||||||
maxTime - Maximum amount of time in ms that is allowed to pass
|
|
||||||
since a document was added before automatically
|
|
||||||
triggering a new commit.
|
|
||||||
openSearcher - if false, the commit causes recent index changes
|
|
||||||
to be flushed to stable storage, but does not cause a new
|
|
||||||
searcher to be opened to make those changes visible.
|
|
||||||
|
|
||||||
If the updateLog is enabled, then it's highly recommended to
|
|
||||||
have some sort of hard autoCommit to limit the log size.
|
|
||||||
-->
|
|
||||||
<autoCommit>
|
|
||||||
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
|
|
||||||
<openSearcher>false</openSearcher>
|
|
||||||
</autoCommit>
|
|
||||||
|
|
||||||
<!-- softAutoCommit is like autoCommit except it causes a
|
|
||||||
'soft' commit which only ensures that changes are visible
|
|
||||||
but does not ensure that data is synced to disk. This is
|
|
||||||
faster and more near-realtime friendly than a hard commit.
|
|
||||||
-->
|
|
||||||
<autoSoftCommit>
|
|
||||||
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
|
|
||||||
</autoSoftCommit>
|
|
||||||
|
|
||||||
<!-- Update Related Event Listeners
|
|
||||||
|
|
||||||
Various IndexWriter related events can trigger Listeners to
|
|
||||||
take actions.
|
|
||||||
|
|
||||||
postCommit - fired after every commit or optimize command
|
|
||||||
postOptimize - fired after every optimize command
|
|
||||||
-->
|
|
||||||
|
|
||||||
</updateHandler>
|
|
||||||
|
|
||||||
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
Query section - these settings control query time things like caches
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
|
|
||||||
<query>
|
|
||||||
<!-- Solr Internal Query Caches
|
|
||||||
|
|
||||||
There are two implementations of cache available for Solr,
|
|
||||||
LRUCache, based on a synchronized LinkedHashMap, and
|
|
||||||
FastLRUCache, based on a ConcurrentHashMap.
|
|
||||||
|
|
||||||
FastLRUCache has faster gets and slower puts in single
|
|
||||||
threaded operation and thus is generally faster than LRUCache
|
|
||||||
when the hit ratio of the cache is high (> 75%), and may be
|
|
||||||
faster under other scenarios on multi-cpu systems.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<!-- Filter Cache
|
|
||||||
|
|
||||||
Cache used by SolrIndexSearcher for filters (DocSets),
|
|
||||||
unordered sets of *all* documents that match a query. When a
|
|
||||||
new searcher is opened, its caches may be prepopulated or
|
|
||||||
"autowarmed" using data from caches in the old searcher.
|
|
||||||
autowarmCount is the number of items to prepopulate. For
|
|
||||||
LRUCache, the autowarmed items will be the most recently
|
|
||||||
accessed items.
|
|
||||||
|
|
||||||
Parameters:
|
|
||||||
class - the SolrCache implementation LRUCache or
|
|
||||||
(LRUCache or FastLRUCache)
|
|
||||||
size - the maximum number of entries in the cache
|
|
||||||
initialSize - the initial capacity (number of entries) of
|
|
||||||
the cache. (see java.util.HashMap)
|
|
||||||
autowarmCount - the number of entries to prepopulate from
|
|
||||||
and old cache.
|
|
||||||
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
|
|
||||||
to occupy. Note that when this option is specified, the size
|
|
||||||
and initialSize parameters are ignored.
|
|
||||||
-->
|
|
||||||
<filterCache class="solr.FastLRUCache"
|
|
||||||
size="512"
|
|
||||||
initialSize="512"
|
|
||||||
autowarmCount="0"/>
|
|
||||||
|
|
||||||
<!-- Query Result Cache
|
|
||||||
|
|
||||||
Caches results of searches - ordered lists of document ids
|
|
||||||
(DocList) based on a query, a sort, and the range of documents requested.
|
|
||||||
Additional supported parameter by LRUCache:
|
|
||||||
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
|
|
||||||
to occupy
|
|
||||||
-->
|
|
||||||
<queryResultCache class="solr.LRUCache"
|
|
||||||
size="512"
|
|
||||||
initialSize="512"
|
|
||||||
autowarmCount="0"/>
|
|
||||||
|
|
||||||
<!-- Document Cache
|
|
||||||
|
|
||||||
Caches Lucene Document objects (the stored fields for each
|
|
||||||
document). Since Lucene internal document ids are transient,
|
|
||||||
this cache will not be autowarmed.
|
|
||||||
-->
|
|
||||||
<documentCache class="solr.LRUCache"
|
|
||||||
size="512"
|
|
||||||
initialSize="512"
|
|
||||||
autowarmCount="0"/>
|
|
||||||
|
|
||||||
<!-- custom cache currently used by block join -->
|
|
||||||
<cache name="perSegFilter"
|
|
||||||
class="solr.search.LRUCache"
|
|
||||||
size="10"
|
|
||||||
initialSize="0"
|
|
||||||
autowarmCount="10"
|
|
||||||
regenerator="solr.NoOpRegenerator" />
|
|
||||||
|
|
||||||
<!-- Lazy Field Loading
|
|
||||||
|
|
||||||
If true, stored fields that are not requested will be loaded
|
|
||||||
lazily. This can result in a significant speed improvement
|
|
||||||
if the usual case is to not load all stored fields,
|
|
||||||
especially if the skipped fields are large compressed text
|
|
||||||
fields.
|
|
||||||
-->
|
|
||||||
<enableLazyFieldLoading>true</enableLazyFieldLoading>
|
|
||||||
|
|
||||||
<!-- Result Window Size
|
|
||||||
|
|
||||||
An optimization for use with the queryResultCache. When a search
|
|
||||||
is requested, a superset of the requested number of document ids
|
|
||||||
are collected. For example, if a search for a particular query
|
|
||||||
requests matching documents 10 through 19, and queryWindowSize is 50,
|
|
||||||
then documents 0 through 49 will be collected and cached. Any further
|
|
||||||
requests in that range can be satisfied via the cache.
|
|
||||||
-->
|
|
||||||
<queryResultWindowSize>20</queryResultWindowSize>
|
|
||||||
|
|
||||||
<!-- Maximum number of documents to cache for any entry in the
|
|
||||||
queryResultCache.
|
|
||||||
-->
|
|
||||||
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
|
|
||||||
|
|
||||||
<!-- Use Cold Searcher
|
|
||||||
|
|
||||||
If a search request comes in and there is no current
|
|
||||||
registered searcher, then immediately register the still
|
|
||||||
warming searcher and use it. If "false" then all requests
|
|
||||||
will block until the first searcher is done warming.
|
|
||||||
-->
|
|
||||||
<useColdSearcher>false</useColdSearcher>
|
|
||||||
|
|
||||||
</query>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Request Dispatcher
|
|
||||||
|
|
||||||
This section contains instructions for how the SolrDispatchFilter
|
|
||||||
should behave when processing requests for this SolrCore.
|
|
||||||
|
|
||||||
-->
|
|
||||||
<requestDispatcher>
|
|
||||||
<httpCaching never304="true" />
|
|
||||||
</requestDispatcher>
|
|
||||||
|
|
||||||
<!-- Request Handlers
|
|
||||||
|
|
||||||
http://wiki.apache.org/solr/SolrRequestHandler
|
|
||||||
|
|
||||||
Incoming queries will be dispatched to a specific handler by name
|
|
||||||
based on the path specified in the request.
|
|
||||||
|
|
||||||
If a Request Handler is declared with startup="lazy", then it will
|
|
||||||
not be initialized until the first request that uses it.
|
|
||||||
|
|
||||||
-->
|
|
||||||
<!-- SearchHandler
|
|
||||||
|
|
||||||
http://wiki.apache.org/solr/SearchHandler
|
|
||||||
|
|
||||||
For processing Search Queries, the primary Request Handler
|
|
||||||
provided with Solr is "SearchHandler" It delegates to a sequent
|
|
||||||
of SearchComponents (see below) and supports distributed
|
|
||||||
queries across multiple shards
|
|
||||||
-->
|
|
||||||
<requestHandler name="/select" class="solr.SearchHandler">
|
|
||||||
<!-- default values for query parameters can be specified, these
|
|
||||||
will be overridden by parameters in the request
|
|
||||||
-->
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="echoParams">explicit</str>
|
|
||||||
<int name="rows">10</int>
|
|
||||||
</lst>
|
|
||||||
</requestHandler>
|
|
||||||
|
|
||||||
<initParams path="/update/**,/select">
|
|
||||||
<lst name="defaults">
|
|
||||||
<str name="df">_text_</str>
|
|
||||||
</lst>
|
|
||||||
</initParams>
|
|
||||||
|
|
||||||
<!-- Response Writers
|
|
||||||
|
|
||||||
http://wiki.apache.org/solr/QueryResponseWriter
|
|
||||||
|
|
||||||
Request responses will be written using the writer specified by
|
|
||||||
the 'wt' request parameter matching the name of a registered
|
|
||||||
writer.
|
|
||||||
|
|
||||||
The "default" writer is the default and will be used if 'wt' is
|
|
||||||
not specified in the request.
|
|
||||||
-->
|
|
||||||
<queryResponseWriter name="xml"
|
|
||||||
default="true"
|
|
||||||
class="solr.XMLResponseWriter" />
|
|
||||||
</config>
|
|
||||||
@ -1,22 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="ISO-8859-1"?>
|
|
||||||
<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">
|
|
||||||
|
|
||||||
<!-- Context configuration file for the Solr web application in Jetty -->
|
|
||||||
|
|
||||||
<Configure class="org.eclipse.jetty.webapp.WebAppContext">
|
|
||||||
<Set name="contextPath">/solr</Set>
|
|
||||||
<Set name="war">/usr/share/solr/web</Set>
|
|
||||||
|
|
||||||
<!-- Set the solr.solr.home system property -->
|
|
||||||
<Call name="setProperty" class="java.lang.System">
|
|
||||||
<Arg type="String">solr.solr.home</Arg>
|
|
||||||
<Arg type="String">/usr/share/solr</Arg>
|
|
||||||
</Call>
|
|
||||||
|
|
||||||
<!-- Enable symlinks -->
|
|
||||||
<!-- <Call name="addAliasCheck">
|
|
||||||
<Arg>
|
|
||||||
<New class="org.eclipse.jetty.server.handler.ContextHandler$ApproveSameSuffixAliases"/>
|
|
||||||
</Arg>
|
|
||||||
</Call>-->
|
|
||||||
</Configure>
|
|
||||||
@ -1,48 +0,0 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
|
|
||||||
<schema name="dovecot" version="2.0">
|
|
||||||
<fieldType name="string" class="solr.StrField" omitNorms="true" sortMissingLast="true"/>
|
|
||||||
<fieldType name="long" class="solr.LongPointField" positionIncrementGap="0"/>
|
|
||||||
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
|
|
||||||
|
|
||||||
<fieldType name="text" class="solr.TextField" autoGeneratePhraseQueries="true" positionIncrementGap="100">
|
|
||||||
<analyzer type="index">
|
|
||||||
<tokenizer class="solr.StandardTokenizerFactory"/>
|
|
||||||
<filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
|
|
||||||
<filter class="solr.FlattenGraphFilterFactory"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
|
|
||||||
<filter class="solr.PorterStemFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
<analyzer type="query">
|
|
||||||
<tokenizer class="solr.StandardTokenizerFactory"/>
|
|
||||||
<filter class="solr.SynonymGraphFilterFactory" expand="true" ignoreCase="true" synonyms="synonyms.txt"/>
|
|
||||||
<filter class="solr.FlattenGraphFilterFactory"/>
|
|
||||||
<filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
|
|
||||||
<filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
|
|
||||||
<filter class="solr.LowerCaseFilterFactory"/>
|
|
||||||
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
|
|
||||||
<filter class="solr.PorterStemFilterFactory"/>
|
|
||||||
</analyzer>
|
|
||||||
</fieldType>
|
|
||||||
|
|
||||||
<field name="id" type="string" indexed="true" required="true" stored="true"/>
|
|
||||||
<field name="uid" type="long" indexed="true" required="true" stored="true"/>
|
|
||||||
<field name="box" type="string" indexed="true" required="true" stored="true"/>
|
|
||||||
<field name="user" type="string" indexed="true" required="true" stored="true"/>
|
|
||||||
|
|
||||||
<field name="hdr" type="text" indexed="true" stored="false"/>
|
|
||||||
<field name="body" type="text" indexed="true" stored="false"/>
|
|
||||||
|
|
||||||
<field name="from" type="text" indexed="true" stored="false"/>
|
|
||||||
<field name="to" type="text" indexed="true" stored="false"/>
|
|
||||||
<field name="cc" type="text" indexed="true" stored="false"/>
|
|
||||||
<field name="bcc" type="text" indexed="true" stored="false"/>
|
|
||||||
<field name="subject" type="text" indexed="true" stored="false"/>
|
|
||||||
|
|
||||||
<!-- Used by Solr internally: -->
|
|
||||||
<field name="_version_" type="long" indexed="true" stored="true"/>
|
|
||||||
|
|
||||||
<uniqueKey>id</uniqueKey>
|
|
||||||
</schema>
|
|
||||||
@ -1,20 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=Apache SOLR
|
|
||||||
After=network.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=forking
|
|
||||||
User=solr
|
|
||||||
Environment=SOLR_INCLUDE=/etc/default/solr.in.sh
|
|
||||||
ExecStart=/usr/local/lib/solr/bin/solr start
|
|
||||||
ExecStop=/usr/local/lib/solr/bin/solr stop
|
|
||||||
Restart=on-failure
|
|
||||||
#ReadWritePaths=/var/lib/solr/
|
|
||||||
#ReadWritePaths=/var/lib/solr/data/
|
|
||||||
LimitNOFILE=65000
|
|
||||||
LimitNPROC=65000
|
|
||||||
TimeoutSec=180s
|
|
||||||
PrivateTmp=true
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
15
lib/howto_build_xapian.txt
Normal file
15
lib/howto_build_xapian.txt
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
# Install required packages
|
||||||
|
apt-get build-dep dovecot-core
|
||||||
|
apt-get install dovecot-dev libxapian-dev git libxapian30 libicu-dev
|
||||||
|
|
||||||
|
# Clone the project
|
||||||
|
git clone https://github.com/grosjo/fts-xapian
|
||||||
|
cd fts-xapian
|
||||||
|
|
||||||
|
# Compile (don't install)
|
||||||
|
autoreconf -vi
|
||||||
|
./configure --with-dovecot=/usr/lib/dovecot
|
||||||
|
make
|
||||||
|
|
||||||
|
# install library found under src/.libs
|
||||||
|
cp src/.libs/lib21_fts_xapian_plugin.so /usr/lib/dovecot/modules
|
||||||
BIN
lib/lib21_fts_xapian_plugin.so
Executable file
BIN
lib/lib21_fts_xapian_plugin.so
Executable file
Binary file not shown.
@ -40,7 +40,6 @@ def get_services():
|
|||||||
{ "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, },
|
{ "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, },
|
||||||
{ "name": "HTTP Web (nginx)", "port": 80, "public": True, },
|
{ "name": "HTTP Web (nginx)", "port": 80, "public": True, },
|
||||||
{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
|
{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
|
||||||
{ "name": "Solr Full Text Search", "port": 8983, "public": False, },
|
|
||||||
]
|
]
|
||||||
|
|
||||||
def run_checks(rounded_values, env, output, pool):
|
def run_checks(rounded_values, env, output, pool):
|
||||||
|
|||||||
86
setup/dovecot-fts-xapian.sh
Normal file
86
setup/dovecot-fts-xapian.sh
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# IMAP search with xapian
|
||||||
|
# --------------------------------
|
||||||
|
#
|
||||||
|
# By default dovecot uses its own Squat search index that has awful performance
|
||||||
|
# on large mailboxes and is obsolete. Dovecot 2.1+ has support for using Lucene
|
||||||
|
# internally but this didn't make it into the Ubuntu packages. Solr uses too
|
||||||
|
# much memory. Same goes for elasticsearch. fts xapian might be a good match
|
||||||
|
# for mail-in-a-box. See https://github.com/grosjo/fts-xapian
|
||||||
|
|
||||||
|
source setup/functions.sh # load our functions
|
||||||
|
source /etc/mailinabox.conf # load global vars
|
||||||
|
|
||||||
|
# Install packages and basic configuation
|
||||||
|
# ---------------------------------------
|
||||||
|
|
||||||
|
echo "Installing fts-xapian..."
|
||||||
|
|
||||||
|
apt_install libxapian30
|
||||||
|
|
||||||
|
# Update the dovecot plugin configuration
|
||||||
|
#
|
||||||
|
# Break-imap-search makes search work the way users expect, rather than the way
|
||||||
|
# the IMAP specification expects.
|
||||||
|
tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
|
||||||
|
mail_plugins="fts fts_xapian" \
|
||||||
|
mail_home="/home/user-data/mail/homes/%d/%n"
|
||||||
|
|
||||||
|
# Install cronjobs to keep FTS up to date.
|
||||||
|
hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/
|
||||||
|
|
||||||
|
# Install files
|
||||||
|
if [ ! -f /usr/lib/dovecot/decode2text.sh ]; then
|
||||||
|
cp -f /usr/share/doc/dovecot-core/examples/decode2text.sh /usr/lib/dovecot
|
||||||
|
fi
|
||||||
|
|
||||||
|
cp -f lib/lib21_fts_xapian_plugin.so /usr/lib/dovecot/modules/
|
||||||
|
|
||||||
|
# Create configuration file
|
||||||
|
cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
|
||||||
|
plugin {
|
||||||
|
plugin = fts fts_xapian
|
||||||
|
|
||||||
|
fts = xapian
|
||||||
|
fts_xapian = partial=3 full=20 verbose=0
|
||||||
|
|
||||||
|
fts_autoindex = yes
|
||||||
|
fts_enforced = yes
|
||||||
|
|
||||||
|
fts_autoindex_exclude = \Trash
|
||||||
|
fts_autoindex_exclude2 = \Junk
|
||||||
|
fts_autoindex_exclude3 = \Spam
|
||||||
|
|
||||||
|
fts_decoder = decode2text
|
||||||
|
}
|
||||||
|
|
||||||
|
service indexer-worker {
|
||||||
|
vsz_limit = 2G
|
||||||
|
}
|
||||||
|
|
||||||
|
service decode2text {
|
||||||
|
executable = script /usr/lib/dovecot/decode2text.sh
|
||||||
|
user = dovecot
|
||||||
|
unix_listener decode2text {
|
||||||
|
mode = 0666
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
restart_service dovecot
|
||||||
|
|
||||||
|
# Kickoff building the index
|
||||||
|
|
||||||
|
# Per doveadm-fts manpage: Scan what mails exist in the full text search index
|
||||||
|
# and compare those to what actually exist in mailboxes.
|
||||||
|
# This removes mails from the index that have already been expunged and makes
|
||||||
|
# sure that the next doveadm index will index all the missing mails (if any).
|
||||||
|
doveadm fts rescan -A
|
||||||
|
|
||||||
|
# Adds unindexed files to the fts database
|
||||||
|
# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing)
|
||||||
|
# * `-A`: All users
|
||||||
|
# * `'*'`: All folders
|
||||||
|
doveadm index -q -A '*'
|
||||||
|
|
||||||
166
setup/solr.sh
166
setup/solr.sh
@ -1,166 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#
|
|
||||||
# IMAP search with lucene via solr
|
|
||||||
# --------------------------------
|
|
||||||
#
|
|
||||||
# By default dovecot uses its own Squat search index that has awful performance
|
|
||||||
# on large mailboxes. Dovecot 2.1+ has support for using Lucene internally but
|
|
||||||
# this didn't make it into the Ubuntu packages, so we use Solr instead to run
|
|
||||||
# Lucene for us.
|
|
||||||
#
|
|
||||||
# Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its
|
|
||||||
# HTTP interface, searching indexed mail and returning results back to dovecot.
|
|
||||||
#
|
|
||||||
# Based on https://forum.iredmail.org/topic17251-dovecot-fts-full-text-search-using-apache-solr-on-ubuntu-1804-lts.html
|
|
||||||
# https://doc.dovecot.org/configuration_manual/fts/solr/ and https://solr.apache.org/guide/8_8/installing-solr.html
|
|
||||||
#
|
|
||||||
# solr-jetty package is removed from Ubuntu 21.04 onward. This installation
|
|
||||||
# therefore depends on manual installation of solr instead of an ubuntu package
|
|
||||||
|
|
||||||
source setup/functions.sh # load our functions
|
|
||||||
source /etc/mailinabox.conf # load global vars
|
|
||||||
|
|
||||||
# Install packages and basic configuation
|
|
||||||
# ---------------------------------------
|
|
||||||
|
|
||||||
echo "Installing Solr..."
|
|
||||||
|
|
||||||
apt_install dovecot-solr default-jre-headless
|
|
||||||
|
|
||||||
VERSION=8.8.2
|
|
||||||
HASH=7c3e2ed31a4412e7dac48d68c3abd52f75684577
|
|
||||||
|
|
||||||
needs_update=0
|
|
||||||
|
|
||||||
if [ ! -f /usr/local/lib/solr/bin/solr ]; then
|
|
||||||
# not installed yet
|
|
||||||
needs_update=1
|
|
||||||
elif [[ "$VERSION" != `/usr/local/lib/solr/bin/solr version` ]]; then
|
|
||||||
# checks if the version is what we want
|
|
||||||
needs_update=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ $needs_update == 1 ]; then
|
|
||||||
# install SOLR
|
|
||||||
wget_verify \
|
|
||||||
"https://www.apache.org/dyn/closer.lua?action=download&filename=lucene/solr/$VERSION/solr-$VERSION.tgz" \
|
|
||||||
$HASH \
|
|
||||||
/tmp/solr-$VERSION.tgz
|
|
||||||
|
|
||||||
tar xzf /tmp/solr-$VERSION.tgz -C /tmp solr-$VERSION/bin/install_solr_service.sh --strip-components=2
|
|
||||||
# install to usr/local, force update, do not start service on installation complete
|
|
||||||
bash /tmp/install_solr_service.sh /tmp/solr-$VERSION.tgz -i /usr/local/lib -f -n
|
|
||||||
|
|
||||||
rm -f /tmp/solr-$VERSION.tgz
|
|
||||||
rm -f /tmp/install_solr_service.sh
|
|
||||||
|
|
||||||
# stop and remove the init.d script
|
|
||||||
rm -f /etc/init.d/solr
|
|
||||||
update-rc.d solr remove
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add security
|
|
||||||
tools/editconf.py /etc/default/solr.in.sh \
|
|
||||||
SOLR_IP_WHITELIST='"127.0.0.1, [::1]"'
|
|
||||||
|
|
||||||
# Change log dir
|
|
||||||
if [ ! -d "/var/log/solr" ]; then
|
|
||||||
mkdir /var/log/solr
|
|
||||||
fi
|
|
||||||
|
|
||||||
chown solr:solr /var/log/solr
|
|
||||||
|
|
||||||
tools/editconf.py /etc/default/solr.in.sh \
|
|
||||||
SOLR_LOGS_DIR="/var/log/solr"
|
|
||||||
|
|
||||||
# Install systemd service
|
|
||||||
cp -f conf/solr/solr.service /lib/systemd/system/solr.service
|
|
||||||
# hide_output systemctl link -f /lib/systemd/system/solr.service
|
|
||||||
|
|
||||||
# Reload systemctl to pickup the above changes
|
|
||||||
hide_output systemctl daemon-reload
|
|
||||||
|
|
||||||
# Make sure service is enabled
|
|
||||||
hide_output systemctl enable solr.service
|
|
||||||
|
|
||||||
# Update the dovecot plugin configuration
|
|
||||||
#
|
|
||||||
# Break-imap-search makes search work the way users expect, rather than the way
|
|
||||||
# the IMAP specification expects.
|
|
||||||
# https://wiki.dovecot.org/Plugins/FTS/Solr
|
|
||||||
# "break-imap-search : Use Solr also for indexing TEXT and BODY searches.
|
|
||||||
# This makes your server non-IMAP-compliant."
|
|
||||||
tools/editconf.py /etc/dovecot/conf.d/10-mail.conf \
|
|
||||||
mail_plugins="fts fts_solr"
|
|
||||||
|
|
||||||
cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
|
|
||||||
plugin {
|
|
||||||
fts = solr
|
|
||||||
fts_autoindex = yes
|
|
||||||
fts_solr = url=http://127.0.0.1:8983/solr/dovecot/
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Install cronjobs to keep FTS up to date.
|
|
||||||
hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/
|
|
||||||
hide_output install -m 644 conf/cron/miab_solr /etc/cron.d/
|
|
||||||
|
|
||||||
# Initialize solr dovecot instance
|
|
||||||
if [ ! -d "/var/solr/data/dovecot" ]; then
|
|
||||||
# Starting solr might take a while
|
|
||||||
echo "Starting solr..."
|
|
||||||
hide_output systemctl restart solr.service
|
|
||||||
|
|
||||||
sudo -u solr /usr/local/lib/solr/bin/solr create -c dovecot
|
|
||||||
rm -f /var/solr/data/dovecot/conf/schema.xml
|
|
||||||
rm -f /var/solr/data/dovecot/conf/managed-schema
|
|
||||||
rm -f /var/solr/data/dovecot/conf/solrconfig.xml
|
|
||||||
cp -f conf/solr/solr-config-7.7.0.xml /var/solr/data/dovecot/conf/solrconfig.xml
|
|
||||||
cp -f conf/solr/solr-schema-7.7.0.xml /var/solr/data/dovecot/conf/schema.xml
|
|
||||||
chown -R solr:solr /var/solr/data/dovecot/conf/*
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Create new rsyslog config for solr
|
|
||||||
cat > /etc/rsyslog.d/10-solr.conf <<EOF
|
|
||||||
# Send solr systemd messages to solr-systemd.log when using systemd
|
|
||||||
:programname, startswith, "solr" {
|
|
||||||
/var/log/solr-systemd.log
|
|
||||||
stop
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Also adjust logrotated to the new file and correct user
|
|
||||||
|
|
||||||
cat > /etc/logrotate.d/solr-systemd <<EOF
|
|
||||||
/var/log/solr-systemd.log {
|
|
||||||
rotate 4
|
|
||||||
weekly
|
|
||||||
missingok
|
|
||||||
notifempty
|
|
||||||
compress
|
|
||||||
delaycompress
|
|
||||||
create 640 syslog adm
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
|
|
||||||
# Restart services to reload solr schema, dovecot plugins and rsyslog changes
|
|
||||||
restart_service dovecot
|
|
||||||
restart_service rsyslog
|
|
||||||
|
|
||||||
# Restarting solr might take a while
|
|
||||||
echo "Restarting solr"
|
|
||||||
hide_output systemctl restart solr.service
|
|
||||||
|
|
||||||
# Kickoff building the index
|
|
||||||
|
|
||||||
# Per doveadm-fts manpage: Scan what mails exist in the full text search index
|
|
||||||
# and compare those to what actually exist in mailboxes.
|
|
||||||
# This removes mails from the index that have already been expunged and makes
|
|
||||||
# sure that the next doveadm index will index all the missing mails (if any).
|
|
||||||
doveadm fts rescan -A
|
|
||||||
|
|
||||||
# Adds unindexed files to the fts database
|
|
||||||
# * `-q`: Queues the indexing to be run by indexer process. (will background the indexing)
|
|
||||||
# * `-A`: All users
|
|
||||||
# * `'*'`: All folders
|
|
||||||
doveadm index -q -A '*'
|
|
||||||
@ -111,7 +111,7 @@ source setup/dns.sh
|
|||||||
source setup/mail-postfix.sh
|
source setup/mail-postfix.sh
|
||||||
source setup/mail-dovecot.sh
|
source setup/mail-dovecot.sh
|
||||||
source setup/mail-users.sh
|
source setup/mail-users.sh
|
||||||
source setup/solr.sh
|
source setup/dovecot-fts-xapian.sh
|
||||||
source setup/dkim.sh
|
source setup/dkim.sh
|
||||||
source setup/spamassassin.sh
|
source setup/spamassassin.sh
|
||||||
source setup/web.sh
|
source setup/web.sh
|
||||||
|
|||||||
@ -53,7 +53,7 @@ tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
|
|||||||
tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
|
tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
|
||||||
default_charset="UTF-8"
|
default_charset="UTF-8"
|
||||||
|
|
||||||
# Set higher timeout since searches with Roundcube and Solr may take longer
|
# Set higher timeout since fts searches with Roundcube may take longer
|
||||||
# than the default 60 seconds. We will also match Roundcube's timeout to the
|
# than the default 60 seconds. We will also match Roundcube's timeout to the
|
||||||
# same value
|
# same value
|
||||||
tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
|
tools/editconf.py /etc/php/$(php_version)/fpm/php.ini -c ';' \
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user