external
/
mailinabox
mirror of https://github.com/mail-in-a-box/mailinabox.git
1
0
Fork 0
Code Issues Releases Wiki Activity

install solr without ubuntu package

This commit is contained in:
github@kiekerjan.isdronken.nl 2021-04-21 22:26:49 +02:00
parent ef5b536f43
commit e946276f15
8 changed files with 442 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf/cron/miab_solr
View File

@ -1,2 +1,2 @@
* 1 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?commit=true >/dev/null 2>&1
30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?optimize=true >/dev/null 2>&1
* 1 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?commit=true >/dev/null 2>&1
30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?optimize=true >/dev/null 2>&1

289
conf/solr/solr-config-7.7.0.xml Normal file
View File

@ -0,0 +1,289 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!-- This is the default config with stuff non-essential to Dovecot removed. -->
<config>
<!-- Controls what version of Lucene various components of Solr
adhere to. Generally, you want to use the latest version to
get all bug fixes and improvements. It is highly recommended
that you fully re-index after changing this setting as it can
affect both how text is indexed and queried.
-->
<luceneMatchVersion>7.7.0</luceneMatchVersion>
<!-- A 'dir' option by itself adds any files found in the directory
to the classpath, this is useful for including all jars in a
directory.
When a 'regex' is specified in addition to a 'dir', only the
files in that directory which completely match the regex
(anchored on both ends) will be included.
If a 'dir' option (with or without a regex) is used and nothing
is found that matches, a warning will be logged.
The examples below can be used to load some solr-contribs along
with their external dependencies.
-->
<lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" />
<!-- Data Directory
Used to specify an alternate directory to hold all index data
other than the default ./data under the Solr home. If
replication is in use, this should match the replication
configuration.
-->
<dataDir>${solr.data.dir:}</dataDir>
<!-- The default high-performance update handler -->
<updateHandler class="solr.DirectUpdateHandler2">
<!-- Enables a transaction log, used for real-time get, durability, and
and solr cloud replica recovery. The log can grow as big as
uncommitted changes to the index, so use of a hard autoCommit
is recommended (see below).
"dir" - the target directory for transaction logs, defaults to the
solr data directory.
"numVersionBuckets" - sets the number of buckets used to keep
track of max version values when checking for re-ordered
updates; increase this value to reduce the cost of
synchronizing access to version buckets during high-volume
indexing, this requires 8 bytes (long) * numVersionBuckets
of heap space per Solr core.
-->
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
<int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
</updateLog>
<!-- AutoCommit
Perform a hard commit automatically under certain conditions.
Instead of enabling autoCommit, consider using "commitWithin"
when adding documents.
http://wiki.apache.org/solr/UpdateXmlMessages
maxDocs - Maximum number of documents to add since the last
commit before automatically triggering a new commit.
maxTime - Maximum amount of time in ms that is allowed to pass
since a document was added before automatically
triggering a new commit.
openSearcher - if false, the commit causes recent index changes
to be flushed to stable storage, but does not cause a new
searcher to be opened to make those changes visible.
If the updateLog is enabled, then it's highly recommended to
have some sort of hard autoCommit to limit the log size.
-->
<autoCommit>
<maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
<openSearcher>false</openSearcher>
</autoCommit>
<!-- softAutoCommit is like autoCommit except it causes a
'soft' commit which only ensures that changes are visible
but does not ensure that data is synced to disk. This is
faster and more near-realtime friendly than a hard commit.
-->
<autoSoftCommit>
<maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
</autoSoftCommit>
<!-- Update Related Event Listeners
Various IndexWriter related events can trigger Listeners to
take actions.
postCommit - fired after every commit or optimize command
postOptimize - fired after every optimize command
-->
</updateHandler>
<!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Query section - these settings control query time things like caches
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
<query>
<!-- Solr Internal Query Caches
There are two implementations of cache available for Solr,
LRUCache, based on a synchronized LinkedHashMap, and
FastLRUCache, based on a ConcurrentHashMap.
FastLRUCache has faster gets and slower puts in single
threaded operation and thus is generally faster than LRUCache
when the hit ratio of the cache is high (> 75%), and may be
faster under other scenarios on multi-cpu systems.
-->
<!-- Filter Cache
Cache used by SolrIndexSearcher for filters (DocSets),
unordered sets of *all* documents that match a query. When a
new searcher is opened, its caches may be prepopulated or
"autowarmed" using data from caches in the old searcher.
autowarmCount is the number of items to prepopulate. For
LRUCache, the autowarmed items will be the most recently
accessed items.
Parameters:
class - the SolrCache implementation LRUCache or
(LRUCache or FastLRUCache)
size - the maximum number of entries in the cache
initialSize - the initial capacity (number of entries) of
the cache. (see java.util.HashMap)
autowarmCount - the number of entries to prepopulate from
and old cache.
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
to occupy. Note that when this option is specified, the size
and initialSize parameters are ignored.
-->
<filterCache class="solr.FastLRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<!-- Query Result Cache
Caches results of searches - ordered lists of document ids
(DocList) based on a query, a sort, and the range of documents requested.
Additional supported parameter by LRUCache:
maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
to occupy
-->
<queryResultCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<!-- Document Cache
Caches Lucene Document objects (the stored fields for each
document). Since Lucene internal document ids are transient,
this cache will not be autowarmed.
-->
<documentCache class="solr.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<!-- custom cache currently used by block join -->
<cache name="perSegFilter"
class="solr.search.LRUCache"
size="10"
initialSize="0"
autowarmCount="10"
regenerator="solr.NoOpRegenerator" />
<!-- Lazy Field Loading
If true, stored fields that are not requested will be loaded
lazily. This can result in a significant speed improvement
if the usual case is to not load all stored fields,
especially if the skipped fields are large compressed text
fields.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<!-- Result Window Size
An optimization for use with the queryResultCache. When a search
is requested, a superset of the requested number of document ids
are collected. For example, if a search for a particular query
requests matching documents 10 through 19, and queryWindowSize is 50,
then documents 0 through 49 will be collected and cached. Any further
requests in that range can be satisfied via the cache.
-->
<queryResultWindowSize>20</queryResultWindowSize>
<!-- Maximum number of documents to cache for any entry in the
queryResultCache.
-->
<queryResultMaxDocsCached>200</queryResultMaxDocsCached>
<!-- Use Cold Searcher
If a search request comes in and there is no current
registered searcher, then immediately register the still
warming searcher and use it. If "false" then all requests
will block until the first searcher is done warming.
-->
<useColdSearcher>false</useColdSearcher>
</query>
<!-- Request Dispatcher
This section contains instructions for how the SolrDispatchFilter
should behave when processing requests for this SolrCore.
-->
<requestDispatcher>
<httpCaching never304="true" />
</requestDispatcher>
<!-- Request Handlers
http://wiki.apache.org/solr/SolrRequestHandler
Incoming queries will be dispatched to a specific handler by name
based on the path specified in the request.
If a Request Handler is declared with startup="lazy", then it will
not be initialized until the first request that uses it.
-->
<!-- SearchHandler
http://wiki.apache.org/solr/SearchHandler
For processing Search Queries, the primary Request Handler
provided with Solr is "SearchHandler" It delegates to a sequent
of SearchComponents (see below) and supports distributed
queries across multiple shards
-->
<requestHandler name="/select" class="solr.SearchHandler">
<!-- default values for query parameters can be specified, these
will be overridden by parameters in the request
-->
<lst name="defaults">
<str name="echoParams">explicit</str>
<int name="rows">10</int>
</lst>
</requestHandler>
<initParams path="/update/**,/select">
<lst name="defaults">
<str name="df">_text_</str>
</lst>
</initParams>
<!-- Response Writers
http://wiki.apache.org/solr/QueryResponseWriter
Request responses will be written using the writer specified by
the 'wt' request parameter matching the name of a registered
writer.
The "default" writer is the default and will be used if 'wt' is
not specified in the request.
-->
<queryResponseWriter name="xml"
default="true"
class="solr.XMLResponseWriter" />
</config>

0
conf/solr-jetty.xml → conf/solr/solr-jetty.xml
View File

48
conf/solr/solr-schema-7.7.0.xml Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<schema name="dovecot" version="2.0">
<fieldType name="string" class="solr.StrField" omitNorms="true" sortMissingLast="true"/>
<fieldType name="long" class="solr.LongPointField" positionIncrementGap="0"/>
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="text" class="solr.TextField" autoGeneratePhraseQueries="true" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
<filter class="solr.FlattenGraphFilterFactory"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.SynonymGraphFilterFactory" expand="true" ignoreCase="true" synonyms="synonyms.txt"/>
<filter class="solr.FlattenGraphFilterFactory"/>
<filter class="solr.StopFilterFactory" words="stopwords.txt" ignoreCase="true"/>
<filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="1" splitOnCaseChange="1" generateWordParts="1" splitOnNumerics="1" catenateAll="1" catenateWords="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
<filter class="solr.PorterStemFilterFactory"/>
</analyzer>
</fieldType>
<field name="id" type="string" indexed="true" required="true" stored="true"/>
<field name="uid" type="long" indexed="true" required="true" stored="true"/>
<field name="box" type="string" indexed="true" required="true" stored="true"/>
<field name="user" type="string" indexed="true" required="true" stored="true"/>
<field name="hdr" type="text" indexed="true" stored="false"/>
<field name="body" type="text" indexed="true" stored="false"/>
<field name="from" type="text" indexed="true" stored="false"/>
<field name="to" type="text" indexed="true" stored="false"/>
<field name="cc" type="text" indexed="true" stored="false"/>
<field name="bcc" type="text" indexed="true" stored="false"/>
<field name="subject" type="text" indexed="true" stored="false"/>
<!-- Used by Solr internally: -->
<field name="_version_" type="long" indexed="true" stored="true"/>
<uniqueKey>id</uniqueKey>
</schema>

21
conf/solr/solr.service Normal file
View File

@ -0,0 +1,21 @@
[Unit]
Description=Apache SOLR
After=network.target
[Service]
Type=forking
User=solr
PIDFile=/var/run/solr-8983.pid
Environment=SOLR_INCLUDE=/etc/default/solr.in.sh
ExecStart=/usr/local/lib/solr/bin/solr start
ExecStop=/usr/local/lib/solr/bin/solr stop
Restart=on-failure
ReadWritePaths=/var/lib/solr/
ReadWritePaths=/var/lib/solr/data/
LimitNOFILE=65000
LimitNPROC=65000
TimeoutSec=180s
PrivateTmp=true
[Install]
WantedBy=multi-user.target

2
management/status_checks.py
View File

@ -40,7 +40,7 @@ def get_services():
{ "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, },
{ "name": "HTTP Web (nginx)", "port": 80, "public": True, },
{ "name": "HTTPS Web (nginx)", "port": 443, "public": True, },
{ "name": "Solr Full Text Search (Jetty)", "port": 8080, "public": False, },
{ "name": "Solr Full Text Search (Jetty)", "port": 8983, "public": False, },
]
def run_checks(rounded_values, env, output, pool):

127
setup/solr.sh
View File

@ -1,8 +1,5 @@
#!/bin/bash
#
# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo)
# with some modifications
#
# IMAP search with lucene via solr
# --------------------------------
#
@ -13,6 +10,12 @@
#
# Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its
# HTTP interface, searching indexed mail and returning results back to dovecot.
#
# Based on https://forum.iredmail.org/topic17251-dovecot-fts-full-text-search-using-apache-solr-on-ubuntu-1804-lts.html
# https://doc.dovecot.org/configuration_manual/fts/solr/ and https://solr.apache.org/guide/8_8/installing-solr.html
#
# solr-jetty package is removed from Ubuntu 21.04 onward. This installation
# therefore depends on manual installation of solr instead of an ubuntu package
source setup/functions.sh # load our functions
source /etc/mailinabox.conf # load global vars
@ -22,14 +25,62 @@ source /etc/mailinabox.conf # load global vars
echo "Installing Solr..."
# Install packages
apt_install solr-jetty dovecot-solr
# TODO remove after testing
set -x
# Solr requires a schema to tell it how to index data, this is provided by dovecot
cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml
apt_install dovecot-solr default-jre-headless
VERSION=8.8.2
HASH=7c3e2ed31a4412e7dac48d68c3abd52f75684577
needs_update=0
if [ ! -f /usr/local/lib/solr/bin/solr ]; then
# not installed yet
needs_update=1
elif [[ "$VERSION" != `/usr/local/lib/solr/bin/solr version` ]]; then
# checks if the version is what we want
needs_update=1
fi
if [ $needs_update == 1 ]; then
# install SOLR
wget_verify \
https://www.apache.org/dyn/closer.lua?action=download&filename=lucene/solr/$VERSION/solr-$VERSION.tgz \
$HASH \
/tmp/solr.tgz
tar xzf /tmp/solr.tgz solr-$VERSION/bin/install_solr_service.sh --strip-components=2
# install to usr/local, force update, do not start service on installation complete
bash /tmp/install_solr_service.sh solr.tgz -i /usr/local/lib -f -n
# TODO uncomment after testing
#rm -f /tmp/solr.tgz
rm -f /tmp/install_solr_service.sh
# stop and remove the init.d script
rm -f /etc/init.d/solr
update-rc.d solr remove
fi
# Install systemd service
if [ ! -f "/lib/systemd/system/solr.service" ]; then
cp -f conf/solr/solr.service /lib/systemd/system/solr.service
hide_output systemctl link -f /lib/systemd/system/solr.service
# Reload systemctl to pickup the above changes
hide_output systemctl daemon-reload
fi
# Make sure service is enabled
hide_output systemctl enable solr.service
# TODO: necessary? Solr requires a schema to tell it how to index data, this is provided by dovecot
# cp -f /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml
# Default config has an error with our config, placing our custom version
cp conf/solr-jetty.xml /etc/solr/solr-jetty.xml
# TBD necessary?
#cp -f conf/solr/solr-jetty.xml /etc/solr/solr-jetty.xml
# Update the dovecot plugin configuration
#
@ -45,7 +96,7 @@ cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF;
plugin {
fts = solr
fts_autoindex = yes
fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/
fts_solr = break-imap-search url=http://127.0.0.1:8983/solr/
}
EOF
@ -53,50 +104,33 @@ EOF
hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/
hide_output install -m 644 conf/cron/miab_solr /etc/cron.d/
# PERMISSIONS
# Initialize solr dovecot instance
if [ ! -d "/var/sorl/data/dovecot" ]; then
sudo -u solr /usr/local/lib/solr/bin/solr create -c dovecot
rm -f /var/solr/data/dovecot/conf/schema.xml
rm -f /var/solr/data/dovecot/conf/managed-schema
rm -f /var/solr/data/dovecot/conf/solrconfig.xml
cp -f conf/solr/solr-config-7.7.0.xml /var/solr/data/dovecot/conf/solrconfig.xml
cp -f conf/solr/solr-schema-7.7.0.xml /var/solr/data/dovecot/conf/schema.xml
fi
# Ensure configuration files are owned by dovecot and not world readable.
chown -R mail:dovecot /etc/dovecot
chmod -R o-rwx /etc/dovecot
# TODO: add security
#SOLR_IP_WHITELIST="127.0.0.1, [::1]"
# Newer updates to jetty9 restrict write directories, this allows for
# jetty to write to solr database directories
mkdir -p /etc/systemd/system/jetty9.service.d/
cat > /etc/systemd/system/jetty9.service.d/solr-permissions.conf << EOF
[Service]
ReadWritePaths=/var/lib/solr/
ReadWritePaths=/var/lib/solr/data/
EOF
# Reload systemctl to pickup the above override.
systemctl daemon-reload
# Fix Logging
# Due to the new systemd security permissions placed when running jetty.
# The log file directory at /var/log/jetty9 is reset to jetty:jetty
# at every program start. This causes syslog to fail to add the
# rsyslog filtered output to this folder. We will move this up a
# directory to /var/log/ since solr-jetty is quite noisy.
# Remove package config file since it points to a folder that
# it does not have permissions to, and is also too far down the
# /etc/rsyslog.d/ order to work anyway.
rm -f /etc/rsyslog.d/jetty9.conf
# Create new rsyslog config for jetty9 for its new location
cat > /etc/rsyslog.d/10-jetty9.conf <<EOF
# Send Jetty messages to jetty-console.log when using systemd
:programname, startswith, "jetty9" {
/var/log/jetty-console.log
# Create new rsyslog config for solr
# TODO check programname
cat > /etc/rsyslog.d/10-solr.conf <<EOF
# Send solr messages to solr-console.log when using systemd
:programname, startswith, "solr" {
/var/log/solr.log
stop
}
EOF
# Also adjust logrotated to the new file and correct user
rm -f /etc/logrotate.d/jetty9
cat > /etc/logrotate.d/jetty9 <<EOF
/var/log/jetty-console.log {
cat > /etc/logrotate.d/solr <<EOF
/var/log/solr.log {
copytruncate
weekly
rotate 12
@ -107,11 +141,10 @@ cat > /etc/logrotate.d/jetty9 <<EOF
}
EOF
# Restart services to reload solr schema, dovecot plugins and rsyslog changes
restart_service jetty9
restart_service dovecot
restart_service rsyslog
hide_output systemctl restart solr.service
# Kickoff building the index

2
setup/start.sh
View File

@ -111,7 +111,7 @@ source setup/dns.sh
source setup/mail-postfix.sh
source setup/mail-dovecot.sh
source setup/mail-users.sh
source setup/solr.sh
#source setup/solr.sh
source setup/dkim.sh
source setup/spamassassin.sh
source setup/web.sh