From e946276f158d205f890f2dc066965d08ca46bc45 Mon Sep 17 00:00:00 2001 From: "github@kiekerjan.isdronken.nl" Date: Wed, 21 Apr 2021 22:26:49 +0200 Subject: [PATCH] install solr without ubuntu package --- conf/cron/miab_solr | 4 +- conf/solr/solr-config-7.7.0.xml | 289 ++++++++++++++++++++++++++++++++ conf/{ => solr}/solr-jetty.xml | 0 conf/solr/solr-schema-7.7.0.xml | 48 ++++++ conf/solr/solr.service | 21 +++ management/status_checks.py | 2 +- setup/solr.sh | 127 ++++++++------ setup/start.sh | 2 +- 8 files changed, 442 insertions(+), 51 deletions(-) create mode 100644 conf/solr/solr-config-7.7.0.xml rename conf/{ => solr}/solr-jetty.xml (100%) create mode 100644 conf/solr/solr-schema-7.7.0.xml create mode 100644 conf/solr/solr.service diff --git a/conf/cron/miab_solr b/conf/cron/miab_solr index 898404cb..cfb102a6 100644 --- a/conf/cron/miab_solr +++ b/conf/cron/miab_solr @@ -1,2 +1,2 @@ -* 1 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?commit=true >/dev/null 2>&1 -30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8080/solr/update?optimize=true >/dev/null 2>&1 +* 1 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?commit=true >/dev/null 2>&1 +30 3 * * * root /usr/bin/curl -s http://127.0.0.1:8983/solr/update?optimize=true >/dev/null 2>&1 diff --git a/conf/solr/solr-config-7.7.0.xml b/conf/solr/solr-config-7.7.0.xml new file mode 100644 index 00000000..3661874d --- /dev/null +++ b/conf/solr/solr-config-7.7.0.xml @@ -0,0 +1,289 @@ + + + + + + + 7.7.0 + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + + + + + + + + + + + true + + + 20 + + + 200 + + + false + + + + + + + + + + + + + + + explicit + 10 + + + + + + _text_ + + + + + + diff --git a/conf/solr-jetty.xml b/conf/solr/solr-jetty.xml similarity index 100% rename from conf/solr-jetty.xml rename to conf/solr/solr-jetty.xml diff --git a/conf/solr/solr-schema-7.7.0.xml b/conf/solr/solr-schema-7.7.0.xml new file mode 100644 index 00000000..601a290c --- /dev/null +++ b/conf/solr/solr-schema-7.7.0.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/conf/solr/solr.service b/conf/solr/solr.service new file mode 100644 index 00000000..599d9c64 --- /dev/null +++ b/conf/solr/solr.service @@ -0,0 +1,21 @@ +[Unit] +Description=Apache SOLR +After=network.target + +[Service] +Type=forking +User=solr +PIDFile=/var/run/solr-8983.pid +Environment=SOLR_INCLUDE=/etc/default/solr.in.sh +ExecStart=/usr/local/lib/solr/bin/solr start +ExecStop=/usr/local/lib/solr/bin/solr stop +Restart=on-failure +ReadWritePaths=/var/lib/solr/ +ReadWritePaths=/var/lib/solr/data/ +LimitNOFILE=65000 +LimitNPROC=65000 +TimeoutSec=180s +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/management/status_checks.py b/management/status_checks.py index 9f78a004..be3def9f 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -40,7 +40,7 @@ def get_services(): { "name": "Mail Filters (Sieve/dovecot)", "port": 4190, "public": True, }, { "name": "HTTP Web (nginx)", "port": 80, "public": True, }, { "name": "HTTPS Web (nginx)", "port": 443, "public": True, }, - { "name": "Solr Full Text Search (Jetty)", "port": 8080, "public": False, }, + { "name": "Solr Full Text Search (Jetty)", "port": 8983, "public": False, }, ] def run_checks(rounded_values, env, output, pool): diff --git a/setup/solr.sh b/setup/solr.sh index e4371a9a..0277073b 100644 --- a/setup/solr.sh +++ b/setup/solr.sh @@ -1,8 +1,5 @@ #!/bin/bash # -# Inspired by the solr.sh from jkaberg (https://github.com/jkaberg/mailinabox-sogo) -# with some modifications -# # IMAP search with lucene via solr # -------------------------------- # @@ -13,6 +10,12 @@ # # Solr runs as a Jetty process. The dovecot solr plugin talks to solr via its # HTTP interface, searching indexed mail and returning results back to dovecot. +# +# Based on https://forum.iredmail.org/topic17251-dovecot-fts-full-text-search-using-apache-solr-on-ubuntu-1804-lts.html +# https://doc.dovecot.org/configuration_manual/fts/solr/ and https://solr.apache.org/guide/8_8/installing-solr.html +# +# solr-jetty package is removed from Ubuntu 21.04 onward. This installation +# therefore depends on manual installation of solr instead of an ubuntu package source setup/functions.sh # load our functions source /etc/mailinabox.conf # load global vars @@ -22,14 +25,62 @@ source /etc/mailinabox.conf # load global vars echo "Installing Solr..." -# Install packages -apt_install solr-jetty dovecot-solr +# TODO remove after testing +set -x -# Solr requires a schema to tell it how to index data, this is provided by dovecot -cp /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml +apt_install dovecot-solr default-jre-headless + +VERSION=8.8.2 +HASH=7c3e2ed31a4412e7dac48d68c3abd52f75684577 + +needs_update=0 + +if [ ! -f /usr/local/lib/solr/bin/solr ]; then + # not installed yet + needs_update=1 +elif [[ "$VERSION" != `/usr/local/lib/solr/bin/solr version` ]]; then + # checks if the version is what we want + needs_update=1 +fi + +if [ $needs_update == 1 ]; then + # install SOLR + wget_verify \ + https://www.apache.org/dyn/closer.lua?action=download&filename=lucene/solr/$VERSION/solr-$VERSION.tgz \ + $HASH \ + /tmp/solr.tgz + + tar xzf /tmp/solr.tgz solr-$VERSION/bin/install_solr_service.sh --strip-components=2 + # install to usr/local, force update, do not start service on installation complete + bash /tmp/install_solr_service.sh solr.tgz -i /usr/local/lib -f -n + + # TODO uncomment after testing + #rm -f /tmp/solr.tgz + rm -f /tmp/install_solr_service.sh + + # stop and remove the init.d script + rm -f /etc/init.d/solr + update-rc.d solr remove +fi + +# Install systemd service +if [ ! -f "/lib/systemd/system/solr.service" ]; then + cp -f conf/solr/solr.service /lib/systemd/system/solr.service + hide_output systemctl link -f /lib/systemd/system/solr.service + + # Reload systemctl to pickup the above changes + hide_output systemctl daemon-reload +fi + +# Make sure service is enabled +hide_output systemctl enable solr.service + +# TODO: necessary? Solr requires a schema to tell it how to index data, this is provided by dovecot +# cp -f /usr/share/dovecot/solr-schema.xml /etc/solr/conf/schema.xml # Default config has an error with our config, placing our custom version -cp conf/solr-jetty.xml /etc/solr/solr-jetty.xml +# TBD necessary? +#cp -f conf/solr/solr-jetty.xml /etc/solr/solr-jetty.xml # Update the dovecot plugin configuration # @@ -45,7 +96,7 @@ cat > /etc/dovecot/conf.d/90-plugin-fts.conf << EOF; plugin { fts = solr fts_autoindex = yes - fts_solr = break-imap-search url=http://127.0.0.1:8080/solr/ + fts_solr = break-imap-search url=http://127.0.0.1:8983/solr/ } EOF @@ -53,50 +104,33 @@ EOF hide_output install -m 755 conf/cron/miab_dovecot /etc/cron.daily/ hide_output install -m 644 conf/cron/miab_solr /etc/cron.d/ -# PERMISSIONS +# Initialize solr dovecot instance +if [ ! -d "/var/sorl/data/dovecot" ]; then + sudo -u solr /usr/local/lib/solr/bin/solr create -c dovecot + rm -f /var/solr/data/dovecot/conf/schema.xml + rm -f /var/solr/data/dovecot/conf/managed-schema + rm -f /var/solr/data/dovecot/conf/solrconfig.xml + cp -f conf/solr/solr-config-7.7.0.xml /var/solr/data/dovecot/conf/solrconfig.xml + cp -f conf/solr/solr-schema-7.7.0.xml /var/solr/data/dovecot/conf/schema.xml +fi -# Ensure configuration files are owned by dovecot and not world readable. -chown -R mail:dovecot /etc/dovecot -chmod -R o-rwx /etc/dovecot +# TODO: add security +#SOLR_IP_WHITELIST="127.0.0.1, [::1]" -# Newer updates to jetty9 restrict write directories, this allows for -# jetty to write to solr database directories -mkdir -p /etc/systemd/system/jetty9.service.d/ -cat > /etc/systemd/system/jetty9.service.d/solr-permissions.conf << EOF -[Service] -ReadWritePaths=/var/lib/solr/ -ReadWritePaths=/var/lib/solr/data/ -EOF - -# Reload systemctl to pickup the above override. -systemctl daemon-reload - -# Fix Logging -# Due to the new systemd security permissions placed when running jetty. -# The log file directory at /var/log/jetty9 is reset to jetty:jetty -# at every program start. This causes syslog to fail to add the -# rsyslog filtered output to this folder. We will move this up a -# directory to /var/log/ since solr-jetty is quite noisy. - -# Remove package config file since it points to a folder that -# it does not have permissions to, and is also too far down the -# /etc/rsyslog.d/ order to work anyway. -rm -f /etc/rsyslog.d/jetty9.conf - -# Create new rsyslog config for jetty9 for its new location -cat > /etc/rsyslog.d/10-jetty9.conf < /etc/rsyslog.d/10-solr.conf < /etc/logrotate.d/jetty9 < /etc/logrotate.d/solr < /etc/logrotate.d/jetty9 <