Updated help page. Not finished yet.
This commit is contained in:
Louwrentius
parent
808d14e08c
commit
ffeaf1ccf6
155
ppss.sh
155
ppss.sh
@ -1,38 +1,37 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#*
|
#
|
||||||
#* PPSS, the Parallel Processing Shell Script
|
# PPSS, the Parallel Processing Shell Script
|
||||||
#*
|
#
|
||||||
#* Copyright (c) 2009, Louwrentius
|
# Copyright (c) 2009, Louwrentius
|
||||||
#* All rights reserved.
|
# All rights reserved.
|
||||||
#*
|
#
|
||||||
#* Redistribution and use in source and binary forms, with or without
|
# Redistribution and use in source and binary forms, with or without
|
||||||
#* modification, are permitted provided that the following conditions are met:
|
# modification, are permitted provided that the following conditions are met:
|
||||||
#* * Redistributions of source code must retain the above copyright
|
# * Redistributions of source code must retain the above copyright
|
||||||
#* notice, this list of conditions and the following disclaimer.
|
# notice, this list of conditions and the following disclaimer.
|
||||||
#* * Redistributions in binary form must reproduce the above copyright
|
# * Redistributions in binary form must reproduce the above copyright
|
||||||
#* notice, this list of conditions and the following disclaimer in the
|
# notice, this list of conditions and the following disclaimer in the
|
||||||
#* documentation and/or other materials provided with the distribution.
|
# documentation and/or other materials provided with the distribution.
|
||||||
#* * Neither the name of the <organization> nor the
|
# * Neither the name of the <organization> nor the
|
||||||
#* names of its contributors may be used to endorse or promote products
|
# names of its contributors may be used to endorse or promote products
|
||||||
#* derived from this software without specific prior written permission.
|
# derived from this software without specific prior written permission.
|
||||||
#*
|
#
|
||||||
#* THIS SOFTWARE IS PROVIDED BY Louwrentius ''AS IS'' AND ANY
|
# THIS SOFTWARE IS PROVIDED BY Louwrentius ''AS IS'' AND ANY
|
||||||
#* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||||
#* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
#* DISCLAIMED. IN NO EVENT SHALL Louwrentius BE LIABLE FOR ANY
|
# DISCLAIMED. IN NO EVENT SHALL Louwrentius BE LIABLE FOR ANY
|
||||||
#* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||||
#* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||||
#* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||||
#* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
#* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||||
#* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
#------------------------------------------------------------------------------
|
||||||
#------------------------------------------------------
|
# It should not be necessary to edit antyhing in this script..
|
||||||
# It should not be necessary to edit antyhing.
|
# Ofcource you can if it is necessary for your needs.
|
||||||
# Ofcource you can if it is necesary for your needs.
|
|
||||||
# Send a patch if your changes may benefit others.
|
# Send a patch if your changes may benefit others.
|
||||||
#------------------------------------------------------
|
#------------------------------------------------------------------------------
|
||||||
|
|
||||||
# Handling control-c for a clean shutdown.
|
# Handling control-c for a clean shutdown.
|
||||||
trap 'kill_process; ' INT
|
trap 'kill_process; ' INT
|
||||||
@ -41,11 +40,12 @@ trap 'kill_process; ' INT
|
|||||||
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
||||||
SCRIPT_VERSION="1.999"
|
SCRIPT_VERSION="1.999"
|
||||||
|
|
||||||
|
# The first argument to this script is always the 'mode'.
|
||||||
MODE="$1"
|
MODE="$1"
|
||||||
shift
|
shift
|
||||||
|
|
||||||
ARGS=$@
|
ARGS=$@
|
||||||
CONFIG="config.cfg"
|
CONFIG="config.cfg"
|
||||||
DAEMON=0
|
|
||||||
HOSTNAME=`hostname`
|
HOSTNAME=`hostname`
|
||||||
ARCH=`uname`
|
ARCH=`uname`
|
||||||
RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS..
|
RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS..
|
||||||
@ -62,12 +62,15 @@ PERCENT="0"
|
|||||||
PID="$$"
|
PID="$$"
|
||||||
LISTENER_PID=""
|
LISTENER_PID=""
|
||||||
IFS_BACKUP="$IFS"
|
IFS_BACKUP="$IFS"
|
||||||
INTERVAL="10" # Polling interval to check if there are running jobs.
|
INTERVAL="30" # Polling interval to check if there are running jobs.
|
||||||
|
|
||||||
SSH_SERVER="" # Remote server or 'master'.
|
SSH_SERVER="" # Remote server or 'master'.
|
||||||
SSH_KEY="" # SSH key for ssh account.
|
SSH_KEY="" # SSH key for ssh account.
|
||||||
SSH_SOCKET="/tmp/PPSS-ssh-socket" # Multiplex multiple SSH connections over 1 master.
|
SSH_SOCKET="/tmp/PPSS-ssh-socket" # Multiplex multiple SSH connections over 1 master.
|
||||||
SSH_OPTS="-o BatchMode=yes -o ControlPath=$SSH_SOCKET -o GlobalKnownHostsFile=./known_hosts -o ControlMaster=auto -o ConnectTimeout=5"
|
SSH_OPTS="-o BatchMode=yes -o ControlPath=$SSH_SOCKET \
|
||||||
|
-o GlobalKnownHostsFile=./known_hosts \
|
||||||
|
-o ControlMaster=auto \
|
||||||
|
-o ConnectTimeout=5"
|
||||||
SSH_MASTER_PID=""
|
SSH_MASTER_PID=""
|
||||||
|
|
||||||
PPSS_HOME_DIR="ppss"
|
PPSS_HOME_DIR="ppss"
|
||||||
@ -86,29 +89,67 @@ showusage () {
|
|||||||
echo "$SCRIPT_NAME"
|
echo "$SCRIPT_NAME"
|
||||||
echo "Version: $SCRIPT_VERSION"
|
echo "Version: $SCRIPT_VERSION"
|
||||||
echo
|
echo
|
||||||
echo "Description: this script processess files or other items in parallel. It is designed to make"
|
echo "PPSS is a Bash shell script that processes a list of items, such as files in a"
|
||||||
echo "use of the multi-core CPUs. It will detect the number of available CPUs and start a thread "
|
echo "directory, in parallel."
|
||||||
echo "for each CPU core. It will also use hyperthreading if available." It has also support for
|
|
||||||
echo "distributed usage, using a Master server in conjunction with (multiple) slaves."
|
|
||||||
echo
|
echo
|
||||||
echo "Usage: $0 [ options ]"
|
echo "Usage: $0 MODE [ options ]"
|
||||||
|
echo " or "
|
||||||
|
echo "Usage: $0 MODE -c <config file>"
|
||||||
|
echo
|
||||||
|
echo "Modes are:"
|
||||||
|
echo
|
||||||
|
echo " standalone For execution of PPSS on a single host."
|
||||||
|
echo " node For execution of PPSS on a node, that is part of a 'cluster'."
|
||||||
|
echo " server Starting PPSS on nodes."
|
||||||
|
echo " config Generate a config file based on the supplied option parameters."
|
||||||
|
echo " deploy Deploy PPSS and related files on the specified nodes."
|
||||||
|
echo " erase Erase PPSS and related files from the specified nodes."
|
||||||
echo
|
echo
|
||||||
echo "Options are:"
|
echo "Options are:"
|
||||||
echo
|
echo
|
||||||
echo -e "\t- c \tCommand to execute. Can be a custom script or just a plain command."
|
echo -e "--command | -c Command to execute. Syntax: '<command> ' including the single quotes."
|
||||||
echo -e "\t- d \tDirectory containing items to be processed."
|
echo -e " Example: -c 'ls -alh '. It is also possible to specify where an item "
|
||||||
echo -e "\t- f \tFile containing items to be processed. (Alternative to -d)"
|
echo -e " must be inserted: 'cp \"\$ITEM\" /somedir'."
|
||||||
echo -e "\t- l \tSpecifies name and location of the logfile."
|
|
||||||
echo -e "\t- p \tSpecifies number of simultaneous processes manually. (optional)"
|
|
||||||
echo -e "\t- j \tEnable or disable hyperthreading. Enabled by default. (optional)"
|
|
||||||
echo
|
|
||||||
echo "Options for distributed usage:"
|
|
||||||
echo
|
echo
|
||||||
echo -e "\t- s \tUsername@server domain name or IP-address of 'PPSS master server'."
|
echo -e "--sourcedir | -d Directory that contains files that must be processed. Individual files"
|
||||||
echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'."
|
echo -e " are fed as an argument to the command that has been specified with -c."
|
||||||
echo -e "\t- t \tTransfer remote item to slave for local processing."
|
echo
|
||||||
echo -e "\t- o \tUpload output back to server into this directory."
|
echo -e "--sourcefile | -f Each single line of the supplied file will be fed as an item to the"
|
||||||
echo -e "\t- b \tDo *not* use scp for item transfer but use cp. "
|
echo -e " command that has been specified with -c."
|
||||||
|
echo
|
||||||
|
echo -e "--config | -c If the mode is config, a config file with the specified name will be"
|
||||||
|
echo -e " generated based on all the options specified. In the other modes".
|
||||||
|
echo -e " this option will result in PPSS reading the config file and start"
|
||||||
|
echo -e " processing items based on the settings of this file."
|
||||||
|
echo
|
||||||
|
echo -e "--enable-ht | -j Enable hyperthreading. Is disabled by default."
|
||||||
|
echo
|
||||||
|
echo -e "--log | -l Sets the name of the log file. The default is ppss-log.txt."
|
||||||
|
echo
|
||||||
|
echo -e "--processes | -p Start the specified number of processes. Ignore the number of available"
|
||||||
|
echo -e " CPU's."
|
||||||
|
echo
|
||||||
|
echo -e "The following options are used for distributed execution of PPSS."
|
||||||
|
echo
|
||||||
|
echo -e "--server | -s Specifies the SSH server that is used for communication between nodes."
|
||||||
|
echo -e " Using SSH, file locks are created, informing other nodes that an item "
|
||||||
|
echo -e " is locked. Also, often items, such as files, reside on this host. SCP "
|
||||||
|
echo -e " is used to transfer files from this host to nodes for local procesing."
|
||||||
|
echo
|
||||||
|
echo -e "--node | -n File containig a list of nodes that act as PPSS clients. One IP / DNS "
|
||||||
|
echo -e " name per line."
|
||||||
|
echo
|
||||||
|
echo -e "--key | -k The SSH key that a node uses to connect to the server."
|
||||||
|
echo
|
||||||
|
echo -e "--user | -u The SSH user name that is used when logging in into the master SSH"
|
||||||
|
echo -e " server."
|
||||||
|
echo
|
||||||
|
echo -e "--script | -s Specifies the script/program that must be copied to the nodes for "
|
||||||
|
echo -e " execution by the nodes through PPSS. Only used in the deploy mode."
|
||||||
|
echo -e " This option should be specified if necessary when generating a config."
|
||||||
|
echo
|
||||||
|
echo -e "--transfer | -t This option specifies that an item will be downloaded by the node "
|
||||||
|
echo -e " from the server or share to the local node for processing."
|
||||||
echo
|
echo
|
||||||
echo -e "Example: encoding some wav files to mp3 using lame:"
|
echo -e "Example: encoding some wav files to mp3 using lame:"
|
||||||
echo
|
echo
|
||||||
@ -310,6 +351,12 @@ do
|
|||||||
--help|-h )
|
--help|-h )
|
||||||
showusage
|
showusage
|
||||||
exit 1;;
|
exit 1;;
|
||||||
|
--homedir|-H)
|
||||||
|
PPSS_HOME_DIR="$2"
|
||||||
|
add_Var_to_config PPSS_HOME_DIR $PPSS_HOME_DIR
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
|
||||||
--enable-ht|-j )
|
--enable-ht|-j )
|
||||||
HYPERTHREADING=yes
|
HYPERTHREADING=yes
|
||||||
add_var_to_config HYPERTHREADING "yes"
|
add_var_to_config HYPERTHREADING "yes"
|
||||||
@ -487,7 +534,7 @@ log () {
|
|||||||
|
|
||||||
echo -e "$LOG_MSG" >> "$LOGFILE"
|
echo -e "$LOG_MSG" >> "$LOGFILE"
|
||||||
|
|
||||||
if [ "$TYPE" == "INFO" ] && [ "$DAEMON" == "0" ]
|
if [ "$TYPE" == "INFO" ]
|
||||||
then
|
then
|
||||||
echo -e "$LOG_MSG"
|
echo -e "$LOG_MSG"
|
||||||
fi
|
fi
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user