sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Rewrote the usage and help screens.

Browse Source
This commit is contained in:
Louwrentius 2009-10-24 18:24:45 +00:00
parent 5b840d33c3
commit b1af50040f
1 changed files with 113 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
ppss.sh
View File

@ -40,9 +40,16 @@ trap 'kill_process; ' INT
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.34" SCRIPT_VERSION="2.34"
# The first argument to this script is always the 'mode'. # The first argument to this script can be a mode.
MODE="$1" MODES="start config stop pause continue deploy status erase kill"
shift for x in $MODES
do
if [ "$x" == "$1" ]
then
MODE="$1"
shift
fi
done
# The working directory of PPSS can be set with # The working directory of PPSS can be set with
# export PPSS_DIR=/path/to/workingdir # export PPSS_DIR=/path/to/workingdir
@ -100,23 +107,83 @@ SCRIPT="" # Custom user script that is executed by
ITEM_ESCAPED="" ITEM_ESCAPED=""
NODE_STATUS="$PPSS_DIR/status.txt" NODE_STATUS="$PPSS_DIR/status.txt"
showusage () { showusage_short () {
echo echo
echo "$SCRIPT_NAME" echo "|P|P|S|S| $SCRIPT_NAME $SCRIPT_VERSION"
echo
echo "usage: $0 [ -d <sourcedir> | -f <sourcefile> ] [ -c '<command> \"%ITEM%\"' ]"
echo " [ -C <configfile> ] [ -j ] [ -l <logfile> ] [ -p <# jobs> ]"
echo
echo "Examples:"
echo " $0 -d /dir/with/some/files -c 'gzip '"
echo " $0 -d /dir/with/some/files -c 'gzip \"%ITEM\"'"
echo " $0 -d /dir/with/some/files -c 'cp \"%ITEM\" /tmp' -p 2"
}
showusage_normal () {
echo
echo "|P|P|S|S| - $SCRIPT_NAME -"
echo "Version: $SCRIPT_VERSION" echo "Version: $SCRIPT_VERSION"
echo echo
echo "PPSS is a Bash shell script that executes commands in parallel on a set " echo "PPSS is a Bash shell script that executes commands in parallel on a set "
echo "of items, such as files, or lines in a file." echo "of items, such as files in a directory, or lines in a file."
echo echo
echo "Usage: $0 MODE [ options ]" echo "This short summary only discuesses options for stand-alone mode. for all "
echo " or " echo "options, run PPSS with the options --help"
echo "Usage: $0 MODE -c <config file>"
echo echo
echo "Modes are:" echo "Usage $0 [ options ]"
echo
echo -e "--command | -c Command to execute. Syntax: '<command> ' including the single quotes."
echo -e " Example: -c 'ls -alh '. It is also possible to specify where an item "
echo -e " must be inserted: 'cp \"\$ITEM\" /somedir'."
echo
echo -e "--sourcedir | -d Directory that contains files that must be processed. Individual files"
echo -e " are fed as an argument to the command that has been specified with -c."
echo
echo -e "--sourcefile | -f Each single line of the supplied file will be fed as an item to the"
echo -e " command that has been specified with -c."
echo
echo -e "--config | -C If the mode is config, a config file with the specified name will be"
echo -e " generated based on all the options specified. In the other modes".
echo -e " this option will result in PPSS reading the config file and start"
echo -e " processing items based on the settings of this file."
echo
echo -e "--enable-ht | -j Enable hyperthreading. Is disabled by default."
echo
echo -e "--log | -l Sets the name of the log file. The default is ppss-log.txt."
echo
echo -e "--processes | -p Start the specified number of processes. Ignore the number of available"
echo -e " CPU's."
echo
echo -e "Example: encoding some wav files to mp3 using lame:"
echo
echo -e "$0 -d /path/to/wavfiles -c 'lame '"
echo
echo -e "Extended usage: use --help"
echo
}
if [ "$#" == "0" ]
then
showusage_short
exit 1
fi
showusage_long () {
echo
echo "|P|P|S|S| - $SCRIPT_NAME -"
echo "Version: $SCRIPT_VERSION"
echo
echo "PPSS is a Bash shell script that executes commands in parallel on a set "
echo "of items, such as files in a directory, or lines in a file."
echo
echo "Usage: $0 [ MODE ] [ options ]"
echo
echo "Modes are optional and mainly used for running in distributed mode. Modes are:"
echo echo
echo " standalone For execution of PPSS on a single host."
echo " node For execution of PPSS on a node, that is part of a 'cluster'."
echo " config Generate a config file based on the supplied option parameters." echo " config Generate a config file based on the supplied option parameters."
echo " deploy Deploy PPSS and related files on the specified nodes." echo " deploy Deploy PPSS and related files on the specified nodes."
echo " erase Erase PPSS and related files from the specified nodes." echo " erase Erase PPSS and related files from the specified nodes."
@ -149,10 +216,6 @@ showusage () {
echo -e "--processes | -p Start the specified number of processes. Ignore the number of available" echo -e "--processes | -p Start the specified number of processes. Ignore the number of available"
echo -e " CPU's." echo -e " CPU's."
echo echo
echo -e "--force | -F Force PPSS to run, even anoter instance of PPSS is already running."
echo " Please note that any running PPSS instance will not exit anymore if"
echo " you start multiple instances, although they will run fine."
echo
echo -e "The following options are used for distributed execution of PPSS." echo -e "The following options are used for distributed execution of PPSS."
echo echo
echo -e "--master | -m Specifies the SSH server that is used for communication between nodes." echo -e "--master | -m Specifies the SSH server that is used for communication between nodes."
@ -191,15 +254,15 @@ showusage () {
echo echo
echo -e "Example: encoding some wav files to mp3 using lame:" echo -e "Example: encoding some wav files to mp3 using lame:"
echo echo
echo -e "$0 standalone -c 'lame ' -d /path/to/wavfiles -j " echo -e "$0 -c 'lame ' -d /path/to/wavfiles -j "
echo echo
echo -e "Running PPSS based on a configuration file." echo -e "Running PPSS based on a configuration file."
echo echo
echo -e "$0 standalone -C config.cfg" echo -e "$0 -C config.cfg"
echo echo
echo -e "Running PPSS on a client as part of a cluster." echo -e "Running PPSS on a client as part of a cluster."
echo echo
echo -e "$0 node -d /somedir -c 'cp "$ITEM" /some/destination' -s 10.0.0.50 -u ppss -t -k ppss-key.key" echo -e "$0 -d /somedir -c 'cp "$ITEM" /some/destination' -s 10.0.0.50 -u ppss -t -k ppss-key.key"
echo echo
} }
@ -212,9 +275,10 @@ kill_process () {
sleep 1 sleep 1
if [ ! -z "$SSH_MASTER_PID" ] if [ ! -z "$SSH_MASTER_PID" ]
then then
kill -9 "$SSH_MASTER_PID" kill -9 "$SSH_MASTER_PID" >> /dev/null 2>&1
fi fi
sleep 1 sleep 1
echo
log INFO "Finished." log INFO "Finished."
} }
@ -380,8 +444,11 @@ do
shift 2 shift 2
;; ;;
--help|-h ) -h )
showusage showusage_normal
exit 1;;
--help)
showusage_long
exit 1;; exit 1;;
--homedir|-H ) --homedir|-H )
if [ ! -z "$2" ] if [ ! -z "$2" ]
@ -402,11 +469,6 @@ do
add_var_to_config LOGFILE "$LOGFILE" add_var_to_config LOGFILE "$LOGFILE"
shift 2 shift 2
;; ;;
--force|-F )
FORCE=yes
add_var_to_config FORCE "$FORCE"
shift 1
;;
--workingdir|-w ) --workingdir|-w )
WORKINGDIR="$2" WORKINGDIR="$2"
add_var_to_config WORKINGDIR "$WORKINGDIR" add_var_to_config WORKINGDIR "$WORKINGDIR"
@ -474,7 +536,7 @@ do
exit 0 exit 0
;; ;;
* ) * )
showusage showusage_normal
exit 1;; exit 1;;
esac esac
done done
@ -516,7 +578,7 @@ init_vars () {
echo echo
log ERROR "No command specified." log ERROR "No command specified."
echo echo
showusage showusage_normal
cleanup cleanup
exit 1 exit 1
fi fi
@ -813,7 +875,7 @@ start_ppss_on_node () {
NODE="$1" NODE="$1"
log INFO "Starting PPSS on node $NODE." log INFO "Starting PPSS on node $NODE."
ssh $SSH_KEY $USER@$NODE "cd $PPSS_DIR ; screen -d -m -S PPSS ./ppss.sh node --config $CONFIG" ssh $SSH_KEY $USER@$NODE "cd $PPSS_DIR ; screen -d -m -S PPSS $0 node --config $CONFIG"
} }
@ -1251,8 +1313,8 @@ start_single_worker () {
# informed that a worker just finished / died. # informed that a worker just finished / died.
# Tis allows the listener to determine if all processes # Tis allows the listener to determine if all processes
# are finished and it is time to stop. # are finished and it is time to stop.
echo
log INFO "Waiting for remaining jobs to finish..." log INFO "Waiting for remaining jobs to finish..."
#echo -en "\033[1A"
echo "$STOP_KEY" > $FIFO echo "$STOP_KEY" > $FIFO
return 1 return 1
else else
@ -1407,6 +1469,7 @@ listen_for_job () {
# The start_single_worker method sends a special signal to # The start_single_worker method sends a special signal to
# inform the listener that a worker is finished. # inform the listener that a worker is finished.
# If all workers are finished, it is time to stop. # If all workers are finished, it is time to stop.
# This mechanism makes PPSS asynchronous.
if [ "$event" == "$STOP_KEY" ] if [ "$event" == "$STOP_KEY" ]
then then
((DIED++)) ((DIED++))
@ -1512,14 +1575,6 @@ show_status () {
main () { main () {
case $MODE in case $MODE in
node|standalone )
init_vars
test_server
get_all_items
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & 2>&1 >> /dev/null
LISTENER_PID=$!
start_all_workers
;;
start ) start )
# This option only starts all nodes. # This option only starts all nodes.
display_header display_header
@ -1607,9 +1662,14 @@ main () {
;; ;;
* ) * )
showusage init_vars
exit 1 test_server
get_all_items
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & 2>&1 >> /dev/null
LISTENER_PID=$!
start_all_workers
;; ;;
esac esac
} }