Cleaned up some code, added some comments. Released as 2.60.

This commit is contained in:
Louwrentius 2010-03-20 22:25:07 +00:00
parent 7df68e7658
commit efd6f72a5e
2 changed files with 28 additions and 20 deletions

46
ppss
View File

@ -25,7 +25,7 @@ trap 'kill_process' SIGINT
# Setting some vars. # Setting some vars.
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.60b2" SCRIPT_VERSION="2.60"
# The first argument to this script can be a mode. # The first argument to this script can be a mode.
MODES="node start config stop pause continue deploy status erase kill ec2" MODES="node start config stop pause continue deploy status erase kill ec2"
@ -110,8 +110,8 @@ showusage_short () {
echo echo
echo "Examples:" echo "Examples:"
echo " $0 -d /dir/with/some/files -c 'gzip '" echo " $0 -d /dir/with/some/files -c 'gzip '"
echo " $0 -d /dir/with/some/files -c 'gzip \"\$ITEM\"' -D 5"
echo " $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2" echo " $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2"
echo " $0 -f <file> -c 'wget -q -P /destination/directory \"\$ITEM\"' -p 10"
} }
showusage_normal () { showusage_normal () {
@ -335,9 +335,11 @@ exec_cmd () {
return $STATUS return $STATUS
} }
# this function makes remote or local checking of existence of items transparent.
does_file_exist () { does_file_exist () {
#
# this function makes remote or local checking of existence of items transparent.
#
FILE="$1" FILE="$1"
`exec_cmd "ls -1 $FILE" >> /dev/null 2>&1` `exec_cmd "ls -1 $FILE" >> /dev/null 2>&1`
if [ "$?" == "0" ] if [ "$?" == "0" ]
@ -350,6 +352,10 @@ does_file_exist () {
check_for_interrupt () { check_for_interrupt () {
#
# PPSS can be interupted with a stop or pause command.
#
does_file_exist "$STOP_SIGNAL" does_file_exist "$STOP_SIGNAL"
if [ "$?" == "0" ] if [ "$?" == "0" ]
then then
@ -373,7 +379,7 @@ check_for_interrupt () {
cleanup () { cleanup () {
#log DEBUG "$FUNCNAME - Cleaning up all temp files and processes." log DEBUG "$FUNCNAME - Cleaning up all temp files and processes."
if [ -e "$FIFO" ] if [ -e "$FIFO" ]
then then
@ -676,8 +682,10 @@ expand_str () {
log () { log () {
# Type 'INFO' is logged to the screen #
# Type 'DSPLY ERROR and WARN' is logged to the screen
# Any other log-type is only logged to the logfile. # Any other log-type is only logged to the logfile.
#
TYPE="$1" TYPE="$1"
MESG="$2" MESG="$2"
@ -864,7 +872,6 @@ erase_ppss () {
else else
log DSPLY "Aborting.." log DSPLY "Aborting.."
fi fi
sleep 1
} }
ec2_get_pending_nodes() { ec2_get_pending_nodes() {
@ -904,7 +911,6 @@ deploy () {
-o ControlMaster=auto \ -o ControlMaster=auto \
-o Cipher=blowfish \ -o Cipher=blowfish \
-o ConnectTimeout=5 " -o ConnectTimeout=5 "
ERROR=0 ERROR=0
set_error () { set_error () {
@ -927,8 +933,6 @@ deploy () {
KEY=`echo $SSH_KEY | cut -d " " -f 2` KEY=`echo $SSH_KEY | cut -d " " -f 2`
sleep 1
ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && mkdir $PPSS_HOME_DIR >> /dev/null 2>&1" ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && mkdir $PPSS_HOME_DIR >> /dev/null 2>&1"
scp -q $SSH_OPTS_NODE $SSH_KEY $0 $USER@$NODE:~/$PPSS_HOME_DIR scp -q $SSH_OPTS_NODE $SSH_KEY $0 $USER@$NODE:~/$PPSS_HOME_DIR
set_error $? set_error $?
@ -1233,8 +1237,6 @@ escape_item () {
sed s/\(/\\\\\\\\\\(/g | \ sed s/\(/\\\\\\\\\\(/g | \
sed s/\)/\\\\\\\\\\)/g ` sed s/\)/\\\\\\\\\\)/g `
} }
#sed s/\\\`/\\\\\\\\\\\\\\\`/g | \
#sed s/\\\//\\\\\\\\\\\\\\ /g | \
download_item () { download_item () {
@ -1458,25 +1460,29 @@ get_item () {
SIZE_OF_ARRAY="${#ARRAY[@]}" SIZE_OF_ARRAY="${#ARRAY[@]}"
#
# Return error if the array is empty. # Return error if the array is empty.
#
if [ "$SIZE_OF_ARRAY" -le "0" ] if [ "$SIZE_OF_ARRAY" -le "0" ]
then then
release_global_lock release_global_lock
return 1 return 1
fi fi
#
# This variable is used to walk thtough all array items. # This variable is used to walk thtough all array items.
#
ARRAY_POINTER=`cat $ARRAY_POINTER_FILE` ARRAY_POINTER=`cat $ARRAY_POINTER_FILE`
#
# Check if all items have been processed. # Check if all items have been processed.
#
if [ "$ARRAY_POINTER" -ge "$SIZE_OF_ARRAY" ] if [ "$ARRAY_POINTER" -ge "$SIZE_OF_ARRAY" ]
then then
release_global_lock release_global_lock
#echo -en "\033[1A"
return 1 return 1
fi fi
# Select an item.
ITEM="${ARRAY[$ARRAY_POINTER]}" ITEM="${ARRAY[$ARRAY_POINTER]}"
if [ -z "$ITEM" ] if [ -z "$ITEM" ]
then then
@ -1492,6 +1498,9 @@ get_item () {
then then
log DEBUG "Item $ITEM is locked." log DEBUG "Item $ITEM is locked."
release_global_lock release_global_lock
#
# Recursion, get_ttem calls itself, until all items are done.
#
get_item get_item
else else
log DEBUG "Got lock on $ITEM, processing." log DEBUG "Got lock on $ITEM, processing."
@ -1504,12 +1513,11 @@ get_item () {
start_single_worker () { start_single_worker () {
# #
# This function sends an item to the fifo. This signals # This function sends an item to the fifo. This signals
# the listener process to execute a 'worker' on this # the listener process to execute a 'worker' on this
# item, using the 'commando' function. # item, using the 'commando' function.
# #
get_item get_item
ERROR=$? ERROR=$?
if [ ! "$ERROR" == "0" ] if [ ! "$ERROR" == "0" ]

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
DEBUG="$1" DEBUG="$1"
VERSION="2.60b2" VERSION="2.60"
TMP_DIR="ppss" TMP_DIR="ppss"
PPSS=./ppss PPSS=./ppss
PPSS_DIR=ppss_dir PPSS_DIR=ppss_dir