sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

PPSS tests fixed.

Browse Source
This commit is contained in:
Louwrentius 2010-06-27 08:59:22 +00:00
parent 6836784b54
commit c1a359340c
2 changed files with 68 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
ppss
View File

@ -26,7 +26,7 @@
trap 'kill_process' SIGINT trap 'kill_process' SIGINT
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.70" SCRIPT_VERSION="2.80"
# #
# The first argument to this script can be a mode. # The first argument to this script can be a mode.
@ -58,12 +58,10 @@ PPSS_HOME_DIR="ppss-home"
SOURCED="$0" SOURCED="$0"
PID="$$" PID="$$"
GLOBAL_LOCK="$PPSS_DIR/PPSS-GLOBAL-LOCK-$PID" # Global lock file used by local PPSS instance.
PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal" # Pause processing if this file is present. PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal" # Pause processing if this file is present.
PAUSE_DELAY="60" # Polling every 1 minutes by default. PAUSE_DELAY="60" # Polling every 1 minutes by default.
STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal" # Stop processing if this file is present. STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal" # Stop processing if this file is present.
GLOBAL_COUNTER="" GLOBAL_COUNTER=1
GLOBAL_COUNTER_FILE="$PPSS_DIR/ppss-input-counter-$PID"
LISTOFITEMS="$PPSS_DIR/INPUT_FILE-$PID" LISTOFITEMS="$PPSS_DIR/INPUT_FILE-$PID"
JOB_LOG_DIR="$PPSS_DIR/job_log" # Directory containing log files of processed items. JOB_LOG_DIR="$PPSS_DIR/job_log" # Directory containing log files of processed items.
LOGFILE="$PPSS_DIR/ppss-log-$PID.txt" # General PPSS log file. Contains lots of info. LOGFILE="$PPSS_DIR/ppss-log-$PID.txt" # General PPSS log file. Contains lots of info.
@ -411,16 +409,6 @@ cleanup () {
rm "$FIFO" rm "$FIFO"
fi fi
if [ -e "$GLOBAL_COUNTER_FILE" ]
then
rm "$GLOBAL_COUNTER_FILE"
fi
if [ -e "$GLOBAL_LOCK" ]
then
rm -rf "$GLOBAL_LOCK"
fi
if [ -e "$SSH_SOCKET" ] if [ -e "$SSH_SOCKET" ]
then then
rm -rf "$SSH_SOCKET" rm -rf "$SSH_SOCKET"
@ -896,12 +884,6 @@ init_vars () {
MIN_JOBS=3 MIN_JOBS=3
fi fi
#
# Create a remote homedir for PPSS
#
echo 1 > $GLOBAL_COUNTER_FILE
FIFO=/tmp/ppss-fifo-$RANDOM-$RANDOM FIFO=/tmp/ppss-fifo-$RANDOM-$RANDOM
if [ ! -e "$FIFO" ] if [ ! -e "$FIFO" ]
@ -991,6 +973,7 @@ init_vars () {
then then
mkdir "$PPSS_LOCAL_OUTPUT" mkdir "$PPSS_LOCAL_OUTPUT"
fi fi
} }
get_status () { get_status () {
@ -1039,13 +1022,17 @@ erase_ppss () {
} }
ec2_get_pending_nodes() { ec2_get_pending_nodes() {
#
# This function has naver been tested by the author of PPSS.
#
RES="$(ec2-describe-instances | grep 'INSTANCE' | awk '{print $4}'| grep pending)" RES="$(ec2-describe-instances | grep 'INSTANCE' | awk '{print $4}'| grep pending)"
echo "$RES" echo "$RES"
} }
ec2_launch_nodes() { ec2_launch_nodes() {
#
# This function has naver been tested by the author of PPSS.
#
ec2run $AMI_ID -n $NUM_NODES -t $INSTANCE_TYPE -k $AWS_KEYPAIR -g $SECURITY_GROUP ec2run $AMI_ID -n $NUM_NODES -t $INSTANCE_TYPE -k $AWS_KEYPAIR -g $SECURITY_GROUP
# #
@ -1603,6 +1590,7 @@ get_all_items () {
fi fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }') SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
if [ "$SIZE_OF_INPUT" -le "0" ] if [ "$SIZE_OF_INPUT" -le "0" ]
then then
log ERROR "Source file/dir seems to be empty." log ERROR "Source file/dir seems to be empty."
@ -1610,7 +1598,6 @@ get_all_items () {
cleanup cleanup
exit 1 exit 1
fi fi
} }
get_item () { get_item () {
@ -1619,30 +1606,42 @@ get_item () {
if [ "$STOP" == "1" ] if [ "$STOP" == "1" ]
then then
log DEBUG "Found stop signal."
return 1
fi
#
# Return error if list size is empty.
#
if [ -z "$SIZE_OF_INPUT" ]
then
log DEBUG "Got no size of input..."
return 1 return 1
fi fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
# #
# Return error if the list is empty. # Return error if the list is empty.
# #
#if [ "$SIZE_OF_INPUT" -le "0" ] if [ "$SIZE_OF_INPUT" -le "0" ]
#then then
# return 1 SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
#fi if [ "$?" = "0" ]
then
return 0
else
log DEBUG "Size of input 0 or less."
return 1
fi
fi
#
# This variable is used to walk thtough all input file items.
#
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
# #
# Check if all items have been processed. # Check if all items have been processed.
# #
#if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ] if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]
#then then
# return 1 log DEBUG "Counter $GLOBAL_COUNTER is greater than sizeof input $SIZE_OF_INPUT."
#fi return 1
fi
ITEM="$(sed -n $GLOBAL_COUNTER\p $LISTOFITEMS)" ITEM="$(sed -n $GLOBAL_COUNTER\p $LISTOFITEMS)"
@ -1652,23 +1651,19 @@ get_item () {
return 1 return 1
else else
((GLOBAL_COUNTER++)) ((GLOBAL_COUNTER++))
echo $GLOBAL_COUNTER > $GLOBAL_COUNTER_FILE
LOCK=""
if [ ! -z "$SSH_SERVER" ] if [ ! -z "$SSH_SERVER" ]
then then
lock_item "$ITEM" lock_item "$ITEM"
LOCK="$?" LOCK="$?"
else if [ ! "$LOCK" = "0" ]
log DEBUG "Item lock disabled - no remote server configured." then
LOCK=0 log DEBUG "Item $ITEM is locked."
fi #
if [ ! "$LOCK" = "0" ] # Recursion, get_ttem calls itself, until all items are done.
then #
log DEBUG "Item $ITEM is locked." get_item
# fi
# Recursion, get_ttem calls itself, until all items are done.
#
get_item
else else
log DEBUG "Got lock on $ITEM, processing." log DEBUG "Got lock on $ITEM, processing."
download_item "$ITEM" download_item "$ITEM"
@ -1867,7 +1862,6 @@ commando () {
RETVAL="$?" RETVAL="$?"
if [ "$RETVAL" = "0" ] if [ "$RETVAL" = "0" ]
then then
echo "$TMP - $RETVAL" >> /tmp/hoeba
eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1 eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
ERROR="$?" ERROR="$?"
MYPID="$!" MYPID="$!"
@ -2027,7 +2021,6 @@ listen_for_job () {
fi fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }') SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
PERCENT=$((100 * $GLOBAL_COUNTER / $SIZE_OF_INPUT )) PERCENT=$((100 * $GLOBAL_COUNTER / $SIZE_OF_INPUT ))
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ] if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
then then

39
ppss-test.sh
View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
DEBUG="$1" DEBUG="$1"
VERSION="2.70" VERSION="2.80"
TMP_DIR="/tmp/ppss" TMP_DIR="/tmp/ppss"
PPSS=./ppss PPSS=./ppss
PPSS_DIR=ppss_dir PPSS_DIR=ppss_dir
@ -14,6 +14,7 @@ cleanup () {
unset RES1 unset RES1
unset RES2 unset RES2
GLOBAL_COUNTER=1
for x in $REMOVEFILES for x in $REMOVEFILES
do do
@ -37,10 +38,15 @@ parseJobStatus () {
echo "$STATUS" echo "$STATUS"
} }
get_contents_of_input_file () { get_item_count_of_input_file () {
RES=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'` if [ -e "$PPSS_DIR/INPUT_FILE-$$" ]
echo "$RES" then
CONTENTS_OF_INPUTFILE=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
echo "$CONTENTS_OF_INPUTFILE"
else
echo "Error, file $PPSS_DIR/INPUT_FILE-$$ does not exist."
fi
} }
oneTimeSetUp () { oneTimeSetUp () {
@ -131,22 +137,21 @@ testMD5 () {
init_get_all_items () { init_get_all_items () {
RECURSION="$2"
DIR="$1" DIR="$1"
RECURSION="$2"
createDirectoryWithSomeFiles createDirectoryWithSomeFiles
create_working_directory create_working_directory
init_vars > /dev/null 2>&1
export SRC_DIR=$DIR export SRC_DIR=$DIR
init_vars > /dev/null 2>&1
get_all_items get_all_items
RES=`get_contents_of_input_file`
} }
testRecursion () { testRecursion () {
init_get_all_items $TMP_DIR/root 1 init_get_all_items $TMP_DIR/root 1
RESULT=`get_item_count_of_input_file`
EXPECTED=32 EXPECTED=32
assertEquals "Recursion not correct." "$EXPECTED" "$RES" assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
rename-ppss-dir $FUNCNAME rename-ppss-dir $FUNCNAME
} }
@ -154,9 +159,10 @@ testRecursion () {
testNoRecursion () { testNoRecursion () {
init_get_all_items $TMP_DIR/root 0 init_get_all_items $TMP_DIR/root 0
RESULT=`get_item_count_of_input_file`
EXPECTED=12 EXPECTED=12
assertEquals "Recursion not correct." "$EXPECTED" "$RES" assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
rename-ppss-dir $FUNCNAME rename-ppss-dir $FUNCNAME
} }
@ -192,23 +198,25 @@ return_all_items () {
while get_item while get_item
do do
RES2="$RES2$ITEM"$'\n' ALL_ITEMS="$ALL_ITEMS$ITEM"$'\n'
done done
echo "$ALL_ITEMS"
} }
testNumberOfItems () { testNumberOfItems () {
createSpecialFilenames createSpecialFilenames
init_get_all_items $TMP_DIR/root 1 RESULT=`init_get_all_items $TMP_DIR/root 1`
RES1=`find $TMP_DIR/root/ ! -type d` RES1=`find $TMP_DIR/root/ ! -type d`
RES1="$RES1"$'\n' # NASTY HACK YUCK
return_all_items RES2=`return_all_items`
echo "$RES1" > a
echo "$RES2" > b
assertEquals "Input file and actual files not the same!" "$RES1" "$RES2" assertEquals "Input file and actual files not the same!" "$RES1" "$RES2"
rename-ppss-dir $FUNCNAME rename-ppss-dir $FUNCNAME
cleanup
} }
testNumberOfLogfiles () { testNumberOfLogfiles () {
@ -223,7 +231,6 @@ testNumberOfLogfiles () {
RES=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'` RES=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'`
assertEquals "Got wrong number of log files." 40 "$RES" assertEquals "Got wrong number of log files." 40 "$RES"
rename-ppss-dir $FUNCNAME rename-ppss-dir $FUNCNAME
cleanup
} }
. ./shunit2 . ./shunit2