PPSS tests fixed.

This commit is contained in:
Louwrentius 2010-06-27 08:59:22 +00:00
parent 6836784b54
commit c1a359340c
2 changed files with 68 additions and 68 deletions

97
ppss
View File

@ -26,7 +26,7 @@
trap 'kill_process' SIGINT
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.70"
SCRIPT_VERSION="2.80"
#
# The first argument to this script can be a mode.
@ -58,12 +58,10 @@ PPSS_HOME_DIR="ppss-home"
SOURCED="$0"
PID="$$"
GLOBAL_LOCK="$PPSS_DIR/PPSS-GLOBAL-LOCK-$PID" # Global lock file used by local PPSS instance.
PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal" # Pause processing if this file is present.
PAUSE_DELAY="60" # Polling every 1 minutes by default.
STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal" # Stop processing if this file is present.
GLOBAL_COUNTER=""
GLOBAL_COUNTER_FILE="$PPSS_DIR/ppss-input-counter-$PID"
GLOBAL_COUNTER=1
LISTOFITEMS="$PPSS_DIR/INPUT_FILE-$PID"
JOB_LOG_DIR="$PPSS_DIR/job_log" # Directory containing log files of processed items.
LOGFILE="$PPSS_DIR/ppss-log-$PID.txt" # General PPSS log file. Contains lots of info.
@ -411,16 +409,6 @@ cleanup () {
rm "$FIFO"
fi
if [ -e "$GLOBAL_COUNTER_FILE" ]
then
rm "$GLOBAL_COUNTER_FILE"
fi
if [ -e "$GLOBAL_LOCK" ]
then
rm -rf "$GLOBAL_LOCK"
fi
if [ -e "$SSH_SOCKET" ]
then
rm -rf "$SSH_SOCKET"
@ -896,12 +884,6 @@ init_vars () {
MIN_JOBS=3
fi
#
# Create a remote homedir for PPSS
#
echo 1 > $GLOBAL_COUNTER_FILE
FIFO=/tmp/ppss-fifo-$RANDOM-$RANDOM
if [ ! -e "$FIFO" ]
@ -991,6 +973,7 @@ init_vars () {
then
mkdir "$PPSS_LOCAL_OUTPUT"
fi
}
get_status () {
@ -1039,13 +1022,17 @@ erase_ppss () {
}
ec2_get_pending_nodes() {
#
# This function has naver been tested by the author of PPSS.
#
RES="$(ec2-describe-instances | grep 'INSTANCE' | awk '{print $4}'| grep pending)"
echo "$RES"
}
ec2_launch_nodes() {
#
# This function has naver been tested by the author of PPSS.
#
ec2run $AMI_ID -n $NUM_NODES -t $INSTANCE_TYPE -k $AWS_KEYPAIR -g $SECURITY_GROUP
#
@ -1603,6 +1590,7 @@ get_all_items () {
fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
if [ "$SIZE_OF_INPUT" -le "0" ]
then
log ERROR "Source file/dir seems to be empty."
@ -1610,7 +1598,6 @@ get_all_items () {
cleanup
exit 1
fi
}
get_item () {
@ -1619,30 +1606,42 @@ get_item () {
if [ "$STOP" == "1" ]
then
log DEBUG "Found stop signal."
return 1
fi
#
# Return error if list size is empty.
#
if [ -z "$SIZE_OF_INPUT" ]
then
log DEBUG "Got no size of input..."
return 1
fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
#
# Return error if the list is empty.
#
#if [ "$SIZE_OF_INPUT" -le "0" ]
#then
# return 1
#fi
#
# This variable is used to walk thtough all input file items.
#
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
if [ "$SIZE_OF_INPUT" -le "0" ]
then
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
if [ "$?" = "0" ]
then
return 0
else
log DEBUG "Size of input 0 or less."
return 1
fi
fi
#
# Check if all items have been processed.
#
#if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]
#then
# return 1
#fi
if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]
then
log DEBUG "Counter $GLOBAL_COUNTER is greater than sizeof input $SIZE_OF_INPUT."
return 1
fi
ITEM="$(sed -n $GLOBAL_COUNTER\p $LISTOFITEMS)"
@ -1652,23 +1651,19 @@ get_item () {
return 1
else
((GLOBAL_COUNTER++))
echo $GLOBAL_COUNTER > $GLOBAL_COUNTER_FILE
LOCK=""
if [ ! -z "$SSH_SERVER" ]
then
lock_item "$ITEM"
LOCK="$?"
else
log DEBUG "Item lock disabled - no remote server configured."
LOCK=0
fi
if [ ! "$LOCK" = "0" ]
then
log DEBUG "Item $ITEM is locked."
#
# Recursion, get_ttem calls itself, until all items are done.
#
get_item
if [ ! "$LOCK" = "0" ]
then
log DEBUG "Item $ITEM is locked."
#
# Recursion, get_ttem calls itself, until all items are done.
#
get_item
fi
else
log DEBUG "Got lock on $ITEM, processing."
download_item "$ITEM"
@ -1867,7 +1862,6 @@ commando () {
RETVAL="$?"
if [ "$RETVAL" = "0" ]
then
echo "$TMP - $RETVAL" >> /tmp/hoeba
eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
ERROR="$?"
MYPID="$!"
@ -2027,7 +2021,6 @@ listen_for_job () {
fi
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
PERCENT=$((100 * $GLOBAL_COUNTER / $SIZE_OF_INPUT ))
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
then

View File

@ -1,7 +1,7 @@
#!/bin/bash
DEBUG="$1"
VERSION="2.70"
VERSION="2.80"
TMP_DIR="/tmp/ppss"
PPSS=./ppss
PPSS_DIR=ppss_dir
@ -14,6 +14,7 @@ cleanup () {
unset RES1
unset RES2
GLOBAL_COUNTER=1
for x in $REMOVEFILES
do
@ -37,10 +38,15 @@ parseJobStatus () {
echo "$STATUS"
}
get_contents_of_input_file () {
get_item_count_of_input_file () {
RES=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
echo "$RES"
if [ -e "$PPSS_DIR/INPUT_FILE-$$" ]
then
CONTENTS_OF_INPUTFILE=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
echo "$CONTENTS_OF_INPUTFILE"
else
echo "Error, file $PPSS_DIR/INPUT_FILE-$$ does not exist."
fi
}
oneTimeSetUp () {
@ -131,22 +137,21 @@ testMD5 () {
init_get_all_items () {
RECURSION="$2"
DIR="$1"
RECURSION="$2"
createDirectoryWithSomeFiles
create_working_directory
init_vars > /dev/null 2>&1
export SRC_DIR=$DIR
init_vars > /dev/null 2>&1
get_all_items
RES=`get_contents_of_input_file`
}
testRecursion () {
init_get_all_items $TMP_DIR/root 1
RESULT=`get_item_count_of_input_file`
EXPECTED=32
assertEquals "Recursion not correct." "$EXPECTED" "$RES"
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
rename-ppss-dir $FUNCNAME
}
@ -154,9 +159,10 @@ testRecursion () {
testNoRecursion () {
init_get_all_items $TMP_DIR/root 0
RESULT=`get_item_count_of_input_file`
EXPECTED=12
assertEquals "Recursion not correct." "$EXPECTED" "$RES"
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
rename-ppss-dir $FUNCNAME
}
@ -192,23 +198,25 @@ return_all_items () {
while get_item
do
RES2="$RES2$ITEM"$'\n'
ALL_ITEMS="$ALL_ITEMS$ITEM"$'\n'
done
echo "$ALL_ITEMS"
}
testNumberOfItems () {
createSpecialFilenames
init_get_all_items $TMP_DIR/root 1
RESULT=`init_get_all_items $TMP_DIR/root 1`
RES1=`find $TMP_DIR/root/ ! -type d`
RES1="$RES1"$'\n' # NASTY HACK YUCK
return_all_items
RES2=`return_all_items`
echo "$RES1" > a
echo "$RES2" > b
assertEquals "Input file and actual files not the same!" "$RES1" "$RES2"
rename-ppss-dir $FUNCNAME
cleanup
}
testNumberOfLogfiles () {
@ -223,7 +231,6 @@ testNumberOfLogfiles () {
RES=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'`
assertEquals "Got wrong number of log files." 40 "$RES"
rename-ppss-dir $FUNCNAME
cleanup
}
. ./shunit2