PPSS tests fixed.
This commit is contained in:
Louwrentius
parent
6836784b54
commit
c1a359340c
83
ppss
83
ppss
@ -26,7 +26,7 @@
|
||||
trap 'kill_process' SIGINT
|
||||
|
||||
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
|
||||
SCRIPT_VERSION="2.70"
|
||||
SCRIPT_VERSION="2.80"
|
||||
|
||||
#
|
||||
# The first argument to this script can be a mode.
|
||||
@ -58,12 +58,10 @@ PPSS_HOME_DIR="ppss-home"
|
||||
SOURCED="$0"
|
||||
|
||||
PID="$$"
|
||||
GLOBAL_LOCK="$PPSS_DIR/PPSS-GLOBAL-LOCK-$PID" # Global lock file used by local PPSS instance.
|
||||
PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal" # Pause processing if this file is present.
|
||||
PAUSE_DELAY="60" # Polling every 1 minutes by default.
|
||||
STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal" # Stop processing if this file is present.
|
||||
GLOBAL_COUNTER=""
|
||||
GLOBAL_COUNTER_FILE="$PPSS_DIR/ppss-input-counter-$PID"
|
||||
GLOBAL_COUNTER=1
|
||||
LISTOFITEMS="$PPSS_DIR/INPUT_FILE-$PID"
|
||||
JOB_LOG_DIR="$PPSS_DIR/job_log" # Directory containing log files of processed items.
|
||||
LOGFILE="$PPSS_DIR/ppss-log-$PID.txt" # General PPSS log file. Contains lots of info.
|
||||
@ -411,16 +409,6 @@ cleanup () {
|
||||
rm "$FIFO"
|
||||
fi
|
||||
|
||||
if [ -e "$GLOBAL_COUNTER_FILE" ]
|
||||
then
|
||||
rm "$GLOBAL_COUNTER_FILE"
|
||||
fi
|
||||
|
||||
if [ -e "$GLOBAL_LOCK" ]
|
||||
then
|
||||
rm -rf "$GLOBAL_LOCK"
|
||||
fi
|
||||
|
||||
if [ -e "$SSH_SOCKET" ]
|
||||
then
|
||||
rm -rf "$SSH_SOCKET"
|
||||
@ -896,12 +884,6 @@ init_vars () {
|
||||
MIN_JOBS=3
|
||||
fi
|
||||
|
||||
#
|
||||
# Create a remote homedir for PPSS
|
||||
#
|
||||
|
||||
echo 1 > $GLOBAL_COUNTER_FILE
|
||||
|
||||
FIFO=/tmp/ppss-fifo-$RANDOM-$RANDOM
|
||||
|
||||
if [ ! -e "$FIFO" ]
|
||||
@ -991,6 +973,7 @@ init_vars () {
|
||||
then
|
||||
mkdir "$PPSS_LOCAL_OUTPUT"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
get_status () {
|
||||
@ -1039,13 +1022,17 @@ erase_ppss () {
|
||||
}
|
||||
|
||||
ec2_get_pending_nodes() {
|
||||
|
||||
#
|
||||
# This function has naver been tested by the author of PPSS.
|
||||
#
|
||||
RES="$(ec2-describe-instances | grep 'INSTANCE' | awk '{print $4}'| grep pending)"
|
||||
echo "$RES"
|
||||
}
|
||||
|
||||
ec2_launch_nodes() {
|
||||
|
||||
#
|
||||
# This function has naver been tested by the author of PPSS.
|
||||
#
|
||||
ec2run $AMI_ID -n $NUM_NODES -t $INSTANCE_TYPE -k $AWS_KEYPAIR -g $SECURITY_GROUP
|
||||
|
||||
#
|
||||
@ -1603,6 +1590,7 @@ get_all_items () {
|
||||
fi
|
||||
|
||||
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
|
||||
|
||||
if [ "$SIZE_OF_INPUT" -le "0" ]
|
||||
then
|
||||
log ERROR "Source file/dir seems to be empty."
|
||||
@ -1610,7 +1598,6 @@ get_all_items () {
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
get_item () {
|
||||
@ -1619,30 +1606,42 @@ get_item () {
|
||||
|
||||
if [ "$STOP" == "1" ]
|
||||
then
|
||||
log DEBUG "Found stop signal."
|
||||
return 1
|
||||
fi
|
||||
|
||||
#
|
||||
# Return error if list size is empty.
|
||||
#
|
||||
if [ -z "$SIZE_OF_INPUT" ]
|
||||
then
|
||||
log DEBUG "Got no size of input..."
|
||||
return 1
|
||||
fi
|
||||
|
||||
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
|
||||
#
|
||||
# Return error if the list is empty.
|
||||
#
|
||||
#if [ "$SIZE_OF_INPUT" -le "0" ]
|
||||
#then
|
||||
# return 1
|
||||
#fi
|
||||
|
||||
#
|
||||
# This variable is used to walk thtough all input file items.
|
||||
#
|
||||
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
|
||||
if [ "$SIZE_OF_INPUT" -le "0" ]
|
||||
then
|
||||
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
|
||||
if [ "$?" = "0" ]
|
||||
then
|
||||
return 0
|
||||
else
|
||||
log DEBUG "Size of input 0 or less."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Check if all items have been processed.
|
||||
#
|
||||
#if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]
|
||||
#then
|
||||
# return 1
|
||||
#fi
|
||||
if [ "$GLOBAL_COUNTER" -gt "$SIZE_OF_INPUT" ]
|
||||
then
|
||||
log DEBUG "Counter $GLOBAL_COUNTER is greater than sizeof input $SIZE_OF_INPUT."
|
||||
return 1
|
||||
fi
|
||||
|
||||
ITEM="$(sed -n $GLOBAL_COUNTER\p $LISTOFITEMS)"
|
||||
|
||||
@ -1652,16 +1651,11 @@ get_item () {
|
||||
return 1
|
||||
else
|
||||
((GLOBAL_COUNTER++))
|
||||
echo $GLOBAL_COUNTER > $GLOBAL_COUNTER_FILE
|
||||
LOCK=""
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
lock_item "$ITEM"
|
||||
LOCK="$?"
|
||||
else
|
||||
log DEBUG "Item lock disabled - no remote server configured."
|
||||
LOCK=0
|
||||
fi
|
||||
if [ ! "$LOCK" = "0" ]
|
||||
then
|
||||
log DEBUG "Item $ITEM is locked."
|
||||
@ -1669,6 +1663,7 @@ get_item () {
|
||||
# Recursion, get_ttem calls itself, until all items are done.
|
||||
#
|
||||
get_item
|
||||
fi
|
||||
else
|
||||
log DEBUG "Got lock on $ITEM, processing."
|
||||
download_item "$ITEM"
|
||||
@ -1867,7 +1862,6 @@ commando () {
|
||||
RETVAL="$?"
|
||||
if [ "$RETVAL" = "0" ]
|
||||
then
|
||||
echo "$TMP - $RETVAL" >> /tmp/hoeba
|
||||
eval "$COMMAND" >> "$ITEM_LOG_FILE" 2>&1
|
||||
ERROR="$?"
|
||||
MYPID="$!"
|
||||
@ -2027,7 +2021,6 @@ listen_for_job () {
|
||||
fi
|
||||
|
||||
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
|
||||
GLOBAL_COUNTER=$(cat $GLOBAL_COUNTER_FILE)
|
||||
PERCENT=$((100 * $GLOBAL_COUNTER / $SIZE_OF_INPUT ))
|
||||
if [ "$DIED" == "0" ] && [ "$FINISHED" == "0" ]
|
||||
then
|
||||
|
||||
39
ppss-test.sh
39
ppss-test.sh
@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
DEBUG="$1"
|
||||
VERSION="2.70"
|
||||
VERSION="2.80"
|
||||
TMP_DIR="/tmp/ppss"
|
||||
PPSS=./ppss
|
||||
PPSS_DIR=ppss_dir
|
||||
@ -14,6 +14,7 @@ cleanup () {
|
||||
|
||||
unset RES1
|
||||
unset RES2
|
||||
GLOBAL_COUNTER=1
|
||||
|
||||
for x in $REMOVEFILES
|
||||
do
|
||||
@ -37,10 +38,15 @@ parseJobStatus () {
|
||||
echo "$STATUS"
|
||||
}
|
||||
|
||||
get_contents_of_input_file () {
|
||||
get_item_count_of_input_file () {
|
||||
|
||||
RES=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
|
||||
echo "$RES"
|
||||
if [ -e "$PPSS_DIR/INPUT_FILE-$$" ]
|
||||
then
|
||||
CONTENTS_OF_INPUTFILE=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
|
||||
echo "$CONTENTS_OF_INPUTFILE"
|
||||
else
|
||||
echo "Error, file $PPSS_DIR/INPUT_FILE-$$ does not exist."
|
||||
fi
|
||||
}
|
||||
|
||||
oneTimeSetUp () {
|
||||
@ -131,22 +137,21 @@ testMD5 () {
|
||||
|
||||
init_get_all_items () {
|
||||
|
||||
RECURSION="$2"
|
||||
DIR="$1"
|
||||
RECURSION="$2"
|
||||
createDirectoryWithSomeFiles
|
||||
create_working_directory
|
||||
init_vars > /dev/null 2>&1
|
||||
export SRC_DIR=$DIR
|
||||
init_vars > /dev/null 2>&1
|
||||
get_all_items
|
||||
RES=`get_contents_of_input_file`
|
||||
}
|
||||
|
||||
testRecursion () {
|
||||
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
|
||||
RESULT=`get_item_count_of_input_file`
|
||||
EXPECTED=32
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RES"
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
|
||||
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
@ -154,9 +159,10 @@ testRecursion () {
|
||||
testNoRecursion () {
|
||||
|
||||
init_get_all_items $TMP_DIR/root 0
|
||||
RESULT=`get_item_count_of_input_file`
|
||||
EXPECTED=12
|
||||
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RES"
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
|
||||
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
@ -192,23 +198,25 @@ return_all_items () {
|
||||
|
||||
while get_item
|
||||
do
|
||||
RES2="$RES2$ITEM"$'\n'
|
||||
ALL_ITEMS="$ALL_ITEMS$ITEM"$'\n'
|
||||
done
|
||||
echo "$ALL_ITEMS"
|
||||
}
|
||||
|
||||
testNumberOfItems () {
|
||||
|
||||
createSpecialFilenames
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
RESULT=`init_get_all_items $TMP_DIR/root 1`
|
||||
|
||||
RES1=`find $TMP_DIR/root/ ! -type d`
|
||||
RES1="$RES1"$'\n' # NASTY HACK YUCK
|
||||
|
||||
return_all_items
|
||||
RES2=`return_all_items`
|
||||
|
||||
echo "$RES1" > a
|
||||
echo "$RES2" > b
|
||||
|
||||
assertEquals "Input file and actual files not the same!" "$RES1" "$RES2"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
cleanup
|
||||
}
|
||||
|
||||
testNumberOfLogfiles () {
|
||||
@ -223,7 +231,6 @@ testNumberOfLogfiles () {
|
||||
RES=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'`
|
||||
assertEquals "Got wrong number of log files." 40 "$RES"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
cleanup
|
||||
}
|
||||
|
||||
. ./shunit2
|
||||
|
||||
Loading…
Reference in New Issue
Block a user