sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Updated version of distributed PPSS - does not work - not tested

Browse Source
This commit is contained in:
Louwrentius 2009-01-24 23:36:18 +00:00
parent 7bb1641075
commit 4a232fcb96
1 changed files with 31 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
trunk/ppss.sh
View File

@ -39,7 +39,7 @@ trap 'kill_process; ' INT
# Setting some vars. Do not change. # Setting some vars. Do not change.
SCRIPT_NAME="Parallel Processing Shell Script" SCRIPT_NAME="Parallel Processing Shell Script"
SCRIPT_VERSION="1.08" SCRIPT_VERSION="1.10"
RUNNING_SIGNAL="$0_is_running" RUNNING_SIGNAL="$0_is_running"
GLOBAL_LOCK="PPSS-$RANDOM-$RANDOM" GLOBAL_LOCK="PPSS-$RANDOM-$RANDOM"
@ -55,6 +55,8 @@ IFS_BACKUP="$IFS"
SSH_SERVER="" # Remote server or 'master'. SSH_SERVER="" # Remote server or 'master'.
SSH_KEY="" # SSH key for ssh account. SSH_KEY="" # SSH key for ssh account.
SSH_OPTS="-o \"BatchMode yes\" \"ControlPath /tmp/master-%r@%h:%p\" -o \"ControlMaster auto\""
SSH_MASTER_PID=""
showusage () { showusage () {
@ -112,6 +114,12 @@ kill_process () {
exit 0 exit 0
fi fi
done done
# The master SSH connection should be killed.
if [ ! -z "$SSH_MASTER_PID" ]
then
kill -9 "$SSH_MASTER_PID"
fi
} }
@ -238,12 +246,19 @@ exec_cmd () {
if [ ! -z "$SSH_SERVER" ] if [ ! -z "$SSH_SERVER" ]
then then
ssh "$SSH_KEY" "$SSH_SERVER" "$CMD" ssh "$SSH_OPTS" "$SSH_KEY" "$SSH_SERVER" "$CMD"
else else
`"$CMD"` "$CMD"
fi fi
} }
# this function makes remote or local checking of existence of items transparent.
does_file_exist () {
FILE="$1"
exec_cmd "if [ -e \"$FILE\" ]; then return 0; else return 1; fi"
}
# Init all vars # Init all vars
init_vars () { init_vars () {
@ -344,11 +359,15 @@ check_status () {
test_server () { test_server () {
# Testing if the remote server works as expected.
if [ ! -z "$SSH_SERVER" ] if [ ! -z "$SSH_SERVER" ]
then then
ssh "$SSH_KEY" "$SSH_SERVER" date >> /dev/null 2>&1 ssh -N -M "$SSH_OPTS" "$SSH_KEY" "$SSH_SERVER" &
SSH_MASTER_PID="$!"
check_status "$?" "$FUNCNAME" "Server $SSH_SERVER could not be reached." check_status "$?" "$FUNCNAME" "Server $SSH_SERVER could not be reached."
exec_cmd "read" &
else else
log DEBUG "No remote server specified, assuming stand-alone mode." log DEBUG "No remote server specified, assuming stand-alone mode."
fi fi
@ -557,7 +576,6 @@ get_item () {
log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items." log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
echo -en "\033[1A" echo -en "\033[1A"
# Check if all items have been processed. # Check if all items have been processed.
if [ "$ARRAY_POINTER" -ge "$SIZE_OF_ARRAY" ] if [ "$ARRAY_POINTER" -ge "$SIZE_OF_ARRAY" ]
then then
@ -579,7 +597,6 @@ get_item () {
release_global_lock release_global_lock
return 0 return 0
fi fi
} }
start_single_worker () { start_single_worker () {
@ -610,7 +627,8 @@ commando () {
LOG_FILE_NAME=`echo $ITEM | sed s/^\\.//g | sed s/^\\.\\.//g | sed s/\\\///g` LOG_FILE_NAME=`echo $ITEM | sed s/^\\.//g | sed s/^\\.\\.//g | sed s/\\\///g`
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME" ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
if [ -e "$ITEM_LOG_FILE" ] does_file_exist "$ITEM_LOG_FILE"
if [ "$0" == "0" ]
then then
log DEBUG "Skipping item $ITEM - already processed." log DEBUG "Skipping item $ITEM - already processed."
else else
@ -619,9 +637,12 @@ commando () {
eval "$EXECME" eval "$EXECME"
fi fi
get_global_lock if [ ! -z "$SSH_SERVER" ]
scp "$SSH_KEY" "$ITEM_LOG_FILE" "$SSH_SERVER:~/$JOB_LOG" then
release_global_lock get_global_lock
scp "$SSH_KEY" "$ITEM_LOG_FILE" "$SSH_SERVER:~/$JOB_LOG"
release_global_lock
fi
start_single_worker start_single_worker
return $? return $?
@ -663,7 +684,6 @@ main () {
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & listen_for_job "$MAX_NO_OF_RUNNING_JOBS" &
LISTENER_PID=$! LISTENER_PID=$!
start_all_workers start_all_workers
} }
# This command starts the that sets the whole framework in motion. # This command starts the that sets the whole framework in motion.
main main