sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Backup, fixing distributed processing. Seems to work now.

Browse Source
This commit is contained in:
Louwrentius 2010-03-07 17:34:12 +00:00
parent 54c898f202
commit 831e008e11
1 changed files with 54 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
ppss
View File

@ -1000,6 +1000,7 @@ test_server () {
ssh -N -M $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER & ssh -N -M $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER &
SSH_MASTER_PID="$!" SSH_MASTER_PID="$!"
log DEBUG "SSH Master pid is $SSH_MASTER_PID" log DEBUG "SSH Master pid is $SSH_MASTER_PID"
log DSPLY "Connected to server: $SSH_SERVER"
else else
log DEBUG "No remote server specified, assuming stand-alone mode." log DEBUG "No remote server specified, assuming stand-alone mode."
fi fi
@ -1198,19 +1199,27 @@ download_item () {
ITEM="$1" ITEM="$1"
VIRTUAL="0" VIRTUAL="0"
ERR_STATE="0"
if [ "$RECURSION" = "1" ] if [ "$RECURSION" = "1" ]
then then
does_file_exist "$ITEM" escape_item "$ITEM"
does_file_exist "$ITEM_ESCAPED"
ERR_STATE="$?"
DOWNLOAD_ITEM="$ITEM" DOWNLOAD_ITEM="$ITEM"
LOCAL_DIR=`dirname "$DOWNLOAD_ITEM"`
else else
does_file_exist "$SRC_DIR/$ITEM" escape_item "$ITEM"
does_file_exist "$SRC_DIR/$ITEM_ESCAPED"
ERR_STATE="$?"
DOWNLOAD_ITEM="$SRC_DIR/$ITEM" DOWNLOAD_ITEM="$SRC_DIR/$ITEM"
fi fi
if [ "$?" == "0" ] if [ "$ERR_STATE" == "0" ]
then then
log DEBUG "$FUNCNAME Remote item $ITEM exists" log DEBUG "$FUNCNAME Remote item $ITEM exists"
VIRTUAL=0 VIRTUAL=0
else else
log DEBUG "$FUNCNAME Remote item $ITEM does NOT exist" log DEBUG "$FUNCNAME Remote item $ITEM does NOT exist"
VIRTUAL=1 VIRTUAL=1
@ -1221,10 +1230,20 @@ download_item () {
log DEBUG "Transfering item $ITEM from source to local disk." log DEBUG "Transfering item $ITEM from source to local disk."
if [ "$SECURE_COPY" == "1" ] && [ ! -z "$SSH_SERVER" ] if [ "$SECURE_COPY" == "1" ] && [ ! -z "$SSH_SERVER" ]
then then
escape_item "$DOWNLOAD_ITEM" if [ "$RECURSION" == "1" ]
log DEBUG "$SSH_SERVER:$ITEM_ESCAPED $PPSS_LOCAL_TMPDIR" then
scp -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER:"$ITEM_ESCAPED" $PPSS_LOCAL_TMPDIR escape_item "$DOWNLOAD_ITEM"
log DEBUG "Exit code of remote transfer is $?" mkdir -p "$PPSS_LOCAL_TMPDIR/$LOCAL_DIR"
log DEBUG "$SSH_SERVER:$ITEM_ESCAPED $PPSS_LOCAL_TMPDIR/$LOCAL_DIR"
scp -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER:"$ITEM_ESCAPED" $PPSS_LOCAL_TMPDIR/"$LOCAL_DIR"
log DEBUG "Exit code of remote transfer is $?"
else
escape_item "$DOWNLOAD_ITEM"
log DEBUG "$SSH_SERVER:$ITEM_ESCAPED $PPSS_LOCAL_TMPDIR"
scp -q $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER:"$ITEM_ESCAPED" $PPSS_LOCAL_TMPDIR
log DEBUG "Exit code of remote transfer is $?"
fi
else else
cp "$ITEM" $PPSS_LOCAL_TMPDIR cp "$ITEM" $PPSS_LOCAL_TMPDIR
log DEBUG "Exit code of local transfer is $?" log DEBUG "Exit code of local transfer is $?"
@ -1242,15 +1261,20 @@ upload_item () {
log DEBUG "Uploading item $OUTPUT_ITEM." log DEBUG "Uploading item $OUTPUT_ITEM."
if [ "$SECURE_COPY" == "1" ] if [ "$SECURE_COPY" == "1" ]
then then
if [ "$RECURSION" = "1" ]
escape_item "$REMOTE_OUTPUT_DIR$ITEMDIR" then
escape_item "$REMOTE_OUTPUT_DIR$ITEMDIR"
else
escape_item "$REMOTE_OUTPUT_DIR"
fi
DIR_ESCAPED="$ITEM_ESCAPED" DIR_ESCAPED="$ITEM_ESCAPED"
exec_cmd "mkdir -p $DIR_ESCAPED"
scp -q $SSH_OPTS $SSH_KEY "$OUTPUT_ITEM"/* $USER@$SSH_SERVER:"$DIR_ESCAPED" scp -q $SSH_OPTS $SSH_KEY "$OUTPUT_ITEM"/* $USER@$SSH_SERVER:"$DIR_ESCAPED" 2>/dev/null
ERROR="$?" ERROR="$?"
if [ ! "$ERROR" == "0" ] if [ ! "$ERROR" == "0" ]
then then
log ERROR "Uploading of $OUTPUT_ITEM via SCP failed." log ERROR "Uploading of $OUTPUT_ITEM_ESCAPED via SCP failed."
else else
log DEBUG "Upload of item $OUTPUT_ITEM success" log DEBUG "Upload of item $OUTPUT_ITEM success"
rm -rf ./"$OUTPUT_ITEM" rm -rf ./"$OUTPUT_ITEM"
@ -1511,10 +1535,12 @@ commando () {
if [ "$RECURSION" == "1" ] if [ "$RECURSION" == "1" ]
then then
does_file_exist "$ITEM" escape_item "$ITEM"
does_file_exist "$ITEM_ESCAPED"
ERR_STATE="$?" ERR_STATE="$?"
else else
does_file_exist "$SRC_DIR/$ITEM" escape_item "$ITEM"
does_file_exist "$SRC_DIR/$ITEM_ESCAPED"
ERR_STATE="$?" ERR_STATE="$?"
fi fi
@ -1573,7 +1599,7 @@ commando () {
else else
if [ "$RECURSION" == "1" ] if [ "$RECURSION" == "1" ]
then then
ITEM="$PPSS_LOCAL_TMPDIR/$DIR_NAME/$ITEM_NO_PATH" ITEM="$PPSS_LOCAL_TMPDIR/$ITEM"
else else
ITEM="$PPSS_LOCAL_TMPDIR/$ITEM_NO_PATH" ITEM="$PPSS_LOCAL_TMPDIR/$ITEM_NO_PATH"
fi fi
@ -1658,7 +1684,7 @@ commando () {
# If part of a cluster, remove the downloaded item after # If part of a cluster, remove the downloaded item after
# it has been processed and uploaded as not to fill up disk space. # it has been processed and uploaded as not to fill up disk space.
# #
if [ "$TRANSFER_TO_SLAVE" == "1" ] if [ "$TRANSFER_TO_SLAVE" == "1" ] && [ "1" == "2" ]
then then
if [ -e "$ITEM" ] if [ -e "$ITEM" ]
then then
@ -1679,7 +1705,6 @@ commando () {
then then
log DEBUG "File transfer is disabled." log DEBUG "File transfer is disabled."
else else
exec_cmd "mkdir -p $ITEM_OUTPUT_DIR"
if [ "$DIR_NAME" == "." ] if [ "$DIR_NAME" == "." ]
then then
DIR_NAME="" DIR_NAME=""
@ -1687,7 +1712,6 @@ commando () {
upload_item "$OUTPUT_DIR" "$DIR_NAME" upload_item "$OUTPUT_DIR" "$DIR_NAME"
fi fi
elapsed "$BEFORE" "$AFTER" >> "$ITEM_LOG_FILE" elapsed "$BEFORE" "$AFTER" >> "$ITEM_LOG_FILE"
echo -e "" >> "$ITEM_LOG_FILE" echo -e "" >> "$ITEM_LOG_FILE"
@ -1751,6 +1775,8 @@ listen_for_job () {
# process as defined by $PID. All processes that have ever been # process as defined by $PID. All processes that have ever been
# spawned, although disowned or backgrounded will be killed... # spawned, although disowned or backgrounded will be killed...
# #
# *** THIS MAY BE DEAD CODE ***
#
PROCLIST=`ps a -o pid,pgid,ppid,command | grep [0-9] | grep $PID | grep -v -i grep` PROCLIST=`ps a -o pid,pgid,ppid,command | grep [0-9] | grep $PID | grep -v -i grep`
#echo "$PROCLIST" > proclist.txt #echo "$PROCLIST" > proclist.txt
oldIFS=$IFS # save the field separator oldIFS=$IFS # save the field separator
@ -1774,10 +1800,6 @@ listen_for_job () {
done done
IFS=$oldIFS IFS=$oldIFS
if [ ! -z "$SSH_MASTER_PID" ]
then
kill "$SSH_MASTER_PID"
fi
break break
else else
commando "$event" & commando "$event" &
@ -1804,16 +1826,24 @@ listen_for_job () {
fi fi
done done
if [ ! -z "$SSH_MASTER_PID" ]
then
log DEBUG "SSH master PID is $SSH_MASTER_PID"
kill "$SSH_MASTER_PID"
else
log DEBUG "SSH master PID is empty."
fi
set_status STOPPED set_status STOPPED
log DEBUG "Listener stopped." log DEBUG "Listener stopped."
if [ ! "$PERCENT" == "100" ] if [ ! "$PERCENT" == "100" ]
then then
echo echo
stop-ppss stop-ppss
log DSPLY "Finished. Consult $JOB_LOG_DIR for job output." log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
log DSPLY "Press ENTER to continue." log DSPLY "Press ENTER to continue."
else else
stop-ppss stop-ppss
log DSPLY "Finished. Consult $JOB_LOG_DIR for job output." log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
fi fi
cleanup cleanup