sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed issues with the distributed computing interface. Clients upload the status to the server. 'ppss status' does not need to poll all clients for their status.

Browse Source
This commit is contained in:
louwrentius@gmail.com 2011-08-07 11:52:48 +00:00
parent 300f84ad14
commit aedad2522b
1 changed files with 38 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
trunk/ppss
View File

@ -1110,8 +1110,9 @@ upload_status () {
set_status () { set_status () {
STATUS="$1" STATUS="$1"
NO_PROCESSED=$(wc -l "$PPSS_HOME_DIR/$LIST_OF_PROCESSED_ITEMS" | cut -d " " -f 1) NO_PROCESSED=$(wc -l "$LIST_OF_PROCESSED_ITEMS" | awk '{ print $1 }' )
NODE=`cat $PPSS_DIR/$NODE_ID` NODE=`cat $PPSS_DIR/$NODE_ID`
echo "$NODE $HOSTNAME $STATUS $NO_PROCESSED" > "$NODE_STATUS_FILE" echo "$NODE $HOSTNAME $STATUS $NO_PROCESSED" > "$NODE_STATUS_FILE"
if [ ! -z "$SSH_SERVER" ] if [ ! -z "$SSH_SERVER" ]
then then
@ -1259,7 +1260,10 @@ deploy () {
ERROR=0 ERROR=0
set_error () { set_error () {
if [ ! "$1" == "0" ] if [ "$ERROR" == "1" ]
then
ERROR=1
elif [ ! "$1" == "0" ]
then then
ERROR=1 ERROR=1
fi fi
@ -1273,7 +1277,9 @@ deploy () {
KEY=`echo $SSH_KEY | cut -d " " -f 2` KEY=`echo $SSH_KEY | cut -d " " -f 2`
ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && mkdir -p $PPSS_HOME_DIR && mkdir -p $PPSS_HOME_DIR/$JOB_LOG_DIR && mkdir -p $PPSS_HOME_DIR/ITEM_LOCK_DIR >> /dev/null 2>&1" ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && mkdir -p $PPSS_HOME_DIR && mkdir -p $PPSS_HOME_DIR/$JOB_LOG_DIR && mkdir -p $PPSS_HOME_DIR/ITEM_LOCK_DIR >> /dev/null 2>&1"
set_error $?
ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && cd $PPSS_HOME_DIR && cd $PPSS_DIR && echo $NODE > $NODE_ID" ssh -q $SSH_OPTS_NODE $SSH_KEY $USER@$NODE "cd ~ && cd $PPSS_HOME_DIR && cd $PPSS_DIR && echo $NODE > $NODE_ID"
set_error $?
scp -q $SSH_OPTS_NODE $SSH_KEY $0 $USER@$NODE:~/$PPSS_HOME_DIR scp -q $SSH_OPTS_NODE $SSH_KEY $0 $USER@$NODE:~/$PPSS_HOME_DIR
set_error $? set_error $?
scp -q $SSH_OPTS_NODE $SSH_KEY $KEY $USER@$NODE:~/$PPSS_HOME_DIR scp -q $SSH_OPTS_NODE $SSH_KEY $KEY $USER@$NODE:~/$PPSS_HOME_DIR
@ -2663,20 +2669,41 @@ start_all_workers () {
#} #}
#get_node_status_from_server () { #get_node_status_from_server () {
get_status_of_node () { get_status_of_nodes () {
RESULT_FILE="$1"
ssh $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER cat "$PPSS_HOME_DIR/$PPSS_NODE_STATUS/*" > "$RESULT_FILE" 2>&1
RES=`ssh $SSH_OPTS $SSH_KEY $USER@$SERVER cat "$PPSS_HOME_DIR/$PPSS_NODE_STATUS/*" 2> /dev/null`
IFS=$'\n' IFS=$'\n'
for x in $RES
for x in `cat $RESULT_FILE`
do do
log DSPLY "$x" IP=`echo $x | awk '{ print $1 }'`
HOST=`echo $x | awk '{ print $2 }'`
STATUS=`echo $x | awk '{ print $3 }'`
RES=`echo $x | awk '{ print $4 }'`
if [ -z "$RES" ]
then
RES="0"
fi
PROCESSED=$((PROCESSED+RES))
LINE=`echo "$IP $HOST $RES $STATUS" | awk '{ printf ("%-16s %-18s % 10s %10s\n",$1,$2,$3,$4) }'`
log DSPLY "$LINE"
done done
log DSPLY "---------------------------------------------------------"
LINE=`echo $PROCESSED | awk '{ printf ("Total processed: % 29s\n",$1) }'`
log DSPLY "$LINE"
rm "$RESULT_FILE"
} }
show_status () { show_status () {
source $CONFIG . $CONFIG
if [ ! -z "$SSH_KEY" ] if [ ! -z "$SSH_KEY" ]
then then
SSH_KEY="-i $SSH_KEY" SSH_KEY="-i $SSH_KEY"
@ -2708,29 +2735,9 @@ show_status () {
log DSPLY "---------------------------------------------------------" log DSPLY "---------------------------------------------------------"
PROCESSED=0 PROCESSED=0
for x in `cat $NODES_FILE` RESULT_FILE="$RADOM$RANDOM.deleteme"
do get_status_of_nodes "$RESULT_FILE"
RES=0
NODE=`get_status_of_node "$x" | awk '{ print $1 }'`
if [ ! "$NODE" == "UNKNOWN" ]
then
STATUS=`get_status_of_node "$x" | awk '{ print $2 }'`
RES=`exec_cmd "grep -i $NODE ~/$PPSS_HOME_DIR/$JOB_LOG_DIR/* 2>/dev/null | wc -l " 1`
if [ ! "$?" ] || [ -z "$RES" ]
then
RES=0
fi
else
STATUS="UNKNOWN"
RES=0
fi
let PROCESSED="$PROCESSED+$RES"
LINE=`echo "$x $NODE $RES $STATUS" | awk '{ printf ("%-16s %-18s % 10s %10s\n",$1,$2,$3,$4) }'`
log DSPLY "$LINE"
done
log DSPLY "---------------------------------------------------------"
LINE=`echo $PROCESSED | awk '{ printf ("Total processed: % 29s\n",$1) }'`
log DSPLY "$LINE"
} }
main () { main () {