sneak
/
ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

backup - broken

This commit is contained in:
Louwrentius 2009-02-08 16:41:34 +00:00
parent f9e1a6fb25
commit 0b0a9b0cc5
1 changed files with 144 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
branches/distributed-ppss/ppss.sh
View File

@ -41,12 +41,18 @@ trap 'kill_process; ' INT
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="1.90" SCRIPT_VERSION="1.90"
MODE="$1"
shift
QUIET="0"
RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS.. RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS..
GLOBAL_LOCK="PPSS-GLOBAL-LOCK" # Global lock file used by local PPSS instance. GLOBAL_LOCK="PPSS-GLOBAL-LOCK" # Global lock file used by local PPSS instance.
PAUSE_SIGNAL="pause.txt" # Not implemented yet (pause processing). PAUSE_SIGNAL="pause_signal" # Not implemented yet (pause processing).
PAUSE_DELAY=20
STOP_SIGNAL="stop_signal"
ARRAY_POINTER_FILE="ppss-array-pointer" # ARRAY_POINTER_FILE="ppss-array-pointer" #
JOB_LOG_DIR="JOB_LOG" # Directory containing log files of processed items. JOB_LOG_DIR="JOB_LOG" # Directory containing log files of processed items.
LOGFILE="ppss-log.txt" # General PPSS log file. Contains lots of info. LOGFILE="ppss-log.txt" # General PPSS log file. Contains lots of info.
STOP=9 # STOP job.
MAX_DELAY=2 MAX_DELAY=2
PERCENT="0" PERCENT="0"
PID="$$" PID="$$"
@ -93,7 +99,7 @@ showusage () {
echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'." echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'."
echo -e "\t- t \tTransfer remote item to slave for local processing." echo -e "\t- t \tTransfer remote item to slave for local processing."
echo -e "\t- o \tUpload output back to server into this directory." echo -e "\t- o \tUpload output back to server into this directory."
echo -e "\t- n \tDo *not* use scp for item transfer but use cp. " echo -e "\t- b \tDo *not* use scp for item transfer but use cp. "
echo echo
echo -e "Example: encoding some wav files to mp3 using lame:" echo -e "Example: encoding some wav files to mp3 using lame:"
echo echo
@ -101,6 +107,8 @@ showusage () {
echo echo
} }
echo mode is $MODE
kill_process () { kill_process () {
kill $LISTENER_PID >> /dev/null 2>&1 kill $LISTENER_PID >> /dev/null 2>&1
@ -158,6 +166,24 @@ does_file_exist () {
fi fi
} }
check_for_interrupt () {
does_file_exist "$STOP_SIGNAL"
if [ "$?" == "0" ]
then
log INFO "STOPPING job. Stop signal found."
STOP="1"
fi
does_file_exist "$PAUSE_SIGNAL"
if [ "$?" == "0" ]
then
log INFO "PAUSE: sleeping for 5 minutes."
sleep $PAUSE_DELAY
check_for_interrupt
fi
}
cleanup () { cleanup () {
@ -190,8 +216,6 @@ cleanup () {
} }
# check if ppss is already running. # check if ppss is already running.
is_running () { is_running () {
@ -205,82 +229,74 @@ is_running () {
} }
# If no arguments are specified, show usage. # If no arguments are specified, show usage.
if [ $# -eq 0 ] #if [ $# -eq 0 ]
then #then
showusage # showusage
exit 1 # exit 1
fi #fi
# If rubbish is givven as an argument, display usage info."
echo $1 | grep -e ^- >> /dev/null
ERROR=$?
if [ ! "$ERROR" == "0" ]
then
showusage
exit 1
fi
# Process any command-line options that are specified." # Process any command-line options that are specified."
while getopts ":c:d:f:i:jhk:l:no:p:s:tv" OPTIONS while getopts ":bc:d:f:i:jhk:l:n:o:p:s:tv" OPTIONS
do do
case $OPTIONS in case $OPTIONS in
f ) n )
INPUT_FILE="$OPTARG" NODES_FILE="$OPTARG"
;; ;;
d )
SRC_DIR="$OPTARG"
;;
c )
COMMAND="$OPTARG"
;;
h ) f )
showusage INPUT_FILE="$OPTARG"
exit 1;; ;;
j ) d )
HYPERTHREADING=yes SRC_DIR="$OPTARG"
;; ;;
l ) c )
LOGFILE="$OPTARG" COMMAND="$OPTARG"
;; ;;
k )
SSH_KEY="-i $OPTARG"
;;
n )
SECURE_COPY=0
;;
o )
REMOTE_OUTPUT_DIR="$OPTARG"
;;
p )
TMP="$OPTARG"
if [ ! -z "$TMP" ]
then
MAX_NO_OF_RUNNING_JOBS="$TMP"
fi
;;
s )
SSH_SERVER="$OPTARG"
;;
t )
TRANSFER_TO_SLAVE="1"
;;
v ) h )
echo "" showusage
echo "$SCRIPT_NAME version $SCRIPT_VERSION" exit 1;;
echo "" j )
exit 0 HYPERTHREADING=yes
;; ;;
* ) l )
showusage LOGFILE="$OPTARG"
exit 1;; ;;
esac k )
done SSH_KEY="-i $OPTARG"
;;
# This function makes local and remote operation transparent. b )
SECURE_COPY=0
;;
o )
REMOTE_OUTPUT_DIR="$OPTARG"
;;
p )
TMP="$OPTARG"
if [ ! -z "$TMP" ]
then
MAX_NO_OF_RUNNING_JOBS="$TMP"
fi
;;
s )
SSH_SERVER="$OPTARG"
;;
t )
TRANSFER_TO_SLAVE="1"
;;
v )
echo ""
echo "$SCRIPT_NAME version $SCRIPT_VERSION"
echo ""
exit 0
;;
* )
showusage
exit 1;;
esac
done
# Init all vars # Init all vars
init_vars () { init_vars () {
@ -328,7 +344,7 @@ init_vars () {
fi fi
does_file_exist "$ITEM_LOCK_DIR" does_file_exist "$ITEM_LOCK_DIR"
if [ ! "$?" == "0" ] if [ ! "$?" == "0" ] && [ ! -z "$SSH_SERVER" ]
then then
log DEBUG "Creating remote item lock dir." log DEBUG "Creating remote item lock dir."
exec_cmd "mkdir $ITEM_LOCK_DIR" exec_cmd "mkdir $ITEM_LOCK_DIR"
@ -601,14 +617,17 @@ upload_item () {
} }
lock_item () { lock_item () {
if [ ! -z "$SSH_SERVER" ]
then
ITEM="$1"
LOCK_FILE_NAME=`echo $ITEM | sed s/^\\\.//g |sed s/^\\\.\\\.//g | sed s/\\\///g`
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
ITEM="$1" exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
LOCK_FILE_NAME=`echo $ITEM | sed s/^\\\.//g |sed s/^\\\.\\\.//g | sed s/\\\///g` ERROR="$?"
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME" return "$ERROR"
fi
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
ERROR="$?"
return "$ERROR"
} }
release_item () { release_item () {
@ -674,6 +693,13 @@ get_all_items () {
get_item () { get_item () {
check_for_interrupt
if [ "$STOP" == "1" ]
then
return 1
fi
get_global_lock get_global_lock
SIZE_OF_ARRAY="${#ARRAY[@]}" SIZE_OF_ARRAY="${#ARRAY[@]}"
@ -770,7 +796,12 @@ commando () {
mv $ITEM $ITEM.error mv $ITEM $ITEM.error
elif [ "$TRANSFER_TO_SLAVE" == "1" ] elif [ "$TRANSFER_TO_SLAVE" == "1" ]
then then
rm $ITEM if [ -e "$ITEM" ]
then
rm $ITEM
else
log DEBUG "ERROR Something went wrong removing item from local work dir."
fi
fi fi
#release_item "$ITEM" #release_item "$ITEM"
@ -814,14 +845,37 @@ start_all_workers () {
main () { main () {
is_running is_running
init_vars
log DEBUG "---------------- START ---------------------" log DEBUG "---------------- START ---------------------"
log INFO "$SCRIPT_NAME version $SCRIPT_VERSION" log INFO "$SCRIPT_NAME version $SCRIPT_VERSION"
test_server
get_all_items case $MODE in
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & client )
LISTENER_PID=$! init_vars
start_all_workers test_server
get_all_items
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" &
LISTENER_PID=$!
start_all_workers
;;
server )
init_vars
echo "server"
;;
stop )
#some stop
;;
deploy )
deploy_ppss
#somefunction
;;
show )
# some show command
;;
* )
showusage
exit 1;;
esac
} }
# This command starts the that sets the whole framework in motion. # This command starts the that sets the whole framework in motion.
main main