backup - broken

This commit is contained in:
Louwrentius 2009-02-08 16:41:34 +00:00
parent 04501ea92a
commit a8bb31a575
1 changed files with 144 additions and 90 deletions

104
ppss.sh
View File

@ -41,12 +41,18 @@ trap 'kill_process; ' INT
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="1.90" SCRIPT_VERSION="1.90"
MODE="$1"
shift
QUIET="0"
RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS.. RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS..
GLOBAL_LOCK="PPSS-GLOBAL-LOCK" # Global lock file used by local PPSS instance. GLOBAL_LOCK="PPSS-GLOBAL-LOCK" # Global lock file used by local PPSS instance.
PAUSE_SIGNAL="pause.txt" # Not implemented yet (pause processing). PAUSE_SIGNAL="pause_signal" # Not implemented yet (pause processing).
PAUSE_DELAY=20
STOP_SIGNAL="stop_signal"
ARRAY_POINTER_FILE="ppss-array-pointer" # ARRAY_POINTER_FILE="ppss-array-pointer" #
JOB_LOG_DIR="JOB_LOG" # Directory containing log files of processed items. JOB_LOG_DIR="JOB_LOG" # Directory containing log files of processed items.
LOGFILE="ppss-log.txt" # General PPSS log file. Contains lots of info. LOGFILE="ppss-log.txt" # General PPSS log file. Contains lots of info.
STOP=9 # STOP job.
MAX_DELAY=2 MAX_DELAY=2
PERCENT="0" PERCENT="0"
PID="$$" PID="$$"
@ -93,7 +99,7 @@ showusage () {
echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'." echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'."
echo -e "\t- t \tTransfer remote item to slave for local processing." echo -e "\t- t \tTransfer remote item to slave for local processing."
echo -e "\t- o \tUpload output back to server into this directory." echo -e "\t- o \tUpload output back to server into this directory."
echo -e "\t- n \tDo *not* use scp for item transfer but use cp. " echo -e "\t- b \tDo *not* use scp for item transfer but use cp. "
echo echo
echo -e "Example: encoding some wav files to mp3 using lame:" echo -e "Example: encoding some wav files to mp3 using lame:"
echo echo
@ -101,6 +107,8 @@ showusage () {
echo echo
} }
echo mode is $MODE
kill_process () { kill_process () {
kill $LISTENER_PID >> /dev/null 2>&1 kill $LISTENER_PID >> /dev/null 2>&1
@ -158,6 +166,24 @@ does_file_exist () {
fi fi
} }
check_for_interrupt () {
does_file_exist "$STOP_SIGNAL"
if [ "$?" == "0" ]
then
log INFO "STOPPING job. Stop signal found."
STOP="1"
fi
does_file_exist "$PAUSE_SIGNAL"
if [ "$?" == "0" ]
then
log INFO "PAUSE: sleeping for 5 minutes."
sleep $PAUSE_DELAY
check_for_interrupt
fi
}
cleanup () { cleanup () {
@ -190,8 +216,6 @@ cleanup () {
} }
# check if ppss is already running. # check if ppss is already running.
is_running () { is_running () {
@ -205,25 +229,21 @@ is_running () {
} }
# If no arguments are specified, show usage. # If no arguments are specified, show usage.
if [ $# -eq 0 ] #if [ $# -eq 0 ]
then #then
showusage # showusage
exit 1 # exit 1
fi #fi
# If rubbish is givven as an argument, display usage info."
echo $1 | grep -e ^- >> /dev/null
ERROR=$?
if [ ! "$ERROR" == "0" ]
then
showusage
exit 1
fi
# Process any command-line options that are specified." # Process any command-line options that are specified."
while getopts ":c:d:f:i:jhk:l:no:p:s:tv" OPTIONS while getopts ":bc:d:f:i:jhk:l:n:o:p:s:tv" OPTIONS
do do
case $OPTIONS in case $OPTIONS in
n )
NODES_FILE="$OPTARG"
;;
f ) f )
INPUT_FILE="$OPTARG" INPUT_FILE="$OPTARG"
;; ;;
@ -246,13 +266,12 @@ do
k ) k )
SSH_KEY="-i $OPTARG" SSH_KEY="-i $OPTARG"
;; ;;
n ) b )
SECURE_COPY=0 SECURE_COPY=0
;; ;;
o ) o )
REMOTE_OUTPUT_DIR="$OPTARG" REMOTE_OUTPUT_DIR="$OPTARG"
;; ;;
p ) p )
TMP="$OPTARG" TMP="$OPTARG"
if [ ! -z "$TMP" ] if [ ! -z "$TMP" ]
@ -279,9 +298,6 @@ do
esac esac
done done
# This function makes local and remote operation transparent.
# Init all vars # Init all vars
init_vars () { init_vars () {
@ -328,7 +344,7 @@ init_vars () {
fi fi
does_file_exist "$ITEM_LOCK_DIR" does_file_exist "$ITEM_LOCK_DIR"
if [ ! "$?" == "0" ] if [ ! "$?" == "0" ] && [ ! -z "$SSH_SERVER" ]
then then
log DEBUG "Creating remote item lock dir." log DEBUG "Creating remote item lock dir."
exec_cmd "mkdir $ITEM_LOCK_DIR" exec_cmd "mkdir $ITEM_LOCK_DIR"
@ -602,6 +618,8 @@ upload_item () {
lock_item () { lock_item () {
if [ ! -z "$SSH_SERVER" ]
then
ITEM="$1" ITEM="$1"
LOCK_FILE_NAME=`echo $ITEM | sed s/^\\\.//g |sed s/^\\\.\\\.//g | sed s/\\\///g` LOCK_FILE_NAME=`echo $ITEM | sed s/^\\\.//g |sed s/^\\\.\\\.//g | sed s/\\\///g`
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME" ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
@ -609,6 +627,7 @@ lock_item () {
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1" exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
ERROR="$?" ERROR="$?"
return "$ERROR" return "$ERROR"
fi
} }
release_item () { release_item () {
@ -674,6 +693,13 @@ get_all_items () {
get_item () { get_item () {
check_for_interrupt
if [ "$STOP" == "1" ]
then
return 1
fi
get_global_lock get_global_lock
SIZE_OF_ARRAY="${#ARRAY[@]}" SIZE_OF_ARRAY="${#ARRAY[@]}"
@ -769,8 +795,13 @@ commando () {
then then
mv $ITEM $ITEM.error mv $ITEM $ITEM.error
elif [ "$TRANSFER_TO_SLAVE" == "1" ] elif [ "$TRANSFER_TO_SLAVE" == "1" ]
then
if [ -e "$ITEM" ]
then then
rm $ITEM rm $ITEM
else
log DEBUG "ERROR Something went wrong removing item from local work dir."
fi
fi fi
#release_item "$ITEM" #release_item "$ITEM"
@ -814,14 +845,37 @@ start_all_workers () {
main () { main () {
is_running is_running
init_vars
log DEBUG "---------------- START ---------------------" log DEBUG "---------------- START ---------------------"
log INFO "$SCRIPT_NAME version $SCRIPT_VERSION" log INFO "$SCRIPT_NAME version $SCRIPT_VERSION"
case $MODE in
client )
init_vars
test_server test_server
get_all_items get_all_items
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" & listen_for_job "$MAX_NO_OF_RUNNING_JOBS" &
LISTENER_PID=$! LISTENER_PID=$!
start_all_workers start_all_workers
;;
server )
init_vars
echo "server"
;;
stop )
#some stop
;;
deploy )
deploy_ppss
#somefunction
;;
show )
# some show command
;;
* )
showusage
exit 1;;
esac
} }
# This command starts the that sets the whole framework in motion. # This command starts the that sets the whole framework in motion.
main main