From 13e6b77c651e8fda20cab076dba293770075c383 Mon Sep 17 00:00:00 2001 From: Louwrentius Date: Thu, 15 Apr 2010 21:46:27 +0000 Subject: [PATCH] Added daemon mode. --- ppss | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/ppss b/ppss index b06d986..25d8f01 100755 --- a/ppss +++ b/ppss @@ -67,11 +67,16 @@ ARCH="`uname`" PPSS_HOME_DIR="ppss-home" PID="$$" +if [ -z "$INPUT_LOCK" ] +then + INPUT_LOCK="INPUT_LOCK" +fi GLOBAL_LOCK="$PPSS_DIR/PPSS-GLOBAL-LOCK-$PID" # Global lock file used by local PPSS instance. PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal" # Pause processing if this file is present. PAUSE_DELAY="60" # Polling every 1 minutes by default. STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal" # Stop processing if this file is present. ARRAY_POINTER_FILE="$PPSS_DIR/ppss-array-pointer-$PID" # Pointer for keeping track of processed items. +ARRAY="" JOB_LOG_DIR="$PPSS_DIR/job_log" # Directory containing log files of processed items. LOGFILE="$PPSS_DIR/ppss-log-$PID.txt" # General PPSS log file. Contains lots of info. QUIET="0" @@ -88,6 +93,7 @@ KILL_KEY="$RANDOM$RANDOM$RANDOM" # This is a signal to st RECURSION="1" # all running processes. START_PPSS=`get_time_in_seconds` STOP_PPSS="" +SIZE_OF_ARRAY="" SSH_SERVER="" # Remote server or 'master'. SSH_KEY="" # SSH key for ssh account. @@ -123,7 +129,7 @@ showusage_short () { echo echo "usage: $0 [ -d | -f ] [ -c ' \"\$ITEM\"' ]" echo " [ -C ] [ -j ] [ -l ] [ -p <# jobs> ]" - echo " [ -q ] [ -D ] [ -h ] [ --help ] [ -r ] " + echo " [ -q ] [ -D ] [ -h ] [ --help ] [ -r ] [ --daemon ]" echo echo "Examples:" echo " $0 -d /dir/with/some/files -c 'gzip '" @@ -173,6 +179,9 @@ showusage_normal () { echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread" echo -e " the load. The delay is only used at the start of all 'threads'." echo + echo -e "--daemon Do not exit after items are professed, but keep looking for new items" + echo -e " and process them. Read the manual how to use this!" + echo echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e " used. If this is not prefered, this can be disabled with this option " echo -e " Only files within the specified directory will be processed." @@ -247,6 +256,9 @@ showusage_long () { echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread" echo -e " the load. The delay is only used at the start of all 'threads'." echo + echo -e "--daemon Do not exit after items are professed, but keep looking for new items" + echo -e " and process them. Read the manual how to use this!" + echo echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e " used. If this is not prefered, this can be disabled with this option." echo -e " Only files within the specified directory will be processed." @@ -517,6 +529,13 @@ do add_var_to_config MAX_DELAY "$MAX_DELAY" shift 2 ;; + --daemon) + DAEMON="1" + QUIET="1" + add_var_to_config DAEMON "$DAEMON" + add_var_to_config QUIET "$QUIET" + shift 1 + ;; --awskeypair|-P) AWS_KEYPAIR="$2" add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR" @@ -1418,8 +1437,42 @@ lock_item () { fi } +get_input_lock () { + + while true + do + exec_cmd "mkdir -p $INPUT_LOCK" + if [ "$?" == "0" ] + then + log DEBUG "Input lock is obtained..." + break + else + log DEBUG "Input lock is present...sleeping.." + sleep 5 + fi + done +} + +release_input_lock () { + + exec_cmd "rm -rf $INPUT_LOCK" + if [ "$?" == "0" ] + then + log DEBUG "Input lock was released..." + return 0 + else + log ERROR "Input lock was already gone...this should never happen..." + return 1 + fi +} + get_all_items () { + if [ "$DAEMON" == "1" ] + then + get_input_lock + fi + count=0 if [ -z "$INPUT_FILE" ] @@ -1496,6 +1549,11 @@ get_all_items () { fi + if [ "$DAEMON" == "1" ] + then + release_input_lock + fi + SIZE_OF_ARRAY="${#ARRAY[@]}" if [ "$SIZE_OF_ARRAY" -le "0" ] then @@ -1624,6 +1682,10 @@ elapsed () { commando () { log DEBUG "-------------------------------------" + if [ "$DAEMON" == "1" ] + then + log INFO "Processing item: $1 in DAEMON MODE" + fi # # This function will start a chain reaction of events. @@ -1882,7 +1944,20 @@ listen_for_job () { ((DIED++)) if [ "$DIED" -ge "$MAX_NO_OF_RUNNING_JOBS" ] then - break + if [ "$DAEMON" == "1" ] + then + # + # In daemon mode, start all over again. + # + DIED=0 + export ARRAY="" + get_all_items + log DEBUG "Found $SIZE_OF_ARRAY items." + start_all_workers + sleep 10 + else + break + fi else RES=$((MAX_NO_OF_RUNNING_JOBS-DIED)) if [ "$RES" == "1" ] && [ "$QUIET" == "0" ] @@ -1907,8 +1982,6 @@ listen_for_job () { # process as defined by $PID. All processes that have ever been # spawned, although disowned or backgrounded will be killed... # - # *** THIS MAY BE DEAD CODE *** - # PROCLIST=`ps a -o pid,pgid,ppid,command | grep [0-9] | grep $PID | grep -v -i grep` #echo "$PROCLIST" > proclist.txt oldIFS=$IFS # save the field separator @@ -1965,6 +2038,7 @@ listen_for_job () { fi done + if [ ! -z "$SSH_MASTER_PID" ] then log DEBUG "SSH master PID is $SSH_MASTER_PID"