Added daemon mode.

ppss

@@ -67,11 +67,16 @@ ARCH="`uname`"
 PPSS_HOME_DIR="ppss-home"
 
 PID="$$"
+if [ -z "$INPUT_LOCK" ]
+then
+    INPUT_LOCK="INPUT_LOCK"
+fi
 GLOBAL_LOCK="$PPSS_DIR/PPSS-GLOBAL-LOCK-$PID"           # Global lock file used by local PPSS instance.
 PAUSE_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/pause_signal"                   # Pause processing if this file is present.
 PAUSE_DELAY="60"                                          # Polling every 1 minutes by default.
 STOP_SIGNAL="$PPSS_HOME_DIR/$PPSS_DIR/stop_signal"                     # Stop processing if this file is present.
 ARRAY_POINTER_FILE="$PPSS_DIR/ppss-array-pointer-$PID"  # Pointer for keeping track of processed items.
+ARRAY=""
 JOB_LOG_DIR="$PPSS_DIR/job_log"                         # Directory containing log files of processed items.
 LOGFILE="$PPSS_DIR/ppss-log-$PID.txt"                     # General PPSS log file. Contains lots of info.
 QUIET="0"
@@ -88,6 +93,7 @@ KILL_KEY="$RANDOM$RANDOM$RANDOM"                        # This is a signal to st
 RECURSION="1"                                           # all running processes.
 START_PPSS=`get_time_in_seconds`
 STOP_PPSS=""
+SIZE_OF_ARRAY=""
 
 SSH_SERVER=""                           # Remote server or 'master'.
 SSH_KEY=""                              # SSH key for ssh account.
@@ -123,7 +129,7 @@ showusage_short () {
     echo
     echo "usage: $0 [ -d <sourcedir> | -f <sourcefile> ]  [ -c '<command> \"\$ITEM\"' ]"
     echo "                 [ -C <configfile> ] [ -j ] [ -l <logfile> ] [ -p <# jobs> ]"
-    echo "                 [ -q ] [ -D <delay> ] [ -h ] [ --help ] [ -r ] " 
+    echo "                 [ -q ] [ -D <delay> ] [ -h ] [ --help ] [ -r ] [ --daemon ]" 
     echo
     echo "Examples:"
     echo "                 $0 -d /dir/with/some/files -c 'gzip '" 
@@ -173,6 +179,9 @@ showusage_normal () {
     echo -e "--delay | -D       Adds an initial random delay to the start of all parallel jobs to spread"
     echo -e "                   the load. The delay is only used at the start of all 'threads'."
     echo
+    echo -e "--daemon           Do not exit after items are professed, but keep looking for new items"
+    echo -e "                   and process them. Read the manual how to use this!"
+    echo
     echo -e "--no-recursion|-r  By default, recursion of directories is enabled when the -d option is "
     echo -e "                   used. If this is not prefered, this can be disabled with this option "
     echo -e "                   Only files within the specified directory will be processed."
@@ -247,6 +256,9 @@ showusage_long () {
     echo -e "--delay | -D       Adds an initial random delay to the start of all parallel jobs to spread"
     echo -e "                   the load. The delay is only used at the start of all 'threads'."
     echo 
+    echo -e "--daemon           Do not exit after items are professed, but keep looking for new items"
+    echo -e "                   and process them. Read the manual how to use this!"
+    echo
     echo -e "--no-recursion|-r  By default, recursion of directories is enabled when the -d option is "
     echo -e "                   used. If this is not prefered, this can be disabled with this option."
     echo -e "                   Only files within the specified directory will be processed."
@@ -517,6 +529,13 @@ do
                         add_var_to_config MAX_DELAY "$MAX_DELAY"
                         shift 2
                         ;;
+             --daemon)
+                        DAEMON="1"
+                        QUIET="1"
+                        add_var_to_config DAEMON "$DAEMON"
+                        add_var_to_config QUIET "$QUIET"
+                        shift 1
+                        ;;
 	  --awskeypair|-P)
 	  		AWS_KEYPAIR="$2"
 			add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR"
@@ -1418,8 +1437,42 @@ lock_item () {
     fi
 }
 
+get_input_lock () {
+
+    while true
+    do 
+        exec_cmd "mkdir -p $INPUT_LOCK"
+        if [ "$?" == "0" ]
+        then
+            log DEBUG "Input lock is obtained..."
+            break
+        else
+            log DEBUG "Input lock is present...sleeping.."
+            sleep 5
+        fi
+    done
+}
+
+release_input_lock () {
+    
+    exec_cmd "rm -rf $INPUT_LOCK"
+    if [ "$?" == "0" ]
+    then
+        log DEBUG "Input lock was released..."
+        return 0
+    else
+        log ERROR "Input lock was already gone...this should never happen..."
+        return 1
+    fi
+}
+
 get_all_items () {
 
+    if [ "$DAEMON" == "1" ]
+    then
+        get_input_lock
+    fi
+
     count=0
 
     if [ -z "$INPUT_FILE" ]
@@ -1496,6 +1549,11 @@ get_all_items () {
   
     fi
 
+    if [ "$DAEMON" == "1" ]
+    then
+        release_input_lock
+    fi
+
     SIZE_OF_ARRAY="${#ARRAY[@]}"
     if [ "$SIZE_OF_ARRAY" -le "0" ]
     then
@@ -1624,6 +1682,10 @@ elapsed () {
 commando () {
     
     log DEBUG "-------------------------------------"
+    if [ "$DAEMON" == "1" ]
+    then
+        log INFO "Processing item: $1 in DAEMON MODE"
+    fi
 
     #
     # This function will start a chain reaction of events.
@@ -1882,7 +1944,20 @@ listen_for_job () {
             ((DIED++))
             if [ "$DIED" -ge "$MAX_NO_OF_RUNNING_JOBS" ] 
             then
+                if [ "$DAEMON" == "1" ] 
+                then
+                    #
+                    # In daemon mode, start all over again.
+                    #
+                    DIED=0
+                    export ARRAY=""
+                    get_all_items
+                    log DEBUG "Found $SIZE_OF_ARRAY items."
+                    start_all_workers
+                    sleep 10
+                else
                     break
+                fi
             else
                 RES=$((MAX_NO_OF_RUNNING_JOBS-DIED))
                 if [ "$RES" == "1" ] && [ "$QUIET" == "0" ]
@@ -1907,8 +1982,6 @@ listen_for_job () {
             # process as defined by $PID. All processes that have ever been
             # spawned, although disowned or backgrounded will be killed...
             #
-            # *** THIS MAY BE DEAD CODE ***
-            #
             PROCLIST=`ps a -o pid,pgid,ppid,command | grep [0-9] | grep $PID | grep -v -i grep`
             #echo "$PROCLIST" > proclist.txt
             oldIFS=$IFS # save the field separator
@@ -1965,6 +2038,7 @@ listen_for_job () {
         fi
     done
 
+
     if [ ! -z "$SSH_MASTER_PID" ]
     then
         log DEBUG "SSH master PID is $SSH_MASTER_PID"