sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Backup, major overhaul of daemon mode and listener process...

Browse Source
This commit is contained in:
Louwrentius 2010-07-17 19:49:19 +00:00
parent caa6519d0f
commit ecc9d3da3b
1 changed files with 244 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

429
ppss
View File

@ -75,7 +75,6 @@ IFS_BACKUP="$IFS"
CPUINFO="/proc/cpuinfo" CPUINFO="/proc/cpuinfo"
PROCESSORS="" PROCESSORS=""
START_KEY="$RANDOM$RANDOM$RANDOM" START_KEY="$RANDOM$RANDOM$RANDOM"
STOP_KEY="$RANDOM$RANDOM$RANDOM" # This is a signal to the listener to stop.
KILL_KEY="$RANDOM$RANDOM$RANDOM" # This is a signal to stop immediately and kill KILL_KEY="$RANDOM$RANDOM$RANDOM" # This is a signal to stop immediately and kill
QUEUE="" QUEUE=""
INOTIFY=0 INOTIFY=0
@ -85,6 +84,9 @@ STOP_PPSS=""
SIZE_OF_INPUT="" SIZE_OF_INPUT=""
LOCAL_LOCKING="1" LOCAL_LOCKING="1"
PROCESSED_ITEMS="" PROCESSED_ITEMS=""
UNPROCESSED_ITEMS=""
ACTIVE_WORKERS="0"
DAEMON_POLLING_INTERVAL="10"
SSH_SERVER="" # Remote server or 'master'. SSH_SERVER="" # Remote server or 'master'.
SSH_KEY="" # SSH key for ssh account. SSH_KEY="" # SSH key for ssh account.
@ -346,20 +348,20 @@ exec_cmd () {
then then
if [ -z "$NOMP" ] if [ -z "$NOMP" ]
then then
log DEBUG "REMOTE EXEC" # log DEBUG "REMOTE EXEC"
log DEBUG "$USER@$SSH_SERVER $CMD" # log DEBUG "$USER@$SSH_SERVER $CMD"
ssh $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER $CMD ssh $SSH_OPTS $SSH_KEY $USER@$SSH_SERVER $CMD
STATUS=$? STATUS=$?
elif [ "$NOMP" == "1" ] elif [ "$NOMP" == "1" ]
then then
log DEBUG "REMOTE EXEC NO MP" # log DEBUG "REMOTE EXEC NO MP"
ssh $SSH_OPTS_NOMP $SSH_KEY $USER@$SSH_SERVER $CMD ssh $SSH_OPTS_NOMP $SSH_KEY $USER@$SSH_SERVER $CMD
STATUS=$? STATUS=$?
fi fi
else else
eval "$CMD" eval "$CMD"
STATUS=$? STATUS=$?
log DEBUG "LOCAL EXEC - status is $STATUS" # log DEBUG "LOCAL EXEC - status is $STATUS"
fi fi
return $STATUS return $STATUS
} }
@ -373,10 +375,10 @@ does_file_exist () {
RES=`exec_cmd "ls -1 $FILE" 2>&1` RES=`exec_cmd "ls -1 $FILE" 2>&1`
if [ "$?" = "0" ] if [ "$?" = "0" ]
then then
log DEBUG "$FILE does exist" #log DEBUG "$FILE does exist"
return 0 return 0
else else
log DEBUG "$FILE does not exist" #log DEBUG "$FILE does not exist"
return 1 return 1
fi fi
} }
@ -514,38 +516,30 @@ process_arguments () {
exit exit
fi fi
fi fi
shift 2 shift 2 ;;
;;
--working-dir|-w ) --working-dir|-w )
PPSS_DIR="$2" PPSS_DIR="$2"
add_var_to_config PPSS_DIR "$PPSS_DIR" add_var_to_config PPSS_DIR "$PPSS_DIR"
shift 2 shift 2 ;;
;;
--node|-n ) --node|-n )
NODES_FILE="$2" NODES_FILE="$2"
add_var_to_config NODES_FILE "$NODES_FILE" add_var_to_config NODES_FILE "$NODES_FILE"
shift 2 shift 2 ;;
;;
--sourcefile|-f ) --sourcefile|-f )
INPUT_FILE="$2" INPUT_FILE="$2"
is_var_empty "$INPUT_FILE" is_var_empty "$INPUT_FILE"
add_var_to_config INPUT_FILE "$INPUT_FILE" add_var_to_config INPUT_FILE "$INPUT_FILE"
shift 2 shift 2 ;;
;;
--sourcedir|-d ) --sourcedir|-d )
SRC_DIR="$2" SRC_DIR="$2"
is_var_empty "$SRC_DIR" is_var_empty "$SRC_DIR"
add_var_to_config SRC_DIR "$SRC_DIR" add_var_to_config SRC_DIR "$SRC_DIR"
shift 2 shift 2 ;;
;;
--delay|-D) --delay|-D)
MAX_DELAY="$2" MAX_DELAY="$2"
add_var_to_config MAX_DELAY "$MAX_DELAY" add_var_to_config MAX_DELAY "$MAX_DELAY"
shift 2 shift 2 ;;
;;
--daemon) --daemon)
DAEMON="1" DAEMON="1"
QUIET="1" QUIET="1"
@ -553,34 +547,32 @@ process_arguments () {
add_var_to_config DAEMON "$DAEMON" add_var_to_config DAEMON "$DAEMON"
add_var_to_config QUIET "$QUIET" add_var_to_config QUIET "$QUIET"
add_var_to_config INOTIFY "$INOTIFY" add_var_to_config INOTIFY "$INOTIFY"
shift 1 shift 1 ;;
;; --interval)
is_var_empty "$2"
DAEMON_POLLING_INTERVAL="$2"
add_var_to_config DAEMON_POLLING_INTERVAL "$DAEMON_POLLING_INTERVAL"
shift 2 ;;
--awskeypair|-P) --awskeypair|-P)
AWS_KEYPAIR="$2" AWS_KEYPAIR="$2"
add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR" add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR"
shift 2 shift 2 ;;
;;
--AMI|-A) --AMI|-A)
AMI_ID="$2" AMI_ID="$2"
add_var_to_config AMI_ID "$AMI_ID" add_var_to_config AMI_ID "$AMI_ID"
shift 2 shift 2 ;;
;;
--type|-T) --type|-T)
INSTANCE_TYPE="$2" INSTANCE_TYPE="$2"
add_var_to_config INSTANCE_TYPE "$INSTANCE_TYPE" add_var_to_config INSTANCE_TYPE "$INSTANCE_TYPE"
shift 2 shift 2 ;;
;;
--security|-G) --security|-G)
SECURITY_GROUP="$2" SECURITY_GROUP="$2"
add_var_to_config SECURITY_GROUP "$SECURITY_GROUP" add_var_to_config SECURITY_GROUP "$SECURITY_GROUP"
shift 2 shift 2 ;;
;;
--instances|-I) --instances|-I)
NUM_NODES="$2" NUM_NODES="$2"
add_var_to_config NUM_NODES "$NUM_NODES" add_var_to_config NUM_NODES "$NUM_NODES"
shift 2 shift 2 ;;
;;
--command|-c ) --command|-c )
COMMAND="$2" COMMAND="$2"
is_var_empty "$COMMAND" is_var_empty "$COMMAND"
@ -589,44 +581,35 @@ process_arguments () {
COMMAND=\'$COMMAND\' COMMAND=\'$COMMAND\'
add_var_to_config COMMAND "$COMMAND" add_var_to_config COMMAND "$COMMAND"
fi fi
shift 2 shift 2 ;;
;;
-h ) -h )
showusage_normal showusage_normal
exit 1;; exit 1 ;;
--help) --help)
showusage_long showusage_long
exit 1;; exit 1 ;;
--homedir|-H ) --homedir|-H )
if [ ! -z "$2" ] is_var_empty "$2"
then PPSS_HOME_DIR="$2"
PPSS_HOME_DIR="$2" add_var_to_config PPSS_DIR $PPSS_HOME_DIR
add_var_to_config PPSS_DIR $PPSS_HOME_DIR shift 2 ;;
shift 2
fi
;;
--disable-ht|-j ) --disable-ht|-j )
HYPERTHREADING=no HYPERTHREADING=no
add_var_to_config HYPERTHREADING $HYPERTHREADING add_var_to_config HYPERTHREADING $HYPERTHREADING
shift 1 shift 1 ;;
;;
--log|-l ) --log|-l )
LOGFILE="$2" LOGFILE="$2"
add_var_to_config LOGFILE "$LOGFILE" add_var_to_config LOGFILE "$LOGFILE"
shift 2 shift 2 ;;
;;
--no-recursion|-r ) --no-recursion|-r )
RECURSION="0" RECURSION="0"
add_var_to_config LOGFILE "$RECURSION" add_var_to_config LOGFILE "$RECURSION"
shift 1 shift 1 ;;
;;
--workingdir|-w ) --workingdir|-w )
WORKINGDIR="$2" WORKINGDIR="$2"
add_var_to_config WORKINGDIR "$WORKINGDIR" add_var_to_config WORKINGDIR "$WORKINGDIR"
shift 2 shift 2 ;;
;;
--key|-k ) --key|-k )
SSH_KEY="$2" SSH_KEY="$2"
is_var_empty "$SSH_KEY" is_var_empty "$SSH_KEY"
@ -635,48 +618,37 @@ process_arguments () {
then then
SSH_KEY="-i $SSH_KEY" SSH_KEY="-i $SSH_KEY"
fi fi
shift 2 shift 2 ;;
;;
--known-hosts | -K ) --known-hosts | -K )
SSH_KNOWN_HOSTS="$2" SSH_KNOWN_HOSTS="$2"
add_var_to_config SSH_KNOWN_HOSTS "$SSH_KNOWN_HOSTS" add_var_to_config SSH_KNOWN_HOSTS "$SSH_KNOWN_HOSTS"
shift 2 shift 2 ;;
;;
--no-scp |-b ) --no-scp |-b )
SECURE_COPY=0 SECURE_COPY=0
add_var_to_config SECURE_COPY "$SECURE_COPY" add_var_to_config SECURE_COPY "$SECURE_COPY"
shift 1 shift 1 ;;
;;
--outputdir|-o ) --outputdir|-o )
REMOTE_OUTPUT_DIR="$2" REMOTE_OUTPUT_DIR="$2"
add_var_to_config REMOTE_OUTPUT_DIR "$REMOTE_OUTPUT_DIR" add_var_to_config REMOTE_OUTPUT_DIR "$REMOTE_OUTPUT_DIR"
shift 2 shift 2 ;;
;;
--processes|-p ) --processes|-p )
TMP="$2" is_var_empty "$2"
if [ ! -z "$TMP" ] MAX_NO_OF_RUNNING_JOBS="$2"
then add_var_to_config MAX_NO_OF_RUNNING_JOBS "$MAX_NO_OF_RUNNING_JOBS"
MAX_NO_OF_RUNNING_JOBS="$TMP" shift 2 ;;
add_var_to_config MAX_NO_OF_RUNNING_JOBS "$MAX_NO_OF_RUNNING_JOBS"
shift 2
fi
;;
--master|-m ) --master|-m )
SSH_SERVER="$2" SSH_SERVER="$2"
add_var_to_config SSH_SERVER "$SSH_SERVER" add_var_to_config SSH_SERVER "$SSH_SERVER"
shift 2 shift 2 ;;
;;
--script|-S ) --script|-S )
SCRIPT="$2" SCRIPT="$2"
add_var_to_config SCRIPT "$SCRIPT" add_var_to_config SCRIPT "$SCRIPT"
shift 2 shift 2 ;;
;;
--download) --download)
DOWNLOAD_TO_NODE="1" DOWNLOAD_TO_NODE="1"
add_var_to_config DOWNLOAD_TO_NODE "$DOWNLOAD_TO_NODE" add_var_to_config DOWNLOAD_TO_NODE "$DOWNLOAD_TO_NODE"
shift 1 shift 1 ;;
;;
--upload) --upload)
if [ -z "$REMOTE_OUTPUT_DIR" ] if [ -z "$REMOTE_OUTPUT_DIR" ]
then then
@ -685,32 +657,28 @@ process_arguments () {
fi fi
UPLOAD_TO_SERVER="1" UPLOAD_TO_SERVER="1"
add_var_to_config UPLOAD_TO_SERVER "$UPLOAD_TO_SERVER" add_var_to_config UPLOAD_TO_SERVER "$UPLOAD_TO_SERVER"
shift 1 shift 1 ;;
;;
--quiet|-q ) --quiet|-q )
QUIET="1" QUIET="1"
add_var_to_config QUIET "$QUIET" add_var_to_config QUIET "$QUIET"
shift 1 shift 1 ;;
;;
--user|-u ) --user|-u )
USER="$2" USER="$2"
add_var_to_config USER "$USER" add_var_to_config USER "$USER"
shift 2 shift 2 ;;
;;
--version|-v ) --version|-v )
echo "" echo ""
echo "$SCRIPT_NAME version $SCRIPT_VERSION" echo "$SCRIPT_NAME version $SCRIPT_VERSION"
echo "" echo ""
exit 0 exit 0 ;;
;;
* ) * )
showusage_short showusage_short
echo echo
echo "Unknown option $1 " echo "Unknown option $1 "
echo echo
exit 1;; exit 1 ;;
esac esac
done done
@ -1063,7 +1031,32 @@ stack_push () {
fi fi
} }
unprocessed_stack_push () {
line="$1"
if [ -z "$PROCESSED_ITEMS" ]
then
UNPROCESSED_ITEMS="$line"
else
UNPROCESSED_ITEMS="$line"$'\n'"$UNPROCESSED_ITEMS"
fi
}
processed_stack_push () {
line="$1"
if [ -z "$PROCESSED_ITEMS" ]
then
PROCESSED_ITEMS="$line"
else
PROCESSED_ITEMS="$line"$'\n'"$PROCESSED_ITEMS"
fi
}
stack_pop () { stack_pop () {
TMP_STACK="" TMP_STACK=""
i=0 i=0
tmp="" tmp=""
@ -1612,24 +1605,36 @@ remove_processed_items_from_input_file () {
# This function removes all items that have already been processed. # This function removes all items that have already been processed.
# Processed items have a lock dir in the PPPSS_ITEM_LOCK_DIR. # Processed items have a lock dir in the PPPSS_ITEM_LOCK_DIR.
# #
UNPROCESSED_ITEMS=""
if [ "$MODE" = "status" ] if [ "$MODE" = "status" ]
then then
return 1 return 1
fi fi
log DEBUG "Running $FUNCNAME...."
if [ ! -e "$LISTOFITEMS" ]
then
echo "$LISTOFITEMS does not exist!"
return 1
else
SIZE=`wc -l "$LISTOFITEMS"`
if [ "$SIZE" = "0" ]
then
echo "$LISTOFITEMS exists but is empty."
return 1
fi
fi
INPUTFILES=`list_all_input_items` INPUTFILES=`list_all_input_items`
RES=`exec_cmd "ls -1 $ITEM_LOCK_DIR"` log DEBUG "$PROCESSED_ITEMS"
rm "$LISTOFITEMS"
for x in $INPUTFILES for x in $INPUTFILES
do do
FILE_IS_PROCESSED=0 FILE_IS_PROCESSED=0
for y in $RES for y in $PROCESSED_ITEMS
do do
TMP=`echo "$x" | $MD5 | awk '{ print $1 }'` if [ "$y" = "$x" ]
if [ "$y" = "$TMP" ]
then then
FILE_IS_PROCESSED=1 FILE_IS_PROCESSED=1
fi fi
@ -1637,22 +1642,20 @@ remove_processed_items_from_input_file () {
if [ "$FILE_IS_PROCESSED" = "0" ] if [ "$FILE_IS_PROCESSED" = "0" ]
then then
echo "$x" >> "$LISTOFITEMS" log DEBUG "ITEM $x is not processed."
unprocessed_stack_push "$x"
else else
log DEBUG "ITEM $x is already processed!" log DEBUG "ITEM $x is already processed!."
fi fi
done done
} echo "$UNPROCESSED_ITEMS" > "$LISTOFITEMS"
get_list_of_new_items () {
echo
} }
get_all_items () { get_all_items () {
if [ "$DAEMON" == "1" ] if [ "$DAEMON" == "1" ]
then then
GLOBAL_COUNTER=1
get_input_lock get_input_lock
fi fi
@ -1677,8 +1680,6 @@ get_all_items () {
check_status "$?" "$FUNCNAME" "Could not list files within remote source directory." check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
fi fi
remove_processed_items_from_input_file
else else
if [ -e "$SRC_DIR" ] if [ -e "$SRC_DIR" ]
then then
@ -1744,6 +1745,7 @@ get_all_items () {
cleanup cleanup
exit 1 exit 1
fi fi
remove_processed_items_from_input_file
} }
get_item () { get_item () {
@ -1759,7 +1761,7 @@ get_item () {
# #
# Return error if list size is empty. # Return error if list size is empty.
# #
if [ -z "$SIZE_OF_INPUT" ] if [ -z "$SIZE_OF_INPUT" ]
then then
log DEBUG "Got no size of input..." log DEBUG "Got no size of input..."
return 1 return 1
@ -1770,14 +1772,7 @@ get_item () {
# #
if [ "$SIZE_OF_INPUT" -le "0" ] if [ "$SIZE_OF_INPUT" -le "0" ]
then then
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }') return 1
if [ "$?" = "0" ]
then
return 0
else
log DEBUG "Size of input 0 or less."
return 1
fi
fi fi
# #
@ -1805,9 +1800,6 @@ get_item () {
if [ ! "$LOCK" = "0" ] if [ ! "$LOCK" = "0" ]
then then
log DEBUG "Item $ITEM is locked." log DEBUG "Item $ITEM is locked."
#
# Recursion, get_ttem calls itself, until all items are done.
#
get_item get_item
else else
log DEBUG "Got lock on $ITEM" log DEBUG "Got lock on $ITEM"
@ -1820,21 +1812,15 @@ get_item () {
fi fi
} }
start_single_worker () { start_new_worker () {
# #
# This function kicks the listener to start a worker process. # This function kicks the listener to start a worker process.
# #
if ! are_we_sourced if ! are_we_sourced
then then
if [ "$DAEMON" = "1" ] && [ "$INOTIFY" = "1" ]
then
echo "$STOP_KEY" >> "$FIFO"
return $?
else
echo "$START_KEY" >> "$FIFO" echo "$START_KEY" >> "$FIFO"
return $? return $?
fi
fi fi
} }
@ -1867,12 +1853,12 @@ commando () {
# This function will start a chain reaction of events. # This function will start a chain reaction of events.
# #
# The commando executes a command on an item and, when finished, # The commando executes a command on an item and, when finished,
# executes the start_single_worker. This function selects a new # executes the start_new_worker. This function selects a new
# item and sends it to the fifo. The listener process receives # item and sends it to the fifo. The listener process receives
# the item and excutes this commando function on the item. # the item and excutes this commando function on the item.
# So in essence, the commando function keeps calling itself # So in essence, the commando function keeps calling itself
# indirectly until no items are left. This will form a single # indirectly until no items are left. This will form a single
# working queue. By executing multiple start_single_worker # working queue. By executing multiple start_new_worker
# functions based on the CPU cores available, parallel processing # functions based on the CPU cores available, parallel processing
# is achieved, with a queue for each core. # is achieved, with a queue for each core.
# #
@ -1962,7 +1948,7 @@ commando () {
if [ -e "$ITEM_LOG_FILE" ] && [ "$DISABLE_SKIPPING" = "0" ] if [ -e "$ITEM_LOG_FILE" ] && [ "$DISABLE_SKIPPING" = "0" ]
then then
log DEBUG "Item is already processed, skipping..." log DEBUG "Item is already processed, skipping..."
start_single_worker start_new_worker
return 0 return 0
fi fi
@ -2089,7 +2075,7 @@ commando () {
fi fi
fi fi
start_single_worker start_new_worker
} }
infanticide () { infanticide () {
@ -2128,18 +2114,34 @@ infanticide () {
run_command () { run_command () {
if [ ! -d "$1" ] && [ ! -z "$1" ] INPUT="$1"
log DEBUG "Current active workers is $ACTIVE_WORKERS"
if [ "$ACTIVE_WORKERS" -le "$MAX_NO_OF_RUNNING_JOBS" ]
then then
commando "$1" & if [ -z "$INPUT" ]
MYPID="$!" then
disown stack_pop
PIDS="$PIDS $MYPID" INPUT="$REGISTER"
((ACTIVE_WORKERS++)) fi
log DEBUG "Increasing active workers to $ACTIVE_WORKERS"
return 0 if [ ! -d "$INPUT" ] && [ ! -z "$INPUT" ]
else then
return 1 commando "$INPUT" &
fi MYPID="$!"
disown
PIDS="$PIDS $MYPID"
((ACTIVE_WORKERS++))
log DEBUG "Increasing active workers to $ACTIVE_WORKERS"
return 0
else
log DEBUG "Item is a directory or empty."
return 1
fi
else
log DEBUG "Strange, this message should never be displaed."
fi
} }
display_jobs_remaining () { display_jobs_remaining () {
@ -2157,15 +2159,6 @@ display_jobs_remaining () {
fi fi
} }
start_as_daemon () {
ACTIVE_WORKERS=0
get_all_items
log DEBUG "Found $SIZE_OF_INPUT items as daemon."
start_all_workers
sleep 10
}
display_progress () { display_progress () {
if [ "$DAEMON" = "0" ] if [ "$DAEMON" = "0" ]
@ -2196,6 +2189,8 @@ display_progress () {
terminate_listener () { terminate_listener () {
log DEBUG "Terminating listener."
if [ ! -z "$SSH_MASTER_PID" ] if [ ! -z "$SSH_MASTER_PID" ]
then then
log DEBUG "SSH master PID is $SSH_MASTER_PID" log DEBUG "SSH master PID is $SSH_MASTER_PID"
@ -2212,7 +2207,7 @@ terminate_listener () {
echo echo
stop-ppss stop-ppss
log DSPLY "Finished. Consult $JOB_LOG_DIR for job output." log DSPLY "Finished. Consult $JOB_LOG_DIR for job output."
log DSPLY "Press ENTER to continue." #log DSPLY "Press ENTER to continue."
else else
echo echo
stop-ppss stop-ppss
@ -2234,79 +2229,142 @@ inotify_listener () {
done done
} }
start_inotify_listener () { is_item_unprocessed () {
ACTIVE_WORKERS=0 VAR="$1"
inotify_listener & STATUS=0
MYPID="$!"
disown if [ -z "$VAR" ]
PIDS="$PIDS $MYPID" then
log DEBUG "$FUNCNAME: something is wrong, no argument received."
return 1
fi
for x in $PROCESSED_ITEMS
do
if [ "$x" = "$VAR" ]
then
STATUS=1
fi
done
log DEBUG "Is item $VAR unprocessed: $STATUS"
return $STATUS
} }
listen_for_job () { daemon_listener () {
FINISHED=0 while true
ACTIVE_WORKERS=$MAX_NO_OF_RUNNING_JOBS do
PIDS="" get_all_items
log DEBUG "Listener started." while get_item
do
if is_item_unprocessed "$ITEM"
then
log DEBUG "Daemon sending item $ITEM to fifo."
echo "$ITEM" >> "$FIFO"
processed_stack_push "$ITEM"
log DEBUG "Processed items is $PROCESSED_ITEMS"
fi
done
sleep "$DAEMON_POLLING_INTERVAL"
done
}
start_daemon_listener () {
daemon_listener &
MYPID="$!"
disown
PIDS="$PIDS $MYPID"
}
start_inotify_listener () {
ACTIVE_WORKERS=0
inotify_listener &
MYPID="$!"
disown
PIDS="$PIDS $MYPID"
}
start_as_daemon () {
if [ "$DAEMON" = "1" ] if [ "$DAEMON" = "1" ]
then then
log DEBUG "Daemon mode enabled."
if [ "$INOTIFY" = "1" ] if [ "$INOTIFY" = "1" ]
then then
log INFO "Linux inotify enabled." log INFO "Linux inotify enabled."
start_inotify_listener start_inotify_listener
else else
start_daemon_listener
log INFO "Linux inotify disabled." log INFO "Linux inotify disabled."
fi fi
else
log DEBUG "Daemon mode disabled."
fi fi
}
decrease_active_workers () {
if [ "$ACTIVE_WORKERS" -gt "0" ]
then
((ACTIVE_WORKERS--))
fi
}
listen_for_job () {
FINISHED=0
ACTIVE_WORKERS="$MAX_NO_OF_RUNNING_JOBS"
PIDS=""
log DEBUG "Listener started."
start_as_daemon
while read event <& 42 while read event <& 42
do do
log DEBUG "Current active workers is $ACTIVE_WORKERS"
decrease_active_workers
if [ "$event" = "$START_KEY" ] if [ "$event" = "$START_KEY" ]
then then
log DEBUG "Got a 'start-key' event" log DEBUG "Got a 'start-key' event"
if get_item
if [ "$DAEMON" = "0" ]
then then
run_command "$ITEM" if get_item
else
((ACTIVE_WORKERS--))
if [ "$DAEMON" == "1" ] && [ "$INOTIFY" = "0" ]
then then
start_as_daemon log DEBUG "Got an item, running command..."
run_command "$ITEM"
else else
break log DEBUG "No more new items..."
if [ "$ACTIVE_WORKERS" = "0" ]
then
break
else
display_jobs_remaining
fi
fi fi
display_jobs_remaining else
fi log DEBUG "Daemon mode: a worker finished..."
elif [ "$event" == "$STOP_KEY" ] decrease_active_workers
then run_command
((ACTIVE_WORKERS--))
log DEBUG "Decrease active workers to $ACTIVE_WORKERS"
if [ "$ACTIVE_WORKERS" -lt "$MAX_NO_OF_RUNNING_JOBS" ]
then
stack_pop
run_command "$REGISTER"
fi fi
elif [ "$event" == "$KILL_KEY" ] elif [ "$event" == "$KILL_KEY" ]
then then
infanticide infanticide
break break
else else
if [ -e "$event" ] log DEBUG "Event is an item!"
then stack_push "$event"
log DEBUG "Event is an item!" log DEBUG "stack after push is $STACK"
log DEBUG "$ACTIVE_WORKERS - $MAX_NO_OF_RUNNING_JOBS" run_command
stack_push "$event"
if [ "$ACTIVE_WORKERS" -lt "$MAX_NO_OF_RUNNING_JOBS" ]
then
stack_pop
run_command "$REGISTER"
log DEBUG "Active workers: $ACTIVE_WORKERS"
fi
fi
fi fi
display_progress display_progress
done done
@ -2321,6 +2379,7 @@ start_all_workers () {
else else
log DSPLY "Starting $MAX_NO_OF_RUNNING_JOBS parallel workers." log DSPLY "Starting $MAX_NO_OF_RUNNING_JOBS parallel workers."
fi fi
if [ "$DAEMON" == "0" ] if [ "$DAEMON" == "0" ]
then then
log DSPLY "---------------------------------------------------------" log DSPLY "---------------------------------------------------------"
@ -2332,7 +2391,7 @@ start_all_workers () {
i=0 i=0
while [ "$i" -lt "$MAX_NO_OF_RUNNING_JOBS" ] while [ "$i" -lt "$MAX_NO_OF_RUNNING_JOBS" ]
do do
start_single_worker start_new_worker
log DEBUG "Starting worker $i" log DEBUG "Starting worker $i"
((i++)) ((i++))