Fixed various bugs, releae of 8.85, see changelog for details.

This commit is contained in:
Louwrentius 2010-11-21 19:40:34 +00:00
parent 8663f482c3
commit ff63e9950b
2 changed files with 92 additions and 12 deletions

102
ppss
View File

@ -26,7 +26,7 @@
trap 'kill_process' SIGINT
SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.84"
SCRIPT_VERSION="2.85"
#
# The first argument to this script can be a mode.
@ -83,6 +83,7 @@ START_PPSS=""
STOP_PPSS=""
SIZE_OF_INPUT=""
LOCAL_LOCKING="1"
LIST_OF_PROCESSED_ITEMS="$PPSS_DIR/LIST_OF_PROCESSED_ITEMS"
PROCESSED_ITEMS=""
UNPROCESSED_ITEMS=""
ACTIVE_WORKERS="0"
@ -475,9 +476,52 @@ is_var_empty () {
fi
}
detect_source_dir_nfs_exported () {
log DEBUG "Executing $FUNCNAME"
if [ -e /etc/exports ]
then
log DEBUG "NFS /etc/exports found."
NFS=0
EXPORTS=`cat /etc/exports | grep ^/ | awk '{ print $1 }'`
for export in $EXPORTS
do
#
# If this for loop matches anything, the SRC_DIR is NFS exported.
# inotify does not play well with NFS. So it must be disabled.
#
DIRECTORY=`dirname "$SRC_DIR"`
while true
do
if [ ! "$DIRECTORY" = "/" ] && [ ! "$DIRECTORY" = "." ]
then
if [ "$export" = "$DIRECTORY" ]
then
NFS=1
break
fi
else
break
fi
DIRECTORY=`dirname "$DIRECTORY"`
done
done
fi
if [ "$NFS" = "1" ]
then
log INFO "Source directory is NFS exported. Disabling inotify."
return 1
else
log INFO "Source directory is NOT NFS exported. Enabling inotify."
return 0
fi
}
detect_inotify () {
if [ -e /usr/bin/inotifywait ] && [ ! "$INOTIFY" = "0" ]
if [ -e /usr/bin/inotifywait ] && [ ! "$INOTIFY" = "0" ] && detect_source_dir_nfs_exported
then
INOTIFY=1
else
@ -882,8 +926,13 @@ log () {
PREFIX="$DATE: ${TYPE_EXP:0:$TYPE_LENGTH}"
PREFIX_SMALL="$DATE: "
LOG_MSG="$PREFIX $MESG"
if [ ! "$TYPE" = "ERROR" ]
then
ECHO_MSG="$PREFIX_SMALL $MESG"
else
ECHO_MSG="$PREFIX_SMALL [ERROR] $MESG"
fi
LOG_MSG="$PREFIX $MESG"
if [ ! -z "$PPSS_DEBUG" ] && [ ! "$PPSS_DEBUG" == "0" ]
then
@ -1681,16 +1730,25 @@ remove_processed_items_from_input_file () {
#
UNPROCESSED_ITEMS=""
if [ -e "$LIST_OF_PROCESSED_ITEMS" ]
then
PROCESSED_ITEMS=`cat $LIST_OF_PROCESSED_ITEMS`
fi
log DEBUG "Running $FUNCNAME"
if [ -z "$PROCESSED_ITEMS" ]
then
log DEBUG "Variable processed_items is empty."
return 1
fi
if [ "$MODE" = "status" ]
then
log DEBUG "Mode is status."
return 1
fi
log DEBUG "Running $FUNCNAME...."
if [ ! -e "$LISTOFITEMS" ]
then
echo "$LISTOFITEMS does not exist!"
@ -1709,6 +1767,8 @@ remove_processed_items_from_input_file () {
oldIFS=$IFS # save the field separator
IFS=$'\n' # new field separator, the end of line
log DEBUG "Now removing processed items from input."
for x in $INPUTFILES
do
FILE_IS_PROCESSED=0
@ -1788,7 +1848,7 @@ get_all_items () {
ITEMS=""
fi
fi
else
else # Using an input file as the source of our items or STDIN.
if [ ! -z "$SSH_SERVER" ] # Are we running stand-alone or as a slave?"
then
log DEBUG "Running as node, input file has been pushed (hopefully)."
@ -1812,7 +1872,18 @@ get_all_items () {
echo "$LINE" >> "$LISTOFITEMS"
done
fi
if [ ! -e "$LISTOFITEMS" ]
then
log ERROR "Input is empty."
infanticide
terminate_listener
cleanup
exit 1
fi
fi
remove_processed_items_from_input_file
if [ "$DAEMON" == "1" ]
then
@ -1821,6 +1892,11 @@ get_all_items () {
SIZE_OF_INPUT=$(wc -l "$LISTOFITEMS" | awk '{ print $1 }')
#if [ "$SIZE_OF_INPUT" -eq "1" ]
#then
# MAX_NO_OF_RUNNING_JOBS=1
#fi
if [ "$SIZE_OF_INPUT" -le "0" ] && [ "$DAEMON" = "0" ]
then
log ERROR "Source file/dir seems to be empty."
@ -1828,7 +1904,7 @@ get_all_items () {
cleanup
exit 1
fi
remove_processed_items_from_input_file
}
get_item () {
@ -2184,6 +2260,8 @@ commando () {
infanticide () {
log DEBUG "Running $FUNCNAME"
#
# This code is run if ctrl+c is pressed. Very important to prevent
# any child processes running after the parent has died. Keeps the system clean.
@ -2232,7 +2310,7 @@ run_command () {
log INFO "Now processing $INPUT"
if [ ! -d "$INPUT" ] && [ ! -z "$INPUT" ]
if [ ! -z "$INPUT" ] && [ ! -d "$INPUT" ]
then
commando "$INPUT" &
MYPID="$!"
@ -2240,10 +2318,11 @@ run_command () {
PIDS="$PIDS $MYPID"
((ACTIVE_WORKERS++))
log DEBUG "Increasing active workers to $ACTIVE_WORKERS"
echo "$INPUT" >> "$LIST_OF_PROCESSED_ITEMS"
return 0
else
log DEBUG "Item is a directory or empty."
return 1
log DEBUG "Item is a directory or is empty."
return 0
fi
else
log DEBUG "Maximum number of workers are bussy, no more additional workers..."
@ -2288,6 +2367,7 @@ show_eta () {
if [ ! "$RUNNING_TIME" -le "0" ] && [ ! "$CURRENT_PROCESSED" = "0" ] && [ "$CURRENT_PROCESSED" -gt "$MAX_NO_OF_RUNNING_JOBS" ]
then
TIME_PER_ITEM=$(( RUNNING_TIME / ( CURRENT_PROCESSED - MAX_NO_OF_RUNNING_JOBS ) ))
log DEBUG "Time per item is $TIME_PER_ITEM seconds."
TOTAL_TIME=$(( ($TIME_PER_ITEM * SIZE_OF_INPUT) + $TIME_PER_ITEM ))
TOTAL_TIME_IN_SECONDS=$((START_TIME+TOTAL_TIME))
if [ "$ARCH" = "Darwin" ]
@ -2335,7 +2415,7 @@ display_progress () {
terminate_listener () {
log DEBUG "Terminating listener."
log DEBUG "Running $FUNCNAME"
if [ ! -z "$SSH_MASTER_PID" ]
then
@ -2520,6 +2600,7 @@ listen_for_job () {
while read event <& 42
do
display_progress
log INFO "Current active workers is $ACTIVE_WORKERS"
if [ "$event" = "$START_KEY" ]
@ -2557,7 +2638,6 @@ listen_for_job () {
stack_push "$event"
run_command
fi
display_progress
done
terminate_listener

View File

@ -1,7 +1,7 @@
#!/bin/bash
DEBUG="$1"
VERSION="2.83"
VERSION="2.84"
TMP_DIR="/tmp/ppss"
PPSS=./ppss
PPSS_DIR=ppss_dir