This commit is contained in:
Louwrentius 2010-07-21 06:24:25 +00:00
parent 2c42f7568e
commit 654b52d91f
2 changed files with 100 additions and 18 deletions

114
ppss
View File

@ -26,7 +26,7 @@
trap 'kill_process' SIGINT trap 'kill_process' SIGINT
SCRIPT_NAME="Distributed Parallel Processing Shell Script" SCRIPT_NAME="Distributed Parallel Processing Shell Script"
SCRIPT_VERSION="2.81" SCRIPT_VERSION="2.83"
# #
# The first argument to this script can be a mode. # The first argument to this script can be a mode.
@ -89,6 +89,7 @@ ACTIVE_WORKERS="0"
DAEMON_POLLING_INTERVAL="10" DAEMON_POLLING_INTERVAL="10"
STAT="" STAT=""
DAEMON_FILE_AGE="4" DAEMON_FILE_AGE="4"
ENABLE_INPUT_LOCK="0"
SSH_SERVER="" # Remote server or 'master'. SSH_SERVER="" # Remote server or 'master'.
SSH_KEY="" # SSH key for ssh account. SSH_KEY="" # SSH key for ssh account.
@ -118,6 +119,7 @@ ITEM_ESCAPED=""
DISABLE_SKIPPING=0 DISABLE_SKIPPING=0
NODE_STATUS="$PPSS_DIR/status.txt" NODE_STATUS="$PPSS_DIR/status.txt"
DAEMON=0 DAEMON=0
EMAIL=""
REGISTER="" # For STACK REGISTER="" # For STACK
STACK="" STACK=""
@ -136,6 +138,7 @@ showusage_short () {
echo " $0 -d /dir/with/some/files -c 'gzip '" echo " $0 -d /dir/with/some/files -c 'gzip '"
echo " $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2" echo " $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2"
echo " $0 -f <file> -c 'wget -q -P /destination/directory \"\$ITEM\"' -p 10" echo " $0 -f <file> -c 'wget -q -P /destination/directory \"\$ITEM\"' -p 10"
echo
} }
showusage_normal () { showusage_normal () {
@ -178,15 +181,19 @@ showusage_normal () {
echo -e "--quiet | -q Shows no output except for a progress indication using percents." echo -e "--quiet | -q Shows no output except for a progress indication using percents."
echo echo
echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread" echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread"
echo -e " the load. The delay is only used at the start of all 'threads'." echo -e " the load. The delay (seconds) is only used at the start of all 'threads'."
echo echo
echo -e "--daemon Do not exit after items are professed, but keep looking for new items" echo -e "--daemon Daemon mode. Do not exit after items are professed, but keep looking "
echo -e " and process them. Read the manual how to use this!" echo -e " for new items and process them. Read the manual how to use this!"
echo -e " See --help for important additional options regarding daemon mode."
echo echo
echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is "
echo -e " used. If this is not prefered, this can be disabled with this option " echo -e " used. If this is not prefered, this can be disabled with this option "
echo -e " Only files within the specified directory will be processed." echo -e " Only files within the specified directory will be processed."
echo echo
echo -e "--email | -e PPSS sends an e-mail if PPSS has finished. It is also used if processing"
echo -e " of an item has failed (configurable, see -h). "
echo
echo -e "--help Extended help, including options for distributed mode and Amazon EC2." echo -e "--help Extended help, including options for distributed mode and Amazon EC2."
echo echo
echo -e "Example: encoding some wav files to mp3 using lame:" echo -e "Example: encoding some wav files to mp3 using lame:"
@ -254,6 +261,24 @@ showusage_long () {
echo -e "--daemon Do not exit after items are professed, but keep looking for new items" echo -e "--daemon Do not exit after items are professed, but keep looking for new items"
echo -e " and process them. Read the manual how to use this!" echo -e " and process them. Read the manual how to use this!"
echo echo
echo -e "--interval Specifies the polling interval when running in daemon mode. Polls every"
echo -e " x seconds for new items to process."
echo
echo -e "--file-age When not using inotify, specify how many seconds must have passed before"
echo -e " a file may be processed to prevent files being processed while being "
echo -e " written to."
echo
echo -e "--disable-inotify If for some reason, inotify must not be used, use this option to disable"
echo -e " usage of inotify. Regular polling will be used."
echo
echo -e "--enable-input-lock When PPSS is run in daemon mode, create a directory INPUT_LOCK to"
echo -e " signal that items are processed and may not be touched by PPSS."
echo -e " Once this directory is removed, PPSS will start processing items."
echo
echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is "
echo -e " used. If this is not prefered, this can be disabled with this option "
echo -e " Only files within the specified directory will be processed."
echo
echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is "
echo -e " used. If this is not prefered, this can be disabled with this option." echo -e " used. If this is not prefered, this can be disabled with this option."
echo -e " Only files within the specified directory will be processed." echo -e " Only files within the specified directory will be processed."
@ -449,7 +474,6 @@ is_var_empty () {
fi fi
} }
detect_inotify () { detect_inotify () {
if [ -e /usr/bin/inotifywait ] && [ ! "$INOTIFY" = "0" ] if [ -e /usr/bin/inotifywait ] && [ ! "$INOTIFY" = "0" ]
@ -546,6 +570,10 @@ process_arguments () {
INOTIFY=0 INOTIFY=0
add_var_to_config INOTIFY "$INOTIFY" add_var_to_config INOTIFY "$INOTIFY"
shift 1 ;; shift 1 ;;
--enable-input-lock)
ENABLE_INPUT_LOCK=1
add_var_to_config ENABLE_INPUT_LOCK "$ENABLE_INPUT_LOCK"
shift 1 ;;
--daemon) --daemon)
DAEMON="1" DAEMON="1"
QUIET="1" QUIET="1"
@ -563,6 +591,11 @@ process_arguments () {
is_var_empty "$2" is_var_empty "$2"
add_var_to_config DAEMON_FILE_AGE "$DAEMON_FILE_AGE" add_var_to_config DAEMON_FILE_AGE "$DAEMON_FILE_AGE"
shift 2 ;; shift 2 ;;
--email|-e)
is_var_empty "$2"
EMAIL="$2"
add_var_to_config EMAIL "$EMAIL"
shift 2 ;;
--awskeypair|-P) --awskeypair|-P)
AWS_KEYPAIR="$2" AWS_KEYPAIR="$2"
add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR" add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR"
@ -1435,13 +1468,9 @@ random_delay () {
fi fi
NUMBER=$RANDOM NUMBER=$RANDOM
let "NUMBER %= $ARGS" let "NUMBER %= $ARGS"
if [ "$ARCH" == "SunOS" ]
then
sleep "$NUMBER" sleep "$NUMBER"
else
sleep "0.$NUMBER"
fi
} }
escape_item () { escape_item () {
@ -1709,7 +1738,7 @@ remove_processed_items_from_input_file () {
get_all_items () { get_all_items () {
if [ "$DAEMON" == "1" ] if [ "$DAEMON" == "1" ] && [ "$INOTIFY" = "0" ] && [ "$ENABLE_INPUT_LOCK" = "1" ]
then then
GLOBAL_COUNTER=1 GLOBAL_COUNTER=1
get_input_lock get_input_lock
@ -1900,7 +1929,24 @@ elapsed () {
MINS="$(expr $(expr $REMAINDER - $SECS) / 60)" MINS="$(expr $(expr $REMAINDER - $SECS) / 60)"
RES=$(printf "Total processing time (hh:mm:ss): %02d:%02d:%02d" $HOURS $MINS $SECS) RES=$(printf "Total processing time (hh:mm:ss): %02d:%02d:%02d" $HOURS $MINS $SECS)
log DSPLY "$RES" echo "$RES"
}
mail_on_error () {
ITEM="$1"
LOGFILE="$2"
if [ "$MAIL_ON_ERROR" = "1" ]
then
cat "$LOGFILE" | mail -s "$HOSTNAME - PPSS: procesing failed for item." "$EMAIL"
if [ "$?" = "0" ]
then
log DEBUG "Error mail sent."
else
log ERROR "Sending of error email failed."
fi
fi
} }
commando () { commando () {
@ -2073,6 +2119,7 @@ commando () {
# Some error logging. Success or fail. # Some error logging. Success or fail.
if [ ! "$ERROR" == "0" ] if [ ! "$ERROR" == "0" ]
then then
mail_on_error "$ITEM" "$ITEM_LOG_FILE"
echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE" echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE"
else else
echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE" echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE"
@ -2182,7 +2229,7 @@ run_command () {
INPUT="$REGISTER" INPUT="$REGISTER"
fi fi
log DEBUG "Now processing $INPUT" log INFO "Now processing $INPUT"
if [ ! -d "$INPUT" ] && [ ! -z "$INPUT" ] if [ ! -d "$INPUT" ] && [ ! -z "$INPUT" ]
then then
@ -2217,6 +2264,31 @@ display_jobs_remaining () {
fi fi
} }
show_eta () {
CURRENT_PROCESSED=$((GLOBAL_COUNTER-MAX_NO_OF_RUNNING_JOBS))
TOTAL="$SIZE_OF_INPUT"
START_TIME=$START_PPSS
NOW=`get_time_in_seconds`
if [ "$CURRENT_PROCESSED" -le "0" ]
then
return 0
else
TIME_PER_ITEM=$(( (NOW-START_TIME)/CURRENT_PROCESSED ))
TOTAL_TIME=$(( ($TIME_PER_ITEM * SIZE_OF_INPUT) + $TIME_PER_ITEM ))
TOTAL_TIME_IN_SECONDS=$((START_TIME+TOTAL_TIME))
if [ "$ARCH" = "Darwin" ]
then
DATE=`date -r $TOTAL_TIME_IN_SECONDS`
else
DATE=`date -d @$TOTAL_TIME_IN_SECONDS`
fi
echo
log DSPLY "ETA: $DATE"
fi
}
display_progress () { display_progress () {
if [ "$DAEMON" = "0" ] if [ "$DAEMON" = "0" ]
@ -2228,6 +2300,7 @@ display_progress () {
if [ "$QUIET" == "0" ] if [ "$QUIET" == "0" ]
then then
log PRCNT "Currently $PERCENT percent complete. Processed $GLOBAL_COUNTER of $SIZE_OF_INPUT items." log PRCNT "Currently $PERCENT percent complete. Processed $GLOBAL_COUNTER of $SIZE_OF_INPUT items."
show_eta
elif [ "$DAEMON" == "0" ] elif [ "$DAEMON" == "0" ]
then then
echo -en "\r$PERCENT%" echo -en "\r$PERCENT%"
@ -2251,7 +2324,6 @@ terminate_listener () {
if [ ! -z "$SSH_MASTER_PID" ] if [ ! -z "$SSH_MASTER_PID" ]
then then
log DEBUG "SSH master PID is $SSH_MASTER_PID"
kill "$SSH_MASTER_PID" kill "$SSH_MASTER_PID"
else else
log DEBUG "SSH master PID is empty." log DEBUG "SSH master PID is empty."
@ -2275,6 +2347,16 @@ terminate_listener () {
then then
echo echo
fi fi
if [ ! -z "$EMAIL" ]
then
echo "PPSS job finished." | mail -s "$HOSTNAME - PPSS has finished." "$EMAIL"
if [ ! "$?" = "0" ]
then
log ERROR "Sending os status mail failed."
fi
fi
cleanup cleanup
} }
@ -2422,7 +2504,7 @@ listen_for_job () {
while read event <& 42 while read event <& 42
do do
log DEBUG "Current active workers is $ACTIVE_WORKERS" log INFO "Current active workers is $ACTIVE_WORKERS"
if [ "$event" = "$START_KEY" ] if [ "$event" = "$START_KEY" ]
then then
@ -2455,7 +2537,7 @@ listen_for_job () {
infanticide infanticide
break break
else else
log DEBUG "Event $event is an item!" log DEBUG "Event is an item."
stack_push "$event" stack_push "$event"
run_command run_command
fi fi

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
DEBUG="$1" DEBUG="$1"
VERSION="2.81" VERSION="2.82"
TMP_DIR="/tmp/ppss" TMP_DIR="/tmp/ppss"
PPSS=./ppss PPSS=./ppss
PPSS_DIR=ppss_dir PPSS_DIR=ppss_dir