From 654b52d91fd41590702cd4154db5e477cc385028 Mon Sep 17 00:00:00 2001 From: Louwrentius Date: Wed, 21 Jul 2010 06:24:25 +0000 Subject: [PATCH] backup --- ppss | 116 +++++++++++++++++++++++++++++++++++++++++++-------- ppss-test.sh | 2 +- 2 files changed, 100 insertions(+), 18 deletions(-) diff --git a/ppss b/ppss index dd02074..80b125f 100755 --- a/ppss +++ b/ppss @@ -26,7 +26,7 @@ trap 'kill_process' SIGINT SCRIPT_NAME="Distributed Parallel Processing Shell Script" -SCRIPT_VERSION="2.81" +SCRIPT_VERSION="2.83" # # The first argument to this script can be a mode. @@ -89,6 +89,7 @@ ACTIVE_WORKERS="0" DAEMON_POLLING_INTERVAL="10" STAT="" DAEMON_FILE_AGE="4" +ENABLE_INPUT_LOCK="0" SSH_SERVER="" # Remote server or 'master'. SSH_KEY="" # SSH key for ssh account. @@ -118,6 +119,7 @@ ITEM_ESCAPED="" DISABLE_SKIPPING=0 NODE_STATUS="$PPSS_DIR/status.txt" DAEMON=0 +EMAIL="" REGISTER="" # For STACK STACK="" @@ -136,6 +138,7 @@ showusage_short () { echo " $0 -d /dir/with/some/files -c 'gzip '" echo " $0 -d /dir/with/some/files -c 'cp \"\$ITEM\" /tmp' -p 2" echo " $0 -f -c 'wget -q -P /destination/directory \"\$ITEM\"' -p 10" + echo } showusage_normal () { @@ -178,15 +181,19 @@ showusage_normal () { echo -e "--quiet | -q Shows no output except for a progress indication using percents." echo echo -e "--delay | -D Adds an initial random delay to the start of all parallel jobs to spread" - echo -e " the load. The delay is only used at the start of all 'threads'." + echo -e " the load. The delay (seconds) is only used at the start of all 'threads'." echo - echo -e "--daemon Do not exit after items are professed, but keep looking for new items" - echo -e " and process them. Read the manual how to use this!" + echo -e "--daemon Daemon mode. Do not exit after items are professed, but keep looking " + echo -e " for new items and process them. Read the manual how to use this!" + echo -e " See --help for important additional options regarding daemon mode." echo echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e " used. If this is not prefered, this can be disabled with this option " echo -e " Only files within the specified directory will be processed." echo + echo -e "--email | -e PPSS sends an e-mail if PPSS has finished. It is also used if processing" + echo -e " of an item has failed (configurable, see -h). " + echo echo -e "--help Extended help, including options for distributed mode and Amazon EC2." echo echo -e "Example: encoding some wav files to mp3 using lame:" @@ -254,6 +261,24 @@ showusage_long () { echo -e "--daemon Do not exit after items are professed, but keep looking for new items" echo -e " and process them. Read the manual how to use this!" echo + echo -e "--interval Specifies the polling interval when running in daemon mode. Polls every" + echo -e " x seconds for new items to process." + echo + echo -e "--file-age When not using inotify, specify how many seconds must have passed before" + echo -e " a file may be processed to prevent files being processed while being " + echo -e " written to." + echo + echo -e "--disable-inotify If for some reason, inotify must not be used, use this option to disable" + echo -e " usage of inotify. Regular polling will be used." + echo + echo -e "--enable-input-lock When PPSS is run in daemon mode, create a directory INPUT_LOCK to" + echo -e " signal that items are processed and may not be touched by PPSS." + echo -e " Once this directory is removed, PPSS will start processing items." + echo + echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " + echo -e " used. If this is not prefered, this can be disabled with this option " + echo -e " Only files within the specified directory will be processed." + echo echo -e "--no-recursion|-r By default, recursion of directories is enabled when the -d option is " echo -e " used. If this is not prefered, this can be disabled with this option." echo -e " Only files within the specified directory will be processed." @@ -449,7 +474,6 @@ is_var_empty () { fi } - detect_inotify () { if [ -e /usr/bin/inotifywait ] && [ ! "$INOTIFY" = "0" ] @@ -546,6 +570,10 @@ process_arguments () { INOTIFY=0 add_var_to_config INOTIFY "$INOTIFY" shift 1 ;; + --enable-input-lock) + ENABLE_INPUT_LOCK=1 + add_var_to_config ENABLE_INPUT_LOCK "$ENABLE_INPUT_LOCK" + shift 1 ;; --daemon) DAEMON="1" QUIET="1" @@ -563,6 +591,11 @@ process_arguments () { is_var_empty "$2" add_var_to_config DAEMON_FILE_AGE "$DAEMON_FILE_AGE" shift 2 ;; + --email|-e) + is_var_empty "$2" + EMAIL="$2" + add_var_to_config EMAIL "$EMAIL" + shift 2 ;; --awskeypair|-P) AWS_KEYPAIR="$2" add_var_to_config AWS_KEYPAIR "$AWS_KEYPAIR" @@ -1435,13 +1468,9 @@ random_delay () { fi NUMBER=$RANDOM + let "NUMBER %= $ARGS" - if [ "$ARCH" == "SunOS" ] - then - sleep "$NUMBER" - else - sleep "0.$NUMBER" - fi + sleep "$NUMBER" } escape_item () { @@ -1709,7 +1738,7 @@ remove_processed_items_from_input_file () { get_all_items () { - if [ "$DAEMON" == "1" ] + if [ "$DAEMON" == "1" ] && [ "$INOTIFY" = "0" ] && [ "$ENABLE_INPUT_LOCK" = "1" ] then GLOBAL_COUNTER=1 get_input_lock @@ -1900,7 +1929,24 @@ elapsed () { MINS="$(expr $(expr $REMAINDER - $SECS) / 60)" RES=$(printf "Total processing time (hh:mm:ss): %02d:%02d:%02d" $HOURS $MINS $SECS) - log DSPLY "$RES" + echo "$RES" +} + +mail_on_error () { + + ITEM="$1" + LOGFILE="$2" + + if [ "$MAIL_ON_ERROR" = "1" ] + then + cat "$LOGFILE" | mail -s "$HOSTNAME - PPSS: procesing failed for item." "$EMAIL" + if [ "$?" = "0" ] + then + log DEBUG "Error mail sent." + else + log ERROR "Sending of error email failed." + fi + fi } commando () { @@ -2073,6 +2119,7 @@ commando () { # Some error logging. Success or fail. if [ ! "$ERROR" == "0" ] then + mail_on_error "$ITEM" "$ITEM_LOG_FILE" echo -e "Status:\t\tFAILURE" >> "$ITEM_LOG_FILE" else echo -e "Status:\t\tSUCCESS" >> "$ITEM_LOG_FILE" @@ -2182,7 +2229,7 @@ run_command () { INPUT="$REGISTER" fi - log DEBUG "Now processing $INPUT" + log INFO "Now processing $INPUT" if [ ! -d "$INPUT" ] && [ ! -z "$INPUT" ] then @@ -2217,6 +2264,31 @@ display_jobs_remaining () { fi } +show_eta () { + + CURRENT_PROCESSED=$((GLOBAL_COUNTER-MAX_NO_OF_RUNNING_JOBS)) + TOTAL="$SIZE_OF_INPUT" + START_TIME=$START_PPSS + NOW=`get_time_in_seconds` + + if [ "$CURRENT_PROCESSED" -le "0" ] + then + return 0 + else + TIME_PER_ITEM=$(( (NOW-START_TIME)/CURRENT_PROCESSED )) + TOTAL_TIME=$(( ($TIME_PER_ITEM * SIZE_OF_INPUT) + $TIME_PER_ITEM )) + TOTAL_TIME_IN_SECONDS=$((START_TIME+TOTAL_TIME)) + if [ "$ARCH" = "Darwin" ] + then + DATE=`date -r $TOTAL_TIME_IN_SECONDS` + else + DATE=`date -d @$TOTAL_TIME_IN_SECONDS` + fi + echo + log DSPLY "ETA: $DATE" + fi +} + display_progress () { if [ "$DAEMON" = "0" ] @@ -2228,6 +2300,7 @@ display_progress () { if [ "$QUIET" == "0" ] then log PRCNT "Currently $PERCENT percent complete. Processed $GLOBAL_COUNTER of $SIZE_OF_INPUT items." + show_eta elif [ "$DAEMON" == "0" ] then echo -en "\r$PERCENT%" @@ -2251,7 +2324,6 @@ terminate_listener () { if [ ! -z "$SSH_MASTER_PID" ] then - log DEBUG "SSH master PID is $SSH_MASTER_PID" kill "$SSH_MASTER_PID" else log DEBUG "SSH master PID is empty." @@ -2275,6 +2347,16 @@ terminate_listener () { then echo fi + + if [ ! -z "$EMAIL" ] + then + echo "PPSS job finished." | mail -s "$HOSTNAME - PPSS has finished." "$EMAIL" + if [ ! "$?" = "0" ] + then + log ERROR "Sending os status mail failed." + fi + fi + cleanup } @@ -2422,7 +2504,7 @@ listen_for_job () { while read event <& 42 do - log DEBUG "Current active workers is $ACTIVE_WORKERS" + log INFO "Current active workers is $ACTIVE_WORKERS" if [ "$event" = "$START_KEY" ] then @@ -2455,7 +2537,7 @@ listen_for_job () { infanticide break else - log DEBUG "Event $event is an item!" + log DEBUG "Event is an item." stack_push "$event" run_command fi diff --git a/ppss-test.sh b/ppss-test.sh index ecd69bc..c38b42a 100755 --- a/ppss-test.sh +++ b/ppss-test.sh @@ -1,7 +1,7 @@ #!/bin/bash DEBUG="$1" -VERSION="2.81" +VERSION="2.82" TMP_DIR="/tmp/ppss" PPSS=./ppss PPSS_DIR=ppss_dir