Compare commits
112 Commits
distribute
...
master
Author | SHA1 | Date |
---|---|---|
Jeffrey Paul | e425c6fd5b | |
louwrentius | 2aec933067 | |
louwrentius | 8e11672952 | |
louwrentius | 2f612846cb | |
louwrentius | baa44063f4 | |
louwrentius | b24d58294e | |
louwrentius | 8fe6ca64db | |
louwrentius | a33f87d0b0 | |
louwrentius | eb61175bf4 | |
louwrentius | 01b944ad9e | |
louwrentius | 66d0d3e4b2 | |
louwrentius | 42c4305ac5 | |
louwrentius | d947d84646 | |
louwrentius | c78896892b | |
louwrentius | ec1d542ab6 | |
louwrentius | 30f92bbae9 | |
louwrentius | 74d1e90282 | |
louwrentius | a06634d450 | |
louwrentius | dbd8f51590 | |
louwrentius | a908682185 | |
louwrentius | 48c80d9257 | |
louwrentius | 828e4d135c | |
louwrentius | 114112241f | |
louwrentius | 625eb7e1e4 | |
louwrentius | b69e1e6d7b | |
louwrentius | edb7c21683 | |
louwrentius | 55f4b1f2e2 | |
louwrentius | eaa18282d1 | |
louwrentius | 2dcb39ecbf | |
louwrentius | 136c1c13ca | |
louwrentius | c344794953 | |
louwrentius | bccd42ab9d | |
louwrentius | cc1aa38f05 | |
louwrentius | 836378eda8 | |
louwrentius | ed4c0bf554 | |
louwrentius | 72e0c983d0 | |
louwrentius | 3c9a4d5ee8 | |
louwrentius | 29713bcc14 | |
louwrentius | 126e71e231 | |
Louwrentius | 2946abc3d6 | |
Louwrentius | c07e95b2a2 | |
Louwrentius | ff63e9950b | |
Louwrentius | 8663f482c3 | |
Louwrentius | 427c97f5af | |
Louwrentius | 654b52d91f | |
Louwrentius | 2c42f7568e | |
Louwrentius | 3ec86b451e | |
Louwrentius | 1575fb28f6 | |
Louwrentius | 20e8cdce95 | |
Louwrentius | 7550b4a5c1 | |
Louwrentius | c4bc6f963a | |
Louwrentius | d18102ba58 | |
Louwrentius | c220353e7e | |
Louwrentius | ecc9d3da3b | |
Louwrentius | caa6519d0f | |
Louwrentius | 211bd40641 | |
Louwrentius | 5614724162 | |
Louwrentius | 55583de5f8 | |
Louwrentius | b6bd8c6c33 | |
Louwrentius | c1a359340c | |
Louwrentius | 6836784b54 | |
Louwrentius | b922bd40e5 | |
Louwrentius | f1ccfd2457 | |
Louwrentius | 8eec8287f0 | |
Louwrentius | f85f7b6d03 | |
Louwrentius | b133b76ddf | |
Louwrentius | 4b9df537a0 | |
Louwrentius | 708803688f | |
Louwrentius | 40b3ed8228 | |
Louwrentius | 9d92350f4b | |
Louwrentius | 0db3781021 | |
Louwrentius | 9feea3fb59 | |
Louwrentius | 7c2d336dba | |
Louwrentius | 61d8a24ed2 | |
Louwrentius | 825d7ed0a1 | |
Louwrentius | 2cae93e7ce | |
Louwrentius | 13e6b77c65 | |
Louwrentius | 7c9da647b5 | |
Louwrentius | 2dcaced13b | |
Louwrentius | 38812a58b6 | |
Louwrentius | cf3b54e61e | |
Louwrentius | 8b7944dad7 | |
Louwrentius | 3f1b96e1bb | |
Louwrentius | bdaab0afe3 | |
Louwrentius | efd6f72a5e | |
Louwrentius | 7df68e7658 | |
Louwrentius | f0ca726f28 | |
Louwrentius | 806f17c2dc | |
Louwrentius | 2c595ce846 | |
Louwrentius | 283c93f871 | |
Louwrentius | ea33b0f0e7 | |
Louwrentius | 0109e1fa3c | |
Louwrentius | 831e008e11 | |
Louwrentius | 54c898f202 | |
Louwrentius | 0f8dced209 | |
Louwrentius | 789db609c9 | |
Louwrentius | d1685d297c | |
Louwrentius | d5ea910e05 | |
Louwrentius | a63973fe73 | |
Louwrentius | cda2c7b5c1 | |
Louwrentius | 53deba3d63 | |
Louwrentius | a2fdbba0c4 | |
Louwrentius | 365705e8c7 | |
Louwrentius | 9790e52c89 | |
Louwrentius | 50de3d4fdf | |
Louwrentius | 4d5324e7c8 | |
Louwrentius | a9b4eac886 | |
Louwrentius | bd97eed1ea | |
Louwrentius | 081337c5b2 | |
Louwrentius | 2356f6bb0a | |
Louwrentius | 83bc8fbf06 | |
Louwrentius | 121f1e5779 |
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
INPUT="$1"
|
||||
|
||||
METATAGS="--export-tags-to="
|
||||
LAMEOPTS=""
|
||||
ERROR_STATUS="0"
|
||||
|
||||
function usage () {
|
||||
|
||||
echo
|
||||
echo "Usage: $0 <flac file name>"
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
function error () {
|
||||
|
||||
ERROR="$1"
|
||||
MSG="$2"
|
||||
|
||||
echo "Error: $MSG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
if [ -z "$INPUT" ]
|
||||
then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ ! -e "$INPUT" ]
|
||||
then
|
||||
echo "File $INPUT does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILETYPE="`file -b "$INPUT" | awk '{ print $1 }'`"
|
||||
if [ ! "$FILETYPE" == "FLAC" ]
|
||||
then
|
||||
echo "File $FILE is not a flac file..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
checkvar () {
|
||||
|
||||
VAR="$1"
|
||||
|
||||
if [ -z "$VAR" ] || [ "$VAR" == "" ]
|
||||
then
|
||||
echo "Unknown"
|
||||
else
|
||||
echo "$VAR"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
METATAGS="TITLE ARTIST ALBUM GENRE COMPOSER CONDUCTOR ENSEMBLE TRACKNUMBER DATE ALBUM ARTIST DISCNUMBER DISC"
|
||||
|
||||
function convert () {
|
||||
|
||||
FILE="$1"
|
||||
META="$FILE.meta"
|
||||
MP3FILE="`echo ${FILE%flac}mp3`"
|
||||
DIR="`dirname "$FILE"`"
|
||||
|
||||
metaflac --export-tags-to="$META" "$FILE"
|
||||
|
||||
ARTIST="`metaflac "$FILE" --show-tag=ARTIST | sed s/.*=//g`"
|
||||
TITLE="`metaflac "$FILE" --show-tag=TITLE | sed s/.*=//g`"
|
||||
ALBUM="`metaflac "$FILE" --show-tag=ALBUM | sed s/.*=//g`"
|
||||
GENRE="`metaflac "$FILE" --show-tag=GENRE | sed s/.*=//g`"
|
||||
TRACKNUMBER="`metaflac "$FILE" --show-tag=TRACKNUMBER | sed s/.*=//g`"
|
||||
|
||||
for x in $METATAGS
|
||||
do
|
||||
declare $x="`grep "$x" "$META" | cut -d "=" -f 2`"
|
||||
|
||||
VAR=$(eval echo " \$$x")
|
||||
VAR="`checkvar $VAR`"
|
||||
done
|
||||
|
||||
|
||||
flac -s -c -d "$FILE" | lame --tt "$TITLE" --tn "$TRACKNUMBER" --tg "$GENRE" --ty "$DATE" --ta "$ARTIST" --tl "$ALBUM" --ty "$YEAR" --preset insane - "$MP3FILE"
|
||||
ERROR_STATUS="$?"
|
||||
if [ -e "$META" ]
|
||||
then
|
||||
rm "$META"
|
||||
fi
|
||||
}
|
||||
|
||||
convert "$INPUT"
|
||||
|
||||
exit "$ERROR_STATUS"
|
|
@ -0,0 +1,283 @@
|
|||
#!/bin/bash
|
||||
|
||||
DEBUG="$1"
|
||||
VERSION="2.98"
|
||||
TMP_DIR="/tmp/ppss"
|
||||
PPSS=./ppss
|
||||
PPSS_DIR=ppss_dir
|
||||
export PPSS_DEBUG=1
|
||||
HOST_ARCH=`uname`
|
||||
SPECIAL_DIR=$TMP_DIR/root/special
|
||||
. "$PPSS"
|
||||
|
||||
cleanup () {
|
||||
|
||||
unset RES1
|
||||
unset RES2
|
||||
GLOBAL_COUNTER=1
|
||||
if [ ! "$DEBUG" = "debug" ]
|
||||
then
|
||||
for x in $REMOVEFILES
|
||||
do
|
||||
if [ -e ./$x ]
|
||||
then
|
||||
rm -r ./$x
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ ! -z "$TMP_DIR" ] && [ -e "$TMP_DIR" ]
|
||||
then
|
||||
rm -rf "$TMP_DIR"
|
||||
fi
|
||||
}
|
||||
|
||||
parseJobStatus () {
|
||||
|
||||
TMP_FILE="$1"
|
||||
|
||||
RES=`grep "Status:" "$JOBLOG/$TMP_FILE"`
|
||||
STATUS=`echo "$RES" | awk '{ print $2 }'`
|
||||
echo "$STATUS"
|
||||
}
|
||||
|
||||
get_item_count_of_input_file () {
|
||||
|
||||
if [ -e "$PPSS_DIR/INPUT_FILE-$$" ]
|
||||
then
|
||||
CONTENTS_OF_INPUTFILE=`cat $PPSS_DIR/INPUT_FILE-$$ | wc -l | awk '{ print $1 }'`
|
||||
echo "$CONTENTS_OF_INPUTFILE"
|
||||
else
|
||||
echo "Error, file $PPSS_DIR/INPUT_FILE-$$ does not exist."
|
||||
fi
|
||||
}
|
||||
|
||||
oneTimeSetUp () {
|
||||
|
||||
JOBLOG=./$PPSS_DIR/job_log
|
||||
INPUTFILENORMAL=test-normal.input
|
||||
INPUTFILESPECIAL_DIR=test-special.input
|
||||
LOCALOUTPUT=ppss_dir/PPSS_LOCAL_OUTPUT
|
||||
REMOVEFILES="$PPSS_DIR test-ppss-*"
|
||||
|
||||
if [ ! -e "$TMP_DIR" ]
|
||||
then
|
||||
mkdir -p "$TMP_DIR"
|
||||
fi
|
||||
|
||||
cleanup
|
||||
}
|
||||
|
||||
testVersion () {
|
||||
|
||||
assertEquals "Version mismatch!" "$VERSION" "$SCRIPT_VERSION"
|
||||
}
|
||||
|
||||
rename-ppss-dir () {
|
||||
|
||||
TEST="$1"
|
||||
|
||||
if [ -e "$PPSS_DIR" ] && [ -d "$PPSS_DIR" ] && [ ! -z "$TEST" ]
|
||||
then
|
||||
mv "$PPSS_DIR" test-ppss-"$TEST"
|
||||
fi
|
||||
}
|
||||
|
||||
oneTimeTearDown () {
|
||||
|
||||
if [ ! "$DEBUG" == "debug" ]
|
||||
then
|
||||
cleanup
|
||||
fi
|
||||
}
|
||||
|
||||
createDirectoryWithSomeFiles () {
|
||||
|
||||
ROOT_DIR=$TMP_DIR/root
|
||||
CHILD_1=$ROOT_DIR/child_1
|
||||
CHILD_2=$ROOT_DIR/child_2
|
||||
|
||||
if [ ! -e "$ROOT_DIR" ]
|
||||
then
|
||||
mkdir -p "$ROOT_DIR"
|
||||
fi
|
||||
|
||||
if [ ! -e "$CHILD_1" ]
|
||||
then
|
||||
mkdir -p "$CHILD_1"
|
||||
fi
|
||||
|
||||
if [ ! -e "$CHILD_2" ]
|
||||
then
|
||||
mkdir -p "$CHILD_2"
|
||||
fi
|
||||
|
||||
for x in {1..10}
|
||||
do
|
||||
touch "$ROOT_DIR/file-$x"
|
||||
touch "$CHILD_1/file-$x"
|
||||
touch "$CHILD_2/file-$x"
|
||||
done
|
||||
|
||||
ln -s /etc/resolve.conf "$ROOT_DIR" 2> /dev/null
|
||||
ln -s /etc/hosts "$ROOT_DIR" 2> /dev/null
|
||||
}
|
||||
|
||||
createSpecialFilenames () {
|
||||
|
||||
ERROR=0
|
||||
mkdir -p "$SPECIAL_DIR"
|
||||
|
||||
touch "$SPECIAL_DIR/a file with spaces"
|
||||
touch "$SPECIAL_DIR/a\\'file\\'with\\'quotes"
|
||||
touch "$SPECIAL_DIR/a{file}with{curly}brackets}"
|
||||
touch "$SPECIAL_DIR/a(file)with(parenthesis)"
|
||||
touch "$SPECIAL_DIR/a\\file\\with\\backslashes"
|
||||
touch "$SPECIAL_DIR/a!file!with!exclamationmarks"
|
||||
touch "$SPECIAL_DIR/a filé with special characters"
|
||||
touch "$SPECIAL_DIR/a\"file\"with\"double\"quotes"
|
||||
}
|
||||
|
||||
testMD5 () {
|
||||
|
||||
export USE_MD5=1
|
||||
init_vars > /dev/null 2>&1
|
||||
ARCH=Darwin
|
||||
set_md5
|
||||
assertEquals "MD5 executable not set properly - $MD5" "$MD5" "md5"
|
||||
ARCH=Linux
|
||||
set_md5
|
||||
assertEquals "MD5 executable not set properly - $MD5" "$MD5" "md5sum"
|
||||
ARCH=$HOST_ARCH
|
||||
}
|
||||
|
||||
init_get_all_items () {
|
||||
|
||||
DIR="$1"
|
||||
TRAVERSAL="$2"
|
||||
createDirectoryWithSomeFiles
|
||||
create_working_directory
|
||||
export SRC_DIR=$DIR
|
||||
init_vars > /dev/null 2>&1
|
||||
get_all_items
|
||||
}
|
||||
|
||||
testRecursion () {
|
||||
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
RESULT=`get_item_count_of_input_file`
|
||||
EXPECTED=32
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
|
||||
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
testNoRecursion () {
|
||||
|
||||
init_get_all_items $TMP_DIR/root 0
|
||||
RESULT=`get_item_count_of_input_file`
|
||||
EXPECTED=12
|
||||
|
||||
assertEquals "Recursion not correct." "$EXPECTED" "$RESULT"
|
||||
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
testGetItem () {
|
||||
|
||||
createSpecialFilenames
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
get_item
|
||||
if [ -z "$ITEM" ]
|
||||
then
|
||||
ERROR=1
|
||||
else
|
||||
ERROR=0
|
||||
fi
|
||||
EXPECTED=0
|
||||
assertEquals "Get item failed." "$EXPECTED" "$ERROR"
|
||||
|
||||
i=1
|
||||
ERROR=0
|
||||
while get_item
|
||||
do
|
||||
((i++))
|
||||
done
|
||||
EXPECTED=40
|
||||
assertEquals "Got wrong number of items." "$EXPECTED" "$i"
|
||||
|
||||
rename-ppss-dir $FUNCNAME
|
||||
cleanup
|
||||
}
|
||||
|
||||
return_all_items () {
|
||||
|
||||
while get_item
|
||||
do
|
||||
ALL_ITEMS="$ALL_ITEMS$ITEM"$'\n'
|
||||
done
|
||||
echo "$ALL_ITEMS"
|
||||
}
|
||||
|
||||
testNumberOfItems () {
|
||||
|
||||
createSpecialFilenames
|
||||
RESULT=`init_get_all_items $TMP_DIR/root 1`
|
||||
|
||||
RES1=`find $TMP_DIR/root/ ! -type d`
|
||||
|
||||
RES2=`return_all_items`
|
||||
|
||||
echo "$RES1" > a
|
||||
echo "$RES2" > b
|
||||
|
||||
assertEquals "Input file and actual files not the same!" "$RES1" "$RES2"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
|
||||
testInvalidProcessingOfitemVariable() {
|
||||
|
||||
createSpecialFilenames
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
COMMAND='echo $ITEM'
|
||||
while get_item
|
||||
do
|
||||
commando "$ITEM"
|
||||
done
|
||||
RESULT=$(grep '$ITEM' $PPSS_DIR/job_log/*)
|
||||
EXPECTED=""
|
||||
assertEquals "Got incorrect processing of ITEM variable." "$EXPECTED" "$RESULT"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
|
||||
testNumberOfLogfiles () {
|
||||
|
||||
createSpecialFilenames
|
||||
init_get_all_items $TMP_DIR/root 1
|
||||
COMMAND='echo hoi'
|
||||
while get_item
|
||||
do
|
||||
commando "$ITEM"
|
||||
done
|
||||
RESULT=`ls -1 $PPSS_DIR/job_log/ | wc -l | awk '{ print $1}'`
|
||||
EXPECTED=40
|
||||
assertEquals "Got wrong number of log files." "$EXPECTED" "$RESULT"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
testUserInputFile () {
|
||||
|
||||
cleanup
|
||||
INPUT_FILE=test-special.input
|
||||
create_working_directory
|
||||
init_vars > /dev/null 2>&1
|
||||
get_all_items
|
||||
RESULT=`return_all_items`
|
||||
ORIGINAL=`cat $INPUT_FILE`
|
||||
assertEquals "User input processing not ok." "$RESULT" "$ORIGINAL"
|
||||
rename-ppss-dir $FUNCNAME
|
||||
}
|
||||
|
||||
. ./shunit2
|
|
@ -0,0 +1,14 @@
|
|||
REMOTE_OUTPUT_DIR=/mnt/mp3
|
||||
SSH_KEY=ppss-key.dsa
|
||||
SSH_KNOWN_HOSTS=known_hosts
|
||||
SRC_DIR=/mnt/wav
|
||||
COMMAND='./wav2mp3.sh "$ITEM" "$OUTPUT_DIR"'
|
||||
NODES_FILE=nodes.txt
|
||||
SSH_SERVER=10.0.1.110
|
||||
USER=ppss
|
||||
SCRIPT=wav2mp3.sh
|
||||
RANDOMIZE=1
|
||||
DOWNLOAD_TO_NODE=0
|
||||
UPLOAD_TO_SERVER=0
|
||||
SECURE_COPY=1
|
||||
PPSS_DEBUG=1
|
851
ppss.sh
851
ppss.sh
|
@ -1,851 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#*
|
||||
#* PPSS, the Parallel Processing Shell Script
|
||||
#*
|
||||
#* Copyright (c) 2009, Louwrentius
|
||||
#* All rights reserved.
|
||||
#*
|
||||
#* Redistribution and use in source and binary forms, with or without
|
||||
#* modification, are permitted provided that the following conditions are met:
|
||||
#* * Redistributions of source code must retain the above copyright
|
||||
#* notice, this list of conditions and the following disclaimer.
|
||||
#* * Redistributions in binary form must reproduce the above copyright
|
||||
#* notice, this list of conditions and the following disclaimer in the
|
||||
#* documentation and/or other materials provided with the distribution.
|
||||
#* * Neither the name of the <organization> nor the
|
||||
#* names of its contributors may be used to endorse or promote products
|
||||
#* derived from this software without specific prior written permission.
|
||||
#*
|
||||
#* THIS SOFTWARE IS PROVIDED BY Louwrentius ''AS IS'' AND ANY
|
||||
#* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
#* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
#* DISCLAIMED. IN NO EVENT SHALL Louwrentius BE LIABLE FOR ANY
|
||||
#* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
#* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
#* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
#* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
#* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
#* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
#------------------------------------------------------
|
||||
# It should not be necessary to edit antyhing.
|
||||
# Ofcource you can if it is necesary for your needs.
|
||||
# Send a patch if your changes may benefit others.
|
||||
#------------------------------------------------------
|
||||
|
||||
# Handling control-c for a clean shutdown.
|
||||
trap 'kill_process; ' INT
|
||||
|
||||
# Setting some vars. Do not change.
|
||||
SCRIPT_NAME="Parallel Processing Shell Script"
|
||||
SCRIPT_VERSION="1.10"
|
||||
|
||||
RUNNING_SIGNAL="$0_is_running" # Prevents running mutiple instances of PPSS..
|
||||
GLOBAL_LOCK="PPSS-GLOBAL-LOCK" # Global lock file used by local PPSS instance.
|
||||
PAUSE_SIGNAL="pause.txt" # Not implemented yet (pause processing).
|
||||
ARRAY_POINTER_FILE="ppss-array-pointer" #
|
||||
JOB_LOG_DIR="JOB_LOG" # Directory containing log files of processed items.
|
||||
LOGFILE="ppss-log.txt" # General PPSS log file. Contains lots of info.
|
||||
MAX_DELAY=2
|
||||
PERCENT="0"
|
||||
PID="$$"
|
||||
LISTENER_PID=""
|
||||
IFS_BACKUP="$IFS"
|
||||
INTERVAL="15" # Polling interval to check if there are running jobs.
|
||||
|
||||
SSH_SERVER="" # Remote server or 'master'.
|
||||
SSH_KEY="" # SSH key for ssh account.
|
||||
SSH_SOCKET="/tmp/PPSS-ssh-socket" # Multiplex multiple SSH connections over 1 master.
|
||||
SSH_OPTS="-o BatchMode=yes -o ControlPath=$SSH_SOCKET -o ControlMaster=auto -o ConnectTimeout=5"
|
||||
SSH_MASTER_PID=""
|
||||
ITEM_LOCK_DIR="PPSS_ITEM_LOCK_DIR" # Remote directory on master used for item locking.
|
||||
PPSS_LOCAL_WORKDIR="PPSS_LOCAL_WORKDIR" # Local directory on slave for local processing.
|
||||
TRANSFER_TO_SLAVE="0" # Transfer item to slave via (s)cp.
|
||||
SECURE_COPY="1" # If set, use SCP, Otherwise, use cp.
|
||||
REMOTE_OUTPUT_DIR="" # Remote directory to which output must be uploaded.
|
||||
|
||||
showusage () {
|
||||
|
||||
echo
|
||||
echo "$SCRIPT_NAME"
|
||||
echo "Version: $SCRIPT_VERSION"
|
||||
echo
|
||||
echo "Description: this script processess files or other items in parallel. It is designed to make"
|
||||
echo "use of the multi-core CPUs. It will detect the number of available CPUs and start a thread "
|
||||
echo "for each CPU core. It will also use hyperthreading if available." It has also support for
|
||||
echo "distributed usage, using a Master server in conjunction with (multiple) slaves."
|
||||
echo
|
||||
echo "Usage: $0 [ options ]"
|
||||
echo
|
||||
echo "Options are:"
|
||||
echo
|
||||
echo -e "\t- c \tCommand to execute. Can be a custom script or just a plain command."
|
||||
echo -e "\t- d \tDirectory containing items to be processed."
|
||||
echo -e "\t- f \tFile containing items to be processed. (Alternative to -d)"
|
||||
echo -e "\t- l \tSpecifies name and location of the logfile."
|
||||
echo -e "\t- p \tSpecifies number of simultaneous processes manually. (optional)"
|
||||
echo -e "\t- j \tEnable or disable hyperthreading. Enabled by default. (optional)"
|
||||
echo
|
||||
echo "Options for distributed usage:"
|
||||
echo
|
||||
echo -e "\t- s \tUsername@server domain name or IP-address of 'PPSS master server'."
|
||||
echo -e "\t- k \tSSH key file used for connection with 'PPSS master server'."
|
||||
echo -e "\t- t \tTransfer remote item to slave for local processing."
|
||||
echo -e "\t- o \tUpload output back to server into this directory."
|
||||
echo -e "\t- n \tDo *not* use scp for item transfer but use cp. "
|
||||
echo
|
||||
echo -e "Example: encoding some wav files to mp3 using lame:"
|
||||
echo
|
||||
echo -e "$0 -c 'lame ' -d /path/to/wavfiles -l logfile -j (wach out for the space in -c)"
|
||||
echo
|
||||
}
|
||||
|
||||
kill_process () {
|
||||
|
||||
kill $LISTENER_PID >> /dev/null 2>&1
|
||||
while true
|
||||
do
|
||||
JOBS=`ps ax | grep -v grep | grep ppss.sh | wc -l`
|
||||
if [ "$JOBS" -gt "2" ]
|
||||
then
|
||||
for x in `ps ax | grep -v grep | grep ppss.sh | awk '{ print $1 }'`
|
||||
do
|
||||
if [ ! "$x" == "$PID" ] && [ ! "$x" == "$$" ]
|
||||
then
|
||||
kill -9 $x >> /dev/null 2>&1
|
||||
fi
|
||||
done
|
||||
sleep 5
|
||||
else
|
||||
cleanup
|
||||
echo -en "\033[1B"
|
||||
# The master SSH connection should be killed.
|
||||
if [ ! -z "$SSH_MASTER_PID" ]
|
||||
then
|
||||
kill -9 "$SSH_MASTER_PID"
|
||||
fi
|
||||
log INFO "Finished."
|
||||
echo ""
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
}
|
||||
|
||||
exec_cmd () {
|
||||
|
||||
CMD="$1"
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
ssh $SSH_OPTS $SSH_KEY $SSH_SERVER $CMD
|
||||
else
|
||||
eval "$CMD"
|
||||
fi
|
||||
}
|
||||
|
||||
# this function makes remote or local checking of existence of items transparent.
|
||||
does_file_exist () {
|
||||
|
||||
FILE="$1"
|
||||
`exec_cmd "ls -1 $FILE >> /dev/null 2>&1"`
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
cleanup () {
|
||||
|
||||
log DEBUG "$FUNCNAME - Cleaning up all temp files and processes."
|
||||
|
||||
if [ -e "$FIFO" ]
|
||||
then
|
||||
rm $FIFO
|
||||
fi
|
||||
|
||||
if [ -e "$ARRAY_POINTER_FILE" ]
|
||||
then
|
||||
rm $ARRAY_POINTER_FILE
|
||||
fi
|
||||
|
||||
if [ -e "$GLOBAL_LOCK" ]
|
||||
then
|
||||
rm -rf $GLOBAL_LOCK
|
||||
fi
|
||||
|
||||
if [ -e "$RUNNING_SIGNAL" ]
|
||||
then
|
||||
rm "$RUNNING_SIGNAL"
|
||||
fi
|
||||
|
||||
if [ -e "$SSH_SOCKET" ]
|
||||
then
|
||||
rm -rf "$SSH_SOCKET"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
# check if ppss is already running.
|
||||
is_running () {
|
||||
|
||||
if [ -e "$RUNNING_SIGNAL" ]
|
||||
then
|
||||
echo
|
||||
log INFO "$0 is already running (lock file exists)."
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# If no arguments are specified, show usage.
|
||||
if [ $# -eq 0 ]
|
||||
then
|
||||
showusage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If rubbish is givven as an argument, display usage info."
|
||||
echo $1 | grep -e ^- >> /dev/null
|
||||
ERROR=$?
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
showusage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Process any command-line options that are specified."
|
||||
while getopts ":c:d:f:i:jhk:l:no:p:s:tv" OPTIONS
|
||||
do
|
||||
case $OPTIONS in
|
||||
f )
|
||||
INPUT_FILE="$OPTARG"
|
||||
;;
|
||||
d )
|
||||
SRC_DIR="$OPTARG"
|
||||
;;
|
||||
c )
|
||||
COMMAND="$OPTARG"
|
||||
;;
|
||||
|
||||
h )
|
||||
showusage
|
||||
exit 1;;
|
||||
j )
|
||||
HYPERTHREADING=yes
|
||||
;;
|
||||
l )
|
||||
LOGFILE="$OPTARG"
|
||||
;;
|
||||
k )
|
||||
SSH_KEY="-i $OPTARG"
|
||||
;;
|
||||
n )
|
||||
SECURE_COPY=0
|
||||
;;
|
||||
o )
|
||||
REMOTE_OUTPUT_DIR="$OPTARG"
|
||||
;;
|
||||
|
||||
p )
|
||||
TMP="$OPTARG"
|
||||
if [ ! -z "$TMP" ]
|
||||
then
|
||||
MAX_NO_OF_RUNNING_JOBS="$TMP"
|
||||
fi
|
||||
;;
|
||||
s )
|
||||
SSH_SERVER="$OPTARG"
|
||||
;;
|
||||
t )
|
||||
TRANSFER_TO_SLAVE="1"
|
||||
;;
|
||||
|
||||
v )
|
||||
echo ""
|
||||
echo "$SCRIPT_NAME version $SCRIPT_VERSION"
|
||||
echo ""
|
||||
exit 0
|
||||
;;
|
||||
* )
|
||||
showusage
|
||||
exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
# This function makes local and remote operation transparent.
|
||||
|
||||
|
||||
# Init all vars
|
||||
init_vars () {
|
||||
|
||||
if [ -e "$LOGFILE" ]
|
||||
then
|
||||
rm $LOGFILE
|
||||
fi
|
||||
|
||||
if [ -z "$COMMAND" ]
|
||||
then
|
||||
echo
|
||||
echo "ERROR - no command specified."
|
||||
echo
|
||||
showusage
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo 0 > $ARRAY_POINTER_FILE
|
||||
|
||||
FIFO=$(pwd)/fifo-$RANDOM-$RANDOM
|
||||
|
||||
if [ ! -e "$FIFO" ]
|
||||
then
|
||||
mkfifo -m 600 $FIFO
|
||||
fi
|
||||
|
||||
exec 42<> $FIFO
|
||||
|
||||
touch $RUNNING_SIGNAL
|
||||
|
||||
if [ -z "$MAX_NO_OF_RUNNING_JOBS" ]
|
||||
then
|
||||
MAX_NO_OF_RUNNING_JOBS=`get_no_of_cpus $HYPERTHREADING`
|
||||
fi
|
||||
|
||||
does_file_exist "$JOB_LOG_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log INFO "Job log directory $JOB_lOG_DIR does not exist. Creating."
|
||||
exec_cmd "mkdir $JOB_LOG_DIR"
|
||||
else
|
||||
log INFO "Job log directory $JOB_LOG_DIR exists, if it contains logs for items, these items will be skipped."
|
||||
fi
|
||||
|
||||
does_file_exist "$ITEM_LOCK_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Creating remote item lock dir."
|
||||
exec_cmd "mkdir $ITEM_LOCK_DIR"
|
||||
fi
|
||||
|
||||
if [ ! -e "$JOB_LOG_DIR" ]
|
||||
then
|
||||
mkdir "$JOB_LOG_DIR"
|
||||
fi
|
||||
|
||||
does_file_exist "$REMOTE_OUTPUT_DIR"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
echo "ERROR: remote output dir $REMOTE_OUTPUT_DIR does not exist."
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ ! -e "$PPSS_LOCAL_WORKDIR" ] && [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
mkdir "$PPSS_LOCAL_WORKDIR"
|
||||
fi
|
||||
}
|
||||
|
||||
expand_str () {
|
||||
|
||||
STR=$1
|
||||
LENGTH=$TYPE_LENGTH
|
||||
SPACE=" "
|
||||
|
||||
while [ "${#STR}" -lt "$LENGTH" ]
|
||||
do
|
||||
STR=$STR$SPACE
|
||||
done
|
||||
|
||||
echo "$STR"
|
||||
}
|
||||
|
||||
log () {
|
||||
|
||||
TYPE="$1"
|
||||
MESG="$2"
|
||||
TMP_LOG=""
|
||||
TYPE_LENGTH=6
|
||||
|
||||
TYPE_EXP=`expand_str "$TYPE"`
|
||||
|
||||
DATE=`date +%b\ %d\ %H:%M:%S`
|
||||
PREFIX="$DATE: ${TYPE_EXP:0:$TYPE_LENGTH} -"
|
||||
|
||||
LOG_MSG="$PREFIX $MESG"
|
||||
|
||||
echo -e "$LOG_MSG" >> "$LOGFILE"
|
||||
|
||||
if [ "$TYPE" == "INFO" ]
|
||||
then
|
||||
echo -e "$LOG_MSG"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
check_status () {
|
||||
|
||||
ERROR="$1"
|
||||
FUNCTION="$2"
|
||||
MESSAGE="$3"
|
||||
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
log INFO "$FUNCTION - $MESSAGE"
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
test_server () {
|
||||
|
||||
# Testing if the remote server works as expected.
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
exec_cmd "date >> /dev/null"
|
||||
check_status "$?" "$FUNCNAME" "Server $SSH_SERVER could not be reached"
|
||||
|
||||
ssh -N -M $SSH_OPTS $SSH_KEY $SSH_SERVER &
|
||||
SSH_MASTER_PID="$!"
|
||||
else
|
||||
log DEBUG "No remote server specified, assuming stand-alone mode."
|
||||
fi
|
||||
}
|
||||
|
||||
get_no_of_cpus () {
|
||||
|
||||
# Use hyperthreading or not?
|
||||
HPT=$1
|
||||
NUMBER=""
|
||||
|
||||
if [ -z "$HPT" ]
|
||||
then
|
||||
HPT=no
|
||||
fi
|
||||
|
||||
got_cpu_info () {
|
||||
|
||||
ERROR="$1"
|
||||
check_status "$ERROR" "$FUNCNAME" "cannot determine number of cpu cores. Please specify a number of parallell processes manually with -p."
|
||||
|
||||
}
|
||||
|
||||
if [ "$HPT" == "yes" ]
|
||||
then
|
||||
if [ `uname` == "Linux" ]
|
||||
then
|
||||
NUMBER=`cat /proc/cpuinfo | grep processor | wc -l`
|
||||
got_cpu_info "$?"
|
||||
|
||||
elif [ `uname` == "Darwin" ]
|
||||
then
|
||||
NUMBER=`sysctl -a hw | grep -w logicalcpu | awk '{ print $2 }'`
|
||||
got_cpu_info "$?"
|
||||
elif [ `uname` == "FreeBSD" ]
|
||||
then
|
||||
NUMBER=`sysctl hw.ncpu | awk '{ print $2 }'`
|
||||
got_cpu_info "$?"
|
||||
else
|
||||
NUMBER=`cat /proc/cpuinfo | grep processor | wc -l`
|
||||
got_cpu_info "$?"
|
||||
fi
|
||||
elif [ "$HPT" == "no" ]
|
||||
then
|
||||
if [ `uname` == "Linux" ]
|
||||
then
|
||||
NUMBER=`cat /proc/cpuinfo | grep "cpu cores" | cut -d ":" -f 2 | uniq | sed -e s/\ //g`
|
||||
got_cpu_info "$?"
|
||||
elif [ `uname` == "Darwin" ]
|
||||
then
|
||||
NUMBER=`sysctl -a hw | grep -w physicalcpu | awk '{ print $2 }'`
|
||||
got_cpu_info "$?"
|
||||
elif [ `uname` == "FreeBSD" ]
|
||||
then
|
||||
NUMBER=`sysctl hw.ncpu | awk '{ print $2 }'`
|
||||
got_cpu_info "$?"
|
||||
else
|
||||
NUMBER=`cat /proc/cpuinfo | grep "cpu cores" | cut -d ":" -f 2 | uniq | sed -e s/\ //g`
|
||||
got_cpu_info "$?"
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
if [ ! -z "$NUMBER" ]
|
||||
then
|
||||
echo "$NUMBER"
|
||||
else
|
||||
log INFO "$FUNCNAME ERROR - number of CPUs not obtained."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
random_delay () {
|
||||
|
||||
ARGS="$1"
|
||||
|
||||
if [ -z "$ARGS" ]
|
||||
then
|
||||
log ERROR "$FUNCNAME Function random delay, no argument specified."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
NUMBER=$RANDOM
|
||||
let "NUMBER %= $ARGS"
|
||||
sleep "$NUMBER"
|
||||
}
|
||||
|
||||
|
||||
global_lock () {
|
||||
|
||||
mkdir $GLOBAL_LOCK > /dev/null 2>&1
|
||||
ERROR="$?"
|
||||
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
get_global_lock () {
|
||||
|
||||
while true
|
||||
do
|
||||
global_lock
|
||||
ERROR="$?"
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
random_delay $MAX_DELAY
|
||||
continue
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
release_global_lock () {
|
||||
|
||||
rm -rf "$GLOBAL_LOCK"
|
||||
}
|
||||
|
||||
are_jobs_running () {
|
||||
|
||||
NUMBER_OF_PROCS=`jobs | wc -l`
|
||||
if [ "$NUMBER_OF_PROCS" -gt "1" ]
|
||||
then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_item () {
|
||||
|
||||
ITEM="$1"
|
||||
ITEM_WITH_PATH="$SRC_DIR/$ITEM"
|
||||
|
||||
if [ "$TRANSFER_TO_SLAVE" == "1" ]
|
||||
then
|
||||
log DEBUG "Transfering item $ITEM to local disk."
|
||||
if [ "$SECURE_COPY" == "1" ]
|
||||
then
|
||||
scp -q $SSH_OPTS $SSH_KEY $SSH_SERVER:$ITEM_WITH_PATH $PPSS_LOCAL_WORKDIR
|
||||
else
|
||||
cp $ITEM_WITH_PATH $PPSS_LOCAL_WORKDIR
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
upload_item () {
|
||||
|
||||
ITEM="$1"
|
||||
|
||||
echo "$ITEM" | grep -i ".error" >> /dev/null 2>&1
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "NOT uploading files with errors ($ITEM)."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -e "$PPSS_LOCAL_WORKDIR/$ITEM" ]
|
||||
then
|
||||
log DEBUG "Uploading item $ITEM."
|
||||
if [ "$SECURE_COPY" == "1" ]
|
||||
then
|
||||
scp -q $SSH_OPTS $SSH_KEY $PPSS_LOCAL_WORKDIR/"$ITEM" $SSH_SERVER:$REMOTE_OUTPUT_DIR
|
||||
ERROR="$?"
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
log DEBUG "ERROR - uploading of $ITEM failed."
|
||||
fi
|
||||
else
|
||||
cp "$PPSS_LOCAL_WORKDIR/$ITEM" $REMOTE_OUTPUT_DIR
|
||||
ERROR="$?"
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
log DEBUG "ERROR - uploading of $ITEM failed."
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log DEBUG "ERROR: item $ITEM does not exist."
|
||||
fi
|
||||
}
|
||||
|
||||
lock_item () {
|
||||
|
||||
ITEM="$1"
|
||||
LOCK_FILE_NAME=`echo $ITEM | sed s/^\\\.//g |sed s/^\\\.\\\.//g | sed s/\\\///g`
|
||||
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
|
||||
|
||||
exec_cmd "mkdir $ITEM_LOCK_FILE >> /dev/null 2>&1"
|
||||
ERROR="$?"
|
||||
return "$ERROR"
|
||||
}
|
||||
|
||||
release_item () {
|
||||
|
||||
ITEM="$1"
|
||||
|
||||
LOCK_FILE_NAME=`echo $ITEM` # | sed s/^\\.//g | sed s/^\\.\\.//g | sed s/\\\///g`
|
||||
ITEM_LOCK_FILE="$ITEM_LOCK_DIR/$LOCK_FILE_NAME"
|
||||
|
||||
exec_cmd "rm -rf ./$ITEM_LOCK_FILE"
|
||||
}
|
||||
|
||||
get_all_items () {
|
||||
|
||||
count=0
|
||||
|
||||
#does_file_exist "$SRC_DIR"
|
||||
#check_status "$0" "$FUNCNAME" "ERROR - source dir $SRC_DIR does not exist."
|
||||
|
||||
if [ -z "$INPUT_FILE" ]
|
||||
then
|
||||
if [ ! -z "$SSH_SERVER" ] # Are we running stand-alone or as a slave?"
|
||||
then
|
||||
ITEMS=`exec_cmd "ls -1 $SRC_DIR"`
|
||||
check_status "$?" "$FUNCNAME" "Could not list files within remote source directory."
|
||||
else
|
||||
ITEMS=`ls -1 $SRC_DIR`
|
||||
fi
|
||||
IFS="
|
||||
"
|
||||
for x in $ITEMS
|
||||
do
|
||||
ARRAY[$count]="$x"
|
||||
((count++))
|
||||
done
|
||||
IFS=$IFS_BACKUP
|
||||
else
|
||||
if [ ! -z "$SSH_SERVER" ] # Are we running stand-alone or as a slave?"
|
||||
then
|
||||
scp -q $SSH_OPTS "$SSH_KEY" "$SSH_SERVER:~/$INPUT_FILE" >> /dev/null 2>&!
|
||||
check_status "$?" "$FUNCNAME" "Could not copy input file."
|
||||
fi
|
||||
|
||||
exec 10<$INPUT_FILE
|
||||
|
||||
while read LINE <&10
|
||||
do
|
||||
ARRAY[$count]=$LINE
|
||||
((count++))
|
||||
done
|
||||
|
||||
fi
|
||||
exec 10>&-
|
||||
|
||||
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
||||
if [ "$SIZE_OF_ARRAY" -le "0" ]
|
||||
then
|
||||
echo "ERROR: source file/dir seems to be empty."
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
get_item () {
|
||||
|
||||
get_global_lock
|
||||
|
||||
SIZE_OF_ARRAY="${#ARRAY[@]}"
|
||||
|
||||
# Return error if the array is empty.
|
||||
if [ "$SIZE_OF_ARRAY" -le "0" ]
|
||||
then
|
||||
release_global_lock
|
||||
return 1
|
||||
fi
|
||||
|
||||
# This variable is used to walk thtough all array items.
|
||||
ARRAY_POINTER=`cat $ARRAY_POINTER_FILE`
|
||||
|
||||
# Gives a status update on the current progress..
|
||||
PERCENT=`echo "100 * $ARRAY_POINTER / $SIZE_OF_ARRAY" | bc`
|
||||
log INFO "Currently $PERCENT percent complete. Processed $ARRAY_POINTER of $SIZE_OF_ARRAY items."
|
||||
echo -en "\033[1A"
|
||||
|
||||
# Check if all items have been processed.
|
||||
if [ "$ARRAY_POINTER" -ge "$SIZE_OF_ARRAY" ]
|
||||
then
|
||||
release_global_lock
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Select an item.
|
||||
ITEM="${ARRAY[$ARRAY_POINTER]}"
|
||||
if [ -z "$ITEM" ]
|
||||
then
|
||||
((ARRAY_POINTER++))
|
||||
echo $ARRAY_POINTER > $ARRAY_POINTER_FILE
|
||||
release_global_lock
|
||||
get_item
|
||||
else
|
||||
((ARRAY_POINTER++))
|
||||
echo $ARRAY_POINTER > $ARRAY_POINTER_FILE
|
||||
lock_item "$ITEM"
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Item $ITEM is locked."
|
||||
release_global_lock
|
||||
get_item
|
||||
else
|
||||
release_global_lock
|
||||
download_item "$ITEM"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
start_single_worker () {
|
||||
|
||||
get_item
|
||||
ERROR=$?
|
||||
if [ ! "$ERROR" == "0" ]
|
||||
then
|
||||
log DEBUG "Item empty, we are probably almost finished."
|
||||
return 1
|
||||
else
|
||||
get_global_lock
|
||||
echo "$ITEM" > $FIFO
|
||||
release_global_lock
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
commando () {
|
||||
|
||||
ITEM="$1"
|
||||
|
||||
if [ -z "$INPUT_FILE" ] && [ "$TRANSFER_TO_SLAVE" == "0" ]
|
||||
then
|
||||
ITEM="$SRC_DIR/$ITEM"
|
||||
else
|
||||
ITEM="$PPSS_LOCAL_WORKDIR/$ITEM"
|
||||
fi
|
||||
|
||||
LOG_FILE_NAME=`echo $ITEM | sed s/^\\\.//g | sed s/^\\\.\\\.//g | sed s/\\\///g`
|
||||
ITEM_LOG_FILE="$JOB_LOG_DIR/$LOG_FILE_NAME"
|
||||
|
||||
does_file_exist "$ITEM_LOG_FILE"
|
||||
if [ "$?" == "0" ]
|
||||
then
|
||||
log DEBUG "Skipping item $ITEM - already processed."
|
||||
else
|
||||
|
||||
EXECME='$COMMAND"$ITEM" > "$ITEM_LOG_FILE" 2>&1'
|
||||
eval "$EXECME"
|
||||
ERROR="$?"
|
||||
|
||||
if [ ! "$ERROR" == "0" ] && [ "$TRANSFER_TO_SLAVE" == "1" ]
|
||||
then
|
||||
mv $ITEM $ITEM.error
|
||||
elif [ "$TRANSFER_TO_SLAVE" == "1" ]
|
||||
then
|
||||
rm $ITEM
|
||||
fi
|
||||
|
||||
#release_item "$ITEM"
|
||||
|
||||
if [ ! -z "$SSH_SERVER" ]
|
||||
then
|
||||
scp -q $SSH_OPTS $SSH_KEY $ITEM_LOG_FILE $SSH_SERVER:~/$JOB_LOG_DIR
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
start_single_worker
|
||||
return $?
|
||||
}
|
||||
|
||||
listen_for_job () {
|
||||
|
||||
log INFO "Listener started."
|
||||
while read event <& 42
|
||||
do
|
||||
commando "$event" &
|
||||
done
|
||||
}
|
||||
|
||||
# This starts an number of parallel workers based on the # of parallel jobs allowed.
|
||||
start_all_workers () {
|
||||
|
||||
log INFO "Starting $MAX_NO_OF_RUNNING_JOBS workers."
|
||||
|
||||
i=0
|
||||
while [ "$i" -lt "$MAX_NO_OF_RUNNING_JOBS" ]
|
||||
do
|
||||
log DEBUG "$FUNCNAME - NO OF WORKERS is $i"
|
||||
start_single_worker
|
||||
((i++))
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
# If this is called, the whole framework will execute.
|
||||
main () {
|
||||
|
||||
is_running
|
||||
init_vars
|
||||
log DEBUG "---------------- START ---------------------"
|
||||
log INFO "$SCRIPT_NAME version $SCRIPT_VERSION"
|
||||
test_server
|
||||
get_all_items
|
||||
listen_for_job "$MAX_NO_OF_RUNNING_JOBS" &
|
||||
LISTENER_PID=$!
|
||||
start_all_workers
|
||||
}
|
||||
# This command starts the that sets the whole framework in motion.
|
||||
main
|
||||
while true
|
||||
do
|
||||
JOBS=`ps ax | grep -v grep | grep ppss.sh | wc -l`
|
||||
if [ "$JOBS" -gt "3" ]
|
||||
then
|
||||
sleep $INTERVAL
|
||||
else
|
||||
echo -en "\033[1B"
|
||||
log INFO "There are no more running jobs, so we must be finished."
|
||||
echo -en "\033[1B"
|
||||
if [ ! -z "$REMOTE_OUTPUT_DIR" ]
|
||||
then
|
||||
log INFO "Transfering all processed items back to server."
|
||||
for x in `ls -1 $PPSS_LOCAL_WORKDIR`
|
||||
do
|
||||
upload_item "$x"
|
||||
done
|
||||
fi
|
||||
log INFO "Killing listener and remainig processes."
|
||||
log INFO "Dying processes may display an error message."
|
||||
kill_process
|
||||
fi
|
||||
done
|
||||
wait
|
|
@ -0,0 +1,26 @@
|
|||
test-a
|
||||
test-b
|
||||
test-c
|
||||
test-d
|
||||
test-e
|
||||
test-f
|
||||
test-g
|
||||
test-h
|
||||
test-i
|
||||
test-j
|
||||
test-k
|
||||
test-l
|
||||
test-m
|
||||
test-n
|
||||
test-o
|
||||
test-p
|
||||
test-q
|
||||
test-r
|
||||
test-s
|
||||
test-t
|
||||
test-u
|
||||
test-v
|
||||
test-w
|
||||
test-x
|
||||
test-y
|
||||
test-z
|
|
@ -0,0 +1,8 @@
|
|||
\'file-!@#$%^&*()_ +=-0987654321~\'
|
||||
\'file-/\<>?:;'{}[]\'
|
||||
file-/\/\:\/!@#$%^&*()_+=-0987654321~
|
||||
file-42>424>424<2424>424?24<24>24
|
||||
file-/\<>?:;'{}[]
|
||||
http://www.google.nl
|
||||
ftp://storage.nl
|
||||
./flac/Bééthoven Overtures CD2/01 - Beethoven, Lv - Leonore I - Op.138.flac
|
|
@ -0,0 +1,124 @@
|
|||
#!/bin/bash
|
||||
|
||||
INPUT="$1"
|
||||
RESOLUTION="$3"
|
||||
SEPARATE="$4"
|
||||
TITLES=0
|
||||
OPTS_HIGHRES="-e x264 -q 20.0 -r 29.97 --pfr -a 1 -E faac -B 160 -6 dpl2 -R Auto -D 0.0 -f mp4 -4 -X 1024 --strict-anamorphic -m"
|
||||
OPTS_LOWRES="-e x264 -q 20.0 -a 1 -E faac -B 128 -6 dpl2 -R 48 -D 0.0 -f mp4 -X 480 -m -x cabac=0:ref=2:me=umh:bframes=0:subme=6:8x8dct=0:trellis=0"
|
||||
OPTS_SOURCE="-e x264 -q 20.0 -a 1,1 -E faac,ac3 -l 576 -B 160,160 -6 dpl2,auto -R Auto,Auto -D 0.0,0.0 -f mp4 --detelecine --decomb --strict-anamorphic -m -x b-adapt=2:rc-lookahead=50"
|
||||
MODE=""
|
||||
HANDBRAKE=HandBrakeCLI
|
||||
DIRNAME=`dirname "$INPUT"`
|
||||
BASENAME=`basename "$INPUT"`
|
||||
OPTS=""
|
||||
OUTPUT_DIR="$2"
|
||||
OUTPUT_FILE_NAME=""
|
||||
|
||||
if [ -z "$INPUT" ]
|
||||
then
|
||||
echo "usage $0 <input file / folder> <output folder> <highres|lowres> <separate>"
|
||||
echo
|
||||
echo "Input either file, VIDEO_TS directory or .ISO"
|
||||
echo
|
||||
echo -e "highres:\t1024 x 576"
|
||||
echo -e "lowres:\t\t480 x 320"
|
||||
echo -e "source:\t\tsame as source."
|
||||
echo
|
||||
echo -e "separate:\tseparate files for episodes of a serie."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -z "$OUTPUT_DIR" ]
|
||||
then
|
||||
if [ ! -e "$OUTPUT_DIR" ] || [ ! -d "$OUTPUT_DIR" ]
|
||||
then
|
||||
echo "Output directory does not exist or is not a directory."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Output to current directory."
|
||||
OUTPUT_DIR="."
|
||||
fi
|
||||
|
||||
|
||||
if [ ! -e "$INPUT" ]
|
||||
then
|
||||
echo "$INPUT does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -d "$INPUT" ]
|
||||
then
|
||||
MODE=DIR
|
||||
else
|
||||
MODE=FILE
|
||||
fi
|
||||
|
||||
echo "Input type is $MODE"
|
||||
|
||||
case "$RESOLUTION" in
|
||||
highres|HIGHRES )
|
||||
OPTS="$OPTS_HIGHRES" ;;
|
||||
lowres|LOWRES )
|
||||
OPTS="$OPTS_LOWRES" ;;
|
||||
source|SOURCE )
|
||||
OPTS="$OPTS_SOURCE" ;;
|
||||
*)
|
||||
echo "Resolution must be 'highres', 'source' or 'lowres'."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
function titles () {
|
||||
|
||||
TITLES=`./$HANDBRAKE -t 0 -i "$INPUT" 2>&1 | grep "+ title" | awk '{ print $3 }' | sed s/://g`
|
||||
echo $TITLES
|
||||
}
|
||||
|
||||
if [ "$MODE" = "FILE" ]
|
||||
then
|
||||
mkdir -p "$OUTPUT_DIR/$DIRNAME"
|
||||
OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/${BASENAME%.*}"
|
||||
elif [ "$MODE" = "DIR" ]
|
||||
then
|
||||
echo "$INPUT" | grep -i video_ts >> /dev/null 2>&1
|
||||
if [ "$?" = "0" ]
|
||||
then
|
||||
INTERMEDIATE2=`basename "$DIRNAME"`
|
||||
mkdir -p "$OUTPUT_DIR/$DIRNAME"
|
||||
OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2"
|
||||
else
|
||||
INTERMEDIATE2="$BASENAME"
|
||||
mkdir -p "$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2"
|
||||
OUTPUT_FILE_NAME="$OUTPUT_DIR/$DIRNAME/$INTERMEDIATE2/$INTERMEDIATE2"
|
||||
fi
|
||||
echo "INTERMEDIATE2 = $INTERMEDIATE2"
|
||||
else
|
||||
echo "Mode is not determined..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$SEPARATE" = "separate" ]
|
||||
then
|
||||
TITLES=`titles $INPUT`
|
||||
echo "TITLES = $TITLES"
|
||||
ERROR=0
|
||||
|
||||
for x in $TITLES
|
||||
do
|
||||
HandBrakeCLI $OPTS -i "$INPUT" -o "$OUTPUT_FILE_NAME-$x.mp4"
|
||||
if [ ! "$?" = "0" ]
|
||||
then
|
||||
ERROR="1"
|
||||
fi
|
||||
done
|
||||
exit "$ERROR"
|
||||
else
|
||||
echo "Creating a single file."
|
||||
HandBrakeCLI $OPTS -i "$INPUT" -o "$OUTPUT_FILE_NAME.mp4"
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
SRC="$1"
|
||||
DEST="$2"
|
||||
|
||||
|
||||
TYPE=`file -b "$SRC"`
|
||||
RES=`echo "$TYPE" | grep "WAVE audio"`
|
||||
if [ ! "$?" == "0" ]
|
||||
then
|
||||
echo "File $SRC is not a wav file..."
|
||||
echo "Type is $TYPE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BASENAME=`basename "$SRC"`
|
||||
MP3FILE="`echo ${BASENAME%wav}mp3`"
|
||||
lame --quiet --preset insane "$SRC" "$DEST/$MP3FILE"
|
||||
exit "$?"
|
Loading…
Reference in New Issue