sneak/ppss
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved algorithm for removing processed items from the input file. It was stupid and is now smart.

Browse Source
This commit is contained in:
louwrentius 2011-09-20 20:44:42 +00:00
parent b69e1e6d7b
commit 625eb7e1e4
1 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
ppss
View File

@ -1805,12 +1805,14 @@ remove_processed_items_from_input_file () {
# #
UNPROCESSED_ITEMS="" UNPROCESSED_ITEMS=""
log DSPLY "Removing processed items from list..."
if [ -e "$LIST_OF_PROCESSED_ITEMS" ] if [ -e "$LIST_OF_PROCESSED_ITEMS" ]
then then
PROCESSED_ITEMS=`cat $LIST_OF_PROCESSED_ITEMS` PROCESSED_ITEMS=`cat $LIST_OF_PROCESSED_ITEMS`
fi fi
log DEBUG "Running $FUNCNAME" log INFO "Running $FUNCNAME"
if [ -z "$PROCESSED_ITEMS" ] if [ -z "$PROCESSED_ITEMS" ]
then then
@ -1844,24 +1846,33 @@ remove_processed_items_from_input_file () {
log DEBUG "Now removing processed items from input." log DEBUG "Now removing processed items from input."
for x in $INPUTFILES STATUS_COUNTER=0
TOTAL=`echo "$INPUTFILES" | wc -l`
for x in $PROCESSED_ITEMS
do do
FILE_IS_PROCESSED=0 FILE_IS_PROCESSED=0
for y in $PROCESSED_ITEMS for y in $INPUT_FILES
do do
if [ "$y" = "$x" ] if [ "$y" == "$x" ]
then then
FILE_IS_PROCESSED=1 FILE_IS_PROCESSED=1
fi fi
((STATUS_COUNTER++))
TMP=$((STATUS_COUNTER % 1000))
if [ "$TMP" == "0" ]
then
log DSPLY "Processed $STATUS_COUNTER of $TOTAL."
fi
done done
if [ "$FILE_IS_PROCESSED" = "0" ] if [ "$FILE_IS_PROCESSED" = "0" ]
then then
log DEBUG "ITEM $x is not processed." #log DEBUG "ITEM $x is not processed."
unprocessed_stack_push "$x" unprocessed_stack_push "$x"
else #else
log DEBUG "ITEM $x is already processed!." #log DEBUG "ITEM $x is already processed!."
fi fi
done done
IFS="$oldIFS" IFS="$oldIFS"
@ -1870,6 +1881,8 @@ remove_processed_items_from_input_file () {
get_all_items () { get_all_items () {
log DSPLY "Creating a list of all items to process..."
if [ "$DAEMON" == "1" ] && [ "$INOTIFY" = "0" ] && [ "$ENABLE_INPUT_LOCK" = "1" ] if [ "$DAEMON" == "1" ] && [ "$INOTIFY" = "0" ] && [ "$ENABLE_INPUT_LOCK" = "1" ]
then then
get_input_lock get_input_lock