#!/bin/bash

source /gpfs/wizard/flight/production/installed/set_pam_env.sh > /dev/null 2>&1;

if [ "$PAM_BIN" == "" ]; then exit; fi

export PRODDIR=/gpfs/wizard/flight/production/

export PRERR=$PRODDIR/preRawreader/

export WRKDIR=$PRODDIR/deamons/wrkdir/

export SCRIPT=$PRODDIR/deamons/script/

export LOGDIR=$PRODDIR/logs/

touch $WRKDIR/rr.rrlist

LOG=$WRKDIR/rr.log

BUSY=$WRKDIR/rrlist.busy

touch $LOG

touch $BUSY

echo -n " DEAMON STARTED AT " >> $LOG;

date >> $LOG;

echo "300" > $WRKDIR/rr.step 
#echo "1" > $WRKDIR/rr.step 

TIME=`tail -1 $WRKDIR/rr.step`;

while [ $TIME -ne 0 ]; do

	ls $PRERR/*.pam &> $WRKDIR/rr.rawlist 2>/dev/null;
#	ls $PRERR/02005001.pam &> $WRKDIR/rr.rawlist 2>/dev/null;

	echo -n "#############>>>> DATE: " >> $LOG;	

	date >> $LOG;
	
	for nfile in `sort $WRKDIR/rr.rawlist`; do

		file=`basename $nfile`
	    
		DONE=0;

		for procfile in `cat $WRKDIR/rr.rrlist`; do

			if [ $file == $procfile ]; then

				DONE=1;

			fi;

		done;

		if [ "$file" == "bad_files" ]; then

		    DONE=1;
		    
		fi;

		while [ $DONE -eq 0 ]; do

                        NOW=`date +%s`

                        CTIME=`date --reference=$nfile +%s`

                        let DELTA=NOW-CTIME

                        while [ $DELTA -lt 300 ]; do

                                echo "File $file is hot! waiting 60 seconds..." >> $LOG;

                                sleep 60;

				NOW=`date +%s`

				CTIME=`date --reference=$nfile +%s`

				let DELTA=NOW-CTIME

                        done;
			
			echo -n "FILE IS COLD NOW: " >> $LOG;	

			date >> $LOG;

               	        echo " Processing file $file!" >> $LOG;

			R=0;

			if [ -s $PRERR/$file ]; then

			    echo " File $file is on disk! " >> $LOG;
				    
			else

			    COPY="";

			    echo " File not found on disk or empty! " >> $LOG;

			    date +%s >> $LOG;

			    R=2;

			fi

			if [ $R -gt 1 ]; then

			    echo -n " PROBLEMS! " >> $LOG;
					
			    echo " File $file not processed yet, I will retry later " >> $LOG;
				    
			else
				
			    R=0;

			    TEST=""
					
			    while [ "$TEST" == "" ] && [ $R -lt 10 ]; do
					
				let R=$R+1;
					    
				NOME=`basename $file .pam`

				QUEUE=ts_wizard

				CHECK=`bhist -q "${QUEUE}" -p`; 

				while [ "$CHECK" != "No matching job found" ]; do
	    
				    sleep 20;

				    CHECK=`bhist -q "${QUEUE}" -p`; 
				    
				    if [ "$CHECK" != "No matching job found" ]; then

					CHECK2=`bhist -q "normal" -p`; 

					if [ "$CHECK2" == "No matching job found" ]; then

					    echo " Using normal queue since ts_wizard is full " >> $LOG;

					    QUEUE="normal";
					    
					    CHECK=`bhist -q "${QUEUE}" -p`;
					    
					fi;
					
				    fi;

				done; 

				SUBO=`bsub -q $QUEUE -oo $LOGDIR/rr.lsf.$NOME.log $SCRIPT/rr.sh $file`;
					    
				echo " Output from LSF is: $SUBO " >> $LOG;
					    
				TEST=`echo $SUBO | grep "is submitted"`;
					    
				if [ "$TEST" == "" ]; then 
					    
				    echo " Send LSF job failure! " >> $LOG;
						
				    echo " Retry n. $R " >> $LOG;
						
				fi
					    
			    done
					
			    if [ $R -ge 10 ]; then
				
				echo -n " PROBLEMS SENDING LSF JOBS! " >> $LOG;
					    
				date >> $LOG;
					    
				echo " File $file not processed yet, I will retry later " >> $LOG;
					    
			    else				    
					    
				CHOK=`tail -1 $BUSY`;
						
				if [ "$CHOK" == "" ]; then
						
				    echo "0" > $BUSY;

				    CHOK=`tail -1 $BUSY`;

				fi

				while [ $CHOK -eq 1 ]; do
					    
				    echo -n " LIST FILE IS BUSY " >> $LOG;
					    
				    date >> $LOG;
					    
				    sleep 1;

				    CHOK=`tail -1 $BUSY `;
		    		    
				done
					    
				echo "1" > $BUSY;

				echo $file >> $WRKDIR/rr.rrlist;
					    
				echo "0" > $BUSY;

 #			    sleep 127;

			    fi;
					
			fi;

			DONE=1;

		done;

	        TIME=`tail -1 $WRKDIR/rr.step `;
		
		if [ $TIME -le 0 ]; then

		    echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

		    date >> $LOG;

		    exit;

 		fi

		while [ $TIME -eq 1 ]; do

		    TIME=`tail -1 $WRKDIR/rr.step `;
		    		    
		    echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

		    date >> $LOG;

		    sleep 120;

 		done

	done;

        TIME=`tail -1 $WRKDIR/rr.step `;
		
	if [ $TIME -le 0 ]; then

	    echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

	    date >> $LOG;

	    exit;

	fi

	while [ $TIME -eq 1 ]; do

	    TIME=`tail -1 $WRKDIR/rr.step `;
		    		    
	    echo -n " TIME = 1! DEAMON PAUSE " >> $LOG;

	    date >> $LOG;

	    sleep 120;

	done

	sleep $TIME;

        TIME=`tail -1 $WRKDIR/rr.step `;

done;

echo -n " EXIT DEAMON, RECEIVED TERM SIGNAL " >> $LOG;

date >> $LOG;
