chewbacca/PamOffLineSW/PamOffLineSW_Main.cpp

//============================================================================
// $Id: PamOffLineSW_Main.cpp,v 1.12 2010/02/12 05:22:04 mocchiut Exp $ 
// Description : Pamela Off-Line Software
//============================================================================

#include <sys/time.h>
#include "PamInclude.h"
#include "StateManager.h"

#include "PacketUser.h"
#include <TArrayL64.h>

extern "C" {
    #include <dirent.h>
}

using namespace PamOffLineSW;

// dbg purposes
bool exitdbg = false;
void exitDBG(){exitdbg=true;}
//function needed only for dbg purposes:
void split_input(char fni[]);

#define LENGTH_DATA_SIMU 1024
#define LENGTH_CADRE_VRL 1024 
#define LENGTH_HEADER_VRL 8
#define LENGTH_DATA_CADRE (LENGTH_CADRE_VRL-LENGTH_HEADER_VRL-1)//1015

//main functions
void readOptions(int argc, char *argv[]);
void init();
void firstLog();
void DB_config();
void mainRead_RealData(char fni[]);
void mainRead_SimulatedData(char fni[]);
void finish();
void deleteAll();

//utilities functions
bool VRL_Header_Check(char * headVRL, int length);
short int CRC_Cadre_Check(char dataVRL[], int length);

//global variables
bool simulated_data = false;
//bool do_vrl_check = false; //EMI
bool do_vrl_check = true; //EMI


const char*  db_host = "";
int        db_port = 0;
const char*  db_name = "";
char conn[100]="";

LogUtil::logLevel loglevel=LogUtil::LOGERROR;
const char* logfilename = "chewbacca.log";
//current route
int route = 999;
//previous route
int old_route = 999;

Long64_t filelength = 0LL;
Long64_t streamposi = 0LL;
Long64_t streamposisaved = 0LL;
Long64_t streamposiorig = 0LL;
Int_t numposi = 10;
Long64_t downcount = 1LL;
TArrayL64 *dwin=new TArrayL64(12);
Long64_t cadcount = 0LL;
Int_t horrorcount = 0;
Int_t hc = 0;

bool skip_cadre = false;
long int iNumGoodCadres=0; //total number of good cadres
        
// global variables used in all the project
namespace PamOffLineSW
{       
        const char*  db_user = "";
        const char*  db_pwd  = "";
        char* pelosconnection = NULL;   
        //marco_new_01
        bool single_connection=false;   
        long int iNumCadres=0; //cadre's number
        unsigned long long int iByte_tot=0;// how many bytes I have read till now       

        bool isCadreGood = true;//if the cadre is good  
        char* fni; //path completo
        short compression = 3;
//      char *outDir = ".";
        const char *outDir = "";

  //    char * nome_output="chewbacca"; //EMI
        const char * nome_output="L0PAM";//EMI
        bool multiFile = 0;     
  //    unsigned long int step_pkt_number=0; // EMI
        unsigned long int step_pkt_number=3000;
  //    unsigned long int step_pkt_obt=0; // EMI
        unsigned long int step_pkt_obt=7200000; // EMI
        LogUtil* mainLogUtil = NULL;
        TSQLServer* sqlServer = NULL;
        unsigned long int max_pkt_number =   16777215;//biggest value before reset
        unsigned long int max_pkt_obt    = 4294967295u;//biggest value before reset

  //    bool is_new_route = false;      // EMI
        bool is_new_route = true;       // EMI ??????????? VA BENE? SECONDO ME SI`
        unsigned int download = 0;
        unsigned int orbit_number=0;
        unsigned int mmm_number = 0;//session_number
        unsigned long int time_Offset=0;
  //    bool tryMerge = false;
        bool tryMerge = true;
        bool candelete = true; //EM
                
        bool do_cont_check=true;//if do_cont_check is false do not use a DB ...

        //TODO: now it is unused
        unsigned long int delta_Time=0;//in seconds
#define TAGVALUELEN 4
  char tag_value[TAGVALUELEN];
}

// main: 
// 
int main(int argc, char *argv[])
{
        // read command line options
        readOptions(argc,argv);

        cout<<"Please wait"<<endl;

        // initialize everything and write few things in the log file
        init();
        
        // take time to measure the performance.
        timeval tv1;
        gettimeofday(&tv1,NULL);
        unsigned long long timems1 =  (unsigned long long)tv1.tv_sec * (unsigned long long)1000 + tv1.tv_usec / 1000; 
        
        if(simulated_data)
        {               
                mainLogUtil->logAll("Using simulated data");
                mainRead_SimulatedData(fni);
        }
        else
        {       
                mainLogUtil->logAll("Using real data");
                mainRead_RealData(fni);
        }
        
        finish();
        
        timeval tv2;
        gettimeofday(&tv2,NULL);
        unsigned long long timems2 =  (unsigned long long)tv2.tv_sec * (unsigned long long)1000 + tv2.tv_usec / 1000; 
        unsigned long long timems = timems2 -timems1;
        
        string msg = "The analisys took: " + mainLogUtil->value2string(timems) + " ms";
        mainLogUtil->logAlways(msg);

        deleteAll();
        
        cout<<"The end "<<endl;
        return 0;
}

void readOptions(int argc, char *argv[])
{   
  if (argc < 2)
  {
    cout << "You have forgotten the file name. \n";
    cout << "Try '-help' for more information. \n";
    exit(1);
  } 
   
  if((!strcmp(argv[1], "-help"))||(!strcmp(argv[1], "--help"))||(!strcmp(argv[1], "-h")))
  {
          // cout << "Usage: PamOffLineSW INPUT_FILE [OPTIONS] \n";
                cout << "Usage: chewbacca INPUT_FILE [OPTIONS] \n";
        cout << "\t  (-help | --help | -h) print this help and exit \n";
        cout << "\t -simu       if the input file contains simulated data instead of real data\n";
        cout << "\t -vrl    if you want to perform the vrl check\n";
        cout << "\t -no-vrl    if you DO NOT want to perform the vrl check\n";
        cout << "\t -gpamela    gpamela data as input\n";
        cout << "\t -filelog     set the log filename. [default: chewbacca.log]\n";
        cout << "\t -loglevel     set the log level. Values: [0,3] (error,warning,info,all) [default:0]\n";
        cout << "\t -c      set the compression level for the generated ROOT file(s). Values: [0,9] [default = 3]\n";
        cout << "\t -root_name      set the root-name for the generated ROOT file(s). [default = chewbacca]\n";
        cout << "\t -outDir      set the output directory for the generated root file(s). [default = .]\n";
        cout << "\t -delta_counter     set the allowed difference in the Packet Counter between two continuos packets. If 0 all packets are considered continuos.  [default = 0]\n";
        cout << "\t -delta_obt     set the allowed difference (ms) in the Packet Orbital Time between two continuos packets. If 0 all packets are considered continuos.  [default = 0]\n";
        cout << "\t -db_host   set the host-name of the DataBase.  [default = localhost]\n";
        cout << "\t -db_port  set the port of the DataBase.  [default = 3306]\n";
        cout << "\t -db_name  set the name of the DataBase.  [default = chewbacca_db]\n";
        cout << "\t -db_user  set the user of the DataBase.  [default = chewbacca_user]\n";
        cout << "\t -db_pwd  set the user of the DataBase.  [default = chewbacca_pwd]\n";
        cout << "\t -max_pkt_number  Maximum value for  Packet Counter, after this value it is resetted.  [default = 2^24 - 1]\n";
        cout << "\t -max_pkt_obt  Maximum value for  Packet OBT, after this value it is resetted.  [default = 2^32 - 1]\n";       
        cout << "\t -orbit_number  Value of the orbital number. If 0 this is retrieved from the input file name.  [default = 0]\n";        
        cout << "\t -session_number  Value of the session number. If 0 this is retrieved from the input file name.  [default = 0]\n";                
        cout << "\t -time_Offset  Value of the timeOffset. If 0 this is retrieved using the orbital number.  [default = 0]\n";
        cout << "\t -tryMerge if you want to try to Merge ROOT files\n";
        cout << "\t -dontMerge if you do not want to try to Merge ROOT files\n";
        cout << "\t -dontDelete if you do not want to delete bad ROOT files from disks\n";
        //marco_new_01
        cout << "\t -single_connection if you want to open only one connection to DB.\n";
        cout << "\t -tag if you want add the 'The owner of the file' - the tag lenght is 4 char\n";
        //       cout << "\t -delta_Time  set the allowed difference (seconds) in the Real Time between two root file. [default = 0]\n";
        //cout << "\t -multi  generate multiple root files \n";
        exit(1);
  }
  
  if(!strcmp(argv[1], "-usage")){
      cout << "\t chewbacca INPUT_FILE [OPTIONS] \n";
      cout << "Try '-help' for more information. \n";
      exit(1);
  }
 
  //http://en.wikipedia.org/wiki/Chewbacca_defense
  //http://www.urbandictionary.com/define.php?term=chewbacca+defense
  if(!strcmp(argv[1], "-defense")){
        cout << "\n Why would a Wookiee, an eight-foot tall Wookiee, want to live on Endor,\n with a bunch of two-foot tall Ewoks? That does not make sense!\n But more important, you have to ask yourself:\n What does this have to do with this case? \n Nothing. Ladies and gentlemen, it has nothing to do with this case! \n It does not make sense!\n Look at me. I'm a lawyer defending a major record company, \n and I'm talkin' about Chewbacca! Does that make sense? \n Ladies and gentlemen, I am not making any sense! \n None of this makes sense! \n And so you have to remember, when you're in that jury room\n deliberatin' and conjugatin' the Emancipation Proclamation,\n does it make sense? \n No! Ladies and gentlemen of this supposed jury, it does not make sense!\n If Chewbacca lives on Endor, you must acquit! \n The defense rests."<<endl;
        cout << "\nTry '-help' for more information. \n";
        exit(1);
    }
 
  //input file
  fni= argv[1];
  
  for (int i = 2; i < argc; i++)
  {       
        //marco_new_01
          if (!strcmp(argv[i], "-single_connection"))
            {
                        single_connection = true;
                continue;
            }


          if (!strcmp(argv[i], "-tryMerge"))
            {
                        tryMerge = true;
                continue;
            }

          if (!strcmp(argv[i], "-dontMerge")) // EMI
            {
                        tryMerge = false;
                continue;
            }

          if (!strcmp(argv[i], "-dontDelete")) // EMI
            {
                        candelete = false;
                continue;
            }
          
          if (!strcmp(argv[i], "-time_Offset")){
                        if (++i >= argc){
                            cerr << "-time_Offset needs arguments. \n";
                            cout << "Try '-help' for more information. \n";
                            exit(1);
                        }
                        if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                                time_Offset = atoll(argv[i]);
                        } else {
                            cerr << "-time_Offset needs an integer value. \n";
                            cout << "Try '-help' for more information. \n";
                                    exit(1);
                                }
                                continue;
                            }
          
          if (!strcmp(argv[i], "-orbit_number")){
                        if (++i >= argc){
                            cerr << "-orbit_number needs arguments. \n";
                            cout << "Try '-help' for more information. \n";
                            exit(1);
                        }
                        if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                                orbit_number = atoi(argv[i]);
                        } else {
                            cerr << "-orbit_number needs an integer value. \n";
                            cout << "Try '-help' for more information. \n";
                                    exit(1);
                                }
                                continue;
                            }
         
          if (!strcmp(argv[i], "-session_number")){
                                if (++i >= argc){
                                    cerr << "-session_number needs arguments. \n";
                                    cout << "Try '-help' for more information. \n";
                                    exit(1);
                                }
                                if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                                        mmm_number = atoi(argv[i]);
                                } else {
                                    cerr << "-session_number needs an integer value. \n";
                                    cout << "Try '-help' for more information. \n";
                                            exit(1);
                                        }
                                        continue;
                                    }
          
          if (!strcmp(argv[i], "-max_pkt_number")){
              if (++i >= argc){
                  cerr << "-max_pkt_number needs arguments. \n";
                  cout << "Try '-help' for more information. \n";
                  exit(1);
              }
              if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                  max_pkt_number = atoll(argv[i]);
              } else {
                  cerr << "-max_pkt_number needs an integer value. \n";
                  cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  }
          
          if (!strcmp(argv[i], "-max_pkt_obt")){
              if (++i >= argc){
                  cerr << "-max_pkt_obt needs arguments. \n";
                  cout << "Try '-help' for more information. \n";
                  exit(1);
              }
              if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                  max_pkt_obt = atoll(argv[i]);
              } else {
                  cerr << "-max_pkt_obt needs an integer value. \n";
                  cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  }
          
          
      if (!strcmp(argv[i], "-simu"))
      {
          simulated_data = true;
          continue;
      }

      if (!strcmp(argv[i], "-gpamela"))
      {
          simulated_data = true;
          do_vrl_check = false;
          nome_output = (char *)gSystem->BaseName(fni);
          tryMerge = false;
          candelete = false;
          continue;
      }
      
      if (!strcmp(argv[i], "-no-vrl")) //EMI
        {
          do_vrl_check = false;
      } 
      if (!strcmp(argv[i], "-vrl"))
      {
          do_vrl_check = true;
          if(simulated_data)
          {
                  cout<<"Impossible to perform VRL check with simulated data. Don't use -simu option"<<endl;
                  cout << "Try '-help' for more information. \n";
                  exit(1);
          }
         
          continue;
       }

      if (!strcmp(argv[i], "-filelog")){
          if (++i >= argc){
              cerr << "-filelog needs arguments. \n";
              cout << "Try '-help' for more information. \n";
              exit(1);
          }
          logfilename=argv[i];
          continue;
      } 
      
      if (!strcmp(argv[i], "-loglevel")){
          if (++i >= argc){
              cerr << "-loglevel needs arguments. \n";
              cout << "Try '-help' for more information. \n";
              exit(1);
          }
          if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) && (atoi(argv[i]) <= 3))) {
              loglevel = (LogUtil::logLevel) atoi(argv[i]);
          } else {
              cerr << "-loglevel needs an integer value beetween 0 and 3. \n";
              cout << "Try '-help' for more information. \n";
              exit(1);
          }
          continue;
      } 

      
      if (!strcmp(argv[i], "-c")){
             if (++i >= argc){
                 cerr << "-c needs arguments. \n";
                 cout << "Try '-help' for more information. \n";
                 exit(1);
             }
             if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) && (atoi(argv[i]) <= 9))) {
                 compression = atoi(argv[i]);
             } else {
                 cerr << "-c needs an integer value beetween 0 and 9. \n";
                 cout << "Try '-help' for more information. \n";
                 exit(1);
             }
             continue;
         }
         
         if (!strcmp(argv[i], "-outDir")){
             if (++i >= argc){
                 cerr << "-outDir needs arguments. \n";
                 cout << "Try '-help' for more information. \n";
                 exit(1);
             }
             DIR* tempdir;
             if ((tempdir = opendir(gSystem->ExpandPathName(argv[i]))) != 0) { // EMI
                 outDir = argv[i];
                 closedir(tempdir);
             } else {
                 cerr << "-outDir needs an existing/accessable directory. \n";
                 cout << "Try '-help' for more information. \n";
                 exit(1);
             }
             continue;
         } 
        
         if (!strcmp(argv[i], "-root_name")){
                     if (++i >= argc){
                         cerr << "-root_name needs arguments. \n";
                         cout << "Try '-help' for more information. \n";
                         exit(1);
                     }
                     
                    nome_output = argv[i];
                     continue;
                 } 
         
         if (!strcmp(argv[i], "-delta_counter")){
                      if (++i >= argc){
                          cerr << "-delta_counter needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                          step_pkt_number = atoi(argv[i]);
                      } else {
                          cerr << "-delta_counter needs an integer value. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  }
         
                if (!strcmp(argv[i], "-delta_obt")){
                      if (++i >= argc){
                          cerr << "-delta_obt needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                          step_pkt_obt = atoi(argv[i]);
                      } else {
                          cerr << "-delta_obt needs an integer value. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  } 
                if (!strcmp(argv[i], "-delta_Time")){
                      if (++i >= argc){
                          cerr << "-delta_Time needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                         delta_Time = atoi(argv[i]);
                      } else {
                          cerr << "-delta_Time needs an integer value. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  } 
        
                if (!strcmp(argv[i], "-db_host")){
                      if (++i >= argc){
                          cerr << "-db_host needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }                      
                      db_host = argv[i];
                      continue;
                  } 
                

                if (!strcmp(argv[i], "-db_port")){
                      if (++i >= argc){
                          cerr << "-db_port needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      if ((isdigit(*argv[i])) && ( (atoi(argv[i]) >= 0) )) {
                          db_port = atoi(argv[i]);
                      } else {
                          cerr << "-db_port needs an integer value. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }
                      continue;
                  } 
                                
                if (!strcmp(argv[i], "-db_name")){
                      if (++i >= argc){
                          cerr << "-db_name needs arguments. \n";
                          cout << "Try '-help' for more information. \n";
                          exit(1);
                      }                      
                     db_name = argv[i];
                      continue;
                  } 
                if (!strcmp(argv[i], "-db_user")){
                              if (++i >= argc){
                                  cerr << "-db_name needs arguments. \n";
                                  cout << "Try '-help' for more information. \n";
                                  exit(1);
                              }                      
                             db_user = argv[i];
                              continue;
                          } 
                
                if (!strcmp(argv[i], "-db_pwd")){
                              if (++i >= argc){
                                  cerr << "-db_name needs arguments. \n";
                                  cout << "Try '-help' for more information. \n";
                                  exit(1);
                              }                      
                             db_pwd = argv[i];
                              continue;
                          } 
               if (!strcmp(argv[i], "-tag")){
                              if (++i >= argc){
                                  cerr << "-tag needs arguments. \n";
                                  cout << "Try '-help' for more information. \n";
                                  exit(1);
                              }
                              int len=strlen(argv[i])>TAGVALUELEN?TAGVALUELEN:strlen(argv[i]);
                              strncpy(tag_value,argv[i],len);
                              tag_value[len]=0;
                              continue;
                          } 
                  
                //TODO: check that multiFile is always 0
       //       if (!strcmp(argv[i], "-multi")){multiFile = 1; cout<<"debug: multi= "<<multiFile<<endl;}         
   }
  
  //TODO: maybe if simulated data I don't need nothig so I can skip the following lines 
  // and use instead something similar to:
  if (simulated_data) {orbit_number=99999; mmm_number=999; download=999; }
  
  //I need the following lines only to retrieve orbit_number and mmm_number (session_number)from the filename
  if((!orbit_number)||(!mmm_number))
  {               
          std::string input = fni;
          std::string fileName;
          int pos = input.find_last_of("/");
          fileName = input.substr(pos+1);
          pos = fileName.find_last_of(".");
          fileName = fileName.substr(0,pos);
        
                if(fileName.length()<8){
                        cout<<"Wrong filename unable to retrieve orbit_number and session_number."<<endl;
                        cout<<"Rename input in format nnnnnmmm.pam or set orbit_number and session_number."<<endl;
                    cout << "Try '-help' for more information. \n";
                        exit(1);
                }
        
          if(!orbit_number){
                  std::string downlink;
                  downlink = fileName.substr(0,5);              
                  orbit_number= atoi(downlink.c_str());         
          }
          
          if(!mmm_number){
                  std::string mmm;
                  mmm = fileName.substr(5,3);
                  mmm_number= atoi(mmm.c_str());
          }
  } 
   
}

void init()
{
        //initial condition of automaton
        StateManager::getInstance().init();
        mainLogUtil = new LogUtil(logfilename,loglevel);
        
        //TODO check if you like this 
        if (!strcmp(outDir,"")) 
        {
                char *pam_out = getenv("PAM_L0");
                if (pam_out) 
                {
                        outDir = pam_out;
                }
                else
                {
                        outDir = ".";
                }
        
        }
        // write some initial things in the Log file 
        firstLog();     
        
        if(do_cont_check){                                      
                DB_config();
                //marco_new_31:spostata logica in PacketUser.cpp
                ////sqlServer = TSQLServer::Connect(connection,db_user,db_pwd);
                ////if ((!sqlServer)||(!(sqlServer->IsConnected()))){cout<<"Can not connect with MYSQL sever"<<endl; exit(1);}  
        }
        
}

void DB_config()
{               
        if( (!strcmp(db_host,""))||(db_port==0)||(!strcmp(db_name,""))) 
        {
                char *pamdbhost = getenv("PAM_DBHOST");
                if (pamdbhost) 
                {
                        pelosconnection = pamdbhost;
                }
                else
                {       
                        if (!strcmp(db_host,"")) 
                        {
                                db_host="localhost";            
                        }
                
                        if (db_port==0) 
                        {
                                db_port=3306;           
                        }
                        
                        if (!strcmp(db_name,"")) 
                        {
                                db_name="chewbacca_db";         
                        }                                                               
                        sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);       
                        pelosconnection=conn;                   
                }
        }
        else{
                sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);       
                pelosconnection=conn;                           
        }
                
        if (!strcmp(db_user,"")) 
        {
                char *pamdbuser = getenv("PAM_DBUSER");
                if (pamdbuser) 
                {
                        db_user = pamdbuser;
                }
                else
                {
                        db_user="chewbacca_user";
                }
        
        }
        
        if(!strcmp(db_pwd,"")) 
        {
                char *pamdbpsw = getenv("PAM_DBPSW");
                if (pamdbpsw) 
                {
                        db_pwd = pamdbpsw;
                }
                else
                {
                        db_pwd="chewbacca_pwd";
                }       
        }
        
        string msg = "Using DB: " + string(pelosconnection);// +" "+ string(db_user) +" "+ string(db_pwd);
        mainLogUtil->logAlways(msg);
}


void firstLog(){
        string msg = "Input file: " + (string)fni;
        mainLogUtil->logAlways(msg);

        msg = "Output directory for the generated root file(s): " + (string)outDir; 
        mainLogUtil->logAlways(msg);    
        msg = "Output directory for the generated root file(s) expanded: " + (string)gSystem->ExpandPathName(outDir); // EMI
        mainLogUtil->logAlways(msg);    //EMI

        msg = "Output root name for the generated root file(s): " + (string)nome_output;
        mainLogUtil->logAlways(msg);    
        
        stringstream oss1;
        oss1.str()="";
        oss1<< "Compression level for the generated root file(s): " <<compression;
        msg = oss1.str();
        mainLogUtil->logInfo(msg);
        
        stringstream oss2;
        oss2.str()="";
        oss2<< "Max value for Packet Counter: " <<max_pkt_number<<" - Max value for Packet OBT: " <<max_pkt_obt;
        msg = oss2.str();
        mainLogUtil->logAll(msg);
        
        if(do_vrl_check)        
        {       
                mainLogUtil->logInfo("VRL check set to true");
        }
        else
        {
                mainLogUtil->logWarning("VRL check skipped");
        }
        
        if(step_pkt_number)     
        {
                stringstream oss;
                oss.str()="";
                oss<<"The allowed difference in the Packet Counter between two continuos packets: "<<step_pkt_number;
                msg = oss.str();
                mainLogUtil->logAlways(msg);
        }
        
        if(step_pkt_obt)        
        {
                stringstream oss;
                oss.str()="";
                oss<<"The allowed difference in the Packet Orbital Time between two continuos packets: " <<step_pkt_obt;
                msg = oss.str();
                mainLogUtil->logAlways(msg);
        }
                
        if((!step_pkt_number)&&(!step_pkt_obt)){
                        mainLogUtil->logAlways("No check about continuity; DB unused. No Merging ...");
                        do_cont_check=false;
                        tryMerge=false;
                        //return;
        } 
        
        /*
                if(do_cont_check){
                sprintf(connection,"mysql://%s:%d/%s",db_host,db_port,db_name);
                string msg = "Using DB: " + (string)connection;
                mainLogUtil->logAlways(msg);    
                }
        */
        
        if(tryMerge)    
        {       
                mainLogUtil->logInfo("Try to Merge ROOT files");
        }
        
        /*
                if(tryMerge){
                        stringstream oss1;
                        oss1.str()="";
                        oss1<<"The allowed difference in the Real Time between two ROOT files: " <<delta_Time<<" s";
                        string msg1=oss1.str();
                        mainLogUtil->logAlways(msg1);
                }
        */
                        
}

void finish()
{       
        PacketUser::getInstance().FinishLastGroup();    
}

void deleteAll()
{       
        if(mainLogUtil){delete mainLogUtil; mainLogUtil = NULL;}
        //marco_new_01: questo non serve sicuramente ...
        if(sqlServer){delete sqlServer; sqlServer = NULL;}
}

//read simulated data
// without vrl header
void mainRead_SimulatedData(char fni[])
{
        //we read LENGTH_DATA_SIMU bytes each time ...
        char ccData[LENGTH_DATA_SIMU]={0}; 
        ifstream fin;   
        fin.open(fni, ifstream::binary);        
        if (!fin) {cout<<"Can not open input file "<<fni<<endl; exit(1);}
        //main read loop
         isCadreGood = true;//all the cadres are good ;-)
         
         while(fin.eof()==0)
         {      
                //reading data in the cadre
                fin.read(ccData,LENGTH_DATA_SIMU);      
                stringstream oss;
        
                if (!fin.good())
                {                                       
        //              if (fin.eof()){break;}//if we reach the end of file
                        stringstream oss;
                        oss<<"Read only "<<fin.gcount()<<" bytes instead of "<<LENGTH_DATA_SIMU<<
                        ". This error occurred after byte: "<<iByte_tot
                        <<" ... use the data read, then ... exit ";                     
                        string msg = oss.str();
                        mainLogUtil->logAll(msg);
                        //I want try to use it ...
                        for(int i=0;i<fin.gcount();i++)
                        {       
                                StateManager::getInstance().getCurrentState()->readInput(ccData[i]);
                                iByte_tot++;
                        }
                        //anyhow then exit                              
                        break;
                }

                //main processing of the data, searching for Pamela Packets
                for(int i=0;i<LENGTH_DATA_SIMU;i++)
                {                               
                        StateManager::getInstance().getCurrentState()->readInput(ccData[i]);
                        iByte_tot++;
                }
                iNumCadres++;
         }                      
                 
        fin.close();
        
        stringstream oss1;
        oss1<<"Total bytes read "<<iByte_tot;   
        string msg = oss1.str();
        mainLogUtil->logInfo(msg);
}
        
//read real data
void mainRead_RealData(char fni[])
{                       
        //header VRL
        char ccHeader_VRL[LENGTH_HEADER_VRL]={0};       
        //data in each cadre, it can contain the PAMELA PKT
        char ccData[LENGTH_DATA_CADRE]={0};             
        //last byte of each cadre, it is used to check CRC of the cadre
        char cCrc_VRL=0; 
                        
        //file di input = fni[]
        ifstream fin;   
        fin.open(fni, ifstream::binary);        
        if (!fin) {cout<<"Can not open input file "<<fni<<endl; exit(1);}


        //
        // EM determine file length, number of downloads (a part small fractions of download due to VRL memory) and interval window for allowed change in download number
        //
        fin.seekg(0,std::ios::end);
        filelength = (Long64_t)fin.tellg();
        //
        //      filelength = 3840214016LL;
        //      filelength = 5763760128LL;
        //      filelength = 15370027008LL;
        //
        downcount = (Long64_t)round((Double_t)filelength/2000000000.);
        if ( downcount < 1LL ) downcount = 1LL;
        Long64_t downsize = (Long64_t)round((Double_t)filelength/(Double_t)downcount);
        Long64_t hwint = Long64_t((Double_t)downsize/25.); 
        //
        if ( downcount > 5LL ) dwin->Set(2LL+2LL*downcount);
        dwin->Reset();
        dwin->AddAt(0LL,0);
        //
        Int_t dco = 1;
        //
        for (Int_t ee=1; ee<(-1+2+2*downcount); ee+=2){
          dwin->AddAt(((downsize*(dco-1))+hwint),ee);     
          dwin->AddAt(((downsize*dco)-hwint),ee+1);       
          dco++;
        };
        //
        fin.seekg(0,std::ios::beg);
        //
        dwin->AddAt(filelength,(-1LL+2LL+2LL*downcount));
        //
        //      printf(" file lenght is %L downcount is %L downsize is %L hwint is %L \n",filelength,downcount,downsize,hwint);
        stringstream ess;
        ess.str() = "";
        ess << " (EM) " << " file lenght is "<< filelength<< " downcount is "<< downcount <<" downsize is "<< downsize <<" hwint is " << hwint;
        string emsg = ess.str();
        mainLogUtil->logInfo(emsg);
        //
        //      cout << " file lenght is "<< filelength<< " downcount is "<< downcount <<" downsize is "<< downsize <<" hwint is " << hwint << endl;
        //      for (Int_t ee=0; ee<TMath::Max((2+2*downcount),12LL); ee++){
        //        cout << " " << ee << " ==> "<< dwin->At(ee) << endl;
        //        //a     printf(" %i ==> %L \n",ee,dwin->At(ee));
        //      };

        //main read loop
         while(fin.eof()==0)
         {              
                isCadreGood = true;             
                //start reading the file
                streamposi = (Long64_t)fin.tellg(); // stream position in the file before reading the vrl header
                if ( iNumCadres > 9 && numposi > 9 ){
                        streamposisaved = streamposi;
                        numposi = 0;                    
                };
                numposi++;
                //
                fin.read(ccHeader_VRL, LENGTH_HEADER_VRL);              
                //              printf(" position in file is %llu \n",streamposi);
                stringstream ss;
                ss.str() = "";
                ss << " position in file is "<<streamposi;
                string sms = ss.str();
                //              mainLogUtil->logError(sms);
                //
                if (!fin.good())
                {       
                        //TODO: check here
                        if (fin.eof())//if we reach the end of file
                        {       
                                break;
                        }
                        
                        stringstream oss;                                               
                        oss<<"The cadre: "<<iNumCadres<<" has something wrong in the lenght of its header VRL."<<
                        " This error occurred after byte: "<<iByte_tot<<" ... exit";
                        string msg = oss.str();
                        mainLogUtil->logError(msg);     
                        
                        if(do_vrl_check){isCadreGood=false;}
                        break;                                  
                }
                skip_cadre = false;
                //check of the VRL header e setta route number
                if(!VRL_Header_Check(ccHeader_VRL,LENGTH_HEADER_VRL))           
                {                               
                        isCadreGood=false;//Check if it is better set it to true ...            
                        stringstream oss;                                               
                        oss<<"The cadre: "<<iNumCadres<<" has something wrong in its header VRL."<<
                        " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully";
                        string msg = oss.str();
                        mainLogUtil->logWarning(msg);
                        //                      printf(" streamposi %llu \n",streamposi);
                } 
                
                //
                // we completely missed more than 10 VRL packets
                //
                if (  hc == 1100 ){
                  fin.seekg(streamposiorig);
                  stringstream os;                                              
                  os<<" NEW (EM) UN-RECOVERABLE SYNC WITH VRL HEADERS, SIG! ";
                  os<<" new position in file, go back to "<<streamposiorig;
                  string ms = os.str();
                  mainLogUtil->logAll(ms);
                  hc++;
                  continue;
                };
                if ( horrorcount > 10 && hc < 1100 ){
                  hc++;
                  //if ( horrorcount > 1 ){
                  stringstream os;                                              
                  os<<" NEW (EM) THIS IS TOO MUCH, LOST SYNC WITH VRL HEADERS! try to recover horrorcount = "<<horrorcount;
                  os<<" new position in file "<<(streamposi-(10LL*1024LL)+1LL);
                  //              os<<" new position in file "<<(streamposi-(1LL*1024LL)+1LL);
                  string ms = os.str();
                  mainLogUtil->logAll(ms);
                  //              printf(" fin.(streamposi-10*1025-7) %llu \n",streamposi-(10LL*1025LL)-7LL);
                  if ( (streamposi-(10LL*1024LL)+1LL) < streamposisaved ){
                          fin.seekg(streamposisaved+1LL);
                          stringstream oss;
                        oss<<" Problems repositioning stream reader... streamposisaved "<<streamposisaved;
                        string msg = oss.str();
                        mainLogUtil->logWarning(msg);

                  } else {
                          fin.seekg(streamposi-(10LL*1024LL)+1LL);
                  };
                  //              fin.seekg(streamposi-(1LL*1024LL)+1LL);
                  horrorcount = 0;
                  continue;
                };


                //we have read and analysed the header VRL of this cadre
                iByte_tot+=fin.gcount();
                
                //reading data in the cadre
                fin.read(ccData,LENGTH_DATA_CADRE);             
                if (!fin.good())
                {               
                        stringstream oss;                                               
                        oss<<"The cadre: "<<iNumCadres<<" has something wrong in its data"<<
                        " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully then ... exit ";
                        string msg = oss.str();
                        mainLogUtil->logWarning(msg);
                        
                        //I am sure that this cadre is not good ...
                        if(do_vrl_check)
                        {
                                isCadreGood=false;
                        }
                        
                        //... but maybe I want try to use it ...
                        for(int i=0;i<fin.gcount();i++)
                        {                               
                                StateManager::getInstance().getCurrentState()->readInput(ccData[i]);
                                iByte_tot++;
                        }
                                        
                        //... anyhow then exit
                        break;                                  
                }
                
                //reading the last byte of the cadre
                fin.get(cCrc_VRL);
                if (!fin.good())
                {
                        //if the last byte representing the CRC_VRL is not present                      
                        //or if we are not able to read it
                        stringstream oss;                                               
                        oss<<"The cadre: "<<iNumCadres<<" has something wrong in its last byte."<<
                        " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully then ... exit ";
                        string msg = oss.str();
                        mainLogUtil->logWarning(msg);

                        //I am sure that this cadre is not good ...
                        if(do_vrl_check)
                        {
                                isCadreGood=false;
                        }
                                                
                        //don't exit now, we need to keep it alive to process data 
                        //we will exit the main reading cicle after, 
                }
                                
                if((do_vrl_check)&&(!skip_cadre))       
                {
                        //check CRC for the cadre               
                        short int CalcCheckSum;
                        CalcCheckSum=CRC_Cadre_Check(ccData,(int)LENGTH_DATA_CADRE);
        
                        if (CalcCheckSum!=(short int)(cCrc_VRL))
                        {
                                isCadreGood=false;                              
                                stringstream oss;                                               
                                oss<<"The cadre: "<<iNumCadres<<" failed the CRC check."<<
                                " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully";
                                string msg = oss.str();
                                mainLogUtil->logError(msg);
                                
                                //non mi fido del contatore del route                                   
                                stringstream oss1;
                                oss1<<"Route number unused = "<<route<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";
                                msg = oss1.str();
                                mainLogUtil->logAll(msg);

                                route=old_route;

                        }
                }
                
                if(!skip_cadre)
                {
                        if(route!=old_route)
                        {       
                                is_new_route=true;
                                cadcount = 1LL;
                                download++;
                                stringstream oss;                                               
                                oss<<"Found a new download n: "<<download<<". Header VRL ends at byte: "<<iByte_tot;
                                string msg = oss.str();
                                mainLogUtil->logAll(msg);
                        }
                        // is_new_route will be reset to false in PacketUser after the packet is used and stored
                        
                        
                        //main processing of the data, searching for Pamela Packets
                        for(int i=0; i<LENGTH_DATA_CADRE; i++)
                        {                       
                          StateManager::getInstance().getCurrentState()->readInput(ccData[i]);
                          iByte_tot++;
                          if(exitdbg==true){cout<<"**** DBG EXIT FOR****"<<endl; break;}
                        }
                }
                else{
                        iByte_tot+=LENGTH_DATA_CADRE;
                }
                
                if(exitdbg==true){cout<<"**** DBG EXIT WHILE***##*#*#"<<endl; break;}

                //taking in account the cCrc_VRL read before    
                iByte_tot++;    
                
                //increment the number of cadre processed
                if(!skip_cadre)
                        iNumCadres++;
        
                if(isCadreGood)
                {
                        if(!skip_cadre)
                                iNumGoodCadres++;
                }
                
                
        //      if(iNumCadres>=0){cout<<"################DBG EXIT ##################"<<endl;  break;}

         }//end reading cycle   

        fin.close();
        
        stringstream oss1;
        oss1<<"Total bytes read "<<iByte_tot
        <<" Total number of cadres processed "<<iNumCadres
        <<" Total number of good cadres "<<iNumGoodCadres;              
        string msg = oss1.str();
        mainLogUtil->logInfo(msg);
}

//function that check the header VRL
bool VRL_Header_Check(char* headVRL, int length)
{       
                
/*      stringstream oss;                                               
        oss<<"Pippooooooo: at byte+ "<<iByte_tot<<" cadre ... route number = "<<(int)headVRL[3];
        string msg = oss.str();
        mainLogUtil->logAll(msg);*/
        
         old_route = route;
         
        //a volte capita che ho header tutti 0 e poi anche la cadre e' tutti zeri da buttare
        //il CRC invece viene buono allora uso skip_cadre
        if(((int)(unsigned char)headVRL[0]==0)&&((int)(unsigned char)headVRL[1]==0)&&((int)(unsigned char)headVRL[2]==0)&&((int)(unsigned char)headVRL[3]==0)&&((int)(unsigned char)headVRL[7]==0))
        {
                skip_cadre=true;
                stringstream oss;                                               
                oss<<"The cadre is all zeros"<<
                " This error occurred after byte: "<<iByte_tot<<" ... Cadre DISCARDED and not computed ";
                string msg = oss.str();
                mainLogUtil->logAll(msg);
                return false;
        }
        
        if(
                        ((int)(unsigned char)headVRL[0]!=CODE_FF)||
                        ((int)(unsigned char)headVRL[1]!=CODE_46)||
                        ((int)(unsigned char)headVRL[2]!=CODE_D5)
        )
        {
                
                stringstream oss;                                               
                oss<<"The cadre does not start with FF 46 D5"<<
                " This error occurred after byte: "<<iByte_tot<<" ... Route number unused= "<<(int)headVRL[3]<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";
                string msg = oss.str();
                mainLogUtil->logAll(msg);

                if ( ((int)(unsigned char)headVRL[0]!=CODE_FF)&&((int)(unsigned char)headVRL[1]!=CODE_46)&& ((int)(unsigned char)headVRL[2]!=CODE_D5) ){
                  stringstream os;                                              
                  os<<" Horror movie, the cadre does not start with FF 46 D5 at all! "<<
                    " This error occurred after byte: "<<iByte_tot<<" ... Route number unused= "<<(int)headVRL[3]<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";
                  string ms = os.str();
                  mainLogUtil->logAll(ms);
                  if ( !hc && !horrorcount ) streamposiorig = streamposi;
                  horrorcount++;
                };

                return false;
        }

        horrorcount = 0;

        // EM qui condizione su posizione file
        Bool_t downchangeallowed = false;       
        for (Int_t ee=0; ee<(-1+2+2*downcount); ee+=2){
          if ( streamposi >= dwin->At(ee) && streamposi < dwin->At(ee+1) ) downchangeallowed = true;
          //      printf(" %i ==> %i \n",ee,dwin->At(ee));
        };
        if ( cadcount < 10000LL && cadcount > 0LL) downchangeallowed = false;
        cadcount++;
        //      cout << " cadcount " << cadcount << endl;
        //      printf("cadcount %L \n",cadcount);
        if ( downchangeallowed ){
          //      printf(" change allowed! \n");

          route = (int)headVRL[3];
          
          if(route!=old_route)
            {   
              stringstream oss;         
              oss<<"Changing ROUTE at byte: "<<iByte_tot<<" route = "<<route<<" old value = "<<old_route<<" (download="<<download<<")";
              string msg = oss.str();
              mainLogUtil->logAll(msg);
            }

        };

        //TODO: check the cadre number and save this info?
        //TODO check that the lenght is 8 
        return true;
}

//function that computes the CRC for each cadre
short int CRC_Cadre_Check(char dataVRL[], int length)
{
        //function used only if(do_vrl_check)
         short int Summ;
         int i;
         int CSum, tmp;
         
         tmp=(int)dataVRL[0];
         CSum=tmp; 
         for (i=1;i<length;i++) {tmp=dataVRL[i]; CSum=CSum^tmp;}
         Summ=(short int) (CSum); 
         return Summ;
}


//TODO: funzione per debug, rimuovere marco
void split_input(char fni[])
{
        cout<<"splitting input file"<<endl;

        system("mkdir splitted");

        ifstream fin;   
        fin.open(fni, ifstream::binary);        
        if (!fin) {cout<<"Can not open input file "<<fni<<endl; exit(1);}
        
        ofstream fout;
        static int num=0;
        
        int cic = 1024; 
        int mille =1000;
        int quanti = 50;
        int tanti =quanti*mille*cic;
        char *  tmp = new char[tanti];
        while(fin.eof()==0)
        { 
                fin.read(tmp,tanti);            
                stringstream fno;
                fno<<"./splitted/splitted_"<<++num;
                fout.open(fno.str().c_str(), ios::binary);
                fout.write(tmp,tanti);
                fout.close();
        }
        
        delete[] tmp;
        fin.close();
cout<<"done, output in folder splitted"<<endl;
}       

