/[PAMELA software]/chewbacca/PamOffLineSW/PamOffLineSW_Main.cpp
ViewVC logotype

Diff of /chewbacca/PamOffLineSW/PamOffLineSW_Main.cpp

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1.4 by mocchiut, Thu Dec 18 12:58:37 2008 UTC revision 1.11 by mocchiut, Fri Feb 12 04:46:53 2010 UTC
# Line 1  Line 1 
1  //============================================================================  //============================================================================
2  // $Id: PamOffLineSW_Main.cpp,v 1.52 2008-09-12 14:03:55 messineo Exp $  // $Id: PamOffLineSW_Main.cpp,v 1.10 2010/02/11 09:02:10 mocchiut Exp $
3  // Description : Pamela Off-Line Software  // Description : Pamela Off-Line Software
4  //============================================================================  //============================================================================
5    
# Line 8  Line 8 
8  #include "StateManager.h"  #include "StateManager.h"
9    
10  #include "PacketUser.h"  #include "PacketUser.h"
11    #include <TArrayL64.h>
12    
13  extern "C" {  extern "C" {
14      #include <dirent.h>      #include <dirent.h>
# Line 42  short int CRC_Cadre_Check(char dataVRL[] Line 43  short int CRC_Cadre_Check(char dataVRL[]
43    
44  //global variables  //global variables
45  bool simulated_data = false;  bool simulated_data = false;
46  bool do_vrl_check = false;  //bool do_vrl_check = false; //EMI
47    bool do_vrl_check = true; //EMI
48    
49  char*  db_host = "";  
50    const char*  db_host = "";
51  int        db_port = 0;  int        db_port = 0;
52  char*  db_name = "";  const char*  db_name = "";
53  char conn[100]="";  char conn[100]="";
54    
55  LogUtil::logLevel loglevel=LogUtil::LOGERROR;  LogUtil::logLevel loglevel=LogUtil::LOGERROR;
56  char* logfilename = "chewbacca.log";  const char* logfilename = "chewbacca.log";
57  //current route  //current route
58  int route = 999;  int route = 999;
59  //previous route  //previous route
60  int old_route = 999;  int old_route = 999;
61    
62    Long64_t filelength = 0LL;
63    Long64_t streamposi = 0LL;
64    Long64_t streamposisaved = 0LL;
65    Int_t numposi = 10;
66    Long64_t downcount = 1LL;
67    TArrayL64 *dwin=new TArrayL64(12);
68    Long64_t cadcount = 0LL;
69    Int_t horrorcount = 0;
70    Int_t hc = 0;
71    
72  bool skip_cadre = false;  bool skip_cadre = false;
73  long int iNumGoodCadres=0; //total number of good cadres  long int iNumGoodCadres=0; //total number of good cadres
74                    
75  // global variables used in all the project  // global variables used in all the project
76  namespace PamOffLineSW  namespace PamOffLineSW
77  {        {      
78          char*  db_user = "";          const char*  db_user = "";
79          char*  db_pwd  = "";          const char*  db_pwd  = "";
80          char* connection = NULL;                  char* pelosconnection = NULL;  
81          //marco_new_01          //marco_new_01
82          bool single_connection=false;            bool single_connection=false;  
83          long int iNumCadres=0; //cadre's number          long int iNumCadres=0; //cadre's number
# Line 74  namespace PamOffLineSW Line 87  namespace PamOffLineSW
87          char* fni; //path completo          char* fni; //path completo
88          short compression = 3;          short compression = 3;
89  //      char *outDir = ".";  //      char *outDir = ".";
90          char *outDir = "";          const char *outDir = "";
91    
92          char * nome_output="chewbacca";    //    char * nome_output="chewbacca"; //EMI
93            const char * nome_output="L0PAM";//EMI
94          bool multiFile = 0;              bool multiFile = 0;    
95          unsigned long int step_pkt_number=0;    //    unsigned long int step_pkt_number=0; // EMI
96          unsigned long int step_pkt_obt=0;          unsigned long int step_pkt_number=3000;
97      //    unsigned long int step_pkt_obt=0; // EMI
98            unsigned long int step_pkt_obt=7200000; // EMI
99          LogUtil* mainLogUtil = NULL;          LogUtil* mainLogUtil = NULL;
100          TSQLServer* sqlServer = NULL;          TSQLServer* sqlServer = NULL;
101          unsigned long int max_pkt_number =   16777215;//biggest value before reset          unsigned long int max_pkt_number =   16777215;//biggest value before reset
102          unsigned long int max_pkt_obt    = 4294967295u;//biggest value before reset          unsigned long int max_pkt_obt    = 4294967295u;//biggest value before reset
103    
104          bool is_new_route = false;          //    bool is_new_route = false;      // EMI
105            bool is_new_route = true;       // EMI ??????????? VA BENE? SECONDO ME SI`
106          unsigned int download = 0;          unsigned int download = 0;
107          unsigned int orbit_number=0;          unsigned int orbit_number=0;
108          unsigned int mmm_number = 0;//session_number          unsigned int mmm_number = 0;//session_number
109          unsigned long int time_Offset=0;          unsigned long int time_Offset=0;
110          bool tryMerge = false;    //    bool tryMerge = false;
111            bool tryMerge = true;
112            bool candelete = true; //EM
113                                    
114          bool do_cont_check=true;//if do_cont_check is false do not use a DB ...          bool do_cont_check=true;//if do_cont_check is false do not use a DB ...
115    
# Line 160  void readOptions(int argc, char *argv[]) Line 179  void readOptions(int argc, char *argv[])
179          cout << "\t  (-help | --help | -h) print this help and exit \n";          cout << "\t  (-help | --help | -h) print this help and exit \n";
180          cout << "\t -simu       if the input file contains simulated data instead of real data\n";          cout << "\t -simu       if the input file contains simulated data instead of real data\n";
181          cout << "\t -vrl    if you want to perform the vrl check\n";          cout << "\t -vrl    if you want to perform the vrl check\n";
182            cout << "\t -no-vrl    if you DO NOT want to perform the vrl check\n";
183            cout << "\t -gpamela    gpamela data as input\n";
184          cout << "\t -filelog     set the log filename. [default: chewbacca.log]\n";          cout << "\t -filelog     set the log filename. [default: chewbacca.log]\n";
185          cout << "\t -loglevel     set the log level. Values: [0,3] (error,warning,info,all) [default:0]\n";          cout << "\t -loglevel     set the log level. Values: [0,3] (error,warning,info,all) [default:0]\n";
186          cout << "\t -c      set the compression level for the generated ROOT file(s). Values: [0,9] [default = 3]\n";          cout << "\t -c      set the compression level for the generated ROOT file(s). Values: [0,9] [default = 3]\n";
# Line 178  void readOptions(int argc, char *argv[]) Line 199  void readOptions(int argc, char *argv[])
199          cout << "\t -session_number  Value of the session number. If 0 this is retrieved from the input file name.  [default = 0]\n";                          cout << "\t -session_number  Value of the session number. If 0 this is retrieved from the input file name.  [default = 0]\n";                
200          cout << "\t -time_Offset  Value of the timeOffset. If 0 this is retrieved using the orbital number.  [default = 0]\n";          cout << "\t -time_Offset  Value of the timeOffset. If 0 this is retrieved using the orbital number.  [default = 0]\n";
201          cout << "\t -tryMerge if you want to try to Merge ROOT files\n";          cout << "\t -tryMerge if you want to try to Merge ROOT files\n";
202            cout << "\t -dontMerge if you do not want to try to Merge ROOT files\n";
203            cout << "\t -dontDelete if you do not want to delete bad ROOT files from disks\n";
204          //marco_new_01          //marco_new_01
205          cout << "\t -single_connection if you want to open only one connection to DB.\n";          cout << "\t -single_connection if you want to open only one connection to DB.\n";
206          cout << "\t -tag if you want add the 'The owner of the file' - the tag lenght is 4 char\n";          cout << "\t -tag if you want add the 'The owner of the file' - the tag lenght is 4 char\n";
# Line 200  void readOptions(int argc, char *argv[]) Line 223  void readOptions(int argc, char *argv[])
223          exit(1);          exit(1);
224      }      }
225    
226      //input file
227      fni= argv[1];
228      
229    for (int i = 2; i < argc; i++)    for (int i = 2; i < argc; i++)
230    {          {      
231          //marco_new_01          //marco_new_01
# Line 215  void readOptions(int argc, char *argv[]) Line 241  void readOptions(int argc, char *argv[])
241                          tryMerge = true;                          tryMerge = true;
242                  continue;                  continue;
243              }              }
244    
245              if (!strcmp(argv[i], "-dontMerge")) // EMI
246                {
247                            tryMerge = false;
248                    continue;
249                }
250    
251              if (!strcmp(argv[i], "-dontDelete")) // EMI
252                {
253                            candelete = false;
254                    continue;
255                }
256                        
257            if (!strcmp(argv[i], "-time_Offset")){            if (!strcmp(argv[i], "-time_Offset")){
258                          if (++i >= argc){                          if (++i >= argc){
# Line 302  void readOptions(int argc, char *argv[]) Line 340  void readOptions(int argc, char *argv[])
340            simulated_data = true;            simulated_data = true;
341            continue;            continue;
342        }        }
343    
344          if (!strcmp(argv[i], "-gpamela"))
345          {
346              simulated_data = true;
347              do_vrl_check = false;
348              nome_output = (char *)gSystem->BaseName(fni);
349              tryMerge = false;
350              candelete = false;
351              continue;
352          }
353                
354          if (!strcmp(argv[i], "-no-vrl")) //EMI
355            {
356              do_vrl_check = false;
357          }
358        if (!strcmp(argv[i], "-vrl"))        if (!strcmp(argv[i], "-vrl"))
359        {        {
360            do_vrl_check = true;            do_vrl_check = true;
# Line 366  void readOptions(int argc, char *argv[]) Line 418  void readOptions(int argc, char *argv[])
418                   exit(1);                   exit(1);
419               }               }
420               DIR* tempdir;               DIR* tempdir;
421               if ((tempdir = opendir(argv[i])) != 0) {               if ((tempdir = opendir(gSystem->ExpandPathName(argv[i]))) != 0) { // EMI
422                   outDir = argv[i];                   outDir = argv[i];
423                   closedir(tempdir);                   closedir(tempdir);
424               } else {               } else {
# Line 506  void readOptions(int argc, char *argv[]) Line 558  void readOptions(int argc, char *argv[])
558         //       if (!strcmp(argv[i], "-multi")){multiFile = 1; cout<<"debug: multi= "<<multiFile<<endl;}                 //       if (!strcmp(argv[i], "-multi")){multiFile = 1; cout<<"debug: multi= "<<multiFile<<endl;}        
559     }     }
560        
   //input file  
   fni= argv[1];  
     
561    //TODO: maybe if simulated data I don't need nothig so I can skip the following lines    //TODO: maybe if simulated data I don't need nothig so I can skip the following lines
562    // and use instead something similar to:    // and use instead something similar to:
563    //if(simulated data){orbit_number=99999; mmm_number=999; download=999; }    if (simulated_data) {orbit_number=99999; mmm_number=999; download=999; }
564        
565    //I need the following lines only to retrieve orbit_number and mmm_number (session_number)from the filename    //I need the following lines only to retrieve orbit_number and mmm_number (session_number)from the filename
566    if((!orbit_number)||(!mmm_number))    if((!orbit_number)||(!mmm_number))
# Line 584  void DB_config() Line 633  void DB_config()
633                  char *pamdbhost = getenv("PAM_DBHOST");                  char *pamdbhost = getenv("PAM_DBHOST");
634                  if (pamdbhost)                  if (pamdbhost)
635                  {                  {
636                          connection = pamdbhost;                          pelosconnection = pamdbhost;
637                  }                  }
638                  else                  else
639                  {                        {      
# Line 603  void DB_config() Line 652  void DB_config()
652                                  db_name="chewbacca_db";                                          db_name="chewbacca_db";        
653                          }                                                                                        }                                                              
654                          sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);                                sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);      
655                          connection=conn;                                                  pelosconnection=conn;                  
656                  }                  }
657          }          }
658          else{          else{
659                  sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);                        sprintf(conn,"mysql://%s:%d/%s",db_host,db_port,db_name);      
660                  connection=conn;                                                  pelosconnection=conn;                          
661          }          }
662                                    
663          if (!strcmp(db_user,""))          if (!strcmp(db_user,""))
# Line 638  void DB_config() Line 687  void DB_config()
687                  }                        }      
688          }          }
689                    
690          string msg = "Using DB: " + string(connection);// +" "+ string(db_user) +" "+ string(db_pwd);          string msg = "Using DB: " + string(pelosconnection);// +" "+ string(db_user) +" "+ string(db_pwd);
691          mainLogUtil->logAlways(msg);          mainLogUtil->logAlways(msg);
692  }  }
693    
# Line 647  void firstLog(){ Line 696  void firstLog(){
696          string msg = "Input file: " + (string)fni;          string msg = "Input file: " + (string)fni;
697          mainLogUtil->logAlways(msg);          mainLogUtil->logAlways(msg);
698    
699          msg = "Output directory for the generated root file(s): " + (string)outDir;          msg = "Output directory for the generated root file(s): " + (string)outDir;
700          mainLogUtil->logAlways(msg);              mainLogUtil->logAlways(msg);    
701            msg = "Output directory for the generated root file(s) expanded: " + (string)gSystem->ExpandPathName(outDir); // EMI
702            mainLogUtil->logAlways(msg);    //EMI
703    
704          msg = "Output root name for the generated root file(s): " + (string)nome_output;          msg = "Output root name for the generated root file(s): " + (string)nome_output;
705          mainLogUtil->logAlways(msg);              mainLogUtil->logAlways(msg);    
# Line 804  void mainRead_RealData(char fni[]) Line 855  void mainRead_RealData(char fni[])
855          ifstream fin;            ifstream fin;  
856          fin.open(fni, ifstream::binary);                  fin.open(fni, ifstream::binary);        
857          if (!fin) {cout<<"Can not open input file "<<fni<<endl; exit(1);}          if (!fin) {cout<<"Can not open input file "<<fni<<endl; exit(1);}
858            
859    
860            //
861            // EM determine file length, number of downloads (a part small fractions of download due to VRL memory) and interval window for allowed change in download number
862            //
863            fin.seekg(0,std::ios::end);
864            filelength = (Long64_t)fin.tellg();
865            //
866            //      filelength = 3840214016LL;
867            //      filelength = 5763760128LL;
868            //      filelength = 15370027008LL;
869            //
870            downcount = (Long64_t)round((Double_t)filelength/2000000000.);
871            if ( downcount < 1LL ) downcount = 1LL;
872            Long64_t downsize = (Long64_t)round((Double_t)filelength/(Double_t)downcount);
873            Long64_t hwint = Long64_t((Double_t)downsize/25.);
874            //
875            if ( downcount > 5LL ) dwin->Set(2LL+2LL*downcount);
876            dwin->Reset();
877            dwin->AddAt(0LL,0);
878            //
879            Int_t dco = 1;
880            //
881            for (Int_t ee=1; ee<(-1+2+2*downcount); ee+=2){
882              dwin->AddAt(((downsize*(dco-1))+hwint),ee);    
883              dwin->AddAt(((downsize*dco)-hwint),ee+1);      
884              dco++;
885            };
886            //
887            fin.seekg(0,std::ios::beg);
888            //
889            dwin->AddAt(filelength,(-1LL+2LL+2LL*downcount));
890            //
891            //      printf(" file lenght is %L downcount is %L downsize is %L hwint is %L \n",filelength,downcount,downsize,hwint);
892            stringstream ess;
893            ess.str() = "";
894            ess << " (EM) " << " file lenght is "<< filelength<< " downcount is "<< downcount <<" downsize is "<< downsize <<" hwint is " << hwint;
895            string emsg = ess.str();
896            mainLogUtil->logInfo(emsg);
897            //
898            //      cout << " file lenght is "<< filelength<< " downcount is "<< downcount <<" downsize is "<< downsize <<" hwint is " << hwint << endl;
899            //      for (Int_t ee=0; ee<TMath::Max((2+2*downcount),12LL); ee++){
900            //        cout << " " << ee << " ==> "<< dwin->At(ee) << endl;
901            //        //a     printf(" %i ==> %L \n",ee,dwin->At(ee));
902            //      };
903    
904          //main read loop          //main read loop
905           while(fin.eof()==0)           while(fin.eof()==0)
906           {                         {              
907                  isCadreGood = true;                              isCadreGood = true;            
908                  //start reading the file                  //start reading the file
909                    streamposi = (Long64_t)fin.tellg(); // stream position in the file before reading the vrl header
910                    if ( iNumCadres > 9 && numposi > 9 ){
911                            streamposisaved = streamposi;
912                            numposi = 0;                    
913                    };
914                    numposi++;
915                    //
916                  fin.read(ccHeader_VRL, LENGTH_HEADER_VRL);                                fin.read(ccHeader_VRL, LENGTH_HEADER_VRL);              
917                    //              printf(" position in file is %llu \n",streamposi);
918                    stringstream ss;
919                    ss.str() = "";
920                    ss << " position in file is "<<streamposi;
921                    string sms = ss.str();
922                    //              mainLogUtil->logError(sms);
923                    //
924                  if (!fin.good())                  if (!fin.good())
925                  {                        {      
926                          //TODO: check here                          //TODO: check here
# Line 838  void mainRead_RealData(char fni[]) Line 948  void mainRead_RealData(char fni[])
948                          " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully";                          " This error occurred after byte: "<<iByte_tot<<" ... use the data carefully";
949                          string msg = oss.str();                          string msg = oss.str();
950                          mainLogUtil->logWarning(msg);                          mainLogUtil->logWarning(msg);
951                            //                      printf(" streamposi %llu \n",streamposi);
952                  }                  }
953                                    
954                    //
955                    // we completely missed more than 10 VRL packets
956                    //
957                    if ( horrorcount > 10 && hc < 500 ){
958                            hc++;
959                    //if ( horrorcount > 1 ){
960                      stringstream os;                                              
961                      os<<" NEW (EM) THIS IS TOO MUCH, LOST SYNC WITH VRL HEADERS! try to recover horrorcount = "<<horrorcount;
962                      os<<" new position in file "<<(streamposi-(10LL*1024LL)+1LL);
963                      //              os<<" new position in file "<<(streamposi-(1LL*1024LL)+1LL);
964                      string ms = os.str();
965                      mainLogUtil->logAll(ms);
966                      //              printf(" fin.(streamposi-10*1025-7) %llu \n",streamposi-(10LL*1025LL)-7LL);
967                      if ( (streamposi-(10LL*1024LL)+1LL) < streamposisaved ){
968                              fin.seekg(streamposisaved+1LL);
969                              stringstream oss;
970                            oss<<" Problems repositioning stream reader... streamposisaved "<<streamposisaved;
971                            string msg = oss.str();
972                            mainLogUtil->logWarning(msg);
973    
974                      } else {
975                              fin.seekg(streamposi-(10LL*1024LL)+1LL);
976                      };
977                      //              fin.seekg(streamposi-(1LL*1024LL)+1LL);
978                      horrorcount = 0;
979                      continue;
980                    };
981    
982    
983                  //we have read and analysed the header VRL of this cadre                  //we have read and analysed the header VRL of this cadre
984                  iByte_tot+=fin.gcount();                  iByte_tot+=fin.gcount();
985                                    
# Line 923  void mainRead_RealData(char fni[]) Line 1063  void mainRead_RealData(char fni[])
1063                          if(route!=old_route)                          if(route!=old_route)
1064                          {                                {      
1065                                  is_new_route=true;                                  is_new_route=true;
1066                                    cadcount = 1LL;
1067                                  download++;                                  download++;
1068                                  stringstream oss;                                                                                stringstream oss;                                              
1069                                  oss<<"Found a new download n: "<<download<<". Header VRL ends at byte: "<<iByte_tot;                                  oss<<"Found a new download n: "<<download<<". Header VRL ends at byte: "<<iByte_tot;
# Line 1010  bool VRL_Header_Check(char* headVRL, int Line 1151  bool VRL_Header_Check(char* headVRL, int
1151                  " This error occurred after byte: "<<iByte_tot<<" ... Route number unused= "<<(int)headVRL[3]<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";                  " This error occurred after byte: "<<iByte_tot<<" ... Route number unused= "<<(int)headVRL[3]<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";
1152                  string msg = oss.str();                  string msg = oss.str();
1153                  mainLogUtil->logAll(msg);                  mainLogUtil->logAll(msg);
1154    
1155                    if ( ((int)(unsigned char)headVRL[0]!=CODE_FF)&&((int)(unsigned char)headVRL[1]!=CODE_46)&& ((int)(unsigned char)headVRL[2]!=CODE_D5) ){
1156                      stringstream os;                                              
1157                      os<<" Horror movie, the cadre does not start with FF 46 D5 at all! "<<
1158                        " This error occurred after byte: "<<iByte_tot<<" ... Route number unused= "<<(int)headVRL[3]<<" Resetting ROUTE to old value = "<<old_route<<" (download="<<download<<")";
1159                      string ms = os.str();
1160                      mainLogUtil->logAll(ms);
1161                      horrorcount++;
1162                    };
1163    
1164                  return false;                  return false;
1165          }          }
1166            
1167          route = (int)headVRL[3];          horrorcount = 0;
1168                    
1169          if(route!=old_route)          // EM qui condizione su posizione file
1170          {                Bool_t downchangeallowed = false;      
1171                  stringstream oss;                        for (Int_t ee=0; ee<(-1+2+2*downcount); ee+=2){
1172                  oss<<"Changing ROUTE at byte: "<<iByte_tot<<" route = "<<route<<" old value = "<<old_route<<" (download="<<download<<")";            if ( streamposi >= dwin->At(ee) && streamposi < dwin->At(ee+1) ) downchangeallowed = true;
1173                  string msg = oss.str();            //      printf(" %i ==> %i \n",ee,dwin->At(ee));
1174                  mainLogUtil->logAll(msg);          };
1175          }          if ( cadcount < 10000LL && cadcount > 0LL) downchangeallowed = false;
1176            cadcount++;
1177            //      cout << " cadcount " << cadcount << endl;
1178            //      printf("cadcount %L \n",cadcount);
1179            if ( downchangeallowed ){
1180              //      printf(" change allowed! \n");
1181    
1182              route = (int)headVRL[3];
1183              
1184              if(route!=old_route)
1185                {  
1186                  stringstream oss;        
1187                  oss<<"Changing ROUTE at byte: "<<iByte_tot<<" route = "<<route<<" old value = "<<old_route<<" (download="<<download<<")";
1188                  string msg = oss.str();
1189                  mainLogUtil->logAll(msg);
1190                }
1191    
1192            };
1193    
1194          //TODO: check the cadre number and save this info?          //TODO: check the cadre number and save this info?
1195          //TODO check that the lenght is 8          //TODO check that the lenght is 8
Legend:
Removed from v.1.4  
changed lines
  Added in v.1.11
  ViewVC Help
Powered by ViewVC 1.1.23