diff --git a/configure.ac b/configure.ac index 0ff3f5b6296fd6fb430226a307317705e1a80486..af10cb5b04d4a2b88dfa91d2a8628620136ddcf4 100644 --- a/configure.ac +++ b/configure.ac @@ -48,6 +48,8 @@ LALAPPS_ENABLE_STATIC_BINARIES LALAPPS_ENABLE_NOMAD +LALAPPS_ENABLE_OPENMP + LALSUITE_ENABLE_LALPULSAR #LALAPPS_ENABLE_OPTPP @@ -101,6 +103,15 @@ if test "x$mpi" = "xtrue"; then fi LALSUITE_ENABLE_MODULE([MPI],[mpi]) +# check for OpenMP +if test "x$openmp" = "xtrue"; then + AC_OPENMP([openmp=true],[openmp=false]) + LIBS="$LIBS $OPENMP_LIBS" + CPPFLAGS="$CPPFLAGS $OPENMP_CFLAGS" +fi +LALSUITE_ENABLE_MODULE([OPENMP],[openmp]) + + # checks for programs AC_PROG_CPP @@ -254,6 +265,7 @@ echo "================================================================= LALPulsar library support is $LALPULSAR_ENABLE_VAL NOMAD library support is $NOMAD_ENABLE_VAL + OPENMP support is $OPENMP_ENABLE_VAL and will be installed under the directory: diff --git a/misc/lalapps.m4 b/misc/lalapps.m4 index b3a20fbfb675c6c3e842d0ff2d3951c603a50488..816152e502f812c92edfccb8d3330a77d42dd558 100644 --- a/misc/lalapps.m4 +++ b/misc/lalapps.m4 @@ -110,6 +110,19 @@ AC_DEFUN([LALAPPS_ENABLE_MPI], ], [ mpi=false ] ) ]) +AC_DEFUN([LALAPPS_ENABLE_OPENMP], +[AC_ARG_ENABLE( + [openmp], + AC_HELP_STRING([--enable-openmp],[compile using OpenMP [default=no]]), + [ case "${enableval}" in + yes) openmp=true;; + no) openmp=false;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-openmp) ;; + esac + ], [ openmp=false ] ) +]) + + AC_DEFUN([LALAPPS_ENABLE_FFTW], [AC_ARG_ENABLE( [fftw], diff --git a/src/FollowUp/Nomad/FStatNomad.cpp b/src/FollowUp/Nomad/FStatNomad.cpp index 45f8c744ca75769e374263a6ddcdda31a3d4dc3e..7f9c3ebdf2a2f99e09d4207ff86cad2d252febf1 100644 --- a/src/FollowUp/Nomad/FStatNomad.cpp +++ b/src/FollowUp/Nomad/FStatNomad.cpp @@ -33,8 +33,13 @@ * *********************************************************************************/ +#ifdef _OPENMP +#include <omp.h> +#endif + #include "FStatNomad.h" + using namespace std; #define EARTHEPHEMERIS "earth00-19-DE405.dat" @@ -225,8 +230,8 @@ bool SCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double n++; } } - if (MFCSearch->FirstPoint()) { - MFCSearch->FirstPoint(FALSE); + if (MFCSearch->FirstPoint(s)) { + MFCSearch->FirstPoint(s,FALSE); } else { if (MFCSearch->UseMetric()) { @@ -256,7 +261,7 @@ bool SCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double } bool FCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval) { - +printf("%d XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX eval_x %d\n",omp_get_thread_num(),s); int n = 0; ExPoint p; NOMAD::Point y(x.size()); @@ -402,8 +407,9 @@ bool FCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double } } - if (MFCSearch->FirstPoint()) { - MFCSearch->FirstPoint(FALSE); + if (MFCSearch->FirstPoint(s)) { +printf("AAAAAAAAAAAAAAAAAAAAAa CHECK s: %d\n",s); + MFCSearch->FirstPoint(s,FALSE); } else { if (MFCSearch->UseMetric()) { @@ -487,6 +493,12 @@ INT4 FCSearch::write_cache(NOMAD::Cache *cache, INT4 cnt, INT4 type) { int main(int argc, char *argv[]) { +#ifdef _OPENMP + printf("CWSeaOptiFol compiled with OpenMP (vers = %d) support.\n",_OPENMP); + int numcpu = omp_get_num_procs(); + printf("Number of CPU cores: %d\n",numcpu); +#endif + MFCSearch = new FCSearch(argc,argv); MFCSearch->XLALInitUserVars(argc,argv); @@ -1950,8 +1962,8 @@ FCSearch::FCSearch(int argc, char *argv[]) { /* ephemeris */ edat = NULL; - bSkipSearch = FALSE; - bFirstPoint = TRUE; + bSkipSearch.reserve(INT_MAX); + bFirstPoint.reserve(INT_MAX); UserVariables = empty_UserInput; // initializes this struct to {0} /* user variables */ diff --git a/src/FollowUp/Nomad/FStatNomad.h b/src/FollowUp/Nomad/FStatNomad.h index 39e8a48bbe0de5a9437f19eb8ec50ea23187101a..ed668e3cfce8b2747d087eba18236e33b6c91fd1 100644 --- a/src/FollowUp/Nomad/FStatNomad.h +++ b/src/FollowUp/Nomad/FStatNomad.h @@ -180,8 +180,8 @@ extern "C" { UserInput_t UserVariables; UserInput_t *uvar; - BOOLEAN bFirstPoint; - BOOLEAN bSkipSearch; + std::vector<BOOLEAN> bFirstPoint; + std::vector<BOOLEAN> bSkipSearch; BOOLEAN DumpCache(){ return uvar->DumpCache; diff --git a/src/FollowUp/Nomad/libFStatNomad.cpp b/src/FollowUp/Nomad/libFStatNomad.cpp index 9ae4aaa592615073f2c5709d066fd247e03ede32..68f4489d8371a3db3e4328a2415487698d703611 100644 --- a/src/FollowUp/Nomad/libFStatNomad.cpp +++ b/src/FollowUp/Nomad/libFStatNomad.cpp @@ -25,6 +25,10 @@ #define __STDC_CONSTANT_MACROS +#ifdef _OPENMP +#include <omp.h> +#endif + #include "libFStatNomad.h" static FstatCandidate empty_FstatCandidate; @@ -139,7 +143,8 @@ INT4 FStatNomad::write_nomad_output(vector<ExPoint> epv,INT4 fstatsearchtype,INT /* add code version ID (only useful for git-derived versions) */ fprintf ( fpOut, "%%%% version: %s\n", VCSInfoString ); - tCell best_cell = Cell(); +// // // TODO +// // // tCell best_cell = Cell(); INT4 n = 0; fprintf( fpOut,"%%%% lower_bound:"); @@ -388,16 +393,17 @@ INT4 FStatNomad::write_nomad_output(vector<ExPoint> epv,INT4 fstatsearchtype,INT } fprintf( fpOut,"\n"); - if (BinarySearch()) { - fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5],best_cell.lower_bound[6],best_cell.lower_bound[7],best_cell.lower_bound[8],best_cell.lower_bound[9],best_cell.lower_bound[10]); - fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5],best_cell.start_point[6],best_cell.start_point[7],best_cell.start_point[8],best_cell.start_point[9],best_cell.start_point[10]); - fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5],best_cell.upper_bound[6],best_cell.upper_bound[7],best_cell.upper_bound[8],best_cell.upper_bound[9],best_cell.upper_bound[10]); - } - else { - fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5]); - fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5]); - fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5]); - } +// // // TODO +// // // if (BinarySearch()) { +// // // fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5],best_cell.lower_bound[6],best_cell.lower_bound[7],best_cell.lower_bound[8],best_cell.lower_bound[9],best_cell.lower_bound[10]); +// // // fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5],best_cell.start_point[6],best_cell.start_point[7],best_cell.start_point[8],best_cell.start_point[9],best_cell.start_point[10]); +// // // fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5],best_cell.upper_bound[6],best_cell.upper_bound[7],best_cell.upper_bound[8],best_cell.upper_bound[9],best_cell.upper_bound[10]); +// // // } +// // // else { +// // // fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5]); +// // // fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5]); +// // // fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5]); +// // // } for (thispoint=startpoint; thispoint<epv.size(); thispoint++) { PulsarDopplerParams XLAL_INIT_DECL(cPoint); @@ -867,7 +873,7 @@ void FStatNomad::SetUpSFTs( LALStatus *status, /**< pointer to LALStatus struc UsefulStageVariables *in, /**< input params */ LALSegList *segList) { - +printf("YYYYYYYYYYYYYYYYYYYYYYYYY SetUpSFTs\n"); static SFTConstraints constraints; REAL8 timebase, tObs, deltaFsft; UINT4 k,numSFT; @@ -2398,6 +2404,8 @@ INT4 FStatNomad::get_SC2F(ExPoint *p) { } INT4 FStatNomad::get_FC2F(ExPoint *p) { +// PulsarDopplerParams cPoint; + PulsarDopplerParams XLAL_INIT_DECL(cPoint); cPoint.Alpha = p->alpha; cPoint.Delta = p->delta; cPoint.fkdot[0] = p->freq; @@ -2415,11 +2423,32 @@ INT4 FStatNomad::get_FC2F(ExPoint *p) { cPoint.period = p->period; } FstatQuantities Fstat_what = FSTATQ_2F; + FstatResults *tFstat = NULL; + PrintPoint(1,p,"get_FC2F:",true); + printf("%d dADRESS: tFstat: %x, cPoint: %x, Fstat_in_vec->data[0]: %x\n",omp_get_thread_num(),&tFstat,&cPoint,&Fstat_in_vec->data[0]); + #pragma omp critical XLALComputeFstat(&tFstat,Fstat_in_vec->data[0],&cPoint,1,Fstat_what); p->stat = -tFstat->twoF[0]; return 0; } +void FStatNomad::CopyFstatVector(FstatInputVector **local_Fstat_in_vec, FstatInputVector *Fstat_in_vec){ + printf("COPY data!!!\n"); +//*local_Fstat_in_vec = Fstat_in_vec; + /* set up vector of Fstat input data structs */ + (*local_Fstat_in_vec) = XLALCreateFstatInputVector( Fstat_in_vec->length ); + if ( (*local_Fstat_in_vec) == NULL ) { +// ABORT ( status, FSTATFCNOMAD_EMEM, FSTATFCNOMAD_MSGEMEM ); + printf("ABORRRRRRRRRRRRRRRRRRRRT!\n"); + } + else { + for (int i=0; i<Fstat_in_vec->length; i++){ + (*local_Fstat_in_vec)->data[i] = Fstat_in_vec->data[i]; + } + } + return; +} + REAL8 FStatNomad::Randomize(REAL8 s, REAL8 u, REAL8 l, INT4 p) { REAL8 r; INT4 c = 0; @@ -2773,15 +2802,29 @@ gsl_vector* FStatNomad::RandomPointInside(gsl_matrix *g, NOMAD::Point start_poin INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsearchtype,vector<INT4> searchtype, INT4 &stcounter, BOOLEAN randomstartpoint) { + for (INT4 s=0; s<numsteps; s++) { + bFirstPoint.push_back(true); + bSkipSearch.push_back(false); + } + +#ifdef _OPENMP + for (int stcounter = 0; stcounter < searchtype.size(); stcounter++) { +#else + BOOLEAN dosearch = TRUE; + for (int stcounter = 0; stcounter < searchtype.size() && dosearch; stcounter++) { +#endif + +// INT4 s; + + #pragma omp parallel for ordered + for (INT4 s=0; s<numsteps; s++) { + NOMAD::Point lower_bound(directions.size()); NOMAD::Point upper_bound(directions.size()); NOMAD::Point start_point(directions.size()); NOMAD::Point scale_point(directions.size()); - BOOLEAN dosearch = TRUE; - - for (stcounter = 0; stcounter < searchtype.size() && dosearch; stcounter++) { - + NOMAD::Parameters p ( out ); p.set_DIMENSION (directions.size()); // number of variables @@ -2834,10 +2877,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear PrintNomadPoint(init_mesh,"init_mesh"); p.set_INITIAL_MESH_SIZE(init_mesh); - - INT4 s; - - for (s=0; s<numsteps; s++) { + if (SaveNomadStat()) { p.set_HISTORY_FILE(string(HistoryFile(fstatsearchtype,0,countstep))); p.set_SOLUTION_FILE(string(SolutionFile(fstatsearchtype,0,countstep))); @@ -2851,7 +2891,6 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear p.set_MIN_MESH_SIZE(MinMeshSize()); p.set_MIN_POLL_SIZE(MinPollSize()); - for (INT4 i = 0; i < directions.size(); i++) { start_point[i] = directions.at(i).startpoint * directions.at(i).scale; lower_bound[i] = directions.at(i).min * directions.at(i).scale; @@ -2887,7 +2926,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear PrintNomadPoint(start_point,"start_point"); PrintNomadPoint(upper_bound,"upper_bound"); - Cell(lower_bound,upper_bound,start_point); +// // // TODO Cell(lower_bound,upper_bound,start_point); p.reset_X0(); @@ -2901,7 +2940,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear // custom evaluator creation: if (fstatsearchtype == SCSEARCH) { - SCFStatNomadEvaluator scev( p ); + SCFStatNomadEvaluator scev( p, s ); NOMAD::Mads mads ( p , &scev ); NOMAD::Evaluator_Control *ec; ec = &mads.get_evaluator_control(); @@ -2911,8 +2950,8 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear } NOMAD::Evaluator *pev; pev = ec->get_evaluator(); - FirstPoint(TRUE); - SkipSearch(false); + FirstPoint(s,true); + SkipSearch(s,false); trials_init(); mads.run(); CacheIn(&ec->get_cache()); @@ -2938,7 +2977,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear } else { - FCFStatNomadEvaluator fcev( p ); + FCFStatNomadEvaluator fcev( p, s ); NOMAD::Mads mads ( p , &fcev ); NOMAD::Evaluator_Control *ec; ec = &mads.get_evaluator_control(); @@ -2948,8 +2987,8 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear } NOMAD::Evaluator *pev; pev = ec->get_evaluator(); - FirstPoint(true); - SkipSearch(false); + FirstPoint(s,true); + SkipSearch(s,false); trials_init(); mads.run(); CacheIn(&ec->get_cache()); @@ -2971,6 +3010,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear epv.push_back(tExPoint); trials_push(); +#ifndef _OPENMP if (BreakASAP()) { if (((maxAccept2F() - tExPoint.stat) > 0) && ((tExPoint.stat - minAccept2F()) > 0)) { LogPrintf(LOG_NORMAL,"Found point loud %f within the expectation [%f,%f], break the search!\n",tExPoint.stat,minAccept2F(),maxAccept2F()); @@ -2983,6 +3023,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear LogPrintf(LOG_NORMAL,"The found 2F %f is outside the expectation [%f,%f]!\n",tExPoint.stat,minAccept2F(),maxAccept2F()); } } +#endif } } } // end loop stcounter @@ -3171,6 +3212,9 @@ FStatNomad::FStatNomad() { twoFi[i] = 0; Nsegi[i] = 0; } +#ifdef OPENMP + omp_init_lock(&fstatlock); +#endif }; FStatNomad::~FStatNomad() {}; diff --git a/src/FollowUp/Nomad/libFStatNomad.h b/src/FollowUp/Nomad/libFStatNomad.h index ea7317eae966355067b0d53e4f34f921cee3a6ca..50741080d69eac1754fa5425e2a6a4cc1d0bc624 100644 --- a/src/FollowUp/Nomad/libFStatNomad.h +++ b/src/FollowUp/Nomad/libFStatNomad.h @@ -69,7 +69,6 @@ #include <lal/DopplerScan.h> #include <lal/UniversalDopplerMetric.h> #include <lal/StringVector.h> -#include <lal/ComputeFstat.h> #include <lal/LALString.h> /* lalapps includes */ @@ -320,6 +319,12 @@ typedef struct DopplerCoordinateSystem coordSys; /**< array of enums describing Doppler-coordinates to compute metric in */ } ConfigVariables; +typedef struct { + REAL8 lower_bound[11]; + REAL8 upper_bound[11]; + REAL8 start_point[11]; +} tCell; + typedef struct { REAL8 alpha; REAL8 delta; @@ -337,14 +342,9 @@ typedef struct { REAL8 cosi; REAL8 psi; REAL8 phi0; + tCell cell; } ExPoint; -typedef struct { - REAL8 lower_bound[11]; - REAL8 upper_bound[11]; - REAL8 start_point[11]; -} tCell; - typedef struct { REAL8 min; REAL8 max; @@ -367,16 +367,20 @@ typedef struct { } FstatCandidate; class SCFStatNomadEvaluator : public NOMAD::Evaluator { public: - SCFStatNomadEvaluator ( const NOMAD::Parameters & p ):NOMAD::Evaluator ( p ) {} + SCFStatNomadEvaluator ( const NOMAD::Parameters & p, int c ):NOMAD::Evaluator ( p ) { s= c; } ~SCFStatNomadEvaluator ( void ) {} bool eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval); +private: + int s; }; class FCFStatNomadEvaluator : public NOMAD::Evaluator { public: - FCFStatNomadEvaluator ( const NOMAD::Parameters & p ):NOMAD::Evaluator ( p ) {} + FCFStatNomadEvaluator ( const NOMAD::Parameters & p, int c ):NOMAD::Evaluator ( p ) { s = c; } ~FCFStatNomadEvaluator ( void ) {} bool eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval); +private: + int s; }; class FStatNomad { @@ -566,20 +570,20 @@ public: return var_UseMetric; }; - BOOLEAN FirstPoint() { - return bFirstPoint; + BOOLEAN FirstPoint(INT4 s) { + return bFirstPoint.at(s); }; - void FirstPoint(BOOLEAN firstpoint) { - bFirstPoint = firstpoint; + void FirstPoint(INT4 s, BOOLEAN firstpoint) { + bFirstPoint.at(s) = firstpoint; }; - BOOLEAN SkipSearch() { - return bSkipSearch; + BOOLEAN SkipSearch(INT4 s) { + return bSkipSearch[s]; }; - void SkipSearch(BOOLEAN skipsearch) { - bSkipSearch = skipsearch; + void SkipSearch(INT4 s, BOOLEAN skipsearch) { + bSkipSearch[s] = skipsearch; }; void setSSBprecision() { @@ -2031,7 +2035,8 @@ protected: var_RestrictionType = v; }; - + void CopyFstatVector(FstatInputVector **local_Fstat_in_vec, FstatInputVector *Fstat_in_vec); + private: INT4 var_Ndet; @@ -2207,8 +2212,8 @@ private: INT4 var_Seed; - BOOLEAN bFirstPoint; - BOOLEAN bSkipSearch; + std::vector<BOOLEAN> bFirstPoint; + std::vector<BOOLEAN> bSkipSearch; BOOLEAN var_SaveNomadStat; INT4 var_SplitSky; @@ -2251,6 +2256,9 @@ private: INT8 *freq_index; +#ifdef OPENMP + omp_lock_t fstatlock; +#endif }; /* ------------------------ Functions -------------------------------- */