Commit 7cc3764e authored by Miroslav Shaltev's avatar Miroslav Shaltev
Browse files

work on openmp support

parent 78ad8b73
......@@ -48,6 +48,8 @@ LALAPPS_ENABLE_STATIC_BINARIES
LALAPPS_ENABLE_NOMAD
LALAPPS_ENABLE_OPENMP
LALSUITE_ENABLE_LALPULSAR
#LALAPPS_ENABLE_OPTPP
......@@ -101,6 +103,15 @@ if test "x$mpi" = "xtrue"; then
fi
LALSUITE_ENABLE_MODULE([MPI],[mpi])
# check for OpenMP
if test "x$openmp" = "xtrue"; then
AC_OPENMP([openmp=true],[openmp=false])
LIBS="$LIBS $OPENMP_LIBS"
CPPFLAGS="$CPPFLAGS $OPENMP_CFLAGS"
fi
LALSUITE_ENABLE_MODULE([OPENMP],[openmp])
# checks for programs
AC_PROG_CPP
......@@ -254,6 +265,7 @@ echo "=================================================================
LALPulsar library support is $LALPULSAR_ENABLE_VAL
NOMAD library support is $NOMAD_ENABLE_VAL
OPENMP support is $OPENMP_ENABLE_VAL
and will be installed under the directory:
......
......@@ -110,6 +110,19 @@ AC_DEFUN([LALAPPS_ENABLE_MPI],
], [ mpi=false ] )
])
AC_DEFUN([LALAPPS_ENABLE_OPENMP],
[AC_ARG_ENABLE(
[openmp],
AC_HELP_STRING([--enable-openmp],[compile using OpenMP [default=no]]),
[ case "${enableval}" in
yes) openmp=true;;
no) openmp=false;;
*) AC_MSG_ERROR(bad value ${enableval} for --enable-openmp) ;;
esac
], [ openmp=false ] )
])
AC_DEFUN([LALAPPS_ENABLE_FFTW],
[AC_ARG_ENABLE(
[fftw],
......
......@@ -33,8 +33,13 @@
*
*********************************************************************************/
#ifdef _OPENMP
#include <omp.h>
#endif
#include "FStatNomad.h"
using namespace std;
#define EARTHEPHEMERIS "earth00-19-DE405.dat"
......@@ -225,8 +230,8 @@ bool SCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double
n++;
}
}
if (MFCSearch->FirstPoint()) {
MFCSearch->FirstPoint(FALSE);
if (MFCSearch->FirstPoint(s)) {
MFCSearch->FirstPoint(s,FALSE);
}
else {
if (MFCSearch->UseMetric()) {
......@@ -256,7 +261,7 @@ bool SCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double
}
bool FCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval) {
printf("%d XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX eval_x %d\n",omp_get_thread_num(),s);
int n = 0;
ExPoint p;
NOMAD::Point y(x.size());
......@@ -402,8 +407,9 @@ bool FCFStatNomadEvaluator::eval_x( NOMAD::Eval_Point & x, const NOMAD::Double
}
}
if (MFCSearch->FirstPoint()) {
MFCSearch->FirstPoint(FALSE);
if (MFCSearch->FirstPoint(s)) {
printf("AAAAAAAAAAAAAAAAAAAAAa CHECK s: %d\n",s);
MFCSearch->FirstPoint(s,FALSE);
}
else {
if (MFCSearch->UseMetric()) {
......@@ -487,6 +493,12 @@ INT4 FCSearch::write_cache(NOMAD::Cache *cache, INT4 cnt, INT4 type) {
int main(int argc, char *argv[]) {
#ifdef _OPENMP
printf("CWSeaOptiFol compiled with OpenMP (vers = %d) support.\n",_OPENMP);
int numcpu = omp_get_num_procs();
printf("Number of CPU cores: %d\n",numcpu);
#endif
MFCSearch = new FCSearch(argc,argv);
MFCSearch->XLALInitUserVars(argc,argv);
......@@ -1950,8 +1962,8 @@ FCSearch::FCSearch(int argc, char *argv[]) {
/* ephemeris */
edat = NULL;
bSkipSearch = FALSE;
bFirstPoint = TRUE;
bSkipSearch.reserve(INT_MAX);
bFirstPoint.reserve(INT_MAX);
UserVariables = empty_UserInput; // initializes this struct to {0}
/* user variables */
......
......@@ -180,8 +180,8 @@ extern "C" {
UserInput_t UserVariables;
UserInput_t *uvar;
BOOLEAN bFirstPoint;
BOOLEAN bSkipSearch;
std::vector<BOOLEAN> bFirstPoint;
std::vector<BOOLEAN> bSkipSearch;
BOOLEAN DumpCache(){
return uvar->DumpCache;
......
......@@ -25,6 +25,10 @@
#define __STDC_CONSTANT_MACROS
#ifdef _OPENMP
#include <omp.h>
#endif
#include "libFStatNomad.h"
static FstatCandidate empty_FstatCandidate;
......@@ -139,7 +143,8 @@ INT4 FStatNomad::write_nomad_output(vector<ExPoint> epv,INT4 fstatsearchtype,INT
/* add code version ID (only useful for git-derived versions) */
fprintf ( fpOut, "%%%% version: %s\n", VCSInfoString );
tCell best_cell = Cell();
// // // TODO
// // // tCell best_cell = Cell();
INT4 n = 0;
fprintf( fpOut,"%%%% lower_bound:");
......@@ -388,16 +393,17 @@ INT4 FStatNomad::write_nomad_output(vector<ExPoint> epv,INT4 fstatsearchtype,INT
}
fprintf( fpOut,"\n");
if (BinarySearch()) {
fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5],best_cell.lower_bound[6],best_cell.lower_bound[7],best_cell.lower_bound[8],best_cell.lower_bound[9],best_cell.lower_bound[10]);
fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5],best_cell.start_point[6],best_cell.start_point[7],best_cell.start_point[8],best_cell.start_point[9],best_cell.start_point[10]);
fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5],best_cell.upper_bound[6],best_cell.upper_bound[7],best_cell.upper_bound[8],best_cell.upper_bound[9],best_cell.upper_bound[10]);
}
else {
fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5]);
fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5]);
fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5]);
}
// // // TODO
// // // if (BinarySearch()) {
// // // fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5],best_cell.lower_bound[6],best_cell.lower_bound[7],best_cell.lower_bound[8],best_cell.lower_bound[9],best_cell.lower_bound[10]);
// // // fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5],best_cell.start_point[6],best_cell.start_point[7],best_cell.start_point[8],best_cell.start_point[9],best_cell.start_point[10]);
// // // fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e %.16e %.16e %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5],best_cell.upper_bound[6],best_cell.upper_bound[7],best_cell.upper_bound[8],best_cell.upper_bound[9],best_cell.upper_bound[10]);
// // // }
// // // else {
// // // fprintf( fpOut, "%%%% nomad_lower: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.lower_bound[0],best_cell.lower_bound[1],best_cell.lower_bound[2],best_cell.lower_bound[3],best_cell.lower_bound[4],best_cell.lower_bound[5]);
// // // fprintf( fpOut, "%%%% nomad_start: %.16f %.16f %.16f %.16e %.16e %.16e\n",best_cell.start_point[0],best_cell.start_point[1],best_cell.start_point[2],best_cell.start_point[3],best_cell.start_point[4],best_cell.start_point[5]);
// // // fprintf( fpOut, "%%%% nomad_upper: %.16f %.16f %.16f %.16e %.16e %.16e\n\n",best_cell.upper_bound[0],best_cell.upper_bound[1],best_cell.upper_bound[2],best_cell.upper_bound[3],best_cell.upper_bound[4],best_cell.upper_bound[5]);
// // // }
for (thispoint=startpoint; thispoint<epv.size(); thispoint++) {
PulsarDopplerParams XLAL_INIT_DECL(cPoint);
......@@ -867,7 +873,7 @@ void FStatNomad::SetUpSFTs( LALStatus *status, /**< pointer to LALStatus struc
UsefulStageVariables *in, /**< input params */
LALSegList *segList)
{
printf("YYYYYYYYYYYYYYYYYYYYYYYYY SetUpSFTs\n");
static SFTConstraints constraints;
REAL8 timebase, tObs, deltaFsft;
UINT4 k,numSFT;
......@@ -2398,6 +2404,8 @@ INT4 FStatNomad::get_SC2F(ExPoint *p) {
}
INT4 FStatNomad::get_FC2F(ExPoint *p) {
// PulsarDopplerParams cPoint;
PulsarDopplerParams XLAL_INIT_DECL(cPoint);
cPoint.Alpha = p->alpha;
cPoint.Delta = p->delta;
cPoint.fkdot[0] = p->freq;
......@@ -2415,11 +2423,32 @@ INT4 FStatNomad::get_FC2F(ExPoint *p) {
cPoint.period = p->period;
}
FstatQuantities Fstat_what = FSTATQ_2F;
FstatResults *tFstat = NULL;
PrintPoint(1,p,"get_FC2F:",true);
printf("%d dADRESS: tFstat: %x, cPoint: %x, Fstat_in_vec->data[0]: %x\n",omp_get_thread_num(),&tFstat,&cPoint,&Fstat_in_vec->data[0]);
#pragma omp critical
XLALComputeFstat(&tFstat,Fstat_in_vec->data[0],&cPoint,1,Fstat_what);
p->stat = -tFstat->twoF[0];
return 0;
}
void FStatNomad::CopyFstatVector(FstatInputVector **local_Fstat_in_vec, FstatInputVector *Fstat_in_vec){
printf("COPY data!!!\n");
//*local_Fstat_in_vec = Fstat_in_vec;
/* set up vector of Fstat input data structs */
(*local_Fstat_in_vec) = XLALCreateFstatInputVector( Fstat_in_vec->length );
if ( (*local_Fstat_in_vec) == NULL ) {
// ABORT ( status, FSTATFCNOMAD_EMEM, FSTATFCNOMAD_MSGEMEM );
printf("ABORRRRRRRRRRRRRRRRRRRRT!\n");
}
else {
for (int i=0; i<Fstat_in_vec->length; i++){
(*local_Fstat_in_vec)->data[i] = Fstat_in_vec->data[i];
}
}
return;
}
REAL8 FStatNomad::Randomize(REAL8 s, REAL8 u, REAL8 l, INT4 p) {
REAL8 r;
INT4 c = 0;
......@@ -2773,15 +2802,29 @@ gsl_vector* FStatNomad::RandomPointInside(gsl_matrix *g, NOMAD::Point start_poin
INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsearchtype,vector<INT4> searchtype, INT4 &stcounter, BOOLEAN randomstartpoint) {
for (INT4 s=0; s<numsteps; s++) {
bFirstPoint.push_back(true);
bSkipSearch.push_back(false);
}
#ifdef _OPENMP
for (int stcounter = 0; stcounter < searchtype.size(); stcounter++) {
#else
BOOLEAN dosearch = TRUE;
for (int stcounter = 0; stcounter < searchtype.size() && dosearch; stcounter++) {
#endif
// INT4 s;
#pragma omp parallel for ordered
for (INT4 s=0; s<numsteps; s++) {
NOMAD::Point lower_bound(directions.size());
NOMAD::Point upper_bound(directions.size());
NOMAD::Point start_point(directions.size());
NOMAD::Point scale_point(directions.size());
BOOLEAN dosearch = TRUE;
for (stcounter = 0; stcounter < searchtype.size() && dosearch; stcounter++) {
NOMAD::Parameters p ( out );
p.set_DIMENSION (directions.size()); // number of variables
......@@ -2834,10 +2877,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
PrintNomadPoint(init_mesh,"init_mesh");
p.set_INITIAL_MESH_SIZE(init_mesh);
INT4 s;
for (s=0; s<numsteps; s++) {
if (SaveNomadStat()) {
p.set_HISTORY_FILE(string(HistoryFile(fstatsearchtype,0,countstep)));
p.set_SOLUTION_FILE(string(SolutionFile(fstatsearchtype,0,countstep)));
......@@ -2851,7 +2891,6 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
p.set_MIN_MESH_SIZE(MinMeshSize());
p.set_MIN_POLL_SIZE(MinPollSize());
for (INT4 i = 0; i < directions.size(); i++) {
start_point[i] = directions.at(i).startpoint * directions.at(i).scale;
lower_bound[i] = directions.at(i).min * directions.at(i).scale;
......@@ -2887,7 +2926,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
PrintNomadPoint(start_point,"start_point");
PrintNomadPoint(upper_bound,"upper_bound");
Cell(lower_bound,upper_bound,start_point);
// // // TODO Cell(lower_bound,upper_bound,start_point);
p.reset_X0();
......@@ -2901,7 +2940,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
// custom evaluator creation:
if (fstatsearchtype == SCSEARCH) {
SCFStatNomadEvaluator scev( p );
SCFStatNomadEvaluator scev( p, s );
NOMAD::Mads mads ( p , &scev );
NOMAD::Evaluator_Control *ec;
ec = &mads.get_evaluator_control();
......@@ -2911,8 +2950,8 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
}
NOMAD::Evaluator *pev;
pev = ec->get_evaluator();
FirstPoint(TRUE);
SkipSearch(false);
FirstPoint(s,true);
SkipSearch(s,false);
trials_init();
mads.run();
CacheIn(&ec->get_cache());
......@@ -2938,7 +2977,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
}
else {
FCFStatNomadEvaluator fcev( p );
FCFStatNomadEvaluator fcev( p, s );
NOMAD::Mads mads ( p , &fcev );
NOMAD::Evaluator_Control *ec;
ec = &mads.get_evaluator_control();
......@@ -2948,8 +2987,8 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
}
NOMAD::Evaluator *pev;
pev = ec->get_evaluator();
FirstPoint(true);
SkipSearch(false);
FirstPoint(s,true);
SkipSearch(s,false);
trials_init();
mads.run();
CacheIn(&ec->get_cache());
......@@ -2971,6 +3010,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
epv.push_back(tExPoint);
trials_push();
#ifndef _OPENMP
if (BreakASAP()) {
if (((maxAccept2F() - tExPoint.stat) > 0) && ((tExPoint.stat - minAccept2F()) > 0)) {
LogPrintf(LOG_NORMAL,"Found point loud %f within the expectation [%f,%f], break the search!\n",tExPoint.stat,minAccept2F(),maxAccept2F());
......@@ -2983,6 +3023,7 @@ INT4 FStatNomad::DoSearch(NOMAD::Display out,vector<ExPoint> &epv,INT4 fstatsear
LogPrintf(LOG_NORMAL,"The found 2F %f is outside the expectation [%f,%f]!\n",tExPoint.stat,minAccept2F(),maxAccept2F());
}
}
#endif
}
}
} // end loop stcounter
......@@ -3171,6 +3212,9 @@ FStatNomad::FStatNomad() {
twoFi[i] = 0;
Nsegi[i] = 0;
}
#ifdef OPENMP
omp_init_lock(&fstatlock);
#endif
};
FStatNomad::~FStatNomad() {};
......
......@@ -69,7 +69,6 @@
#include <lal/DopplerScan.h>
#include <lal/UniversalDopplerMetric.h>
#include <lal/StringVector.h>
#include <lal/ComputeFstat.h>
#include <lal/LALString.h>
/* lalapps includes */
......@@ -320,6 +319,12 @@ typedef struct
DopplerCoordinateSystem coordSys; /**< array of enums describing Doppler-coordinates to compute metric in */
} ConfigVariables;
typedef struct {
REAL8 lower_bound[11];
REAL8 upper_bound[11];
REAL8 start_point[11];
} tCell;
typedef struct {
REAL8 alpha;
REAL8 delta;
......@@ -337,14 +342,9 @@ typedef struct {
REAL8 cosi;
REAL8 psi;
REAL8 phi0;
tCell cell;
} ExPoint;
typedef struct {
REAL8 lower_bound[11];
REAL8 upper_bound[11];
REAL8 start_point[11];
} tCell;
typedef struct {
REAL8 min;
REAL8 max;
......@@ -367,16 +367,20 @@ typedef struct {
} FstatCandidate;
class SCFStatNomadEvaluator : public NOMAD::Evaluator {
public:
SCFStatNomadEvaluator ( const NOMAD::Parameters & p ):NOMAD::Evaluator ( p ) {}
SCFStatNomadEvaluator ( const NOMAD::Parameters & p, int c ):NOMAD::Evaluator ( p ) { s= c; }
~SCFStatNomadEvaluator ( void ) {}
bool eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval);
private:
int s;
};
class FCFStatNomadEvaluator : public NOMAD::Evaluator {
public:
FCFStatNomadEvaluator ( const NOMAD::Parameters & p ):NOMAD::Evaluator ( p ) {}
FCFStatNomadEvaluator ( const NOMAD::Parameters & p, int c ):NOMAD::Evaluator ( p ) { s = c; }
~FCFStatNomadEvaluator ( void ) {}
bool eval_x( NOMAD::Eval_Point & x, const NOMAD::Double & h_max, bool & count_eval);
private:
int s;
};
class FStatNomad {
......@@ -566,20 +570,20 @@ public:
return var_UseMetric;
};
BOOLEAN FirstPoint() {
return bFirstPoint;
BOOLEAN FirstPoint(INT4 s) {
return bFirstPoint.at(s);
};
void FirstPoint(BOOLEAN firstpoint) {
bFirstPoint = firstpoint;
void FirstPoint(INT4 s, BOOLEAN firstpoint) {
bFirstPoint.at(s) = firstpoint;
};
BOOLEAN SkipSearch() {
return bSkipSearch;
BOOLEAN SkipSearch(INT4 s) {
return bSkipSearch[s];
};
void SkipSearch(BOOLEAN skipsearch) {
bSkipSearch = skipsearch;
void SkipSearch(INT4 s, BOOLEAN skipsearch) {
bSkipSearch[s] = skipsearch;
};
void setSSBprecision() {
......@@ -2031,7 +2035,8 @@ protected:
var_RestrictionType = v;
};
void CopyFstatVector(FstatInputVector **local_Fstat_in_vec, FstatInputVector *Fstat_in_vec);
private:
INT4 var_Ndet;
......@@ -2207,8 +2212,8 @@ private:
INT4 var_Seed;
BOOLEAN bFirstPoint;
BOOLEAN bSkipSearch;
std::vector<BOOLEAN> bFirstPoint;
std::vector<BOOLEAN> bSkipSearch;
BOOLEAN var_SaveNomadStat;
INT4 var_SplitSky;
......@@ -2251,6 +2256,9 @@ private:
INT8 *freq_index;
#ifdef OPENMP
omp_lock_t fstatlock;
#endif
};
/* ------------------------ Functions -------------------------------- */
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment