escript  Revision_
EsysMPI.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2020 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14 * Development from 2019 by School of Earth and Environmental Sciences
15 **
16 *****************************************************************************/
17 
18 #ifndef __ESCRIPT_ESYSMPI_H__
19 #define __ESCRIPT_ESYSMPI_H__
20 
21 #include "system_dep.h"
22 
23 #include <escript/DataTypes.h>
24 
25 #include <ctime>
26 #include <sstream>
27 
28 #include <boost/shared_ptr.hpp>
29 
30 #ifdef _OPENMP
31 #include <omp.h>
32 #endif
33 
34 #ifdef ESYS_MPI
35 #include <mpi.h>
36 
37 #ifdef ESYS_INDEXTYPE_LONG
38 #define MPI_DIM_T MPI_LONG
39 #else
40 #define MPI_DIM_T MPI_INT
41 #endif
42 
43 #else
44  typedef int MPI_Comm;
45  typedef int MPI_Request;
46  typedef int MPI_Op;
47  typedef int MPI_Status;
48  #define MPI_INT 6
49  #define MPI_DOUBLE 11
50  #define MPI_COMM_WORLD 91
51  #define MPI_COMM_NULL 0
52 
53  // MPI_Op replacements for non-MPI - these values are arbitrary
54  #define MPI_SUM 100
55  #define MPI_MIN 101
56  #define MPI_MAX 102
57 
58  #define MPI_OP_NULL 17
59  // end MPI_op
60 
61 #endif // ESYS_MPI
62 
63 namespace escript {
64 
69 inline int getSubWorldTag()
70 {
71  return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
72 }
73 
74 class JMPI_;
75 
76 typedef boost::shared_ptr<JMPI_> JMPI;
77 
81 JMPI makeInfo(MPI_Comm comm, bool owncom=false);
82 
84 {
85 public:
86  ~JMPI_();
87 
89  DataTypes::dim_t setDistribution(DataTypes::index_t min_id,
90  DataTypes::index_t max_id,
91  DataTypes::index_t* distribution);
92 
94  void split(DataTypes::dim_t N, DataTypes::dim_t* local_N,
95  DataTypes::index_t* offset);
96 
99  inline int mod_rank(int k) const
100  {
101  int out=0;
102 #ifdef ESYS_MPI
103  if (size > 1) {
104  const int q = k/size;
105  if (k > 0) {
106  out=k-size*q;
107  } else if (k < 0) {
108  out=k-size*(q-1);
109  }
110  }
111 #endif
112  return out;
113  }
114 
116  inline std::string appendRankToFileName(const std::string& fileName) const
117  {
118 #ifdef ESYS_MPI
119  if (size > 1) {
120  std::stringstream ss;
121  ss << fileName << '.';
122  ss.fill('0');
123  ss.width(4);
124  ss << rank;
125  return ss.str();
126  }
127 #endif
128  return fileName;
129  }
130 
132  inline int counter() const
133  {
134  return msg_tag_counter;
135  }
136 
138  inline void incCounter(int i=1)
139  {
140  msg_tag_counter+=i;
141  // there is no particular significance here other than being 7 digits
142  // and prime (because why not). It just needs to be big.
143  msg_tag_counter %= 1010201;
144  }
145 
147  inline void setCounter(int value)
148  {
149  msg_tag_counter = value%1010201;
150  }
151 
153  inline bool isValid() const
154  {
155  return comm!=MPI_COMM_NULL;
156  }
157 
158  int size;
159  int rank;
161 
162 private:
163  JMPI_(MPI_Comm comm, bool owncomm);
164  friend ESCRIPT_DLL_API JMPI makeInfo(MPI_Comm comm, bool owncom);
165 #pragma clang diagnostic push
166 #pragma clang diagnostic ignored "-Wunused-private-field"
167  bool ownscomm;
168 #pragma clang diagnostic pop
170 };
171 
172 // Does not cope with nested calls
174 {
175 public:
176  NoCOMM_WORLD();
177  ~NoCOMM_WORLD();
178  static bool active();
179 };
180 
183 bool checkResult(int input, int& output, const JMPI& comm);
184 
189 bool shipString(const char* src, char** dest, MPI_Comm& comm);
190 
192 inline double gettime()
193 {
194  double out;
195 #ifdef ESYS_MPI
196  out = MPI_Wtime();
197 #else
198 #ifdef _OPENMP
199  out=omp_get_wtime();
200 #else
201  out=((double) clock())/CLOCKS_PER_SEC;
202 #endif
203 #endif
204  return out;
205 }
206 
207 } // namespace escript
208 
209 #endif // __ESCRIPT_ESYSMPI_H__
210 
#define MPI_COMM_NULL
Definition: EsysMPI.h:51
int MPI_Op
Definition: EsysMPI.h:46
int MPI_Request
Definition: EsysMPI.h:45
int MPI_Comm
Definition: EsysMPI.h:44
int MPI_Status
Definition: EsysMPI.h:47
Definition: EsysMPI.h:84
int mod_rank(int k) const
Definition: EsysMPI.h:99
void incCounter(int i=1)
increments the message tag counter by i
Definition: EsysMPI.h:138
bool isValid() const
returns true if this has a valid MPI communicator
Definition: EsysMPI.h:153
void setCounter(int value)
sets the message tag counter to value
Definition: EsysMPI.h:147
int msg_tag_counter
Definition: EsysMPI.h:169
bool ownscomm
Definition: EsysMPI.h:167
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition: EsysMPI.h:116
int counter() const
returns the current value of the message tag counter
Definition: EsysMPI.h:132
int size
Definition: EsysMPI.h:158
MPI_Comm comm
Definition: EsysMPI.h:160
int rank
Definition: EsysMPI.h:159
Definition: EsysMPI.h:174
NoCOMM_WORLD()
Definition: EsysMPI.cpp:224
~NoCOMM_WORLD()
Definition: EsysMPI.cpp:232
static bool active()
Definition: EsysMPI.cpp:237
#define ESCRIPT_DLL_API
Definition: escriptcore/src/system_dep.h:30
index_t dim_t
Definition: DataTypes.h:66
int index_t
type for array/matrix indices used both globally and on each rank
Definition: DataTypes.h:61
Definition: AbstractContinuousDomain.cpp:23
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition: EsysMPI.cpp:110
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition: EsysMPI.h:69
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:29
double gettime()
returns the current ticks for timing
Definition: EsysMPI.h:192
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: EsysMPI.cpp:164
boost::shared_ptr< JMPI_ > JMPI
Definition: EsysMPI.h:74
static dim_t N
Definition: SparseMatrix_saveHB.cpp:37