40 #ifndef Foam_UPstream_H 41 #define Foam_UPstream_H 57 namespace PstreamDetail {}
133 const label numProcs,
134 const label myProcID,
148 const label proci = 0
181 void reset(
const label
procID,
const label numProcs);
207 static bool haveThreads_;
219 static label intraHostComm_;
222 static label interHostComm_;
252 static label getAvailableCommIndex(
const label parentIndex);
255 static void allocateCommunicatorComponents
257 const label parentIndex,
263 static void freeCommunicatorComponents(
const label index);
267 static bool allocateHostCommunicatorPairs();
329 if (communicator >= 0)
worldComm = communicator;
387 const bool withComponents =
true 400 const bool withComponents =
true 408 const bool withComponents =
true 420 const label parentCommunicator =
worldComm 448 const label parentComm,
461 const label parentComm,
487 void reset(label
parent,
const labelRange& subRanks)
514 operator label() const
noexcept {
return comm_; }
524 static label
procNo(
const label comm,
const int baseProcID);
531 const label currentComm,
532 const int currentProcID
541 static bool init(
int& argc,
char**& argv,
const bool needsThread);
554 const label communicator,
555 UPstream::Request* req =
nullptr 570 const int fromProcNo,
793 return procIDs_[communicator].
size();
800 return myProcNo_[communicator];
806 return myProcNo_[communicator] ==
masterNo();
813 return myProcNo_[communicator] >= 0;
819 return myProcNo_[communicator] >
masterNo();
835 static label
parent(
const label communicator)
837 return parentComm_(communicator);
841 static List<int>&
procID(
const label communicator)
843 return procIDs_[communicator];
891 static const List<commsStruct>&
961 static void shutdown(
int errNo = 0);
967 static void exit(
int errNo = 1);
970 #undef Pstream_CommonRoutines 971 #define Pstream_CommonRoutines(Native) \ 975 static void allToAll \ 978 const UList<Native>& sendData, \ 980 UList<Native>& recvData, \ 981 const label communicator = worldComm \ 992 static void allToAllConsensus \ 995 const UList<Native>& sendData, \ 997 UList<Native>& recvData, \ 1000 const label communicator = worldComm \ 1008 static void allToAllConsensus \ 1011 const Map<Native>& sendData, \ 1013 Map<Native>& recvData, \ 1016 const label communicator = worldComm \ 1021 static Map<Native> allToAllConsensus \ 1024 const Map<Native>& sendData, \ 1027 const label communicator = worldComm \ 1030 Map<Native> recvData; \ 1031 allToAllConsensus(sendData, recvData, tag, communicator); \ 1038 #undef Pstream_CommonRoutines 1043 #undef Pstream_CommonRoutines 1044 #define Pstream_CommonRoutines(Native) \ 1047 static void mpiGather \ 1050 const Native* sendData, \ 1055 const label communicator = worldComm \ 1059 static void mpiScatter \ 1062 const Native* sendData, \ 1067 const label communicator = worldComm \ 1072 static void mpiAllGather \ 1078 const label communicator = worldComm \ 1082 static void gather \ 1084 const Native* sendData, \ 1087 const UList<int>& recvCounts, \ 1088 const UList<int>& recvOffsets, \ 1089 const label communicator = worldComm \ 1093 static void scatter \ 1095 const Native* sendData, \ 1096 const UList<int>& sendCounts, \ 1097 const UList<int>& sendOffsets, \ 1100 const label communicator = worldComm \ 1111 #undef Pstream_CommonRoutines 1121 const T& localValue,
1133 const T& localValue,
1159 const std::streamsize bufSize,
1203 return nProcs(communicator) - 1;
1255 return (value_ == rhs.value_);
1261 return (value_ != rhs.value_);
1291 return reinterpret_cast<const void*
>(value_);
1316 Ostream&
operator<<(Ostream&,
const UPstream::commsStruct&);
1323 UPstream::commsStruct&
1327 const UPstream::commsStruct&
static bool floatTransfer
Should compact transfer be used in which floats replace doubles reducing the bandwidth requirement at...
const labelList & below() const noexcept
The procIDs of the processors directly below.
static const word & myWorld()
My world.
commsStruct() noexcept
Default construct with above == -1.
void size(const label n)
Older name for setAddressableSize.
sendModes
Different MPI-send modes (ignored for commsTypes::blocking)
"blocking" : (MPI_Bsend, MPI_Recv)
static void removeRequests(const label pos, label len=-1)
Non-blocking comms: cancel/free outstanding requests (from position onwards) and remove from internal...
static const Enum< commsTypes > commsTypeNames
Enumerated names for the communication types.
static const labelList & worldIDs() noexcept
The indices into allWorlds for all processes.
~communicator()
Free allocated communicator.
label release() noexcept
Release ownership of the communicator, return old value.
label comm() const noexcept
The communicator label.
void reset() noexcept
Reset to default constructed value (MPI_REQUEST_NULL)
static int incrMsgType(int val=1) noexcept
Increment the message tag for standard messages.
static void clearHostComms()
Remove any existing intra and inter host communicators.
commsTypes
Communications types.
static void freeCommunicator(const label communicator, const bool withComponents=true)
Free a previously allocated communicator.
static label nRequests() noexcept
Number of outstanding requests (on the internal list of requests)
An interval of (signed) integers defined by a start and a size.
static rangeType allProcs(const label communicator=worldComm)
Range of process indices for all processes.
A range or interval of labels defined by a start and a size.
IntRange< int > rangeType
Int ranges are used for MPI ranks (processes)
static int maxCommsSize
Optional maximum message size (bytes)
static constexpr int firstSlave() noexcept
Process index of first sub-process.
static int nProcsSimpleSum
Number of processors to change from linear to tree communication.
void wait()
Same as calling UPstream::waitRequest()
communicator() noexcept
Default construct (a placeholder communicator)
static List< T > listGatherValues(const T &localValue, const label communicator=worldComm)
Gather individual values into list locations.
static bool initNull()
Special purpose initialisation function.
static bool & parRun() noexcept
Test if this a parallel run.
static void cancelRequest(const label i)
Non-blocking comms: cancel and free outstanding request. Corresponds to MPI_Cancel() + MPI_Request_fr...
static bool finishedRequest(const label i)
Non-blocking comms: has request i finished? Corresponds to MPI_Test()
static label procNo(const label comm, const int baseProcID)
Return processor number in communicator (given physical processor number) (= reverse of baseProcNo) ...
static List< T > allGatherValues(const T &localValue, const label communicator=worldComm)
Allgather individual values into list locations.
static void reduceOr(bool &value, const label communicator=worldComm)
Logical (or) reduction (MPI_AllReduce)
static const List< commsStruct > & whichCommunication(const label communicator=worldComm)
Communication schedule for linear/tree all-to-master (proc 0). Chooses based on the value of UPstream...
static int & msgType() noexcept
Message tag of standard messages.
static constexpr label commSelf() noexcept
Communicator within the current rank only.
static void printGraph(Ostream &os, const UList< UPstream::commsStruct > &comms, const label proci=0)
Print un-directed graph in graphviz dot format.
static void freeRequest(UPstream::Request &req)
Non-blocking comms: free outstanding request. Corresponds to MPI_Request_free()
static label commIntraHost()
Demand-driven: Intra-host communicator (respects any local worlds)
void reset()
Reset to default constructed state.
static int myProcNo(const label communicator=worldComm)
Rank of this process in the communicator (starting from masterNo()). Can be negative if the process i...
static label worldComm
Communicator for all ranks. May differ from commGlobal() if local worlds are in use.
static void freeRequests(UList< UPstream::Request > &requests)
Non-blocking comms: free outstanding requests. Corresponds to MPI_Request_free()
label above() const noexcept
The procID of the processor directly above.
bool good() const noexcept
True if not equal to MPI_REQUEST_NULL.
static void waitRequests()
Wait for all requests to finish.
static bool hasHostComms()
Test for presence of any intra or inter host communicators.
static int tuning_NBX_
Tuning parameters for non-blocking exchange (NBX)
static void shutdown(int errNo=0)
Shutdown (finalize) MPI as required.
UList< label > labelUList
A UList of labels.
ClassName("UPstream")
Declare name of the class and its debug switch.
static label allocateInterHostCommunicator(const label parentCommunicator=worldComm)
Allocate an inter-host communicator.
void reset()
Free allocated communicator.
Various functions to operate on Lists.
class FOAM_DEPRECATED_FOR(2017-05, "Foam::Enum") NamedEnum
T & operator[](const label i)
Return element of UList.
static bool finishedRequestPair(label &req0, label &req1)
Non-blocking comms: have both requests finished? Corresponds to pair of MPI_Test() ...
static int nProcsNonblockingExchange
Number of processors to change to nonBlocking consensual exchange (NBX). Ignored for zero or negative...
static label commWarn(const label communicator) noexcept
Alter communicator debugging setting. Warns for use of any communicator differing from specified...
dimensionedScalar pos(const dimensionedScalar &ds)
static label nProcs(const label communicator=worldComm)
Number of ranks in parallel run (for given communicator). It is 1 for serial run. ...
bool operator!=(const Request &rhs) const noexcept
Test for inequality.
static int nPollProcInterfaces
Number of polling cycles in processor updates.
static label parent(const label communicator)
The parent communicator.
static void exit(int errNo=1)
Shutdown (finalize) MPI as required and exit program with errNo.
const void * pointer() const noexcept
Return as pointer value.
static void reduceAnd(bool &value, const label communicator=worldComm)
Logical (and) reduction (MPI_AllReduce)
static bool is_rank(const label communicator=worldComm)
True if process corresponds to any rank (master or sub-rank) in the given communicator.
"scheduled" : (MPI_Send, MPI_Recv)
static const List< commsStruct > & treeCommunication(const label communicator=worldComm)
Communication schedule for tree all-to-master (proc 0)
static bool init(int &argc, char **&argv, const bool needsThread)
Initialisation function called from main.
An opaque wrapper for MPI_Request with a vendor-independent representation independent of any <mpi...
label nProcs() const
The number of processors addressed by the structure.
static constexpr int masterNo() noexcept
Relative rank for the master process - is always 0.
void free()
Same as calling UPstream::freeRequest()
static bool is_parallel(const label communicator=worldComm)
True if parallel algorithm or exchange is required.
static label allocateIntraHostCommunicator(const label parentCommunicator=worldComm)
Allocate an intra-host communicator.
static label warnComm
Debugging: warn for use of any communicator differing from warnComm.
Structure for communicating between processors.
const labelList & allBelow() const noexcept
The procIDs of all processors below (so not just directly below)
value_type value() const noexcept
Return raw value.
static label commInterHost()
Demand-driven: Inter-host communicator (respects any local worlds)
static bool waitAnyRequest(const label pos, label len=-1)
Wait until any request (from position onwards) has finished. Corresponds to MPI_Waitany() ...
commsTypes commsType() const noexcept
Get the communications type of the stream.
An Ostream is an abstract base class for all output systems (streams, files, token lists...
bool operator==(const commsStruct &) const
static void resetRequests(const label n)
Truncate outstanding requests to given length, which is expected to be in the range [0 to nRequests()...
UPstream(const commsTypes commsType) noexcept
Construct for given communication type.
static bool is_subrank(const label communicator=worldComm)
True if process corresponds to a sub-rank in the given communicator.
OBJstream os(runTime.globalPath()/outputName)
bool operator!=(const commsStruct &) const
static const List< commsStruct > & linearCommunication(const label communicator=worldComm)
Communication schedule for linear all-to-master (proc 0)
static void printCommTree(const label communicator)
Debugging: print the communication tree.
void T(FieldField< Field, Type > &f1, const FieldField< Field, Type > &f2)
static void waitRequestPair(label &req0, label &req1)
Non-blocking comms: wait for both requests to finish. Corresponds to pair of MPI_Wait() ...
static void addRequest(UPstream::Request &req)
Transfer the (wrapped) MPI request to the internal global list.
bool good() const noexcept
True if communicator is non-negative (ie, was allocated)
void cancel()
Same as calling UPstream::cancelRequest()
static label commWorld() noexcept
Communicator for all ranks (respecting any local worlds)
Wrapper class for allocating/freeing communicators. Always invokes allocateCommunicatorComponents() a...
const labelList & allNotBelow() const noexcept
The procIDs of all processors not below myProcNo. The inverse set of allBelow without myProcNo...
static void abort()
Call MPI_Abort with no other checks or cleanup.
Ostream & operator<<(Ostream &, const boundaryPatch &p)
Write boundaryPatch as dictionary entries (without surrounding braces)
List< word > wordList
List of word.
Request() noexcept
Default construct as MPI_REQUEST_NULL.
static commsTypes defaultCommsType
Default commsType.
static void cancelRequests(UList< UPstream::Request > &requests)
Non-blocking comms: cancel and free outstanding requests. Corresponds to MPI_Cancel() + MPI_Request_f...
static int lastSlave(const label communicator=worldComm)
Process index of last sub-process.
static List< int > & procID(const label communicator)
The list of ranks within a given communicator.
static label nComms() noexcept
Number of currently defined communicators.
std::intptr_t value_type
Storage for MPI_Request (as integer or pointer)
static bool broadcast(char *buf, const std::streamsize bufSize, const label communicator, const int rootProcNo=masterNo())
Broadcast buffer contents to all processes in given communicator. The sizes must match on all process...
static std::pair< int, int > probeMessage(const UPstream::commsTypes commsType, const int fromProcNo, const int tag=UPstream::msgType(), const label communicator=worldComm)
Probe for an incoming message.
const dimensionedScalar c
Speed of light in a vacuum.
static bool master(const label communicator=worldComm)
True if process corresponds to the master rank in the communicator.
"nonBlocking" : (MPI_Isend, MPI_Irecv)
void operator=(const communicator &)=delete
No copy assignment.
static bool haveThreads() noexcept
Have support for threads.
friend Ostream & operator<<(Ostream &, const commsStruct &)
static rangeType subProcs(const label communicator=worldComm)
Range of process indices for sub-processes.
List< label > labelList
A List of labels.
static const int mpiBufferSize
MPI buffer-size (bytes)
static const wordList & allWorlds() noexcept
All worlds.
bool finished()
Same as calling UPstream::finishedRequest()
static T listScatterValues(const UList< T > &allValues, const label communicator=worldComm)
Scatter individual values from list locations.
#define Pstream_CommonRoutines(Native)
static void waitRequest(const label i)
Wait until request i has finished. Corresponds to MPI_Wait()
Inter-processor communications stream.
static int baseProcNo(label comm, int procID)
Return physical processor number (i.e. processor number in worldComm) given communicator and processo...
static void addValidParOptions(HashTable< string > &validParOptions)
Add the valid option this type of communications library adds/requires on the command line...
static label allocateCommunicator(const label parent, const labelRange &subRanks, const bool withComponents=true)
Allocate new communicator with contiguous sub-ranks on the parent communicator.
static label myWorldID()
My worldID.
static bool waitSomeRequests(const label pos, label len=-1, DynamicList< int > *indices=nullptr)
Wait until some requests (from position onwards) have finished. Corresponds to MPI_Waitsome() ...
static void barrier(const label communicator, UPstream::Request *req=nullptr)
Impose a synchronisation barrier (optionally non-blocking)
static bool finishedRequests(const label pos, label len=-1)
Non-blocking comms: have all requests (from position onwards) finished? Corresponds to MPI_Testall() ...
static constexpr label commGlobal() noexcept
Communicator for all ranks, irrespective of any local worlds.