53 { commsTypes::blocking,
"blocking" },
54 { commsTypes::scheduled,
"scheduled" },
55 { commsTypes::nonBlocking,
"nonBlocking" },
110 hostIDs[proci] = uniqDigests.
find(dig);
112 if (hostIDs[proci] < 0)
115 hostIDs[proci] = -(uniqDigests.
size() + 1);
123 hostIDs.data_bytes(),
124 hostIDs.size_bytes(),
137 void Foam::UPstream::setParRun(
const label nProcs,
const bool haveThreads)
143 labelRange singleProc(1);
164 <<
"problem : comm:" << comm
175 <<
"problem : comm:" << comm
198 <<
"problem : comm:" << comm
211 <<
"problem : comm:" << comm
222 Pout<<
"UPstream::setParRun :" 230 Foam::label Foam::UPstream::getAvailableCommIndex(
const label parentIndex)
233 if (!freeComms_.empty())
236 index = freeComms_.back();
237 freeComms_.pop_back();
240 myProcNo_[index] = -1;
241 parentComm_[index] = parentIndex;
243 procIDs_[index].clear();
244 linearCommunication_[index].clear();
245 treeCommunication_[index].clear();
250 index = parentComm_.size();
252 myProcNo_.push_back(-1);
253 parentComm_.push_back(parentIndex);
255 procIDs_.emplace_back();
256 linearCommunication_.emplace_back();
257 treeCommunication_.emplace_back();
266 const label parentIndex,
267 const labelRange& subRanks,
268 const bool withComponents
271 const label index = getAvailableCommIndex(parentIndex);
275 Pout<<
"Allocating communicator " << index <<
nl 276 <<
" parent : " << parentIndex <<
nl 277 <<
" procs : " << subRanks <<
nl 288 auto& procIds = procIDs_[index];
289 procIds.resize_nocopy(subRanks.size());
291 label numSubRanks = 0;
292 for (
const label subRanki : subRanks)
294 procIds[numSubRanks] = subRanki;
299 linearCommunication_[index].clear();
300 treeCommunication_[index].clear();
302 if (withComponents && parRun())
304 allocateCommunicatorComponents(parentIndex, index);
327 const label parentIndex,
329 const bool withComponents
332 const label index = getAvailableCommIndex(parentIndex);
336 Pout<<
"Allocating communicator " << index <<
nl 337 <<
" parent : " << parentIndex <<
nl 350 auto& procIds = procIDs_[index];
351 procIds.resize_nocopy(subRanks.size());
353 label numSubRanks = 0;
354 bool monotonicOrder =
true;
355 for (
const label subRanki : subRanks)
361 if (monotonicOrder && numSubRanks)
363 monotonicOrder = (procIds[numSubRanks-1] < subRanki);
366 procIds[numSubRanks] = subRanki;
372 auto last = procIds.begin() + numSubRanks;
374 last = std::unique(procIds.begin(), last);
375 numSubRanks = label(last - procIds.begin());
378 procIds.resize(numSubRanks);
381 linearCommunication_[index].clear();
382 treeCommunication_[index].clear();
384 if (withComponents && parRun())
386 allocateCommunicatorComponents(parentIndex, index);
409 const label parentCommunicator
414 DynamicList<label> subRanks(hostIDs.size());
420 if (hostIDs[proci] < 0)
422 subRanks.push_back(proci);
426 return allocateCommunicator(parentCommunicator, subRanks);
432 const label parentCommunicator
437 DynamicList<label> subRanks(hostIDs.size());
441 if (myHostId < 0) myHostId = -(myHostId + 1);
445 int id = hostIDs[proci];
446 if (
id < 0)
id = -(
id + 1);
450 subRanks.push_back(proci);
454 return allocateCommunicator(parentCommunicator, subRanks);
458 bool Foam::UPstream::allocateHostCommunicatorPairs()
461 const label parentCommunicator = worldComm;
469 if (interHostComm_ >= 0 || intraHostComm_ >= 0)
473 <<
"Host communicator(s) already created!" <<
endl 478 interHostComm_ = getAvailableCommIndex(parentCommunicator);
479 intraHostComm_ = getAvailableCommIndex(parentCommunicator);
482 if (intraHostComm_ < interHostComm_)
484 std::swap(intraHostComm_, interHostComm_);
493 Pout<<
"Allocating host communicators " 494 << interHostComm_ <<
", " << intraHostComm_ <<
nl 495 <<
" parent : " << parentCommunicator <<
nl 501 DynamicList<int> subRanks(hostIDs.size());
509 if (hostIDs[proci] < 0)
511 subRanks.push_back(proci);
514 hostIDs[proci] = -(hostIDs[proci] + 1);
518 const label index = interHostComm_;
521 procIDs_[index] = subRanks;
526 allocateCommunicatorComponents(parentCommunicator, index);
530 linearCommunication_[index].clear();
531 treeCommunication_[index].clear();
537 if (myHostId < 0) myHostId = -(myHostId + 1);
542 int id = hostIDs[proci];
543 if (
id < 0)
id = -(
id + 1);
547 subRanks.push_back(proci);
551 const label index = intraHostComm_;
554 procIDs_[index] = subRanks;
559 allocateCommunicatorComponents(parentCommunicator, index);
563 linearCommunication_[index].clear();
564 treeCommunication_[index].clear();
573 const label communicator,
574 const bool withComponents
578 if (communicator < 0)
584 if (interHostComm_ == communicator) interHostComm_ = -1;
585 if (intraHostComm_ == communicator) intraHostComm_ = -1;
589 Pout<<
"Communicators : Freeing communicator " << communicator
590 <<
" parent: " << parentComm_[communicator]
591 <<
" myProcNo: " << myProcNo_[communicator]
595 if (withComponents && parRun())
597 freeCommunicatorComponents(communicator);
600 myProcNo_[communicator] = -1;
601 parentComm_[communicator] = -1;
603 linearCommunication_[communicator].clear();
616 procID = parentRanks[procID];
629 int procID = baseProcID;
636 return parentRanks.find(procID);
643 const label currentComm,
644 const int currentProcID
655 if (linearCommunication_[communicator].empty())
657 linearCommunication_[communicator] =
661 return linearCommunication_[communicator];
668 if (treeCommunication_[communicator].empty())
670 treeCommunication_[communicator] =
674 return treeCommunication_[communicator];
684 commsStruct::printGraph(
Info(), comms);
695 if (intraHostComm_ < 0)
697 allocateHostCommunicatorPairs();
699 return intraHostComm_;
709 if (interHostComm_ < 0)
711 allocateHostCommunicatorPairs();
713 return interHostComm_;
719 return (intraHostComm_ >= 0 || interHostComm_ >= 0);
726 freeCommunicator(intraHostComm_,
true);
727 freeCommunicator(interHostComm_,
true);
733 bool Foam::UPstream::parRun_(
false);
735 bool Foam::UPstream::haveThreads_(
false);
737 int Foam::UPstream::msgType_(1);
751 Foam::UPstream::linearCommunication_(16);
754 Foam::UPstream::treeCommunication_(16);
757 Foam::label Foam::UPstream::intraHostComm_(-1);
758 Foam::label Foam::UPstream::interHostComm_(-1);
829 "nPollProcInterfaces",
849 class addcommsTypeToOpt
855 addcommsTypeToOpt(
const char*
name)
860 virtual ~addcommsTypeToOpt() =
default;
874 addcommsTypeToOpt addcommsTypeToOpt_(
"commsType");
registerOptSwitch("fa:geometryOrder", int, faMesh::geometryOrder_)
static bool floatTransfer
Should compact transfer be used in which floats replace doubles reducing the bandwidth requirement at...
Abstract base class for registered object with I/O. Used in debug symbol registration.
prefixOSstream Perr
OSstream wrapped stderr (std::cerr) with parallel prefix.
EnumType read(Istream &is) const
Read a word from Istream and return the corresponding enumeration.
void size(const label n)
Older name for setAddressableSize.
static const Enum< commsTypes > commsTypeNames
Enumerated names for the communication types.
errorManipArg< error, int > exit(error &err, const int errNo=1)
static void clearHostComms()
Remove any existing intra and inter host communicators.
static void writeData(Ostream &os, const Type &val)
void resize(const label len)
Adjust allocated size of list.
commsTypes
Communications types.
error FatalError
Error stream (stdout output on all processes), with additional 'FOAM FATAL ERROR' header text and sta...
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
static void freeCommunicator(const label communicator, const bool withComponents=true)
Free a previously allocated communicator.
static List< int > getHostGroupIds(const label parentCommunicator)
A range or interval of labels defined by a start and a size.
An Istream is an abstract base class for all input systems (streams, files, token lists etc)...
constexpr char nl
The newline '\n' character (0x0a)
static int maxCommsSize
Optional maximum message size (bytes)
static int nProcsSimpleSum
Number of processors to change from linear to tree communication.
Ostream & endl(Ostream &os)
Add newline and flush stream.
static label procNo(const label comm, const int baseProcID)
Return processor number in communicator (given physical processor number) (= reverse of baseProcNo) ...
static const List< commsStruct > & whichCommunication(const label communicator=worldComm)
Communication schedule for linear/tree all-to-master (proc 0). Chooses based on the value of UPstream...
static constexpr label commSelf() noexcept
Communicator within the current rank only.
static label commIntraHost()
Demand-driven: Intra-host communicator (respects any local worlds)
static int myProcNo(const label communicator=worldComm)
Rank of this process in the communicator (starting from masterNo()). Can be negative if the process i...
static label worldComm
Communicator for all ranks. May differ from commGlobal() if local worlds are in use.
char * data_bytes() noexcept
Return pointer to the underlying array serving as data storage,.
const char * cdata_bytes() const noexcept
Raw digest char data (20 bytes) - const access. For consistency with other objects, these are not unsigned.
static bool hasHostComms()
Test for presence of any intra or inter host communicators.
static int tuning_NBX_
Tuning parameters for non-blocking exchange (NBX)
UList< label > labelUList
A UList of labels.
static label allocateInterHostCommunicator(const label parentCommunicator=worldComm)
Allocate an inter-host communicator.
#define forAll(list, i)
Loop across all elements in list.
Useful combination of include files which define Sin, Sout and Serr and the use of IO streams general...
static int nProcsNonblockingExchange
Number of processors to change to nonBlocking consensual exchange (NBX). Ignored for zero or negative...
Functions used by OpenFOAM that are specific to POSIX compliant operating systems and need to be repl...
static label nProcs(const label communicator=worldComm)
Number of ranks in parallel run (for given communicator). It is 1 for serial run. ...
static int nPollProcInterfaces
Number of polling cycles in processor updates.
static label parent(const label communicator)
The parent communicator.
word name(const expressions::valueTypeCode typeCode)
A word representation of a valueTypeCode. Empty for expressions::valueTypeCode::INVALID.
A 1D vector of objects of type <T> that resizes itself as necessary to accept the new objects...
static const List< commsStruct > & treeCommunication(const label communicator=worldComm)
Communication schedule for tree all-to-master (proc 0)
void sort(UList< T > &list)
Sort the list.
static constexpr int masterNo() noexcept
Relative rank for the master process - is always 0.
const string & prefix() const noexcept
Return the stream prefix.
static label allocateIntraHostCommunicator(const label parentCommunicator=worldComm)
Allocate an intra-host communicator.
static void mpiGather(const char *sendData, char *recvData, int count, const label communicator=worldComm)
Receive identically-sized char data from all ranks.
static label warnComm
Debugging: warn for use of any communicator differing from warnComm.
string hostName()
Return the system's host name, as per hostname(1)
label find(const T &val) const
Find index of the first occurrence of the value.
static label commInterHost()
Demand-driven: Inter-host communicator (respects any local worlds)
An Ostream is an abstract base class for all output systems (streams, files, token lists...
int optimisationSwitch(const char *name, const int deflt=0)
Lookup optimisation switch or add default value.
int debug
Static debugging option.
OBJstream os(runTime.globalPath()/outputName)
static const List< commsStruct > & linearCommunication(const label communicator=worldComm)
Communication schedule for linear all-to-master (proc 0)
defineTypeNameAndDebug(combustionModel, 0)
static void printCommTree(const label communicator)
Debugging: print the communication tree.
void push_back(const T &val)
Copy append an element to the end of this list.
Wrapper class for allocating/freeing communicators. Always invokes allocateCommunicatorComponents() a...
const Foam::label nPredefinedComm
static commsTypes defaultCommsType
Default commsType.
Enum is a wrapper around a list of names/values that represent particular enumeration (or int) values...
static List< int > & procID(const label communicator)
The list of ranks within a given communicator.
static label nComms() noexcept
Number of currently defined communicators.
static bool broadcast(char *buf, const std::streamsize bufSize, const label communicator, const int rootProcNo=masterNo())
Broadcast buffer contents to all processes in given communicator. The sizes must match on all process...
static bool master(const label communicator=worldComm)
True if process corresponds to the master rank in the communicator.
static constexpr unsigned max_size() noexcept
The dimensioned size of the digest is always 20 bytes.
messageStream Info
Information stream (stdout output on master, null elsewhere)
dictionary & optimisationSwitches()
The OptimisationSwitches sub-dictionary in the central controlDict(s).
Functions to compute SHA1 message digest according to the NIST specification FIPS-180-1.
static bool haveThreads() noexcept
Have support for threads.
static const int mpiBufferSize
MPI buffer-size (bytes)
void addOptimisationObject(const char *name, simpleRegIOobject *obj)
Register optimisation switch read/write object.
prefixOSstream Pout
OSstream wrapped stdout (std::cout) with parallel prefix.
static int baseProcNo(label comm, int procID)
Return physical processor number (i.e. processor number in worldComm) given communicator and processo...
static label allocateCommunicator(const label parent, const labelRange &subRanks, const bool withComponents=true)
Allocate new communicator with contiguous sub-ranks on the parent communicator.
A class representing the concept of 1 (one) that can be used to avoid manipulating objects known to b...
FlatOutput::OutputAdaptor< Container, Delimiters > flatOutput(const Container &obj, Delimiters delim)
Global flatOutput() function with specified output delimiters.
static constexpr label commGlobal() noexcept
Communicator for all ranks, irrespective of any local worlds.