Implementation details for UPstream/Pstream/MPI etc. More...

Functions
template<class Type >
std::size_t	maxTransferCount (const std::size_t max_bytes=std::size_t(0)) noexcept
	Number of elements corresponding to max byte transfer. More...

std::size_t	maxTransferBytes (const int64_t max_bytes) noexcept
	Upper limit on number of transfer bytes. More...

template<class Type >
void	exchangeBuffers (const UList< std::pair< int, stdFoam::span< const Type >>> &sends, const UList< std::pair< int, stdFoam::span< Type >>> &recvs, const int tag, const label comm, const int64_t maxComms_bytes=UPstream::maxCommsSize)
	Exchange of contiguous data, with or without chunking. Setup sends and receives, each specified as [rank, span] tuple. More...

template<class Container , class Type >
void	exchangeContainer (const UList< Container > &sendBufs, UList< Container > &recvBufs, const int tag, const label comm, const bool wait, const int64_t maxComms_bytes=UPstream::maxCommsSize)
	Exchange contiguous data using point-to-point communication. Sends sendBufs, receives into recvBufs. More...

template<class Container , class Type >
void	exchangeContainer (const Map< Container > &sendBufs, Map< Container > &recvBufs, const int tag, const label comm, const bool wait, const int64_t maxComms_bytes=UPstream::maxCommsSize)
	Exchange contiguous data using point-to-point communication. Sends sendBufs, receives into recvBufs. More...

template<class Container , class Type >
void	exchangeConsensus (const UList< Container > &sendBufs, UList< Container > &recvBufs, labelUList &recvSizes, const int tag, const label comm)
	Exchange contiguous data using non-blocking consensus exchange (NBX) with optional tracking of the receive sizes. More...

template<class Container , class Type >
void	exchangeConsensus (const Map< Container > &sendBufs, Map< Container > &recvBufs, const int tag, const label comm)
	Exchange contiguous data using non-blocking consensus exchange (NBX) More...

template<class Type >
void	broadcast0 (Type *values, int count, MPI_Datatype datatype, const label comm)

template<class Type >
void	reduce0 (Type *values, int count, MPI_Datatype datatype, MPI_Op optype, const label comm)

template<class Type >
void	allReduce (Type values, int count, MPI_Datatype datatype, MPI_Op optype, const label comm, UPstream::Request req=nullptr, label *requestID=nullptr)

template<class Type >
void	allToAll (const UList< Type > &sendData, UList< Type > &recvData, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	allToAllv (const Type sendData, const UList< int > &sendCounts, const UList< int > &sendOffsets, Type recvData, const UList< int > &recvCounts, const UList< int > &recvOffsets, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	allToAllConsensus (const UList< Type > &sendData, UList< Type > &recvData, MPI_Datatype datatype, const int tag, const label comm)

template<class Type >
void	allToAllConsensus (const Map< Type > &sendData, Map< Type > &recvData, MPI_Datatype datatype, const int tag, const label comm)

template<class Type >
void	gather (const Type sendData, Type recvData, int count, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	scatter (const Type sendData, Type recvData, int count, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	gatherv (const Type sendData, int sendCount, Type recvData, const UList< int > &recvCounts, const UList< int > &recvOffsets, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	scatterv (const Type sendData, const UList< int > &sendCounts, const UList< int > &sendOffsets, Type recvData, int recvCount, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label requestID=nullptr)

template<class Type >
void	allGather (Type allData, int count, MPI_Datatype datatype, const label comm, UPstream::Request req=nullptr, label *requestID=nullptr)

Detailed Description

Implementation details for UPstream/Pstream/MPI etc.

Function Documentation

◆ maxTransferCount()

std::size_t Foam::PstreamDetail::maxTransferCount ( const std::size_t max_bytes = std::size_t(0) )

inlinenoexcept

Number of elements corresponding to max byte transfer.

Normal upper limit is INT_MAX since MPI sizes are limited to <int>.

Definition at line 55 of file PstreamExchange.C.

◆ maxTransferBytes()

std::size_t Foam::PstreamDetail::maxTransferBytes ( const int64_t max_bytes )

inlinenoexcept

Upper limit on number of transfer bytes.

Max bytes is normally INT_MAX since MPI sizes are limited to <int>. Negative values indicate a subtraction from INT_MAX.

Definition at line 78 of file PstreamExchange.C.

Referenced by exchangeBuffers(), and exchangeContainer().

Here is the caller graph for this function:

◆ exchangeBuffers()

void Foam::PstreamDetail::exchangeBuffers	(	const UList< std::pair< int, stdFoam::span< const Type >>> &	sends,
		const UList< std::pair< int, stdFoam::span< Type >>> &	recvs,
		const int	tag,
		const label	comm,
		const int64_t	maxComms_bytes = `UPstream::maxCommsSize`
	)

Exchange of contiguous data, with or without chunking. Setup sends and receives, each specified as [rank, span] tuple.

The serial list of tuples can be populated from other lists, from maps of data or subsets of lists/maps etc.

Any waiting for requests is done outside by the caller

Definition at line 100 of file PstreamExchange.C.

References Foam::abort(), stdFoam::end(), Foam::FatalError, FatalErrorInFunction, maxTransferBytes(), UPstream::myProcNo(), Foam::nl, UPstream::nonBlocking, UIPstream::read(), and UOPstream::write().

Here is the call graph for this function:

◆ exchangeContainer() [1/2]

void Foam::PstreamDetail::exchangeContainer	(	const UList< Container > &	sendBufs,
		UList< Container > &	recvBufs,
		const int	tag,
		const label	comm,
		const bool	wait,
		const int64_t	maxComms_bytes = `UPstream::maxCommsSize`
	)

Exchange contiguous data using point-to-point communication. Sends sendBufs, receives into recvBufs.

Data provided and received as container all of which have been properly sized before calling

No internal guards or resizing.

Parameters

wait	Wait for requests to complete

Definition at line 293 of file PstreamExchange.C.

References Foam::abort(), Foam::ensightOutput::debug, UList< T >::empty(), stdFoam::end(), Foam::FatalError, FatalErrorInFunction, forAll, maxTransferBytes(), UPstream::myProcNo(), Foam::nl, UPstream::nonBlocking, UPstream::nRequests(), Foam::Perr, UIPstream::read(), UPstream::waitRequests(), and UOPstream::write().

Here is the call graph for this function:

◆ exchangeContainer() [2/2]

void Foam::PstreamDetail::exchangeContainer	(	const Map< Container > &	sendBufs,
		Map< Container > &	recvBufs,
		const int	tag,
		const label	comm,
		const bool	wait,
		const int64_t	maxComms_bytes = `UPstream::maxCommsSize`
	)

Exchange contiguous data using point-to-point communication. Sends sendBufs, receives into recvBufs.

Data provided and received as container all of which have been properly sized before calling

No internal guards or resizing.

Parameters

wait	Wait for requests to complete

Definition at line 508 of file PstreamExchange.C.

References Foam::constant::physicoChemical::b, Foam::ensightOutput::debug, forAllConstIters(), forAllIters, UPstream::myProcNo(), Foam::nl, UPstream::nRequests(), Foam::Perr, HashTable< T, label, Hash< label > >::size(), Foam::sort(), and UPstream::waitRequests().

Here is the call graph for this function:

◆ exchangeConsensus() [1/2]

void Foam::PstreamDetail::exchangeConsensus	(	const UList< Container > &	sendBufs,
		UList< Container > &	recvBufs,
		labelUList &	recvSizes,
		const int	tag,
		const label	comm
	)

Exchange contiguous data using non-blocking consensus exchange (NBX) with optional tracking of the receive sizes.

No internal guards or resizing - data containers are all properly sized before calling.

Parameters

[in]	sendBufs	The send buffers list (size: numProc)
[out]	recvBufs	The recv buffers list (size: numProc)
[out]	recvSizes	The recv sizes (size: 0 or numProc). This parameter can be an empty list, in which case the receive sizes are not returned.
	tag	The message tag
	comm	The communicator

Definition at line 61 of file PstreamExchangeConsensus.C.

References Foam::abort(), UPstream::barrier(), Foam::BitOps::count(), Foam::FatalError, FatalErrorInFunction, UPstream::finishedRequest(), UPstream::finishedRequests(), UPstream::is_parallel(), UPstream::is_rank(), UPstream::myProcNo(), UPstream::nonBlocking, UPstream::nProcs(), UPstream::probeMessage(), UIPstream::read(), UPstream::scheduled, UList< T >::size(), UPstream::sync, UPstream::tuning_NBX_, and UOPstream::write().

Here is the call graph for this function:

◆ exchangeConsensus() [2/2]

void Foam::PstreamDetail::exchangeConsensus	(	const Map< Container > &	sendBufs,
		Map< Container > &	recvBufs,
		const int	tag,
		const label	comm
	)

Exchange contiguous data using non-blocking consensus exchange (NBX)

No internal guards - the sending Map corresponds to a segment of 0-numProc.

Parameters

[in]	sendBufs	The send buffers map (addr: 0-numProc)
[out]	recvBufs	The recv buffers map
	tag	The message tag
	comm	The communicator

Definition at line 247 of file PstreamExchangeConsensus.C.

References UPstream::barrier(), Foam::BitOps::count(), HashTable< T, label, Hash< label > >::find(), UPstream::finishedRequest(), UPstream::finishedRequests(), forAllConstIters(), forAllIters, UPstream::is_parallel(), UPstream::is_rank(), UPstream::myProcNo(), UPstream::nonBlocking, UPstream::nProcs(), UPstream::probeMessage(), UIPstream::read(), UPstream::scheduled, HashTable< T, label, Hash< label > >::size(), UPstream::sync, UPstream::tuning_NBX_, and UOPstream::write().

Here is the call graph for this function:

◆ broadcast0()

void Foam::PstreamDetail::broadcast0	(	Type *	values,
		int	count,
		MPI_Datatype	datatype,
		const label	comm
	)

◆ reduce0()

void Foam::PstreamDetail::reduce0	(	Type *	values,
		int	count,
		MPI_Datatype	datatype,
		MPI_Op	optype,
		const label	comm
	)

◆ allReduce()

void Foam::PstreamDetail::allReduce	(	Type *	values,
		int	count,
		MPI_Datatype	datatype,
		MPI_Op	optype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

◆ allToAll()

void Foam::PstreamDetail::allToAll	(	const UList< Type > &	sendData,
		UList< Type > &	recvData,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

Referenced by Pstream::exchangeSizes().

Here is the caller graph for this function:

◆ allToAllv()

void Foam::PstreamDetail::allToAllv	(	const Type *	sendData,
		const UList< int > &	sendCounts,
		const UList< int > &	sendOffsets,
		Type *	recvData,
		const UList< int > &	recvCounts,
		const UList< int > &	recvOffsets,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

◆ allToAllConsensus() [1/2]

void Foam::PstreamDetail::allToAllConsensus	(	const UList< Type > &	sendData,
		UList< Type > &	recvData,
		MPI_Datatype	datatype,
		const int	tag,
		const label	comm
	)

Referenced by Pstream::exchangeSizes().

Here is the caller graph for this function:

◆ allToAllConsensus() [2/2]

void Foam::PstreamDetail::allToAllConsensus	(	const Map< Type > &	sendData,
		Map< Type > &	recvData,
		MPI_Datatype	datatype,
		const int	tag,
		const label	comm
	)

◆ gather()

void Foam::PstreamDetail::gather	(	const Type *	sendData,
		Type *	recvData,
		int	count,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

Referenced by globalIndex::gather(), globalIndex::gatherInplace(), globalIndex::gatherInplaceOp(), globalIndex::gatherOp(), globalIndex::mpiGather(), GAMGAgglomeration::procAgglomerateLduAddressing(), GAMGAgglomeration::procAgglomerateRestrictAddressing(), GAMGAgglomeration::restrictField(), and decomposedBlockData::writeObject().

Here is the caller graph for this function:

◆ scatter()

void Foam::PstreamDetail::scatter	(	const Type *	sendData,
		Type *	recvData,
		int	count,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

Referenced by GAMGAgglomeration::procAgglomerateLduAddressing(), GAMGAgglomeration::prolongField(), and globalIndex::scatter().

Here is the caller graph for this function:

◆ gatherv()

void Foam::PstreamDetail::gatherv	(	const Type *	sendData,
		int	sendCount,
		Type *	recvData,
		const UList< int > &	recvCounts,
		const UList< int > &	recvOffsets,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

◆ scatterv()

void Foam::PstreamDetail::scatterv	(	const Type *	sendData,
		const UList< int > &	sendCounts,
		const UList< int > &	sendOffsets,
		Type *	recvData,
		int	recvCount,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

◆ allGather()

void Foam::PstreamDetail::allGather	(	Type *	allData,
		int	count,
		MPI_Datatype	datatype,
		const label	comm,
		UPstream::Request *	req = `nullptr`,
		label *	requestID = `nullptr`
	)

Functions

Detailed Description

Function Documentation

◆ maxTransferCount()

◆ maxTransferBytes()

◆ exchangeBuffers()

◆ exchangeContainer() [1/2]

◆ exchangeContainer() [2/2]

◆ exchangeConsensus() [1/2]

◆ exchangeConsensus() [2/2]

◆ broadcast0()

◆ reduce0()

◆ allReduce()

◆ allToAll()

◆ allToAllv()

◆ allToAllConsensus() [1/2]

◆ allToAllConsensus() [2/2]

◆ gather()

◆ scatter()

◆ gatherv()

◆ scatterv()

◆ allGather()