PstreamGlobals.H

Go to the documentation of this file.

/*---------------------------------------------------------------------------*\
  =========                 |
  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   \\    /   O peration     |
    \\  /    A nd           | www.openfoam.com
     \\/     M anipulation  |
-------------------------------------------------------------------------------
    Copyright (C) 2013-2015 OpenFOAM Foundation
    Copyright (C) 2022-2025 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
    This file is part of OpenFOAM.

    OpenFOAM is free software: you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    for more details.

    You should have received a copy of the GNU General Public License
    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.

Namespace
    Foam::PstreamGlobals

Description
    Global functions and variables for working with parallel streams,
    but principally for MPI.

SourceFiles
    PstreamGlobals.C

\*---------------------------------------------------------------------------*/

#ifndef Foam_PstreamGlobals_H
#define Foam_PstreamGlobals_H

#include "DynamicList.H"
#include "FixedList.H"
#include "UPstream.H"  // For UPstream::Request
#include "openfoam_mpi.H"

// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

namespace Foam
{
namespace PstreamGlobals
{

// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //

// Track if MPI_Comm_free is needed for communicator index in MPICommunicators_
extern DynamicList<bool> pendingMPIFree_;

// Current communicators, which may be allocated or predefined
// (eg, MPI_COMM_SELF, MPI_COMM_WORLD)
extern DynamicList<MPI_Comm> MPICommunicators_;

//- Outstanding non-blocking operations.
extern DynamicList<MPI_Request> outstandingRequests_;

// The fundamental count for each UPstream::dataTypes entry
typedef Foam::FixedList<int, 17> DataTypeCountLookupTable;

//- Fundamental count for each valid UPstream::dataTypes entry
//- Indexed by UPstream::dataTypes enum
extern DataTypeCountLookupTable dataTypesCount_;

// For UPstream::dataTypes lookup, includes space for last 'invalid' entry
typedef Foam::FixedList<MPI_Datatype, 17> DataTypeLookupTable;

//- MPI data types corresponding to fundamental and OpenFOAM types.
//- Indexed by UPstream::dataTypes enum
extern DataTypeLookupTable MPIdataTypes_;

// For UPstream::opCodes lookup, includes space for last 'invalid' entry
typedef Foam::FixedList<MPI_Op, 13> OpCodesLookupTable;

//- MPI operation types, indexed by UPstream::opCodes enum
extern OpCodesLookupTable MPIopCodes_;



// * * * * * * * * * * * * * * * Communicators * * * * * * * * * * * * * * * //

//- Initialize bookkeeping for MPI communicator index
void initCommunicator(const label index);

//- Fatal if communicator is outside the allocated range
inline void checkCommunicator(int comm, int rank)
{
    if (FOAM_UNLIKELY(comm < 0 || comm >= MPICommunicators_.size()))
    {
        FatalErrorInFunction
            << "rank:" << rank << " : illegal communicator "
            << comm << nl
            << "Communicator should be within range [0,"
            << MPICommunicators_.size()
            << ')' << Foam::abort(FatalError);
    }
}

//- True if warn communicator is active and not equal to given communicator
inline bool warnCommunicator(int comm) noexcept
{
    return (UPstream::warnComm >= 0 && comm != UPstream::warnComm);
}


// * * * * * * * * * * * * * * * * Data Types  * * * * * * * * * * * * * * * //

//- Create mapping into MPIdataTypes_ and define user data types
void initDataTypes();

//- Free any user data types
void deinitDataTypes();

//- Debugging only: check if data type mappings are non-null
bool checkDataTypes();

//- Debugging only: print data type names (all or just user-defined)
void printDataTypes(bool all = false);

//- Lookup of dataTypes enumeration as an MPI_Datatype
inline MPI_Datatype getDataType(UPstream::dataTypes id)
{
    return MPIdataTypes_[static_cast<int>(id)];
}

//- Fatal if data type is not valid
inline void checkDataType(UPstream::dataTypes id)
{
    if (id == UPstream::dataTypes::invalid)
    {
        FatalErrorInFunction
            << "Invalid data type"
            << Foam::abort(FatalError);
    }
}

//- Return MPI internal name for specified MPI_Datatype
std::string dataType_name(MPI_Datatype datatype);

//- Return MPI internal name for dataTypes enumeration
inline std::string dataType_name(UPstream::dataTypes id)
{
    return dataType_name(MPIdataTypes_[static_cast<int>(id)]);
}


// * * * * * * * * * * * * * * * * Op Codes  * * * * * * * * * * * * * * * * //

//- Create mapping into MPIopCodes_
void initOpCodes();

//- Free any user-defined op codes
void deinitOpCodes();

//- Debugging only: check if op code mappings are non-null
bool checkOpCodes();

//- Lookup of opCodes enumeration as an MPI_Op
inline MPI_Op getOpCode(UPstream::opCodes id)
{
    return MPIopCodes_[static_cast<int>(id)];
}

//- Fatal if opcode is not valid
inline void checkOpCode(UPstream::opCodes id)
{
    if (id == UPstream::opCodes::invalid)
    {
        FatalErrorInFunction
            << "Invalid operation code"
            << Foam::abort(FatalError);
    }
}


// * * * * * * * * * * * * * * * * Requests  * * * * * * * * * * * * * * * * //

//- Reset UPstream::Request to MPI_REQUEST_NULL
//  Does not affect the stack of outstanding requests
inline void reset_request(UPstream::Request* req) noexcept
{
    if (req) *req = UPstream::Request(MPI_REQUEST_NULL);
}


//- Transcribe MPI_Request to UPstream::Request
//- (does not affect the stack of outstanding requests)
//- or else push onto list of outstanding requests
inline void push_request
(
    MPI_Request request,
    UPstream::Request* req = nullptr
)
{
    if (req)
    {
        // Transcribe as UPstream::Request
        *req = UPstream::Request(request);
    }
    else if (MPI_REQUEST_NULL != request)
    {
        // Push onto list of requests
        PstreamGlobals::outstandingRequests_.push_back(request);
    }
}


// * * * * * * * * * * * * * * Convenience Methods * * * * * * * * * * * * * //

//- Broadcast a single int64 value.
//
//  Ensures consistent data types. Used within the following:
//  - UIPBstream::bufferIPCrecv()
//  - UOPBstream::bufferIPCsend()
//  - UOPBstream::send(Foam::zero, ...)

inline bool broadcast_int64(int64_t& value, int comm)
{
    return
    (
        MPI_SUCCESS
     == MPI_Bcast(&value, 1, MPI_INT64_T, 0, MPICommunicators_[comm])
    );
}


// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

} // End namespace PstreamGlobals
} // End namespace Foam

// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

#endif

// ************************************************************************* //