MpiContainer.h

Go to the documentation of this file.

//Copyright (c) 2013-2020, The MercuryDPM Developers Team. All rights reserved.
//For the list of developers, see <http://www.MercuryDPM.org/Team>.
//
//Redistribution and use in source and binary forms, with or without
//modification, are permitted provided that the following conditions are met:
//  * Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//  * Neither the name MercuryDPM nor the
//    names of its contributors may be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
//THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
//ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
//WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
//DISCLAIMED. IN NO EVENT SHALL THE MERCURYDPM DEVELOPERS TEAM BE LIABLE FOR ANY
//DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
//(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
//LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
//ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
//(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
//SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef MPICONTAINER_H_
#define MPICONTAINER_H_

#include <cstddef>
#include <functional>
#include <vector>
#include "Math/Vector.h"

#ifdef MERCURY_USE_MPI
#include <mpi.h>
#endif

#ifdef MERCURY_FORCE_ASSERTS
#define MERCURY_ASSERTS true
#else
#ifdef MERCURY_NO_ASSERTS
#define MERCURY_ASSERTS false
#else
#ifdef NDEBUG
#define MERCURY_ASSERTS false
#else
#define MERCURY_ASSERTS true
#endif
#endif
#endif

class SpeciesHandler;

enum MercuryMPIType
{
    PARTICLE = 0, POSITION = 1, VELOCITY = 2, FORCE = 3, INTERACTION = 4, SUPERQUADRIC = 5
};

enum MercuryMPITag
{
    PARTICLE_COUNT = 0,
    PARTICLE_DATA = 1,
    POSITION_DATA = 2,
    PERIODIC_POSITION_DATA = 3,
    VELOCITY_DATA = 4,
    INTERACTION_COUNT = 5,
    INTERACTION_DATA = 6,
    PERIODIC_COMPLEXITY = 7,
    PARTICLE_INDEX = 8,
    SUPERQUADRIC_DATA = 9
};

namespace Detail
{
#ifdef MERCURY_USE_MPI
//convert integral data to the corresponding MPI type
template<typename T>
typename std::enable_if<std::is_integral<T>::value, MPI_Datatype>::type
toMPIType(T t)
{
    MPI_Datatype type;
    MPI_Type_match_size(MPI_TYPECLASS_INTEGER, sizeof(T), &type);
    return type;
}

//convert floating point data to the corresponding MPI type
template<typename T>
typename std::enable_if<std::is_floating_point<T>::value, MPI_Datatype>::type
toMPIType(T t)
{
    MPI_Datatype type;
    MPI_Type_match_size(MPI_TYPECLASS_REAL,sizeof(T),&type);
    return type;
}
#endif
} /*detail*/

void initialiseMPI();

class MPIContainer final
{
public:
    static MPIContainer& Instance()
    {
        static MPIContainer theInstance;
        return theInstance;
    }
    
    void initialiseMercuryMPITypes(const SpeciesHandler& speciesHandler);
    
    void sync()
    {
#ifdef MERCURY_USE_MPI
        MPI_Waitall(pending_.size(),pending_.data(),MPI_STATUSES_IGNORE);
        pending_.clear();
        MPI_Barrier(communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    send(T& t, int to, int tag)
    {
#if MERCURY_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Isend(&t, 1, Detail::toMPIType(t), to, tag, communicator_, &request);
        pending_.push_back(request);

#endif
    }

    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    send(T* t, int count, int to, int tag)
    {
#if MERCURY_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Isend(t, count, Detail::toMPIType(*t), to, tag, communicator_, &request);
        pending_.push_back(request);

#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    receive(T& t, int from, int tag)
    {
#if MERCURY_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data from self!");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Irecv(&t, 1, Detail::toMPIType(t), from, tag, communicator_, &request);
        pending_.push_back(request);

#endif
    }

    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    receive(T* t, int count, int from, int tag)
    {
#if MERCURY_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data fromself!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Irecv(&t, count, Detail::toMPIType(*t), from, tag, communicator_, &request);
        pending_.push_back(request);

#endif
    }
    
    
    template<typename T>
    void send(T* t, MercuryMPIType type, int count, int to, int tag)
    {
        //std::cout << "[Process: " << processorID_ << "]: QUEUING SEND REQUEST with tag: " << tag << std::endl;
#if MERCURY_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Isend(t, count, dataTypes_[type], to, tag, communicator_, &request);
        pending_.push_back(request);


#endif
    }
    
    template<typename T>
    void receive(T* t, MercuryMPIType type, int count, int from, int tag)
    {
        //std::cout << "[Process: " << processorID_ << "]: QUEUING RECEIVE REQUEST with tag: " << tag << std::endl;
#if MERCURY_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Request request;
        MPI_Irecv(t, count, dataTypes_[type], from, tag, communicator_, &request);
        pending_.push_back(request);

#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    directSend(T& t, int count, int to, int tag)
    {
#if MERCURY_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Ssend(&t, count, Detail::toMPIType(t), to, tag, communicator_);
#endif
    }
    
    
    template<typename T>
    void directSend(T* t, MercuryMPIType type, int count, int to, int tag)
    {
#if MERCURY_ASSERTS
        if (to == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Sending data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Sending zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Ssend(t,count,dataTypes_[type], to, tag, communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    directReceive(T& t, int count, int from, int tag)
    {
#if MERCURY_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data from self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Recv(&t, count, Detail::toMPIType(t), from, tag,communicator_, MPI_STATUS_IGNORE);
#endif
    }
    
    template<typename T>
    void directReceive(T* t, MercuryMPIType type, int count, int from, int tag)
    {
#if MERCURY_ASSERTS
        if (from == processorID_)
        {
            logger(FATAL, "[MPI FATAL]: Receiving data to self!");
        }
        
        if (count == 0)
        {
            logger(WARN, "[MPI ERROR]: Receiving zero data");
        }
#endif
#ifdef MERCURY_USE_MPI
        MPI_Recv(t, count, dataTypes_[type], from, tag, communicator_, MPI_STATUS_IGNORE);
#endif
    }
    
    template<typename T>
    void gather(T& send_t, T* receive_t)
    {
#ifdef MERCURY_USE_MPI
        MPI_Gather(&send_t, 1, Detail::toMPIType(send_t), receive_t, 1, Detail::toMPIType(send_t), 0, communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    broadcast(T& t, int fromProcessor = 0)
    {
#ifdef MERCURY_USE_MPI
        MPI_Bcast(&t,1,Detail::toMPIType(t),fromProcessor,communicator_);
#endif
    }
    
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    broadcast(T* t, int size, int fromProcessor)
    {
#ifdef MERCURY_USE_MPI
        MPI_Bcast((void *)t,size,Detail::toMPIType(t[0]),fromProcessor,communicator_);

#endif
    }
    
    template<typename T>
    void broadcast(T* t, MercuryMPIType type, int fromProcessor = 0)
    {
#ifdef MERCURY_USE_MPI
        MPI_Bcast((void *)t,1,dataTypes_[type],fromProcessor,communicator_);
#endif
    }
    
#ifdef MERCURY_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    reduce(T& t, MPI_Op operation, int id = 0)
    {

        if(id == getProcessorID())
        {
            MPI_Reduce(MPI_IN_PLACE, &t, 1, Detail::toMPIType(t), operation, id, communicator_);
        }
        else
        {
            MPI_Reduce(&t, nullptr, 1, Detail::toMPIType(t), operation, id, communicator_);
        }
    }
#endif
    
#ifdef MERCURY_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    allReduce(T& send_t, T& receive_t, MPI_Op operation)
    {
        MPI_Allreduce(&send_t, &receive_t, 1, Detail::toMPIType(send_t), operation,communicator_);  
    }
#endif
    
#ifdef MERCURY_USE_MPI
    template<typename T>
    typename std::enable_if<std::is_scalar<T>::value, void>::type
    allGather(T& send_t, int send_count, std::vector<T>& receive_t, int receive_count)
    {
        MPI_Allgather(&send_t, send_count, Detail::toMPIType(send_t),
                                receive_t.data(), receive_count, Detail::toMPIType(receive_t[0]),communicator_);  
    }
#endif
    
    std::size_t getProcessorID();
    
    std::size_t getNumberOfProcessors() const;
    
#ifdef MERCURY_USE_MPI
    MPI_Comm& getComm();
#endif
    
    template<typename T>
    void createMercuryMPIType(T t, MercuryMPIType type)
    {
#ifdef MERCURY_USE_MPI
        MPI_Datatype MPIType;
        MPI_Type_contiguous(sizeof(T), MPI_BYTE, &MPIType);
        MPI_Type_commit(&MPIType);
        dataTypes_.push_back(MPIType);
#endif
    }
    
    
    void deleteMercuryMPITypes()
    {
#ifdef MERCURY_USE_MPI
        for(MPI_Datatype type : dataTypes_)
        {
            MPI_Type_free(&type);
        }
#endif
    }
    
    MPIContainer(const MPIContainer& orig) = delete;

private:
    
    MPIContainer();
    
    //std::size_t processorID_;
    int processorID_;
    
    int numberOfProcessors_;

#ifdef MERCURY_USE_MPI

    std::vector<MPI_Request> pending_;
    
    MPI_Comm communicator_;
    std::vector<MPI_Datatype> dataTypes_;

#endif

};


#endif /* MPICONTAINER_H_ */