Skip to content

Commit

Permalink
Merge pull request #1454 from williamfgc/iter_bug
Browse files Browse the repository at this point in the history 
Allow 64-bit count Isend Irecv
  • Loading branch information
@williamfgc
williamfgc authored May 28, 2019
2 parents d4238ee + 218a6db commit 55390f2
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 36 deletions.
48 changes: 23 additions & 25 deletions source/adios2/helper/adiosMPIFunctions.tcc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,31 +259,35 @@ std::vector<MPI_Request> Isend64<char>(const char *buffer, const size_t count,
int dest, int tag, MPI_Comm mpiComm,
const std::string &hint)
{
const size_t batches = count / DefaultMaxFileBatchSize;
std::vector<MPI_Request> requests(batches + 1);

if (batches > 1)
std::vector<MPI_Request> requests(1);

if (count > DefaultMaxFileBatchSize)
{
const size_t batches = count / DefaultMaxFileBatchSize;
requests.resize(batches);

size_t position = 0;
for (size_t b = 0; b < batches; ++b)
{
int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
CheckMPIReturn(
MPI_Isend(const_cast<char *>(buffer + position), batchSize,
MPI_CHAR, dest, tag, mpiComm, &requests[b]),
"in call to Isend64 batch " + std::to_string(b) + "/" +
std::to_string(batches) + " " + hint + "\n");
CheckMPIReturn(MPI_Isend(const_cast<char *>(buffer + position),
batchSize, MPI_CHAR, dest, tag, mpiComm,
&requests[b]),
"in call to Isend64 batch " + std::to_string(b) +
" " + hint + "\n");

position += DefaultMaxFileBatchSize;
}
const size_t remainder = count % DefaultMaxFileBatchSize;
if (remainder > 0)
{
requests.resize(batches + 1);
int batchSize = static_cast<int>(remainder);
CheckMPIReturn(MPI_Isend(const_cast<char *>(buffer + position),
batchSize, MPI_CHAR, dest, tag, mpiComm,
&requests[batches - 1]),
"in call to Isend64 last batch " + hint + "\n");
&requests[batches]),
"in call to Isend64 remainder batch " + hint + "\n");
}
}
else
Expand All @@ -302,38 +306,32 @@ std::vector<MPI_Request> Irecv64<char>(char *buffer, const size_t count,
int source, int tag, MPI_Comm mpiComm,
const std::string &hint)
{
const size_t batches = count / DefaultMaxFileBatchSize;
std::vector<MPI_Request> requests(batches + 1);
std::vector<MPI_Request> requests(1);

if (requests.size() != batches + 1)
{
throw std::runtime_error(
"ERROR: number of Irecv requests doesn't match number of batches = "
"count/DefaultMaxFileBatchSize\n");
}

if (batches > 1)
if (count > DefaultMaxFileBatchSize)
{
const size_t batches = count / DefaultMaxFileBatchSize;
requests.resize(batches);
size_t position = 0;
for (size_t b = 0; b < batches; ++b)
{
int batchSize = static_cast<int>(DefaultMaxFileBatchSize);
CheckMPIReturn(MPI_Irecv(buffer + position, batchSize, MPI_CHAR,
source, tag, mpiComm, &requests[b]),
"in call to Irecv64 batch " + std::to_string(b) +
"/" + std::to_string(batches) + " " + hint +
"\n");
" " + hint + "\n");

position += DefaultMaxFileBatchSize;
}

const size_t remainder = count % DefaultMaxFileBatchSize;
if (remainder > 0)
{
requests.resize(batches + 1);
int batchSize = static_cast<int>(remainder);
CheckMPIReturn(MPI_Irecv(buffer + position, batchSize, MPI_CHAR,
source, tag, mpiComm,
&requests[batches - 1]),
"in call to Irecv64 last batch " + hint + "\n");
source, tag, mpiComm, &requests[batches]),
"in call to Irecv64 remainder batch " + hint + "\n");
}
}
else
Expand Down
32 changes: 21 additions & 11 deletions source/adios2/toolkit/aggregator/mpi/MPIChain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
* Created on: Feb 21, 2018
* Author: William F Godoy godoywf@ornl.gov
*/

#include "MPIChain.h"

#include "adios2/ADIOSMPI.h"
Expand Down Expand Up @@ -58,13 +57,19 @@ std::vector<std::vector<MPI_Request>> MPIChain::IExchange(BufferSTL &bufferSTL,
", aggregation Isend size at iteration " +
std::to_string(step) + "\n");

const std::vector<MPI_Request> requestsISend64 = helper::Isend64(
sendBuffer.m_Buffer.data(), sendBuffer.m_Position, m_Rank - 1, 1,
m_Comm,
", aggregation Isend64 data at iteration " + std::to_string(step));
// only send data if buffer larger than 0
if (sendBuffer.m_Position > 0)
{

const std::vector<MPI_Request> requestsISend64 =
helper::Isend64(sendBuffer.m_Buffer.data(),
sendBuffer.m_Position, m_Rank - 1, 1, m_Comm,
", aggregation Isend64 data at iteration " +
std::to_string(step));

requests[0].insert(requests[0].end(), requestsISend64.begin(),
requestsISend64.end());
requests[0].insert(requests[0].end(), requestsISend64.begin(),
requestsISend64.end());
}
}
// receive size, resize receiving buffer and receive data
if (receiver)
Expand All @@ -89,10 +94,15 @@ std::vector<std::vector<MPI_Request>> MPIChain::IExchange(BufferSTL &bufferSTL,
"in aggregation, when resizing receiving buffer to size " +
std::to_string(bufferSize));

requests[1] = helper::Irecv64(
receiveBuffer.m_Buffer.data(), receiveBuffer.m_Position, m_Rank + 1,
1, m_Comm,
", aggregation Irecv64 data at iteration " + std::to_string(step));
// only receive data if buffer is larger than 0
if (bufferSize > 0)
{
requests[1] =
helper::Irecv64(receiveBuffer.m_Buffer.data(),
receiveBuffer.m_Position, m_Rank + 1, 1, m_Comm,
", aggregation Irecv64 data at iteration " +
std::to_string(step));
}
}

return requests;
Expand Down

0 comments on commit 55390f2

Please sign in to comment.