Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Base modifications to make BP5 GPU aware on the Read side #3013

Merged
merged 3 commits into from
Jan 29, 2022
Merged

Base modifications to make BP5 GPU aware on the Read side #3013

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca9a72f
Adding memory space information to the read functions
anagainaru Jan 24, 2022
4e24af7
Splitting the read logic for device/host allocated buffers
anagainaru Jan 24, 2022
75bcdc8
Update the cuda example to use GPU buffers on the Read side
anagainaru Jan 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion examples/cuda/cudaWriteRead.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ int BPRead(const std::string fname, const size_t N, int nSteps)
adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);

unsigned int step = 0;
float *gpuSimData;
cudaMalloc(&gpuSimData, N * sizeof(float));
cudaMemset(gpuSimData, 0, N);
for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++step)
{
auto data = io.InquireVariable<float>("data");
Expand All @@ -77,8 +80,10 @@ int BPRead(const std::string fname, const size_t N, int nSteps)
const adios2::Box<adios2::Dims> sel(start, count);
data.SetSelection(sel);

bpReader.Get(data, simData.data());
data.SetMemorySpace(adios2::MemorySpace::CUDA);
bpReader.Get(data, gpuSimData, adios2::Mode::Deferred);
bpReader.EndStep();
cudaMemcpy(simData.data(), gpuSimData, N, cudaMemcpyDeviceToHost);
std::cout << "Simualation step " << step << " : ";
std::cout << simData.size() << " elements: " << simData[1] << std::endl;
}
Expand Down
27 changes: 21 additions & 6 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "adios2/core/Engine.h"
#include "adios2/core/IO.h"
#include "adios2/core/VariableBase.h"
#include "adios2/helper/adiosFunctions.h"

#include "BP5Deserializer.h"
#include "BP5Deserializer.tcc"
Expand Down Expand Up @@ -927,6 +928,7 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
Req.Count = variable.m_Count;
Req.Start = variable.m_Start;
Req.Step = Step;
Req.MemSpace = variable.m_MemorySpace;
Req.Data = DestData;
PendingRequests.push_back(Req);
}
Expand Down Expand Up @@ -1168,14 +1170,14 @@ void BP5Deserializer::FinalizeGets(std::vector<ReadRequest> Requests)
ExtractSelectionFromPartialRM(
ElementSize, DimCount, GlobalDimensions, RankOffset,
RankSize, SelOffset, SelSize, IncomingData,
(char *)Req.Data);
(char *)Req.Data, Req.MemSpace);
}
else
{
ExtractSelectionFromPartialCM(
ElementSize, DimCount, GlobalDimensions, RankOffset,
RankSize, SelOffset, SelSize, IncomingData,
(char *)Req.Data);
(char *)Req.Data, Req.MemSpace);
}
}
}
Expand Down Expand Up @@ -1254,12 +1256,25 @@ static int FindOffsetCM(size_t Dims, const size_t *Size, const size_t *Index)
* *******************************
*/

void BP5Deserializer::MemCopyData(char *OutData, const char *InData,
size_t Size, MemorySpace MemSpace)
{
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::CudaMemCopyToBuffer(OutData, 0, InData, Size);
return;
}
#endif
memcpy(OutData, InData, Size);
}

// Row major version
void BP5Deserializer::ExtractSelectionFromPartialRM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData)
const char *InData, char *OutData, MemorySpace MemSpace)
{
size_t BlockSize;
size_t SourceBlockStride = 0;
Expand Down Expand Up @@ -1341,7 +1356,7 @@ void BP5Deserializer::ExtractSelectionFromPartialRM(
size_t i;
for (i = 0; i < BlockCount; i++)
{
memcpy(OutData, InData, BlockSize * ElementSize);
MemCopyData(OutData, InData, BlockSize * ElementSize, MemSpace);
InData += SourceBlockStride;
OutData += DestBlockStride;
}
Expand All @@ -1353,7 +1368,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData)
const char *InData, char *OutData, MemorySpace MemSpace)
{
int BlockSize;
int SourceBlockStride = 0;
Expand Down Expand Up @@ -1442,7 +1457,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM(
OutData += DestBlockStartOffset;
for (int i = 0; i < BlockCount; i++)
{
memcpy(OutData, InData, BlockSize * ElementSize);
MemCopyData(OutData, InData, BlockSize * ElementSize, MemSpace);
InData += SourceBlockStride;
OutData += DestBlockStride;
}
Expand Down
29 changes: 15 additions & 14 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,20 +171,20 @@ class BP5Deserializer : virtual public BP5Base
bool GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec, void *DestData,
size_t Step, size_t WriterRank);
void ExtractSelectionFromPartialRM(int ElementSize, size_t Dims,
const size_t *GlobalDims,
const size_t *PartialOffsets,
const size_t *PartialCounts,
const size_t *SelectionOffsets,
const size_t *SelectionCounts,
const char *InData, char *OutData);
void ExtractSelectionFromPartialCM(int ElementSize, size_t Dims,
const size_t *GlobalDims,
const size_t *PartialOffsets,
const size_t *PartialCounts,
const size_t *SelectionOffsets,
const size_t *SelectionCounts,
const char *InData, char *OutData);
void MemCopyData(char *OutData, const char *InData, size_t Size,
MemorySpace MemSpace);
void ExtractSelectionFromPartialRM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData,
MemorySpace MemSpace = MemorySpace::Host);
void ExtractSelectionFromPartialCM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData,
MemorySpace MemSpace = MemorySpace::Host);

enum RequestTypeEnum
{
Expand All @@ -200,6 +200,7 @@ class BP5Deserializer : virtual public BP5Base
size_t BlockID;
Dims Start;
Dims Count;
MemorySpace MemSpace;
void *Data;
};
std::vector<BP5ArrayRequest> PendingRequests;
Expand Down