Skip to content

Commit

Permalink
Merge pull request #3013 from anagainaru/bp5-cuda
Browse files Browse the repository at this point in the history
  • Loading branch information
@eisenhauer
eisenhauer authored Jan 29, 2022
2 parents 46a0c3b + 75bcdc8 commit 52e497e
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 21 deletions.
7 changes: 6 additions & 1 deletion examples/cuda/cudaWriteRead.cu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ int BPRead(const std::string fname, const size_t N, int nSteps)
adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);

unsigned int step = 0;
float *gpuSimData;
cudaMalloc(&gpuSimData, N * sizeof(float));
cudaMemset(gpuSimData, 0, N);
for (; bpReader.BeginStep() == adios2::StepStatus::OK; ++step)
{
auto data = io.InquireVariable<float>("data");
Expand All @@ -77,8 +80,10 @@ int BPRead(const std::string fname, const size_t N, int nSteps)
const adios2::Box<adios2::Dims> sel(start, count);
data.SetSelection(sel);

bpReader.Get(data, simData.data());
data.SetMemorySpace(adios2::MemorySpace::CUDA);
bpReader.Get(data, gpuSimData, adios2::Mode::Deferred);
bpReader.EndStep();
cudaMemcpy(simData.data(), gpuSimData, N, cudaMemcpyDeviceToHost);
std::cout << "Simualation step " << step << " : ";
std::cout << simData.size() << " elements: " << simData[1] << std::endl;
}
Expand Down
27 changes: 21 additions & 6 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "adios2/core/Engine.h"
#include "adios2/core/IO.h"
#include "adios2/core/VariableBase.h"
#include "adios2/helper/adiosFunctions.h"

#include "BP5Deserializer.h"
#include "BP5Deserializer.tcc"
Expand Down Expand Up @@ -970,6 +971,7 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
Req.Count = variable.m_Count;
Req.Start = variable.m_Start;
Req.Step = Step;
Req.MemSpace = variable.m_MemorySpace;
Req.Data = DestData;
PendingRequests.push_back(Req);
}
Expand Down Expand Up @@ -1212,14 +1214,14 @@ void BP5Deserializer::FinalizeGets(std::vector<ReadRequest> Requests)
ExtractSelectionFromPartialRM(
ElementSize, DimCount, GlobalDimensions, RankOffset,
RankSize, SelOffset, SelSize, IncomingData,
(char *)Req.Data);
(char *)Req.Data, Req.MemSpace);
}
else
{
ExtractSelectionFromPartialCM(
ElementSize, DimCount, GlobalDimensions, RankOffset,
RankSize, SelOffset, SelSize, IncomingData,
(char *)Req.Data);
(char *)Req.Data, Req.MemSpace);
}
}
}
Expand Down Expand Up @@ -1298,12 +1300,25 @@ static int FindOffsetCM(size_t Dims, const size_t *Size, const size_t *Index)
* *******************************
*/

void BP5Deserializer::MemCopyData(char *OutData, const char *InData,
size_t Size, MemorySpace MemSpace)
{
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::CudaMemCopyToBuffer(OutData, 0, InData, Size);
return;
}
#endif
memcpy(OutData, InData, Size);
}

// Row major version
void BP5Deserializer::ExtractSelectionFromPartialRM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData)
const char *InData, char *OutData, MemorySpace MemSpace)
{
size_t BlockSize;
size_t SourceBlockStride = 0;
Expand Down Expand Up @@ -1385,7 +1400,7 @@ void BP5Deserializer::ExtractSelectionFromPartialRM(
size_t i;
for (i = 0; i < BlockCount; i++)
{
memcpy(OutData, InData, BlockSize * ElementSize);
MemCopyData(OutData, InData, BlockSize * ElementSize, MemSpace);
InData += SourceBlockStride;
OutData += DestBlockStride;
}
Expand All @@ -1397,7 +1412,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData)
const char *InData, char *OutData, MemorySpace MemSpace)
{
int BlockSize;
int SourceBlockStride = 0;
Expand Down Expand Up @@ -1486,7 +1501,7 @@ void BP5Deserializer::ExtractSelectionFromPartialCM(
OutData += DestBlockStartOffset;
for (int i = 0; i < BlockCount; i++)
{
memcpy(OutData, InData, BlockSize * ElementSize);
MemCopyData(OutData, InData, BlockSize * ElementSize, MemSpace);
InData += SourceBlockStride;
OutData += DestBlockStride;
}
Expand Down
29 changes: 15 additions & 14 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,20 +179,20 @@ class BP5Deserializer : virtual public BP5Base
bool GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec, void *DestData,
size_t Step, size_t WriterRank);
void ExtractSelectionFromPartialRM(int ElementSize, size_t Dims,
const size_t *GlobalDims,
const size_t *PartialOffsets,
const size_t *PartialCounts,
const size_t *SelectionOffsets,
const size_t *SelectionCounts,
const char *InData, char *OutData);
void ExtractSelectionFromPartialCM(int ElementSize, size_t Dims,
const size_t *GlobalDims,
const size_t *PartialOffsets,
const size_t *PartialCounts,
const size_t *SelectionOffsets,
const size_t *SelectionCounts,
const char *InData, char *OutData);
void MemCopyData(char *OutData, const char *InData, size_t Size,
MemorySpace MemSpace);
void ExtractSelectionFromPartialRM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData,
MemorySpace MemSpace = MemorySpace::Host);
void ExtractSelectionFromPartialCM(
int ElementSize, size_t Dims, const size_t *GlobalDims,
const size_t *PartialOffsets, const size_t *PartialCounts,
const size_t *SelectionOffsets, const size_t *SelectionCounts,
const char *InData, char *OutData,
MemorySpace MemSpace = MemorySpace::Host);

enum RequestTypeEnum
{
Expand All @@ -208,6 +208,7 @@ class BP5Deserializer : virtual public BP5Base
size_t BlockID;
Dims Start;
Dims Count;
MemorySpace MemSpace;
void *Data;
};
std::vector<BP5ArrayRequest> PendingRequests;
Expand Down

0 comments on commit 52e497e

Please sign in to comment.