From bc3cc55e236f84740461633e54fe130fa97f2151 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 31 Jan 2023 14:22:47 -0800 Subject: [PATCH 1/3] ADIOS2: Fix BTD Resize w/ Empty Ranks For ADIOS2 BP4 and BP5 writes, setting an update of a variable's (openPMD record's) shape is not sufficient to drop all meta-data on disk. In situations where the last write to a BTD stage only adds further particles from a few ranks, we need to pad with zero-block writes so ADIOS2 `Put` gets called. Backend details: - BP4 (as of ADIOS 2.8): last MPI rank's `Put` meta-data wins - BP5 (as of ADIOS 2.8): everyone has to write an empty block --- Source/Diagnostics/WarpXOpenPMD.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 2cee7db218e..7afea475f4a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -754,7 +754,16 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 - if (numParticleOnTile == 0) continue; + // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 + // unless we append in ADIOS2: + // https://github.com/ECP-WarpX/WarpX/issues/3389 + // https://github.com/ornladios/ADIOS2/issues/3455 + // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins + // BP5 (ADIOS 2.8): everyone has to write an empty block + bool write_empty_blocks = false; + if (m_Series->backend() == "ADIOS2") + write_empty_blocks = isBTD; + if (numParticleOnTile == 0 && !write_empty_blocks) continue; // get position and particle ID from aos // note: this implementation iterates the AoS 4x... From 82440cafba693da0050d79f1b9802c2a87b660bc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 31 Jan 2023 18:35:03 -0800 Subject: [PATCH 2/3] [Draft] Start Duplicating All Put Logic Jeeeeeze. --- Source/Diagnostics/WarpXOpenPMD.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 7afea475f4a..631d7127410 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -744,11 +744,14 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, m_Series->flush(); // dump individual particles + bool contributed_particles = false; for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { uint64_t offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed if (isBTD) offset += ParticleFlushOffset; + // int num_tiles = pc->numLocalTilesAtLevel(currentLevel); for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { + // for (auto pti = pc->MakeMFIter(currentLevel); pti.isValid(); ++pti) { // does not skip empty tiles but lacks all particle methods auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); @@ -765,6 +768,8 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, write_empty_blocks = isBTD; if (numParticleOnTile == 0 && !write_empty_blocks) continue; + contributed_particles = true; + // get position and particle ID from aos // note: this implementation iterates the AoS 4x... // if we flush late as we do now, we can also copy out the data in one go @@ -842,8 +847,18 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, write_int_comp, int_comp_names); offset += numParticleOnTile64; - } + } // pti + } // currentLevel + + // work-around for BTD particle resize + // https://github.com/ECP-WarpX/WarpX/issues/3389 + // https://github.com/ornladios/ADIOS2/issues/3455 + // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins + // BP5 (ADIOS 2.8): everyone has to write an empty block + if (!contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { + // do empty Put calls for ADIOS2. } + m_Series->flush(); } From d1967a5e26d912f3eea0ab6903f4b25d7a4e9f2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 1 Feb 2023 15:44:43 -0800 Subject: [PATCH 3/3] Better Work-Around Compact and general, including MR situations. --- Source/Diagnostics/WarpXOpenPMD.cpp | 61 ++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 631d7127410..63707576fa0 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -744,29 +744,19 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, m_Series->flush(); // dump individual particles - bool contributed_particles = false; + bool contributed_particles = false; // did the local MPI rank contribute particles? for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { uint64_t offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed if (isBTD) offset += ParticleFlushOffset; - // int num_tiles = pc->numLocalTilesAtLevel(currentLevel); for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { - // for (auto pti = pc->MakeMFIter(currentLevel); pti.isValid(); ++pti) { // does not skip empty tiles but lacks all particle methods auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - // unless we append in ADIOS2: - // https://github.com/ECP-WarpX/WarpX/issues/3389 - // https://github.com/ornladios/ADIOS2/issues/3455 - // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins - // BP5 (ADIOS 2.8): everyone has to write an empty block - bool write_empty_blocks = false; - if (m_Series->backend() == "ADIOS2") - write_empty_blocks = isBTD; - if (numParticleOnTile == 0 && !write_empty_blocks) continue; + if (numParticleOnTile == 0) continue; contributed_particles = true; @@ -850,13 +840,54 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, } // pti } // currentLevel - // work-around for BTD particle resize + // work-around for BTD particle resize in ADIOS2 + // + // This issues an empty ADIOS2 Put to make sure the new global shape + // meta-data is committed for each variable. + // + // Refs.: // https://github.com/ECP-WarpX/WarpX/issues/3389 // https://github.com/ornladios/ADIOS2/issues/3455 // BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins // BP5 (ADIOS 2.8): everyone has to write an empty block - if (!contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { - // do empty Put calls for ADIOS2. + if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { + for( auto & [record_name, record] : currSpecies ) { + for( auto & [comp_name, comp] : record ) { + if (comp.constant()) continue; + + auto dtype = comp.getDatatype(); + switch (dtype) { + case openPMD::Datatype::FLOAT : + [[fallthrough]]; + case openPMD::Datatype::DOUBLE : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::UINT : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::ULONG : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + case openPMD::Datatype::ULONGLONG : { + auto empty_data = std::make_shared(); + comp.storeChunk(empty_data, {uint64_t(0)}, {uint64_t(0)}); + break; + } + default : { + std::string msg = "WarpX openPMD ADIOS2 work-around has unknown dtype: "; + msg += datatypeToString(dtype); + amrex::Abort(msg); + break; + } + } + } + } } m_Series->flush();