just reuse BndInfo

src/bvals/comms/bnd_info.cpp

@@ -276,6 +276,10 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                  IndexRangeType idx_range_type) {
   allocated = v->IsAllocated();
   alloc_status = v->GetAllocationStatus();
+
+  tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb); 
+  var_id = v->GetUniqueID();
+
 
   buf = combuf->buffer();
   same_to_same = pmb->gid == nb.gid && nb.offsets.IsCell();
@@ -307,24 +311,33 @@ BndInfo::BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
 
 BndInfo BndInfo::GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                 std::shared_ptr<Variable<Real>> v,
+                                BoundaryType b_type,
                                 CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   auto idx_range_type = IndexRangeType::BoundaryInteriorSend;
   // Test if the neighbor block is not offset from this block (i.e. is a
   // parent or daughter block of pmb), and change the IndexRangeType
   // accordingly
   if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorSend;
-  return BndInfo(pmb, nb, v, buf, idx_range_type);
+  BndInfo out(pmb, nb, v, buf, idx_range_type);
+  out.rank_send = Globals::my_rank; 
+  out.rank_recv = nb.rank;
+  out.extra_id = static_cast<int>(b_type); 
+  return out;
 }
 
 BndInfo BndInfo::GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                std::shared_ptr<Variable<Real>> v,
+                               BoundaryType b_type,
                                CommBuffer<buf_pool_t<Real>::owner_t> *buf) {
   auto idx_range_type = IndexRangeType::BoundaryExteriorRecv;
   // Test if the neighbor block is not offset from this block (i.e. is a
   // parent or daughter block of pmb), and change the IndexRangeType
   // accordingly
   if (nb.offsets.IsCell()) idx_range_type = IndexRangeType::InteriorRecv;
   BndInfo out(pmb, nb, v, buf, idx_range_type);
+  out.rank_recv = Globals::my_rank; 
+  out.rank_send = nb.rank;
+  out.extra_id = static_cast<int>(b_type); 
 
   auto buf_state = buf->GetState();
   if (buf_state == BufferState::received) {

src/bvals/comms/bnd_info.hpp

@@ -49,11 +49,33 @@ enum class IndexRangeType {
 };
 
 struct BndInfo {
+  // Information for identifying the buffer with a communication 
+  // channel, variable, and the ranks it is communicated across
+  int tag;
+  int var_id; 
+  int extra_id;
+  int rank_send;
+  int rank_recv;
+  BoundaryType bound_type;
+
+  // MeshData partition id of the *sender*
+  // not set by constructors and only necessary for coalesced comms
+  int partition;
+
   int ntopological_elements = 1;
   using TE = TopologicalElement;
   TE topo_idx[3]{TE::CC, TE::CC, TE::CC};
   SpatiallyMaskedIndexer6D idxer[3];
   forest::LogicalCoordinateTransformation lcoord_trans;
+
+  KOKKOS_FORCEINLINE_FUNCTION
+  int size() const { 
+    int s = 0;
+    for (int n = 0; n < ntopological_elements; ++n) { 
+      s += idxer[n].size();
+    }
+    return s;
+  }
 
   CoordinateDirection dir{CoordinateDirection::X0DIR};
   bool allocated = true;
@@ -76,9 +98,11 @@ struct BndInfo {
   // kinds of boundary types and operations.
   static BndInfo GetSendBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                 std::shared_ptr<Variable<Real>> v,
+                                BoundaryType b_type,
                                 CommBuffer<buf_pool_t<Real>::owner_t> *buf);
   static BndInfo GetSetBndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                std::shared_ptr<Variable<Real>> v,
+                               BoundaryType b_type,
                                CommBuffer<buf_pool_t<Real>::owner_t> *buf);
 };
 

src/bvals/comms/bvals_utils.hpp

@@ -203,6 +203,7 @@ inline auto CheckReceiveBufferCacheForRebuild(std::shared_ptr<MeshData<Real>> md
 
 using F_BND_INFO = std::function<BndInfo(MeshBlock *pmb, const NeighborBlock &nb,
                                          std::shared_ptr<Variable<Real>> v,
+                                         BoundaryType b_type,
                                          CommBuffer<buf_pool_t<Real>::owner_t> *buf)>;
 
 using F_PRORES_INFO = std::function<ProResInfo(MeshBlock *pmb, const NeighborBlock &nb,
@@ -230,7 +231,7 @@ inline void RebuildBufferCache(std::shared_ptr<MeshData<Real>> md, int nbound,
   ForEachBoundary<BOUND_TYPE>(md, [&](auto pmb, sp_mbd_t rc, nb_t &nb, const sp_cv_t v) {
     // bnd_info
     const std::size_t ibuf = cache.idx_vec[ibound];
-    cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, cache.buf_vec[ibuf]);
+    cache.bnd_info_h(ibuf) = BndInfoCreator(pmb, nb, v, BOUND_TYPE, cache.buf_vec[ibuf]);
 
     // subsets ordering is same as in cache.bnd_info
     // RefinementFunctions_t owns all relevant functionality, so

src/bvals/comms/combined_buffers.hpp

@@ -29,77 +29,104 @@
 #include "utils/communication_buffer.hpp"
 
 namespace parthenon {
-
-struct BufferStructure {
-  // These first five variables should be enough information to
-  // uniquely identify the buffer
-  int tag;    // Tag defining communication channel between blocks
-              // (which subsumes send_gid, recv_gid, location_on_block)
-              // within a given MPI rank pair
-  int var_id; // We use an int for the Uid_t since we will be sending via MPI
-  int extra_id;
-  int rank_send; // MPI rank of sender
-  int rank_recv; // MPI rank of receiver
-
-  // Other information that could be useful to sending messages
-  int size;                 // Size of the buffer
-  Mesh::channel_key_t key;  // Actual key
-  bool currently_allocated; // Current allocation status of the buffer
-  int partition;            // Partition of sender
-  BoundaryType btype;       // Type of boundary this was registered for
-
-  static BufferStructure Send(int partition, const MeshBlock *const pmb,
-                              const NeighborBlock &nb,
-                              const std::shared_ptr<Variable<Real>> &var,
-                              BoundaryType b_type) {
-    BufferStructure out;
-    out.tag = pmb->pmy_mesh->tag_map.GetTag(pmb, nb);
-    out.var_id = var->GetUniqueID();
-    out.extra_id = static_cast<int>(b_type);
-    out.rank_send = Globals::my_rank;
-    out.rank_recv = nb.rank;
-
-    out.key = SendKey(pmb, nb, var, b_type);
-    out.size = GetBufferSize(pmb, nb, var);
-    out.currently_allocated = true;
-    out.partition = partition;
-    out.btype = b_type;
-    return out;
-  }
-};
-
 // Structure containing the information required for sending coalesced
 // messages between ranks
 struct CombinedBuffersRank {
-  using coalesced_message_structure_t = std::vector<BufferStructure>;
+  using coalesced_message_structure_t = std::vector<BndInfo>;
 
   // Rank that these buffers communicate with
   int other_rank;
 
   // map from partion id to coalesced message structure for communication
   // from this rank to other_rank
   std::map<int, coalesced_message_structure_t> combined_send_info;
-  std::map<int, ParArray1D<Real>> combined_send_buffers;
+  std::map<int, BufArray1D<Real>> combined_send_buffers;
 
   // map from neighbor partition id to coalesced message structures that
   // this rank can receive from other_rank. We will use the partition id
   // as the mpi tag
   std::map<int, coalesced_message_structure_t> combined_recv_info;
-  std::map<int, ParArray1D<Real>> combined_recv_buffers;
+  std::map<int, BufArray1D<Real>> combined_recv_buffers;
 
-  void AddSendBuffer(int partition, const MeshBlock *const &pmb, const NeighborBlock &nb,
+  void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb,
                      const std::shared_ptr<Variable<Real>> &var, BoundaryType b_type) {
-    combined_send_info[partition].push_back(
-        BufferStructure::Send(partition, pmb, nb, var, b_type));
+    auto bnd = BndInfo::GetSendBndInfo(pmb, nb, var, b_type, nullptr);
+    bnd.partition = partition;
+    combined_send_info[partition].push_back(bnd);
+  }
+
+  void ResolveSendBuffersAndSendInfo() {
+    // First calculate the total size of the message
+    int total_buffers{0};
+    for (auto &[partition, buf_struct_vec] : combined_send_info)
+      total_buffers += buf_struct_vec.size();
+    int total_partitions = combined_send_info.size(); 
+    int nglobal{3};
+    int nper_part{3};
+    int nper_buf{4};
+    int mesg_size = nglobal + nper_part * total_partitions + nper_buf * total_buffers;
+    std::vector<int> message(mesg_size);
+
+    // First store the number of buffers in each partition 
+    int p{0};
+    for (auto &[partition, buf_struct_vec] : combined_send_info) { 
+      message[nglobal + nper_part * p] = partition; // Used as the comm tag
+      message[nglobal + nper_part * p + 1] = buf_struct_vec.size();
+      p++;
+    }
+
+    // Now store the buffer information for each partition,
+    // the total size of the message associated with each
+    // partition
+    int b{0};
+    p = 0;
+    const int start = nglobal + nper_part * total_partitions;
+    std::map<int, int> combined_buf_size;
+    for (auto &[partition, buf_struct_vec] : combined_send_info) {
+      int total_size{0};
+      for (auto &buf_struct : buf_struct_vec) {
+        const int size = buf_struct.size();
+        message[start + 4 * b + 0] = buf_struct.tag;
+        message[start + 4 * b + 1] = buf_struct.var_id;
+        message[start + 4 * b + 2] = buf_struct.extra_id;
+        message[start + 4 * b + 3] = size;
+        b++;
+        total_size += size;
+      }
+      combined_buf_size[partition] = total_size;
+      message[nglobal + nper_part * p + 2] = total_size;
+      ++p;
+    }
+
+    // Send message to other rank
+    // TODO(LFR): Actually do this send
+
+    // Allocate the combined buffers 
+    int total_size{0}; 
+    for (auto &[partition, size] : combined_buf_size)
+      total_size += size;
+    using buf_t = BufArray1D<Real>;
+    buf_t alloc_at_once("shared combined buffer", total_size);
+    int current_position{0};
+    for (auto &[partition, size] : combined_buf_size) {
+      combined_send_buffers[partition] = buf_t(alloc_at_once,
+                                               std::make_pair(current_position,
+                                               current_position + size));
+      current_position += size;
+    }
+  }
+
+  bool ReceiveFinished() { 
+    return true;
   }
 };
 
 struct CombinedBuffers {
   // Combined buffers for each rank
-  std::vector<CombinedBuffersRank> combined_buffers;
-  void AddSendBuffer(int partition, const MeshBlock *const pmb, const NeighborBlock &nb,
+  std::map<std::pair<int, BoundaryType>, CombinedBuffersRank> combined_buffers;
+  void AddSendBuffer(int partition, MeshBlock *pmb, const NeighborBlock &nb,
                      const std::shared_ptr<Variable<Real>> &var, BoundaryType b_type) {
-    combined_buffers[nb.rank].AddSendBuffer(partition, pmb, nb, var, b_type);
+    combined_buffers[{nb.rank, b_type}].AddSendBuffer(partition, pmb, nb, var, b_type);
   }
 };