Exposed BufferSet/MemoryCopy in internal header, added unit tests.

apache · Apr 5, 2022 · 1ec16ea · 1ec16ea
1 parent 08ec640
commit 1ec16ea
Show file tree

Hide file tree

Showing 3 changed files with 273 additions and 30 deletions.
diff --git a/src/runtime/hexagon/hexagon/hexagon_buffer.cc b/src/runtime/hexagon/hexagon/hexagon_buffer.cc
@@ -205,34 +205,8 @@ void HexagonBuffer::SetStorageScope(Optional<String> scope) {
   }
 }
 
-struct BufferSet {
-  BufferSet(void* const* buffers, size_t num_regions, size_t region_size_bytes)
-      : buffers(buffers), num_regions(num_regions), region_size_bytes(region_size_bytes) {}
-
-  size_t TotalBytes() const { return num_regions * region_size_bytes; }
-
-  void* const* buffers;
-  size_t num_regions;
-  size_t region_size_bytes;
-};
-
-struct MemoryCopy {
-  MemoryCopy(void* dest, void* src, size_t num_bytes)
-      : dest(dest), src(src), num_bytes(num_bytes) {}
-
-  bool IsDirectlyBefore(const MemoryCopy& other) {
-    void* src_end = static_cast<unsigned char*>(src);
-    void* dest_end = static_cast<unsigned char*>(dest);
-    return (src_end == other.src) && (dest_end == other.dest);
-  }
-
-  void* dest;
-  void* src;
-  size_t num_bytes;
-};
-
-void hexagon_buffer_copy_across_regions(const BufferSet& dest, const BufferSet& src,
-                                        size_t bytes_to_copy) {
+std::vector<MemoryCopy> BufferSet::MemoryCopies(const BufferSet& dest, const BufferSet& src,
+                                                size_t bytes_to_copy) {
   CHECK_LE(bytes_to_copy, src.TotalBytes());
   CHECK_LE(bytes_to_copy, dest.TotalBytes());
 
@@ -265,8 +239,10 @@ void hexagon_buffer_copy_across_regions(const BufferSet& dest, const BufferSet&
     }
   }
 
-  // If regions are contiguously allocated, we can reduce the number
-  // of copies required by merging adjacent copies.
+  return micro_copies;
+}
+
+std::vector<MemoryCopy> MemoryCopy::MergeAdjacent(std::vector<MemoryCopy> micro_copies) {
   std::sort(micro_copies.begin(), micro_copies.end(),
             [](const MemoryCopy& a, const MemoryCopy& b) { return a.src < b.src; });
 
@@ -279,6 +255,19 @@ void hexagon_buffer_copy_across_regions(const BufferSet& dest, const BufferSet&
     }
   }
 
+  return macro_copies;
+}
+
+void hexagon_buffer_copy_across_regions(const BufferSet& dest, const BufferSet& src,
+                                        size_t bytes_to_copy) {
+  // First, determine all copies that do not cross boundaries in
+  // either source or destination region.
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, bytes_to_copy);
+
+  // If regions are contiguously allocated, we can reduce the number
+  // of copies required by merging adjacent copies.
+  auto macro_copies = MemoryCopy::MergeAdjacent(std::move(micro_copies));
+
   // Finally, do the memory copies.
   for (const auto& copy : macro_copies) {
     int error_code = hexagon_user_dma_1d_sync(copy.dest, copy.src, copy.num_bytes);

diff --git a/src/runtime/hexagon/hexagon/hexagon_buffer.h b/src/runtime/hexagon/hexagon/hexagon_buffer.h
@@ -161,6 +161,42 @@ class HexagonBuffer {
   StorageScope storage_scope_;
 };
 
+/*! \brief Structure used to track/coalesce memory copies */
+struct MemoryCopy {
+  static std::vector<MemoryCopy> MergeAdjacent(std::vector<MemoryCopy> micro_copies);
+
+  MemoryCopy(void* dest, void* src, size_t num_bytes)
+      : dest(dest), src(src), num_bytes(num_bytes) {}
+
+  bool IsDirectlyBefore(const MemoryCopy& other) {
+    void* src_end = static_cast<unsigned char*>(src);
+    void* dest_end = static_cast<unsigned char*>(dest);
+    return (src_end == other.src) && (dest_end == other.dest);
+  }
+
+  void* dest;
+  void* src;
+  size_t num_bytes;
+};
+
+/*!
+ */
+struct BufferSet {
+  // Determine all copies that do not cross boundaries in either
+  // source or destination region.
+  static std::vector<MemoryCopy> MemoryCopies(const BufferSet& dest, const BufferSet& src,
+                                              size_t bytes_to_copy);
+
+  BufferSet(void* const* buffers, size_t num_regions, size_t region_size_bytes)
+      : buffers(buffers), num_regions(num_regions), region_size_bytes(region_size_bytes) {}
+
+  size_t TotalBytes() const { return num_regions * region_size_bytes; }
+
+      void* const* buffers;
+  size_t num_regions;
+  size_t region_size_bytes;
+};
+
 }  // namespace hexagon
 }  // namespace runtime
 }  // namespace tvm

diff --git a/tests/cpp/runtime/hexagon_buffer.cc b/tests/cpp/runtime/hexagon_buffer.cc
@@ -47,6 +47,224 @@ TEST(HexagonBuffer, invalid_scope) {
   EXPECT_THROW(HexagonBuffer hb(8 /* nbytes */, 8 /* alignment */, scope), InternalError);
 }
 
+TEST(HexagonBuffer, micro_copies_corresponding_regions) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 16);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 16);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 32);
+  EXPECT_EQ(micro_copies.size(), 2);
+  for (size_t i = 0; i < micro_copies.size(); i++) {
+    EXPECT_EQ(micro_copies[i].src, ptr(16 * i));
+    EXPECT_EQ(micro_copies[i].dest, ptr(64 + 16 * i));
+    EXPECT_EQ(micro_copies[i].num_bytes, 16);
+  }
+}
+
+TEST(HexagonBuffer, micro_copies_src_bigger) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 16);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(72), ptr(80), ptr(88)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 8);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 32);
+  EXPECT_EQ(micro_copies.size(), 4);
+  for (size_t i = 0; i < micro_copies.size(); i++) {
+    EXPECT_EQ(micro_copies[i].src, ptr(8 * i));
+    EXPECT_EQ(micro_copies[i].dest, ptr(64 + 8 * i));
+    EXPECT_EQ(micro_copies[i].num_bytes, 8);
+  }
+}
+
+TEST(HexagonBuffer, micro_copies_dest_bigger) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(8), ptr(16), ptr(24)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 8);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 16);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 32);
+  EXPECT_EQ(micro_copies.size(), 4);
+  for (size_t i = 0; i < micro_copies.size(); i++) {
+    EXPECT_EQ(micro_copies[i].src, ptr(8 * i));
+    EXPECT_EQ(micro_copies[i].dest, ptr(64 + 8 * i));
+    EXPECT_EQ(micro_copies[i].num_bytes, 8);
+  }
+}
+
+TEST(HexagonBuffer, micro_copies_src_overlaps_dest_region) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 16);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(76)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 12);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 24);
+  EXPECT_EQ(micro_copies.size(), 3);
+
+  // First region of source, first region of dest
+  EXPECT_EQ(micro_copies[0].src, ptr(0));
+  EXPECT_EQ(micro_copies[0].dest, ptr(64));
+  EXPECT_EQ(micro_copies[0].num_bytes, 12);
+
+  // First region of source, second region of dest
+  EXPECT_EQ(micro_copies[1].src, ptr(12));
+  EXPECT_EQ(micro_copies[1].dest, ptr(76));
+  EXPECT_EQ(micro_copies[1].num_bytes, 4);
+
+  // Second region of source, second region of dest
+  EXPECT_EQ(micro_copies[2].src, ptr(16));
+  EXPECT_EQ(micro_copies[2].dest, ptr(80));
+  EXPECT_EQ(micro_copies[2].num_bytes, 8);
+}
+
+TEST(HexagonBuffer, micro_copies_dest_overlaps_src_region) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(12)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 12);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 16);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 24);
+  EXPECT_EQ(micro_copies.size(), 3);
+
+  // First region of source, first region of dest
+  EXPECT_EQ(micro_copies[0].src, ptr(0));
+  EXPECT_EQ(micro_copies[0].dest, ptr(64));
+  EXPECT_EQ(micro_copies[0].num_bytes, 12);
+
+  // Second region of source, first region of dest
+  EXPECT_EQ(micro_copies[1].src, ptr(12));
+  EXPECT_EQ(micro_copies[1].dest, ptr(76));
+  EXPECT_EQ(micro_copies[1].num_bytes, 4);
+
+  // Second region of source, second region of dest
+  EXPECT_EQ(micro_copies[2].src, ptr(16));
+  EXPECT_EQ(micro_copies[2].dest, ptr(80));
+  EXPECT_EQ(micro_copies[2].num_bytes, 8);
+}
+
+TEST(HexagonBuffer, micro_copies_discontiguous_regions) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  // Stride of 16, but only first 11 bytes in each region belong to
+  // this buffer.
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 11);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 13);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 16);
+  EXPECT_EQ(micro_copies.size(), 3);
+
+  // First region of source, first region of dest
+  EXPECT_EQ(micro_copies[0].src, ptr(0));
+  EXPECT_EQ(micro_copies[0].dest, ptr(64));
+  EXPECT_EQ(micro_copies[0].num_bytes, 11);
+
+  // Second region of source, first region of dest
+  EXPECT_EQ(micro_copies[1].src, ptr(16));
+  EXPECT_EQ(micro_copies[1].dest, ptr(75));
+  EXPECT_EQ(micro_copies[1].num_bytes, 2);
+
+  // Second region of source, second region of dest
+  EXPECT_EQ(micro_copies[2].src, ptr(18));
+  EXPECT_EQ(micro_copies[2].dest, ptr(80));
+  EXPECT_EQ(micro_copies[2].num_bytes, 3);
+}
+
+TEST(HexagonBuffer, micro_copies_invalid_size) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+
+  {
+    BufferSet src(src_ptr.data(), 1, 16);
+    BufferSet dest(dest_ptr.data(), 2, 16);
+    EXPECT_THROW(BufferSet::MemoryCopies(dest, src, 24), InternalError);
+  }
+
+  {
+    BufferSet src(src_ptr.data(), 2, 16);
+    BufferSet dest(dest_ptr.data(), 1, 16);
+    EXPECT_THROW(BufferSet::MemoryCopies(dest, src, 24), InternalError);
+  }
+}
+
+TEST(HexagonBuffer, macro_copies_adjacent_corresponding_regions_merged) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 16);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 16);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 32);
+  auto macro_copies = MemoryCopy::MergeAdjacent(std::move(micro_copies));
+
+  ASSERT_EQ(macro_copies.size(), 1);
+  EXPECT_EQ(macro_copies[0].src, ptr(0));
+  EXPECT_EQ(macro_copies[0].dest, ptr(64));
+  EXPECT_EQ(macro_copies[0].num_bytes, 32);
+}
+
+TEST(HexagonBuffer, macro_copies_discontiguous_regions_not_merged) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(16)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 12);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 12);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 24);
+  auto macro_copies = MemoryCopy::MergeAdjacent(std::move(micro_copies));
+
+  ASSERT_EQ(macro_copies.size(), 2);
+
+  EXPECT_EQ(macro_copies[0].src, ptr(0));
+  EXPECT_EQ(macro_copies[0].dest, ptr(64));
+  EXPECT_EQ(macro_copies[0].num_bytes, 12);
+
+  EXPECT_EQ(macro_copies[1].src, ptr(16));
+  EXPECT_EQ(macro_copies[1].dest, ptr(80));
+  EXPECT_EQ(macro_copies[1].num_bytes, 12);
+}
+
+TEST(HexagonBuffer, macro_copies_overlapping_regions_merged) {
+  auto ptr = [](auto val) { return reinterpret_cast<void*>(val); };
+
+  std::vector<void*> src_ptr{ptr(0), ptr(12)};
+  BufferSet src(src_ptr.data(), src_ptr.size(), 12);
+
+  std::vector<void*> dest_ptr{ptr(64), ptr(80)};
+  BufferSet dest(dest_ptr.data(), dest_ptr.size(), 16);
+
+  auto micro_copies = BufferSet::MemoryCopies(dest, src, 24);
+  auto macro_copies = MemoryCopy::MergeAdjacent(std::move(micro_copies));
+
+  ASSERT_EQ(macro_copies.size(), 1);
+  EXPECT_EQ(macro_copies[0].src, ptr(0));
+  EXPECT_EQ(macro_copies[0].dest, ptr(64));
+  EXPECT_EQ(macro_copies[0].num_bytes, 24);
+}
+
 TEST(HexagonBuffer, copy_from) {
   Optional<String> scope("global");
   HexagonBuffer hb(8 /* nbytes */, 8 /* alignment */, scope);