feat(barretenberg): Graph methods for circuit analysis (part 2)

barretenberg/cpp/src/barretenberg/boomerang_value_detection/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s stdlib_blake3s)
+barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s stdlib_blake3s stdlib_poseidon2 stdlib_primitives)

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph.hpp

@@ -3,6 +3,7 @@
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #include <list>
 #include <set>
+#include <typeinfo>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
@@ -16,6 +17,40 @@
  * constrained properly. if number of connected components > 1, it means that there were missed some connections between
  * variables.
  */
+
+namespace cdg {
+
+/*
+ * we add a new feature for static analyzer, now it contains gates where it found every variable. This may be helpful,
+ * if we want to do functions that remove false-positive variables from the analyzer using selectors in the gate + some
+ * additional knowledge about this variable, for example, tau or range tags. this info contains in unordered map with
+ * key as std::pair<uint32_t, size_t>, where uint32_t -- real variable index and size_t -- index of UltraTraceBlock in
+ * Reference Array with all TraceBlocks, that Ultra Circuit Builder contains inside. But there was a problem with
+ * unordered map -- it doesn't have default hash function and function for checking equivalence for std::pair as a key,
+ * so we had to implement it ourselves. We decided to choose approach based on function hash_combine from boost library
+ * for C++, and it's not so difficult to hash 2 elements in pair and check their equivalence.
+ */
+using UltraBlock = bb::UltraTraceBlock;
+using KeyPair = std::pair<uint32_t, size_t>;
+
+struct KeyHasher {
+    size_t operator()(const KeyPair& pair) const
+    {
+        size_t combined_hash = 0;
+        auto hash_combiner = [](size_t lhs, size_t rhs) { return lhs ^ (rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2)); };
+        combined_hash = hash_combiner(combined_hash, std::hash<uint32_t>()(pair.first));
+        combined_hash = hash_combiner(combined_hash, std::hash<size_t>()(pair.second));
+        return combined_hash;
+    }
+};
+
+struct KeyEquals {
+    bool operator()(const KeyPair& p1, const KeyPair& p2) const
+    {
+        return (p1.first == p2.first && p1.second == p2.second);
+    }
+};
+
 template <typename FF> class Graph_ {
   public:
     Graph_() = default;
@@ -30,19 +65,39 @@ template <typename FF> class Graph_ {
     {
         return ultra_circuit_constructor.real_variable_index[variable_index];
     };
+    size_t find_block_index(bb::UltraCircuitBuilder& ultra_builder, const UltraBlock& block);
     void process_gate_variables(bb::UltraCircuitBuilder& ultra_circuit_constructor,
-                                std::vector<uint32_t>& gate_variables);
-
+                                std::vector<uint32_t>& gate_variables,
+                                size_t gate_index,
+                                size_t blk_idx);
     std::unordered_map<uint32_t, size_t> get_variables_gate_counts() { return this->variables_gate_counts; };
 
-    std::vector<uint32_t> get_arithmetic_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                  size_t index);
+    std::vector<std::vector<uint32_t>> get_arithmetic_gate_connected_component(
+        bb::UltraCircuitBuilder& ultra_circuit_builder, size_t index, size_t block_idx, UltraBlock& blk);
     std::vector<uint32_t> get_elliptic_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                size_t index);
+                                                                size_t index,
+                                                                size_t block_idx,
+                                                                UltraBlock& blk);
     std::vector<uint32_t> get_plookup_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                               size_t index);
+                                                               size_t index,
+                                                               size_t block_idx,
+                                                               UltraBlock& blk);
     std::vector<uint32_t> get_sort_constraint_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
-                                                                  size_t index);
+                                                                  size_t index,
+                                                                  size_t block_idx,
+                                                                  UltraBlock& blk);
+    std::vector<uint32_t> get_poseido2s_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
+                                                                 size_t index,
+                                                                 size_t block_idx,
+                                                                 UltraBlock& blk);
+    std::vector<uint32_t> get_auxiliary_gate_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
+                                                                 size_t index,
+                                                                 size_t block_idx,
+                                                                 UltraBlock& blk);
+    std::vector<uint32_t> get_rom_table_connected_component(bb::UltraCircuitBuilder& ultra_circuit_builder,
+                                                            const bb::UltraCircuitBuilder::RomTranscript& rom_array);
+    std::vector<uint32_t> get_ram_table_connected_component(bb::UltraCircuitBuilder& ultra_builder,
+                                                            const bb::UltraCircuitBuilder::RamTranscript& ram_array);
 
     void add_new_edge(const uint32_t& first_variable_index, const uint32_t& second_variable_index);
     std::vector<uint32_t> get_variable_adjacency_list(const uint32_t& variable_index)
@@ -89,6 +144,7 @@ template <typename FF> class Graph_ {
                                                 const std::unordered_set<uint32_t>& decompose_variables);
     void remove_unnecessary_plookup_variables(bb::UltraCircuitBuilder& ultra_circuit_builder,
                                               std::unordered_set<uint32_t>& variables_in_on_gate);
+    void remove_unnecessary_range_constrains_variables(bb::UltraCircuitBuilder& ultra_builder);
     std::unordered_set<uint32_t> show_variables_in_one_gate(bb::UltraCircuitBuilder& ultra_circuit_builder);
 
     void remove_unnecessary_aes_plookup_variables(std::unordered_set<uint32_t>& variables_in_one_gate,
@@ -99,11 +155,13 @@ template <typename FF> class Graph_ {
                                                      bb::UltraCircuitBuilder& ultra_circuit_builder,
                                                      bb::plookup::BasicTableId& table_id,
                                                      size_t gate_index);
+    void remove_record_witness_variables(bb::UltraCircuitBuilder& ultra_builder);
 
     void print_graph();
     void print_connected_components();
     void print_variables_gate_counts();
     void print_variables_edge_counts();
+    void print_variables_in_one_gate(bb::UltraCircuitBuilder& ultra_builder);
     ~Graph_() = default;
 
   private:
@@ -114,6 +172,13 @@ template <typename FF> class Graph_ {
         variables_gate_counts; // we use this data structure to count, how many gates use every variable
     std::unordered_map<uint32_t, size_t>
         variables_degree; // we use this data structure to count, how many every variable have edges
+    std::unordered_map<KeyPair, std::vector<size_t>, KeyHasher, KeyEquals>
+        variable_gates; // we use this data structure to store gates and TraceBlocks for every variables, where static
+                        // analyzer found them in the circuit.
+    std::unordered_set<uint32_t> variables_in_one_gate;
+    std::unordered_set<uint32_t> fixed_variables;
 };
 
-using Graph = Graph_<bb::fr>;
+using Graph = Graph_<bb::fr>;
+
+} // namespace cdg

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description.test.cpp

@@ -11,7 +11,7 @@
 #include <gtest/gtest.h>
 
 using namespace bb;
-
+using namespace cdg;
 /**
  * @brief this test checks graph description of the circuit with arithmetic gates
     the number of connected components = the number of pair (i, j), 0<=i, j <16, i.e 256
@@ -38,10 +38,8 @@ TEST(boomerang_ultra_circuit_constructor, test_graph_for_arithmetic_gates)
 
     Graph graph = Graph(circuit_constructor);
     auto connected_components = graph.find_connected_components();
-    auto num_connected_components = connected_components.size();
-    auto variables_in_one_gate = graph.show_variables_in_one_gate(circuit_constructor);
-    bool result = num_connected_components == 256;
-    EXPECT_EQ(result, true);
+    [[maybe_unused]] auto variables_in_one_gate = graph.show_variables_in_one_gate(circuit_constructor);
+    EXPECT_EQ(connected_components.size(), 256);
 }
 
 /**

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_aes128.test.cpp

@@ -12,19 +12,23 @@
 
 using namespace bb;
 using namespace bb::stdlib;
+using namespace cdg;
 
 using Builder = UltraCircuitBuilder;
-typedef stdlib::field_t<UltraCircuitBuilder> field_pt;
-typedef stdlib::witness_t<bb::UltraCircuitBuilder> witness_pt;
+using field_pt = stdlib::field_t<UltraCircuitBuilder>;
+using witness_pt = stdlib::witness_t<bb::UltraCircuitBuilder>;
 
-bool check_in_vector(const std::vector<field_pt>& input_vector, const uint32_t& real_var_index)
+/**
+ static analyzer usually prints input and output variables as variables in one gate. In tests these variables
+ are not dangerous and usually we can filter them by adding gate for fixing witness. Then these variables will be
+ in 2 gates, and static analyzer won't print them. functions fix_vector_witness doest it for vector in_field
+ */
+
+void fix_vector_witness(std::vector<field_pt>& input_vector)
 {
-    for (const auto& elem : input_vector) {
-        if (elem.witness_index == real_var_index) {
-            return true;
-        }
+    for (auto& elem : input_vector) {
+        elem.fix_witness();
     }
-    return false;
 }
 
 /**
@@ -41,7 +45,7 @@ TEST(boomerang_stdlib_aes, test_graph_for_aes_64_bytes)
                     0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
                     0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 };
 
-    const auto convert_bytes = [](uint8_t* data) {
+    auto convert_bytes = [](uint8_t* data) {
         uint256_t converted(0);
         for (uint64_t i = 0; i < 16; ++i) {
             uint256_t to_add = uint256_t((uint64_t)(data[i])) << uint256_t((15 - i) * 8);
@@ -59,17 +63,21 @@ TEST(boomerang_stdlib_aes, test_graph_for_aes_64_bytes)
         witness_pt(&builder, fr(convert_bytes(in + 48))),
     };
 
+    fix_vector_witness(in_field);
+
     field_pt key_field(witness_pt(&builder, fr(convert_bytes(key))));
     field_pt iv_field(witness_pt(&builder, fr(convert_bytes(iv))));
+    key_field.fix_witness();
+    iv_field.fix_witness();
 
-    const auto result = stdlib::aes128::encrypt_buffer_cbc(in_field, iv_field, key_field);
+    auto result = stdlib::aes128::encrypt_buffer_cbc(in_field, iv_field, key_field);
+    fix_vector_witness(result);
 
     Graph graph = Graph(builder);
     auto connected_components = graph.find_connected_components();
-    auto num_connected_components = connected_components.size();
-    bool graph_result = num_connected_components == 1;
-
-    EXPECT_EQ(graph_result, true);
+    EXPECT_EQ(connected_components.size(), 1);
+    auto variables_in_one_gate = graph.show_variables_in_one_gate(builder);
+    EXPECT_EQ(variables_in_one_gate.size(), 0);
 }
 
 /**
@@ -88,7 +96,7 @@ TEST(boomerang_stdlib_aes, test_variable_gates_count_for_aes128cbc)
                     0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef,
                     0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 };
 
-    const auto convert_bytes = [](uint8_t* data) {
+    auto convert_bytes = [](uint8_t* data) {
         uint256_t converted(0);
         for (uint64_t i = 0; i < 16; ++i) {
             uint256_t to_add = uint256_t((uint64_t)(data[i])) << uint256_t((15 - i) * 8);
@@ -106,18 +114,19 @@ TEST(boomerang_stdlib_aes, test_variable_gates_count_for_aes128cbc)
         witness_pt(&builder, fr(convert_bytes(in + 48))),
     };
 
+    fix_vector_witness(in_field);
+
     field_pt key_field(witness_pt(&builder, fr(convert_bytes(key))));
     field_pt iv_field(witness_pt(&builder, fr(convert_bytes(iv))));
+    key_field.fix_witness();
+    iv_field.fix_witness();
 
-    const auto result = stdlib::aes128::encrypt_buffer_cbc(in_field, iv_field, key_field);
+    auto result = stdlib::aes128::encrypt_buffer_cbc(in_field, iv_field, key_field);
+    fix_vector_witness(result);
 
     Graph graph = Graph(builder);
+    auto connected_components = graph.find_connected_components();
+    EXPECT_EQ(connected_components.size(), 1);
     std::unordered_set<uint32_t> variables_in_one_gate = graph.show_variables_in_one_gate(builder);
-    for (const auto& elem : variables_in_one_gate) {
-        bool result1 = check_in_vector(in_field, elem);
-        bool result2 = check_in_vector(result, elem);
-        bool check =
-            (result1 == 1) || (result2 == 1) || (elem == key_field.witness_index) || (elem == iv_field.witness_index);
-        EXPECT_EQ(check, true);
-    }
-}
+    EXPECT_EQ(variables_in_one_gate.size(), 0);
+}