Merge pull request hyperledger-archives#1885 from aludvik/use-rust-bl…

…ockstore

Implement and integrate Rust block store

integration/sawtooth_integration/docker/test_devmode_engine_liveness.yaml

@@ -154,6 +154,8 @@ services:
       - 8800
       - 5005
     working_dir: /root
+    environment:
+      RUST_BACKTRACE: full
     command: "bash -c \"\
         sawadm keygen && \
         sawset genesis \
@@ -193,6 +195,8 @@ services:
       - 4004
       - 8800
       - 5005
+    environment:
+      RUST_BACKTRACE: full
     command: "bash -c \"\
         sawadm keygen && \
         sawtooth-validator -v \
@@ -222,6 +226,8 @@ services:
       - 4004
       - 8800
       - 5005
+    environment:
+      RUST_BACKTRACE: full
     command: "bash -c \"\
         sawadm keygen && \
         sawtooth-validator -v \
@@ -251,6 +257,8 @@ services:
       - 4004
       - 8800
       - 5005
+    environment:
+      RUST_BACKTRACE: full
     command: "bash -c \"\
         sawadm keygen && \
         sawtooth-validator -v \
@@ -280,6 +288,8 @@ services:
       - 4004
       - 8800
       - 5005
+    environment:
+      RUST_BACKTRACE: full
     command: "bash -c \"\
         sawadm keygen && \
         sawtooth-validator -v \

integration/sawtooth_integration/tests/test_state_verifier.py

@@ -24,7 +24,6 @@
 from sawtooth_signing import create_context
 from sawtooth_signing import CryptoFactory
 
-from sawtooth_validator.database.dict_database import DictDatabase
 from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase
 from sawtooth_validator.state.merkle import MerkleDatabase
 from sawtooth_validator.journal.block_store import BlockStore
@@ -75,8 +74,7 @@ def populate_blockstore(blockstore, signer, state_roots):
             signer=signer)
         blocks.append(block)
 
-    for block in blocks:
-        blockstore[block.header_signature] = block
+    blockstore.put_blocks([blkw.block for blkw in blocks])
 
 
 def make_intkey_address(name):
@@ -193,10 +191,14 @@ def tearDown(self):
         shutil.rmtree(self._temp_dir)
 
     def test_state_verifier(self):
-        blockstore = BlockStore(DictDatabase(
-            indexes=BlockStore.create_index_configuration()))
+        block_store_db = NativeLmdbDatabase(
+            os.path.join(
+                self._temp_dir, 'test_state_verifier_block_store_db.lmdb'),
+            indexes=BlockStore.create_index_configuration())
+        blockstore = BlockStore(block_store_db)
         global_state_db = NativeLmdbDatabase(
-            os.path.join(self._temp_dir, 'test_state_verifier.lmdb'),
+            os.path.join(
+                self._temp_dir, 'test_state_verifier_global_state.lmdb'),
             indexes=MerkleDatabase.create_index_configuration())
 
         precalculated_state_roots = [

validator/sawtooth_validator/ffi.py

@@ -20,6 +20,8 @@
 import os
 import sys
 
+from sawtooth_validator.protobuf.block_pb2 import Block
+
 
 LOGGER = logging.getLogger(__name__)
 
@@ -185,3 +187,37 @@ def callback_wrapper(*args):
         return sender.send(*args)
 
     return callback_wrapper
+
+
+class BlockIterator(OwnedPointer):
+
+    def __init__(self, check_return_code, initialized_ptr=None):
+        super().__init__("{}_drop".format(self.name), initialized_ptr)
+        self._check_return_code = check_return_code
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if not self.pointer:
+            raise StopIteration()
+
+        (vec_ptr, vec_len, vec_cap) = prepare_vec_result()
+
+        self._check_return_code(
+            LIBRARY.call(
+                "{}_next".format(self.name),
+                self.pointer,
+                ctypes.byref(vec_ptr),
+                ctypes.byref(vec_len),
+                ctypes.byref(vec_cap)))
+
+        # Check if NULL
+        if not vec_ptr:
+            raise StopIteration()
+
+        payload = from_rust_vec(vec_ptr, vec_len, vec_cap)
+        block = Block()
+        block.ParseFromString(payload)
+
+        return block

validator/sawtooth_validator/journal/block_manager.py

@@ -17,7 +17,6 @@
 from enum import IntEnum
 
 from sawtooth_validator.ffi import OwnedPointer
-from sawtooth_validator.protobuf.block_pb2 import Block
 from sawtooth_validator import ffi
 
 
@@ -69,11 +68,11 @@ def __init__(self):
         _libexec("block_manager_new",
                  ctypes.byref(self.pointer))
 
-    def add_store(self, name, block_store):
-        _pylibexec("block_manager_add_store",
-                   self.pointer,
-                   ctypes.c_char_p(name.encode()),
-                   ctypes.py_object(block_store))
+    def add_commit_store(self, block_store):
+        _libexec(
+            "block_manager_add_commit_store",
+            self.pointer,
+            block_store.pointer)
 
     def put(self, branch):
         c_put_items = (ctypes.POINTER(_PutEntry) * len(branch))()
@@ -134,7 +133,10 @@ def _pylibexec(name, *args):
 
 
 def _exec(library, name, *args):
-    res = library.call(name, *args)
+    _check_error(library.call(name, *args))
+
+
+def _check_error(res):
     if res == ErrorCode.Success:
         return
 
@@ -156,86 +158,51 @@ def _exec(library, name, *args):
         raise Exception("There was an unknown error: {}".format(res))
 
 
-class _BlockIterator:
-
-    def __del__(self):
-        if self._c_iter_ptr:
-            _libexec("{}_drop".format(self.name), self._c_iter_ptr)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        if not self._c_iter_ptr:
-            raise StopIteration()
-
-        (vec_ptr, vec_len, vec_cap) = ffi.prepare_vec_result()
-
-        _libexec("{}_next".format(self.name),
-                 self._c_iter_ptr,
-                 ctypes.byref(vec_ptr),
-                 ctypes.byref(vec_len),
-                 ctypes.byref(vec_cap))
-
-        # Check if NULL
-        if not vec_ptr:
-            raise StopIteration()
-
-        payload = ffi.from_rust_vec(vec_ptr, vec_len, vec_cap)
-        block = Block()
-        block.ParseFromString(payload)
-
-        return block
-
-
-class _GetBlockIterator(_BlockIterator):
+class _GetBlockIterator(ffi.BlockIterator):
 
     name = "block_manager_get_iterator"
 
     def __init__(self, block_manager_ptr, block_ids):
+        super().__init__(_check_error)
 
         c_block_ids = (ctypes.c_char_p * len(block_ids))()
         for i, block_id in enumerate(block_ids):
             c_block_ids[i] = ctypes.c_char_p(block_id.encode())
 
-        self._c_iter_ptr = ctypes.c_void_p()
-
         _libexec("{}_new".format(self.name),
                  block_manager_ptr,
                  c_block_ids,
                  ctypes.c_size_t(len(block_ids)),
-                 ctypes.byref(self._c_iter_ptr))
+                 ctypes.byref(self.pointer))
 
 
-class _BranchDiffIterator(_BlockIterator):
+class _BranchDiffIterator(ffi.BlockIterator):
 
     name = "block_manager_branch_diff_iterator"
 
     def __init__(self, block_manager_ptr, tip, exclude):
+        super().__init__(_check_error)
 
         c_tip = ctypes.c_char_p(tip.encode())
         c_exclude = ctypes.c_char_p(exclude.encode())
 
-        self._c_iter_ptr = ctypes.c_void_p()
-
         _libexec("{}_new".format(self.name),
                  block_manager_ptr,
                  c_tip,
                  c_exclude,
-                 ctypes.byref(self._c_iter_ptr))
+                 ctypes.byref(self.pointer))
 
 
-class _BranchIterator(_BlockIterator):
+class _BranchIterator(ffi.BlockIterator):
 
     name = "block_manager_branch_iterator"
 
     def __init__(self, block_manager_ptr, tip):
+        super().__init__(_check_error)
 
         c_tip = ctypes.c_char_p(tip.encode())
 
-        self._c_iter_ptr = ctypes.c_void_p()
-
         _libexec("{}_new".format(self.name),
                  block_manager_ptr,
                  c_tip,
-                 ctypes.byref(self._c_iter_ptr))
+                 ctypes.byref(self.pointer))