[3.2] Implement JSON Snapshot Reader

.gitmodules

@@ -18,6 +18,9 @@
 [submodule "libraries/eos-vm"]
 	path = libraries/eos-vm
 	url = https://github.com/eosnetworkfoundation/mandel-eos-vm
+[submodule "libraries/rapidjson"]
+	path = libraries/rapidjson
+	url = https://github.com/Tencent/rapidjson/
 [submodule "eosio-wasm-spec-tests"]
 	path = eosio-wasm-spec-tests
 	url = https://github.com/eosnetworkfoundation/mandel-wasm-spec-tests

libraries/chain/CMakeLists.txt

@@ -131,6 +131,7 @@ target_link_libraries( eosio_chain PUBLIC fc chainbase Logging IR WAST WASM Runt
 target_include_directories( eosio_chain
                             PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include"
                                    "${CMAKE_CURRENT_SOURCE_DIR}/../wasm-jit/Include"
+                                   "${CMAKE_CURRENT_SOURCE_DIR}/../rapidjson/include"
                             )
 
 add_library(eosio_chain_wrap INTERFACE )

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -5,6 +5,7 @@
 #include <fc/variant_object.hpp>
 #include <boost/core/demangle.hpp>
 #include <ostream>
+#include <memory>
 
 namespace eosio { namespace chain {
    /**
@@ -272,19 +273,13 @@ namespace eosio { namespace chain {
          read_section(detail::snapshot_section_traits<T>::section_name(), f);
       }
 
-      template<typename T>
-      bool has_section(const std::string& suffix = std::string()) {
-         return has_section(suffix + detail::snapshot_section_traits<T>::section_name());
-      }
-
       virtual void validate() const = 0;
 
       virtual void return_to_header() = 0;
 
       virtual ~snapshot_reader(){};
 
       protected:
-         virtual bool has_section( const std::string& section_name ) = 0;
          virtual void set_section( const std::string& section_name ) = 0;
          virtual bool read_row( detail::abstract_snapshot_row_reader& row_reader ) = 0;
          virtual bool empty( ) = 0;
@@ -313,7 +308,6 @@ namespace eosio { namespace chain {
          explicit variant_snapshot_reader(const fc::variant& snapshot);
 
          void validate() const override;
-         bool has_section( const string& section_name ) override;
          void set_section( const string& section_name ) override;
          bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
          bool empty ( ) override;
@@ -342,15 +336,29 @@ namespace eosio { namespace chain {
          std::streampos          header_pos;
          std::streampos          section_pos;
          uint64_t                row_count;
+   };
 
+   class ostream_json_snapshot_writer : public snapshot_writer {
+      public:
+         explicit ostream_json_snapshot_writer(std::ostream& snapshot);
+
+         void write_start_section( const std::string& section_name ) override;
+         void write_row( const detail::abstract_snapshot_row_writer& row_writer ) override;
+         void write_end_section() override;
+         void finalize();
+
+         static const uint32_t magic_number = 0x30510550;
+
+      private:
+         detail::ostream_wrapper snapshot;
+         uint64_t                row_count;
    };
 
    class istream_snapshot_reader : public snapshot_reader {
       public:
          explicit istream_snapshot_reader(std::istream& snapshot);
 
          void validate() const override;
-         bool has_section( const string& section_name ) override;
          void set_section( const string& section_name ) override;
          bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
          bool empty ( ) override;
@@ -366,6 +374,24 @@ namespace eosio { namespace chain {
          uint64_t       cur_row;
    };
 
+   class istream_json_snapshot_reader : public snapshot_reader {
+      public:
+         explicit istream_json_snapshot_reader(const fc::path& p);
+         ~istream_json_snapshot_reader();
+
+         void validate() const override;
+         void set_section( const string& section_name ) override;
+         bool read_row( detail::abstract_snapshot_row_reader& row_reader ) override;
+         bool empty ( ) override;
+         void clear_section() override;
+         void return_to_header() override;
+
+      private:
+         bool validate_section() const;
+
+         std::unique_ptr<struct istream_json_snapshot_reader_impl> impl;
+   };
+
    class integrity_hash_snapshot_writer : public snapshot_writer {
       public:
          explicit integrity_hash_snapshot_writer(fc::sha256::encoder&  enc);

libraries/chain/snapshot.cpp

@@ -1,6 +1,15 @@
 #include <eosio/chain/snapshot.hpp>
 #include <eosio/chain/exceptions.hpp>
 #include <fc/scoped_exit.hpp>
+#include <fc/io/json.hpp>
+
+#define RAPIDJSON_NAMESPACE eosio_rapidjson
+#include <rapidjson/document.h>
+#include <rapidjson/filereadstream.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+using namespace eosio_rapidjson;
 
 namespace eosio { namespace chain {
 
@@ -76,17 +85,6 @@ void variant_snapshot_reader::validate() const {
    }
 }
 
-bool variant_snapshot_reader::has_section( const string& section_name ) {
-   const auto& sections = snapshot["sections"].get_array();
-   for( const auto& section: sections ) {
-      if (section["name"].as_string() == section_name) {
-         return true;
-      }
-   }
-
-   return false;
-}
-
 void variant_snapshot_reader::set_section( const string& section_name ) {
    const auto& sections = snapshot["sections"].get_array();
    for( const auto& section: sections ) {
@@ -190,6 +188,45 @@ void ostream_snapshot_writer::finalize() {
    snapshot.write((char*)&end_marker, sizeof(end_marker));
 }
 
+ostream_json_snapshot_writer::ostream_json_snapshot_writer(std::ostream& snapshot)
+      :snapshot(snapshot)
+      ,row_count(0)
+{
+   snapshot << "{\n";
+   // write magic number
+   auto totem = magic_number;
+   snapshot << "\"magic_number\":" << fc::json::to_string(totem, fc::time_point::maximum()) << "\n";
+
+   // write version
+   auto version = current_snapshot_version;
+   snapshot << ",\"version\":" << fc::json::to_string(version, fc::time_point::maximum()) << "\n";
+}
+
+void ostream_json_snapshot_writer::write_start_section( const std::string& section_name )
+{
+   row_count = 0;
+   snapshot.inner << "," << fc::json::to_string(section_name, fc::time_point::maximum()) << ":{\n\"rows\":[\n";
+}
+
+void ostream_json_snapshot_writer::write_row( const detail::abstract_snapshot_row_writer& row_writer ) {
+   const auto yield = [&](size_t s) {};
+
+   if(row_count != 0) snapshot.inner << ",";
+   snapshot.inner << fc::json::to_string(row_writer.to_variant(), yield) << "\n";
+   ++row_count;
+}
+
+void ostream_json_snapshot_writer::write_end_section( ) {
+   snapshot.inner << "],\n\"num_rows\":" << row_count << "\n}\n";
+   row_count = 0;
+}
+
+void ostream_json_snapshot_writer::finalize() {
+   snapshot.inner << "}\n";
+   snapshot.inner.flush();
+}
+
+
 istream_snapshot_reader::istream_snapshot_reader(std::istream& snapshot)
 :snapshot(snapshot)
 ,header_pos(snapshot.tellg())
@@ -246,44 +283,6 @@ bool istream_snapshot_reader::validate_section() const {
    return true;
 }
 
-bool istream_snapshot_reader::has_section( const string& section_name ) {
-   auto restore_pos = fc::make_scoped_exit([this,pos=snapshot.tellg()](){
-      snapshot.seekg(pos);
-   });
-
-   const std::streamoff header_size = sizeof(ostream_snapshot_writer::magic_number) + sizeof(current_snapshot_version);
-
-   auto next_section_pos = header_pos + header_size;
-
-   while (true) {
-      snapshot.seekg(next_section_pos);
-      uint64_t section_size = 0;
-      snapshot.read((char*)&section_size,sizeof(section_size));
-      if (section_size == std::numeric_limits<uint64_t>::max()) {
-         break;
-      }
-
-      next_section_pos = snapshot.tellg() + std::streamoff(section_size);
-
-      uint64_t ignore = 0;
-      snapshot.read((char*)&ignore,sizeof(ignore));
-
-      bool match = true;
-      for(auto c : section_name) {
-         if(snapshot.get() != c) {
-            match = false;
-            break;
-         }
-      }
-
-      if (match && snapshot.get() == 0) {
-         return true;
-      }
-   }
-
-   return false;
-}
-
 void istream_snapshot_reader::set_section( const string& section_name ) {
    auto restore_pos = fc::make_scoped_exit([this,pos=snapshot.tellg()](){
       snapshot.seekg(pos);
@@ -346,6 +345,91 @@ void istream_snapshot_reader::return_to_header() {
    clear_section();
 }
 
+struct istream_json_snapshot_reader_impl {
+   uint64_t num_rows;
+   uint64_t cur_row;
+   eosio_rapidjson::Document doc;
+   std::string sec_name;
+};
+
+istream_json_snapshot_reader::~istream_json_snapshot_reader() = default;
+
+istream_json_snapshot_reader::istream_json_snapshot_reader(const fc::path& p)
+   : impl{new istream_json_snapshot_reader_impl{0, 0, {}, {}}}
+{
+   FILE* fp = fopen(p.string().c_str(), "rb");
+   EOS_ASSERT(fp, snapshot_exception, "Failed to JSON snapshot: ${file}", ("file", p));
+   auto close = fc::make_scoped_exit( [&fp]() { fclose( fp ); } );
+   char readBuffer[65536];
+   eosio_rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer));
+   impl->doc.ParseStream(is);
+}
+
+void istream_json_snapshot_reader::validate() const {
+   try {
+      // validate totem
+      auto expected_totem = ostream_json_snapshot_writer::magic_number;
+      EOS_ASSERT(impl->doc.HasMember("magic_number"), snapshot_exception, "magic_number section not found" );
+      auto actual_totem = impl->doc["magic_number"].GetInt();
+      EOS_ASSERT( actual_totem == expected_totem, snapshot_exception, "JSON snapshot has unexpected magic number" );
+
+      // validate version
+      auto expected_version = current_snapshot_version;
+      EOS_ASSERT(impl->doc.HasMember("version"), snapshot_exception, "version section not found" );
+      auto actual_version = impl->doc["version"].GetInt();
+      EOS_ASSERT( actual_version == expected_version, snapshot_exception,
+                  "JSON snapshot is an unsupported version.  Expected : ${expected}, Got: ${actual}",
+                  ("expected", expected_version)( "actual", actual_version ) );
+
+   } catch( const std::exception& e ) {  \
+      snapshot_exception fce(FC_LOG_MESSAGE( warn, "JSON snapshot validation threw IO exception (${what})",("what",e.what())));
+      throw fce;
+   }
+}
+
+bool istream_json_snapshot_reader::validate_section() const {
+   return true;
+}
+
+void istream_json_snapshot_reader::set_section( const string& section_name ) {
+   EOS_ASSERT( impl->doc.HasMember( section_name.c_str() ), snapshot_exception, "JSON snapshot has no section ${sec}", ("sec", section_name) );
+   EOS_ASSERT( impl->doc[section_name.c_str()].HasMember( "num_rows" ), snapshot_exception, "JSON snapshot ${sec} num_rows not found", ("sec", section_name) );
+   EOS_ASSERT( impl->doc[section_name.c_str()].HasMember( "rows" ), snapshot_exception, "JSON snapshot ${sec} rows not found", ("sec", section_name) );
+   EOS_ASSERT( impl->doc[section_name.c_str()]["rows"].IsArray(), snapshot_exception, "JSON snapshot ${sec} rows is not an array", ("sec_name", section_name) );
+
+   impl->sec_name = section_name;
+   impl->num_rows = impl->doc[section_name.c_str()]["num_rows"].GetInt();
+   ilog( "reading ${section_name}, num_rows: ${num_rows}", ("section_name", section_name)( "num_rows", impl->num_rows ) );
+}
+
+bool istream_json_snapshot_reader::read_row( detail::abstract_snapshot_row_reader& row_reader ) {
+   EOS_ASSERT( impl->cur_row < impl->num_rows, snapshot_exception, "JSON snapshot ${sect}'s cur_row ${cur_row} >= num_rows ${num_rows}",
+               ("sect_name", impl->sec_name)( "cur_row", impl->cur_row )( "num_rows", impl->num_rows ) );
+
+   const eosio_rapidjson::Value& rows = impl->doc[impl->sec_name.c_str()]["rows"];
+   eosio_rapidjson::StringBuffer buffer;
+   eosio_rapidjson::Writer<eosio_rapidjson::StringBuffer> writer( buffer );
+   rows[impl->cur_row].Accept( writer );
+
+   const auto& row = fc::json::from_string( buffer.GetString() );
+   row_reader.provide( row );
+   return ++impl->cur_row < impl->num_rows;
+}
+
+bool istream_json_snapshot_reader::empty ( ) {
+   return impl->num_rows == 0;
+}
+
+void istream_json_snapshot_reader::clear_section() {
+   impl->num_rows = 0;
+   impl->cur_row = 0;
+   impl->sec_name = "";
+}
+
+void istream_json_snapshot_reader::return_to_header() {
+   clear_section();
+}
+
 integrity_hash_snapshot_writer::integrity_hash_snapshot_writer(fc::sha256::encoder& enc)
 :enc(enc)
 {

libraries/testing/include/eosio/testing/snapshot_suites.hpp

@@ -106,5 +106,62 @@ struct buffered_snapshot_suite {
    }
 };
 
-using snapshot_suites = boost::mpl::list<variant_snapshot_suite, buffered_snapshot_suite>;
+
+struct json_snapshot_suite {
+   using writer_t = ostream_json_snapshot_writer;
+   using reader_t = istream_json_snapshot_reader;
+   using write_storage_t = std::ostringstream;
+   using snapshot_t = std::string;
+   using read_storage_t = std::istringstream;
+
+   struct writer : public writer_t {
+      writer( const std::shared_ptr<write_storage_t>& storage )
+      :writer_t(*storage)
+      ,storage(storage)
+      {
+
+      }
+
+      std::shared_ptr<write_storage_t> storage;
+   };
+
+   struct reader : public reader_t {
+      explicit reader(const fc::path& p)
+      :reader_t(p)
+      {}
+      ~reader() {
+         remove("temp.bin.json");
+      }
+   };
+
+
+   static auto get_writer() {
+      return std::make_shared<writer>(std::make_shared<write_storage_t>());
+   }
+
+   static auto finalize(const std::shared_ptr<writer>& w) {
+      w->finalize();
+      return w->storage->str();
+   }
+
+   static auto get_reader( const snapshot_t& buffer) {
+      std::ofstream fs("temp.bin.json");
+      fs << buffer;
+      fs.close();
+      fc::path p("temp.bin.json");
+      return std::make_shared<reader>(p);
+   }
+
+   static snapshot_t load_from_file(const std::string& filename) {
+      snapshot_input_file<snapshot::json_snapshot> file(filename);
+      return file.read_as_string();
+   }
+
+   static void write_to_file( const std::string& basename, const snapshot_t& snapshot ) {
+      snapshot_output_file<snapshot::json_snapshot> file(basename);
+      file.write<snapshot_t>(snapshot);
+   }
+};
+
+using snapshot_suites = boost::mpl::list<variant_snapshot_suite, buffered_snapshot_suite, json_snapshot_suite>;