From 4998ad7365e9f3b54a39f4d5b7ca1dfa927a8153 Mon Sep 17 00:00:00 2001 From: Wouter van Oortmerssen Date: Wed, 29 Jul 2015 17:49:02 -0700 Subject: [PATCH] Added support for adding new tables/strings to an existing FlatBuffer. As part of the reflection support. Change-Id: Ie0a8e233bca7dffa4cff7e564660035d97ff8902 Tested: on Linux. Bug:22637258 --- include/flatbuffers/flatbuffers.h | 17 +++++++ include/flatbuffers/reflection.h | 74 ++++++++++++++++++++++--------- tests/test.cpp | 40 ++++++++++++++++- 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/include/flatbuffers/flatbuffers.h b/include/flatbuffers/flatbuffers.h index 0b9cf58530c..c1e75bd90cb 100644 --- a/include/flatbuffers/flatbuffers.h +++ b/include/flatbuffers/flatbuffers.h @@ -293,11 +293,21 @@ template class Vector { const_iterator end() const { return const_iterator(Data(), size()); } // Change elements if you have a non-const pointer to this object. + // Scalars only. See reflection.h, and the documentation. void Mutate(uoffset_t i, T val) { assert(i < size()); WriteScalar(data() + i, val); } + // Change an element of a vector of tables (or strings). + // "val" points to the new table/string, as you can obtain from + // e.g. reflection::AddFlatBuffer(). + void MutateOffset(uoffset_t i, const uint8_t *val) { + assert(i < size()); + assert(sizeof(T) == sizeof(uoffset_t)); + WriteScalar(data() + i, val - (Data() + i * sizeof(uoffset_t))); + } + // The raw data in little endian format. Use with care. const uint8_t *Data() const { return reinterpret_cast(&length_ + 1); @@ -1035,6 +1045,13 @@ class Table { return true; } + bool SetPointer(voffset_t field, const uint8_t *val) { + auto field_offset = GetOptionalFieldOffset(field); + if (!field_offset) return false; + WriteScalar(data_ + field_offset, val - (data_ + field_offset)); + return true; + } + uint8_t *GetVTable() { return data_ - ReadScalar(data_); } bool CheckField(voffset_t field) const { diff --git a/include/flatbuffers/reflection.h b/include/flatbuffers/reflection.h index 60e2426cc14..9dd5ef96833 100644 --- a/include/flatbuffers/reflection.h +++ b/include/flatbuffers/reflection.h @@ -68,19 +68,19 @@ inline const String *GetFieldS(const Table &table, } // Get a field, if you know it's a vector. -template const Vector *GetFieldV(const Table &table, - const reflection::Field &field) { +template Vector *GetFieldV(const Table &table, + const reflection::Field &field) { assert(field.type()->base_type() == reflection::Vector && sizeof(T) == GetTypeSize(field.type()->element())); - return table.GetPointer *>(field.offset()); + return table.GetPointer *>(field.offset()); } // Get a field, if you know it's a table. -inline const Table *GetFieldT(const Table &table, - const reflection::Field &field) { +inline Table *GetFieldT(const Table &table, + const reflection::Field &field) { assert(field.type()->base_type() == reflection::Obj || field.type()->base_type() == reflection::Union); - return table.GetPointer(field.offset()); + return table.GetPointer(field.offset()); } // Get any field as a 64bit int, regardless of what it is (bool/int/float/str). @@ -227,27 +227,27 @@ inline void SetAnyFieldS(Table *table, const reflection::Field &field, // a vector into a relative offset, such that it is not affected by resizes. template class pointer_inside_vector { public: - pointer_inside_vector(const T *ptr, const std::vector &vec) - : offset_(reinterpret_cast(ptr) - - reinterpret_cast(vec.data())), + pointer_inside_vector(T *ptr, std::vector &vec) + : offset_(reinterpret_cast(ptr) - + reinterpret_cast(vec.data())), vec_(vec) {} - const T *operator*() const { - return reinterpret_cast( - reinterpret_cast(vec_.data()) + offset_); + T *operator*() const { + return reinterpret_cast( + reinterpret_cast(vec_.data()) + offset_); } - const T *operator->() const { + T *operator->() const { return operator*(); } void operator=(const pointer_inside_vector &piv); private: size_t offset_; - const std::vector &vec_; + std::vector &vec_; }; // Helper to create the above easily without specifying template args. -template pointer_inside_vector piv( - const T *ptr, const std::vector &vec) { +template pointer_inside_vector piv(T *ptr, + std::vector &vec) { return pointer_inside_vector(ptr, vec); } @@ -412,12 +412,10 @@ inline void SetString(const reflection::Schema &schema, const std::string &val, flatbuf->data() + sizeof(uoffset_t)); if (delta) { + // Clear the old string, since we don't want parts of it remaining. + memset(flatbuf->data() + start, 0, str->Length()); // Different size, we must expand (or contract). ResizeContext(schema, start, delta, flatbuf, root_table); - if (delta < 0) { - // Clear the old string, since we don't want parts of it remaining. - memset(flatbuf->data() + start, 0, str->Length()); - } } // Copy new data. Safe because we created the right amount of space. memcpy(flatbuf->data() + start, val.c_str(), val.size() + 1); @@ -438,6 +436,12 @@ void ResizeVector(const reflection::Schema &schema, uoffset_t newsize, T val, auto start = static_cast(vec_start + sizeof(uoffset_t) + sizeof(T) * vec->size()); if (delta_bytes) { + if (delta_elem < 0) { + // Clear elements we're throwing away, since some might remain in the + // buffer. + memset(flatbuf->data() + start + delta_elem * sizeof(T), 0, + -delta_elem * sizeof(T)); + } ResizeContext(schema, start, delta_bytes, flatbuf, root_table); WriteScalar(flatbuf->data() + vec_start, newsize); // Length field. // Set new elements to "val". @@ -453,6 +457,36 @@ void ResizeVector(const reflection::Schema &schema, uoffset_t newsize, T val, } } +// Adds any new data (in the form of a new FlatBuffer) to an existing +// FlatBuffer. This can be used when any of the above methods are not +// sufficient, in particular for adding new tables and new fields. +// This is potentially slightly less efficient than a FlatBuffer constructed +// in one piece, since the new FlatBuffer doesn't share any vtables with the +// existing one. +// The return value can now be set using Vector::MutateOffset or SetFieldT +// below. +inline const uint8_t *AddFlatBuffer(std::vector &flatbuf, + const uint8_t *newbuf, size_t newlen) { + // Align to sizeof(uoffset_t) past sizeof(largest_scalar_t) since we're + // going to chop off the root offset. + while ((flatbuf.size() & (sizeof(uoffset_t) - 1)) || + !(flatbuf.size() & (sizeof(largest_scalar_t) - 1))) { + flatbuf.push_back(0); + } + auto insertion_point = static_cast(flatbuf.size()); + // Insert the entire FlatBuffer minus the root pointer. + flatbuf.insert(flatbuf.end(), newbuf + sizeof(uoffset_t), + newbuf + newlen - sizeof(uoffset_t)); + auto root_offset = ReadScalar(newbuf) - sizeof(uoffset_t); + return flatbuf.data() + insertion_point + root_offset; +} + +inline bool SetFieldT(Table *table, const reflection::Field &field, + const uint8_t *val) { + assert(sizeof(uoffset_t) == GetTypeSize(field.type()->base_type())); + return table->SetPointer(field.offset(), val); +} + // Generic copying of tables from a FlatBuffer into a FlatBuffer builder. // Can be used to do any kind of merging/selecting you may want to do out // of existing buffers. Also useful to reconstruct a whole buffer if the diff --git a/tests/test.cpp b/tests/test.cpp index 46f9a2d57d9..eb7c487b226 100644 --- a/tests/test.cpp +++ b/tests/test.cpp @@ -370,8 +370,44 @@ void ReflectionTest(uint8_t *flatbuf, size_t length) { // rinventory still valid, so lets read from it. TEST_EQ(rinventory->Get(10), 50); - // Using reflection, we can also copy tables and other things out of - // other FlatBuffers into a new one, either part or whole. + // For reflection uses not covered already, there is a more powerful way: + // we can simply generate whatever object we want to add/modify in a + // FlatBuffer of its own, then add that to an existing FlatBuffer: + // As an example, let's add a string to an array of strings. + // First, find our field: + auto &testarrayofstring_field = *fields->LookupByKey("testarrayofstring"); + // Find the vector value: + auto rtestarrayofstring = flatbuffers::piv( + flatbuffers::GetFieldV>( + **rroot, testarrayofstring_field), + resizingbuf); + // It's a vector of 2 strings, to which we add one more, initialized to + // offset 0. + flatbuffers::ResizeVector>( + schema, 3, 0, *rtestarrayofstring, &resizingbuf); + // Here we just create a buffer that contans a single string, but this + // could also be any complex set of tables and other values. + flatbuffers::FlatBufferBuilder stringfbb; + stringfbb.Finish(stringfbb.CreateString("hank")); + // Add the contents of it to our existing FlatBuffer. + // We do this last, so the pointer doesn't get invalidated (since it is + // at the end of the buffer): + auto string_ptr = flatbuffers::AddFlatBuffer(resizingbuf, + stringfbb.GetBufferPointer(), + stringfbb.GetSize()); + // Finally, set the new value in the vector. + rtestarrayofstring->MutateOffset(2, string_ptr); + TEST_EQ_STR(rtestarrayofstring->Get(0)->c_str(), "bob"); + TEST_EQ_STR(rtestarrayofstring->Get(2)->c_str(), "hank"); + // As an additional test, also set it on the name field. + // Note: unlike the name change above, this just overwrites the offset, + // rather than changing the string in-place. + SetFieldT(*rroot, name_field, string_ptr); + TEST_EQ_STR(GetFieldS(**rroot, name_field)->c_str(), "hank"); + + // Using reflection, rather than mutating binary FlatBuffers, we can also copy + // tables and other things out of other FlatBuffers into a FlatBufferBuilder, + // either part or whole. flatbuffers::FlatBufferBuilder fbb; auto root_offset = flatbuffers::CopyTable(fbb, schema, *root_table, *flatbuffers::GetAnyRoot(flatbuf));