From e54e05e738ec18bd06799c381ca3ea09bfdf3161 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Fri, 31 Aug 2018 15:05:27 +1000 Subject: [PATCH 01/19] add Julia support to flatbuffers --- CMakeLists.txt | 1 + docs/source/Compiler.md | 2 + docs/source/FlatBuffers.md | 2 + docs/source/JuliaUsage.md | 57 ++ docs/source/Support.md | 36 +- docs/source/Tutorial.md | 134 ++- docs/source/doxyfile | 1 + docs/source/doxygen_layout.xml | 2 + grpc/README.md | 11 + include/flatbuffers/idl.h | 48 +- samples/Monster.jl | 7 + samples/MyGame/MyGame.jl | 7 + samples/MyGame/Sample/Color.jl | 10 + samples/MyGame/Sample/Equipment.jl | 11 + samples/MyGame/Sample/Monster.jl | 27 + samples/MyGame/Sample/Sample.jl | 11 + samples/MyGame/Sample/Vec3.jl | 15 + samples/MyGame/Sample/Weapon.jl | 17 + samples/julia_sample.sh | 48 ++ samples/sample_binary.jl | 49 ++ src/flatc_main.cpp | 4 + src/idl_gen_cpp.cpp | 2 +- src/idl_gen_general.cpp | 4 +- src/idl_gen_go.cpp | 2 +- src/idl_gen_julia.cpp | 815 ++++++++++++++++++ src/idl_gen_lobster.cpp | 2 +- src/idl_gen_lua.cpp | 2 +- src/idl_gen_php.cpp | 2 +- src/idl_gen_python.cpp | 2 +- src/idl_gen_rust.cpp | 4 +- src/idl_gen_text.cpp | 6 +- src/idl_parser.cpp | 19 +- tests/MonsterTest.jl | 7 + tests/MyGame/Example/Ability.jl | 14 + tests/MyGame/Example/AnyAmbiguousAliases.jl | 13 + tests/MyGame/Example/AnyUniqueAliases.jl | 14 + tests/MyGame/Example/Any_.jl | 14 + tests/MyGame/Example/Color.jl | 10 + tests/MyGame/Example/Example.jl | 18 + tests/MyGame/Example/Monster.jl | 84 ++ tests/MyGame/Example/Referrable.jl | 16 + tests/MyGame/Example/Stat.jl | 18 + tests/MyGame/Example/Test.jl | 14 + .../MyGame/Example/TestSimpleTableWithEnum.jl | 16 + tests/MyGame/Example/TypeAliases.jl | 29 + tests/MyGame/Example/Vec3.jl | 18 + tests/MyGame/Example2/Example2.jl | 7 + tests/MyGame/Example2/Monster.jl | 12 + tests/MyGame/InParentNamespace.jl | 12 + tests/MyGame/MyGame.jl | 9 + tests/generate_code.sh | 6 +- tests/namespace_test/NamespaceA/NamespaceA.jl | 9 + .../NamespaceA/NamespaceB/EnumInNestedNS.jl | 10 + .../NamespaceA/NamespaceB/NamespaceB.jl | 9 + .../NamespaceA/NamespaceB/StructInNestedNS.jl | 14 + .../NamespaceA/NamespaceB/TableInNestedNS.jl | 16 + .../NamespaceA/SecondTableInA.jl | 17 + .../NamespaceA/TableInFirstNS.jl | 19 + tests/namespace_test/NamespaceC/NamespaceC.jl | 7 + tests/namespace_test/NamespaceC/TableInC.jl | 18 + tests/namespace_test/NamespaceTest1.jl | 7 + tests/namespace_test/NamespaceTest2.jl | 8 + tests/union_vector/UnionVector.jl | 11 + tests/union_vector/UnionVector/Attacker.jl | 16 + tests/union_vector/UnionVector/BookReader.jl | 13 + tests/union_vector/UnionVector/Character.jl | 16 + tests/union_vector/UnionVector/Movie.jl | 21 + tests/union_vector/UnionVector/Rapunzel.jl | 13 + 68 files changed, 1851 insertions(+), 64 deletions(-) create mode 100644 docs/source/JuliaUsage.md create mode 100644 samples/Monster.jl create mode 100644 samples/MyGame/MyGame.jl create mode 100644 samples/MyGame/Sample/Color.jl create mode 100644 samples/MyGame/Sample/Equipment.jl create mode 100644 samples/MyGame/Sample/Monster.jl create mode 100644 samples/MyGame/Sample/Sample.jl create mode 100644 samples/MyGame/Sample/Vec3.jl create mode 100644 samples/MyGame/Sample/Weapon.jl create mode 100755 samples/julia_sample.sh create mode 100644 samples/sample_binary.jl create mode 100644 src/idl_gen_julia.cpp create mode 100644 tests/MonsterTest.jl create mode 100644 tests/MyGame/Example/Ability.jl create mode 100644 tests/MyGame/Example/AnyAmbiguousAliases.jl create mode 100644 tests/MyGame/Example/AnyUniqueAliases.jl create mode 100644 tests/MyGame/Example/Any_.jl create mode 100644 tests/MyGame/Example/Color.jl create mode 100644 tests/MyGame/Example/Example.jl create mode 100644 tests/MyGame/Example/Monster.jl create mode 100644 tests/MyGame/Example/Referrable.jl create mode 100644 tests/MyGame/Example/Stat.jl create mode 100644 tests/MyGame/Example/Test.jl create mode 100644 tests/MyGame/Example/TestSimpleTableWithEnum.jl create mode 100644 tests/MyGame/Example/TypeAliases.jl create mode 100644 tests/MyGame/Example/Vec3.jl create mode 100644 tests/MyGame/Example2/Example2.jl create mode 100644 tests/MyGame/Example2/Monster.jl create mode 100644 tests/MyGame/InParentNamespace.jl create mode 100644 tests/MyGame/MyGame.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceA.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/SecondTableInA.jl create mode 100644 tests/namespace_test/NamespaceA/TableInFirstNS.jl create mode 100644 tests/namespace_test/NamespaceC/NamespaceC.jl create mode 100644 tests/namespace_test/NamespaceC/TableInC.jl create mode 100644 tests/namespace_test/NamespaceTest1.jl create mode 100644 tests/namespace_test/NamespaceTest2.jl create mode 100644 tests/union_vector/UnionVector.jl create mode 100644 tests/union_vector/UnionVector/Attacker.jl create mode 100644 tests/union_vector/UnionVector/BookReader.jl create mode 100644 tests/union_vector/UnionVector/Character.jl create mode 100644 tests/union_vector/UnionVector/Movie.jl create mode 100644 tests/union_vector/UnionVector/Rapunzel.jl diff --git a/CMakeLists.txt b/CMakeLists.txt index 389596e4f3f..9aa2356678a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,7 @@ set(FlatBuffers_Compiler_SRCS src/idl_gen_general.cpp src/idl_gen_go.cpp src/idl_gen_js_ts.cpp + src/idl_gen_julia.cpp src/idl_gen_php.cpp src/idl_gen_python.cpp src/idl_gen_lobster.cpp diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md index 420019cdce2..f42663570fa 100644 --- a/docs/source/Compiler.md +++ b/docs/source/Compiler.md @@ -45,6 +45,8 @@ For any schema input files, one or more generators can be specified: - `--rust`, `-r` : Generate Rust code. +- `--julia`: Generate Julia code. + For any data input files: - `--binary`, `-b` : If data is contained in this file, generate a diff --git a/docs/source/FlatBuffers.md b/docs/source/FlatBuffers.md index 7cc93b92c44..967ee3f3328 100644 --- a/docs/source/FlatBuffers.md +++ b/docs/source/FlatBuffers.md @@ -146,6 +146,8 @@ sections provide a more in-depth usage guide. own programs. - How to [use the generated Rust code](@ref flatbuffers_guide_use_rust) in your own programs. +- How to [use the generated Julia code](@ref flatbuffers_guide_use_julia) in your + own programs. - [Support matrix](@ref flatbuffers_support) for platforms/languages/features. - Some [benchmarks](@ref flatbuffers_benchmarks) showing the advantage of using FlatBuffers. diff --git a/docs/source/JuliaUsage.md b/docs/source/JuliaUsage.md new file mode 100644 index 00000000000..cd2a80c7905 --- /dev/null +++ b/docs/source/JuliaUsage.md @@ -0,0 +1,57 @@ +Use in Julia {#flatbuffers_guide_use_julia} +============== +## Before you get started +Before diving into the FlatBuffers usage in Julia, it should be noted that the +[Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to general +FlatBuffers usage in all of the supported languages (including Julia). This +page is designed to cover the nuances of FlatBuffers usage, specific to +Julia. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Julia package +The `flatc` backend for Julia makes heavy use of the [FlatBuffers.jl package](https://github.com/JuliaData/FlatBuffers.jl). +Documentation for this package may be found [here](http://juliadata.github.io/FlatBuffers.jl/stable). +You can also browse the source code on the [FlatBuffers.jl GitHub page](https://github.com/JuliaData/FlatBuffers.jl). + +## Testing the FlatBuffers Julia library +The code to test the Julia library can be found at [FlatBuffers.jl/test/flatc.jl](https://github.com/JuliaData/FlatBuffers.jl/tree/master/test/flatc.jl). To run the tests, run `julia test/flatc.jl` in the top-level directory. +To obtain Julia itself, go to the [Julia homepage](http://julialang.org) +for binaries or source code for your platform. + +## Using the FlatBuffers Julia library +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Julia.* + +There is support for both reading and writing FlatBuffers in Julia. +To use FlatBuffers in your own code, first generate Julia modules from your +schema with the `--julia` option to `flatc`. Then you can include both +FlatBuffers and the generated code to read or write a FlatBuffer. + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.julia} + import FlatBuffers + + include("MyGame/MyGame.jl") + import .MyGame.Example.Monster + + monster = open("monsterdata_test.mon", "r") do f Monster(f) end +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Now you can access values like this: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.julia} + hp = monster.hp + pos = monster.pos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +More detailed documentation for using the FlatBuffers Julia package +may be found [here](http://juliadata.github.io/FlatBuffers.jl/stable). + +## Speed +The FlatBuffers Julia package has not been thoroughly optimized for speed. +For example, instead of referring to memory in-place, values are instead +loaded with `unsafe_load`. It does however support deduplication of vtables, which +saves roughly 30% of time spent writing, when compared to writing objects +with duplicate vtables. + +
\ No newline at end of file diff --git a/docs/source/Support.md b/docs/source/Support.md index c8ac7f7e89d..fd2f01f1e72 100644 --- a/docs/source/Support.md +++ b/docs/source/Support.md @@ -18,23 +18,23 @@ In general: NOTE: this table is a start, it needs to be extended. -Feature | C++ | Java | C# | Go | Python | JS | TS | C | PHP | Dart | Lobster | Rust ------------------------------- | ------ | ------ | ------ | ------ | ------ | --------- | --------- | ------ | --- | ------- | ------- | ---- -Codegen for all basic features | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | WiP | Yes | Yes | Yes -JSON parsing | Yes | No | No | No | No | No | No | Yes | No | No | Yes | No -Simple mutation | Yes | Yes | Yes | Yes | No | No | No | No | No | No | No | No -Reflection | Yes | No | No | No | No | No | No | Basic | No | No | No | No -Buffer verifier | Yes | No | No | No | No | No | No | Yes | No | No | No | No -Testing: basic | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | ? | Yes | Yes | Yes -Testing: fuzz | Yes | No | No | Yes | Yes | No | No | No | ? | No | No | Yes -Performance: | Superb | Great | Great | Great | Ok | ? | ? | Superb | ? | ? | Great | Superb -Platform: Windows | VS2010 | Yes | Yes | ? | ? | ? | Yes | VS2010 | ? | Yes | Yes | Yes -Platform: Linux | GCC282 | Yes | ? | Yes | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes -Platform: OS X | Xcode4 | ? | ? | ? | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes -Platform: Android | NDK10d | Yes | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? -Platform: iOS | ? | ? | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? -Engine: Unity | ? | ? | Yes | ? | ? | ? | ? | ? | ? | ? | No | ? -Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | evanw/ev* | kr* | mik* | ch* | dnfield | aard* | rw +Feature | C++ | Java | C# | Go | Python | JS | TS | C | PHP | Dart | Lobster | Rust | Julia | +------------------------------ | ------ | ------ | ------ | ------ | ------ | --------- | --------- | ------ | --- | ------- | ------- | ---- | ---- | +Codegen for all basic features | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | WiP | Yes | Yes | Yes | Yes | +JSON parsing | Yes | No | No | No | No | No | No | Yes | No | No | Yes | No | No | +Simple mutation | Yes | Yes | Yes | Yes | No | No | No | No | No | No | No | No | No | +Reflection | Yes | No | No | No | No | No | No | Basic | No | No | No | No | No | +Buffer verifier | Yes | No | No | No | No | No | No | Yes | No | No | No | No | No | +Testing: basic | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Testing: fuzz | Yes | No | No | Yes | Yes | No | No | No | ? | No | No | Yes | Yes | +Performance: | Superb | Great | Great | Great | Ok | ? | ? | Superb | ? | ? | Great | Superb | ? | +Platform: Windows | VS2010 | Yes | Yes | ? | ? | ? | Yes | VS2010 | ? | Yes | Yes | Yes | Yes | +Platform: Linux | GCC282 | Yes | ? | Yes | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Platform: OS X | Xcode4 | ? | ? | ? | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Platform: Android | NDK10d | Yes | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | ? | +Platform: iOS | ? | ? | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | ? | +Engine: Unity | ? | ? | Yes | ? | ? | ? | ? | ? | ? | ? | No | ? | ? | +Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | evanw/ev* | kr* | mik* | ch* | dnfield | aard* | rw | jq*/rk* | * aard = aardappel (previously: gwvo) * ev = evolutional @@ -42,5 +42,7 @@ Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | ev * mik = mikkelfj * ch = chobie * kr = krojew + * jq = quinnj + * rk = rkat
diff --git a/docs/source/Tutorial.md b/docs/source/Tutorial.md index f2eaa2ea72d..2344a9c99f0 100644 --- a/docs/source/Tutorial.md +++ b/docs/source/Tutorial.md @@ -34,6 +34,7 @@ Please select your desired language for our quest: Lua Lobster Rust + Julia \endhtmlonly @@ -148,7 +149,9 @@ For your chosen language, please cross-reference with:
[sample_binary.rs](https://github.com/google/flatbuffers/blob/master/samples/sample_binary.rs)
- +
+[sample_binary.jl](https://github.com/google/flatbuffers/blob/master/samples/sample_binary.jl) +
## Writing the Monsters' FlatBuffer Schema @@ -353,6 +356,12 @@ Please be aware of the difference between `flatc` and `flatcc` tools. ./../flatc --rust monster.fbs ~~~ +
+~~~{.sh} + cd flatbuffers/sample + ./../flatc --julia monster.fbs +~~~ +
For a more complete guide to using the `flatc` compiler, please read the [Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) @@ -504,6 +513,18 @@ The first step is to import/include the library, generated files, etc. Weapon, WeaponArgs}; ~~~ +
+~~~{.jl} + import FlatBuffers + + # Generated by `flatc`. + import .MyGame.Sample.Color + import .MyGame.Sample.Equipment + import .MyGame.Sample.Monster + import .MyGame.Sample.Vec3 + import .MyGame.Sample.Weapon +~~~ +
Now we are ready to start building some buffers. In order to start, we need to create an instance of the `FlatBufferBuilder`, which will contain the buffer @@ -602,6 +623,13 @@ which will grow automatically if needed: let mut builder = flatbuffers::FlatBufferBuilder::new_with_capacity(1024); ~~~ +
+~~~{julia} + # types are not built up gradually in Julia, + # instead they are written to the buffer all at once + buf = IOBuffer() +~~~ +
After creating the `builder`, we can start serializing our data. Before we make our `orc` Monster, lets create some `Weapon`s: a `Sword` and an `Axe`. @@ -838,6 +866,12 @@ our `orc` Monster, lets create some `Weapon`s: a `Sword` and an `Axe`. }); ~~~ +
+~~~{.jl} + sword = Weapon(;name="Sword", damage=3) + axe = Weapon(;name="Axe", damage=5) +~~~ +
Now let's create our monster, the `orc`. For this `orc`, lets make him `red` with rage, positioned at `(1.0, 2.0, 3.0)`, and give him @@ -1018,6 +1052,20 @@ traversal. This is generally easy to do on any tree structures. let inventory = builder.create_vector(&[0u8, 1, 2, 3, 4, 5, 6, 7, 8, 9]); ~~~ +
+~~~{.jl} + # we don't need to incrementally build things up here, just make the monster + orc = Monster(; + name="Orc", + pos=Vec3(1.0, 2.0, 3.0), + hp=300, + inventory=collect(1:10), + color=MyGame.Sample.ColorRed, + weapons=[sword, axe], + equipped=axe + ) +~~~ +
We serialized two built-in data types (`string` and `vector`) and captured their return values. These values are offsets into the serialized data, @@ -1280,6 +1328,11 @@ for the `path` field above: // let path = builder.create_vector(&[&x, &y]); ~~~ +
+~~~{.jl} + path = Vec3[Vec3(1.0, 2.0, 3.0), Vec3(4.0, 5.0, 6.0)] +~~~ +
We have now serialized the non-scalar components of the orc, so we can serialize the monster itself: @@ -1536,6 +1589,11 @@ can serialize the monster itself: }); ~~~ +
+~~~{.rs} + FlatBuffers.serialize(io, orc) +~~~ +
Note how we create `Vec3` struct in-line in the table. Unlike tables, structs are simple combinations of scalars that are always stored inline, just like @@ -1698,6 +1756,11 @@ Here is a repetition these lines, to help highlight them more clearly: monster_builder.add_equipped(axe.as_union_value()); // Union data ~~~ +
+ ~~~{.jl} + # this section is unnecessary in Julia + ~~~ +
After you have created your buffer, you will have the offset to the root of the data in the `orc` variable, so you can finish the buffer by calling the @@ -1787,6 +1850,11 @@ appropriate `finish` method. builder.finish(orc, None); ~~~ +
+~~~{.rs} + # this section is unnecessary in Julia +~~~ +
The buffer is now ready to be stored somewhere, sent over the network, be compressed, or whatever you'd like to do with it. You can access the buffer @@ -1903,6 +1971,11 @@ like so: let buf = builder.finished_data(); // Of type `&[u8]` ~~~ +
+~~~{.jl} + buf = take!(io) +~~~ +
Now you can write the bytes to a file, send them over the network.. @@ -2051,6 +2124,17 @@ import './monster_my_game.sample_generated.dart' as myGame; Weapon, WeaponArgs}; ~~~ +
+~~~{.py} + import FlatBuffers + + # Generated by `flatc`. + import .MyGame.Sample.Any + import .MyGame.Sample.Color + import .MyGame.Sample.Monster + import .MyGame.Sample.Vec3 +~~~ +
Then, assuming you have a buffer of bytes received from disk, network, etc., you can create start accessing the buffer like so: @@ -2186,6 +2270,13 @@ myGame.Monster monster = new myGame.Monster(data); let monster = get_root_as_monster(buf); ~~~ +
+~~~{.jl} + # where x is either an IO or an AbstractVector{UInt8} containing the data + monster = Monster(x) +~~~ +
+ If you look in the generated files from the schema compiler, you will see it generated accessors for all non-`deprecated` fields. For example: @@ -2286,6 +2377,13 @@ accessors for all non-`deprecated` fields. For example: let name = monster.name(); ~~~ +
+~~~{.rs} + hp = monster.hp + mana = monster.mana + name = monster.name +~~~ +
These should hold `300`, `150`, and `"Orc"` respectively. @@ -2403,6 +2501,14 @@ To access sub-objects, in the case of our `pos`, which is a `Vec3`: let z = pos.z(); ~~~ +
+~~~{.jl} + pos = monster.pos + x = pos.x + y = pos.y + z = pos.z +~~~ +
`x`, `y`, and `z` will contain `1.0`, `2.0`, and `3.0`, respectively. @@ -2497,6 +2603,11 @@ FlatBuffers `vector`. let third_item = inv[2]; ~~~ +
+~~~{.jl} + third_item = monster.inventory[3] +~~~ +
For `vector`s of `table`s, you can access the elements like any other vector, except your need to handle the result as a FlatBuffer `table`: @@ -2603,6 +2714,13 @@ except your need to handle the result as a FlatBuffer `table`: let second_weapon_damage = wep2.damage(); ~~~ +
+~~~{.jl} + # no tables are needed in Julia + second_weapon_name = monster.weapons[2].name + second_weapon_damage = monster.weapons[2].damage +~~~ +
Last, we can access our `Equipped` FlatBuffer `union`. Just like when we created the `union`, we need to get both parts of the `union`: the type and the data. @@ -2776,6 +2894,11 @@ We can access the type to dynamically cast the data as needed (since the let weapon_damage = equipped.damage(); ~~~ +
+~~~{.jl} + equipped = monster.equipped +~~~ +
## Mutating FlatBuffers @@ -2871,6 +2994,11 @@ mutators like so: ~~~ +
+~~~{.rs} + +~~~ +
We use the somewhat verbose term `mutate` instead of `set` to indicate that this is a special use case, not to be confused with the default way of constructing @@ -2997,5 +3125,7 @@ For your chosen language, see:
[Use in Rust](@ref flatbuffers_guide_use_rust)
- +
+[Use in Julia](@ref flatbuffers_guide_use_julia) +

diff --git a/docs/source/doxyfile b/docs/source/doxyfile index 6ba3c108cc9..9b1d6163eca 100644 --- a/docs/source/doxyfile +++ b/docs/source/doxyfile @@ -761,6 +761,7 @@ INPUT = "FlatBuffers.md" \ "LuaUsage.md" \ "LobsterUsage.md" \ "RustUsage.md" \ + "JuliaUsage.md" \ "Support.md" \ "Benchmarks.md" \ "WhitePaper.md" \ diff --git a/docs/source/doxygen_layout.xml b/docs/source/doxygen_layout.xml index a2325075f2d..5f9dc73aa51 100644 --- a/docs/source/doxygen_layout.xml +++ b/docs/source/doxygen_layout.xml @@ -47,6 +47,8 @@ title="Use in Lobster"/> + diff --git a/grpc/README.md b/grpc/README.md index 544651d6024..b9ffd47a60e 100644 --- a/grpc/README.md +++ b/grpc/README.md @@ -28,4 +28,15 @@ the GRPC libraries for this to compile. This test will build using the 1. `ln -s ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.6 ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1` 2. `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${GRPC_INSTALL_PATH}/lib` +3. `make test ARGS=-V` + +### macOS + +1. Fix the dynamic library paths in grpc ``` +for l in ${GRPC_INSTALL_PATH}/lib/*.dylib; do + for dep in ${GRPC_INSTALL_PATH}/lib/*.dylib; do + install_name_tool -change ${dep##*/} $dep $l; done + done +done``` +2. `for l in libgrpc_unsecure libgrpc++_unsecure libgpr; do install_name_tool -change $l.dylib ${GRPC_INSTALL_PATH}/lib/$l.dylib grpctest; done` 3. `make test ARGS=-V` diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 18e1db9d54b..5fc30cbf81e 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -47,24 +47,24 @@ namespace flatbuffers { // of type tokens. // clang-format off #define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \ - TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8) \ - TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8) /* begin scalar/int */ \ - TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool) \ - TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8) \ - TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8) \ - TD(SHORT, "short", int16_t, short, int16, short, int16, i16) \ - TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16) \ - TD(INT, "int", int32_t, int, int32, int, int32, i32) \ - TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32) \ - TD(LONG, "long", int64_t, long, int64, long, int64, i64) \ - TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64) /* end int */ \ - TD(FLOAT, "float", float, float, float32, float, float32, f32) /* begin float */ \ - TD(DOUBLE, "double", double, double, float64, double, float64, f64) /* end float/scalar */ + TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8, UInt8) \ + TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8, UInt8) /* begin scalar/int */ \ + TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool, Bool) \ + TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8, Int8) \ + TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8, UInt8) \ + TD(SHORT, "short", int16_t, short, int16, short, int16, i16, Int16) \ + TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16, UInt16) \ + TD(INT, "int", int32_t, int, int32, int, int32, i32, Int32) \ + TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32, UInt32) \ + TD(LONG, "long", int64_t, long, int64, long, int64, i64, Int64) \ + TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64, UInt64) /* end int */ \ + TD(FLOAT, "float", float, float, float32, float, float32, f32, Float32) /* begin float */ \ + TD(DOUBLE, "double", double, double, float64, double, float64, f64, Float64) /* end float/scalar */ #define FLATBUFFERS_GEN_TYPES_POINTER(TD) \ - TD(STRING, "string", Offset, int, int, StringOffset, int, unused) \ - TD(VECTOR, "", Offset, int, int, VectorOffset, int, unused) \ - TD(STRUCT, "", Offset, int, int, int, int, unused) \ - TD(UNION, "", Offset, int, int, int, int, unused) + TD(STRING, "string", Offset, int, int, StringOffset, int, unused, Int) \ + TD(VECTOR, "", Offset, int, int, VectorOffset, int, unused, Int) \ + TD(STRUCT, "", Offset, int, int, int, int, unused, Int) \ + TD(UNION, "", Offset, int, int, int, int, unused, Int) // The fields are: // - enum @@ -75,13 +75,14 @@ namespace flatbuffers { // - C# / .Net type. // - Python type. // - Rust type. +// - Julia type. // using these macros, we can now write code dealing with types just once, e.g. /* switch (type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ // do something specific to CTYPE here FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -99,14 +100,14 @@ __extension__ // Stop GCC complaining about trailing comma with -Wpendantic. #endif enum BaseType { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ BASE_TYPE_ ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPEL) \ static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \ "define largest_scalar_t as " #CTYPE); FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -427,6 +428,7 @@ struct IDLOptions { kLua = 1 << 12, kLobster = 1 << 13, kRust = 1 << 14, + kJulia = 1 << 15, kMAX }; @@ -842,6 +844,12 @@ extern bool GeneratePython(const Parser &parser, const std::string &path, const std::string &file_name); +// Generate Julia files from the definitions in the Parser object. +// See idl_gen_julia.cpp. +extern bool GenerateJulia(const Parser &parser, + const std::string &path, + const std::string &file_name); + // Generate Lobster files from the definitions in the Parser object. // See idl_gen_lobster.cpp. extern bool GenerateLobster(const Parser &parser, diff --git a/samples/Monster.jl b/samples/Monster.jl new file mode 100644 index 00000000000..aac20e5f01b --- /dev/null +++ b/samples/Monster.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Monster + +module Monster + include("MyGame/MyGame.jl") +end diff --git a/samples/MyGame/MyGame.jl b/samples/MyGame/MyGame.jl new file mode 100644 index 00000000000..6379ae45d93 --- /dev/null +++ b/samples/MyGame/MyGame.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +module MyGame + include("Sample/Sample.jl") +end diff --git a/samples/MyGame/Sample/Color.jl b/samples/MyGame/Sample/Color.jl new file mode 100644 index 00000000000..e6108b38686 --- /dev/null +++ b/samples/MyGame/Sample/Color.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +@enum Color::Int8 begin + ColorRed = 0 + ColorGreen = 1 + ColorBlue = 2 +end + diff --git a/samples/MyGame/Sample/Equipment.jl b/samples/MyGame/Sample/Equipment.jl new file mode 100644 index 00000000000..dfeb2e26d37 --- /dev/null +++ b/samples/MyGame/Sample/Equipment.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@UNION(Equipment, ( + Nothing, + Weapon, +)) + diff --git a/samples/MyGame/Sample/Monster.jl b/samples/MyGame/Sample/Monster.jl new file mode 100644 index 00000000000..b5904138ac7 --- /dev/null +++ b/samples/MyGame/Sample/Monster.jl @@ -0,0 +1,27 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Monster + pos::Union{Vec3, Nothing} = nothing + mana::Int16 = 150 + hp::Int16 = 100 + name::Union{String, Nothing} = nothing + inventory::Union{Vector{UInt8}, Nothing} = nothing + color::Color = 2 + weapons::Union{Vector{Weapon}, Nothing} = nothing + equipped_type::UInt8 = 0 + equipped::Equipment = nothing +end +FlatBuffers.@ALIGN(Monster, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000E, 0x00000010, 0x00000012, 0x00000014, + 0x00000016 +] +FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/samples/MyGame/Sample/Sample.jl b/samples/MyGame/Sample/Sample.jl new file mode 100644 index 00000000000..07e66c9dfd2 --- /dev/null +++ b/samples/MyGame/Sample/Sample.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +module Sample + include("Color.jl") + include("Weapon.jl") + include("Equipment.jl") + include("Vec3.jl") + include("Monster.jl") +end diff --git a/samples/MyGame/Sample/Vec3.jl b/samples/MyGame/Sample/Vec3.jl new file mode 100644 index 00000000000..57e72494a7a --- /dev/null +++ b/samples/MyGame/Sample/Vec3.jl @@ -0,0 +1,15 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 +end +FlatBuffers.@ALIGN(Vec3, 4) + +Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) +Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/samples/MyGame/Sample/Weapon.jl b/samples/MyGame/Sample/Weapon.jl new file mode 100644 index 00000000000..c94e9b14ca0 --- /dev/null +++ b/samples/MyGame/Sample/Weapon.jl @@ -0,0 +1,17 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Weapon + name::Union{String, Nothing} = nothing + damage::Int16 = 0 +end +FlatBuffers.@ALIGN(Weapon, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Weapon} = [ + 0x00000004, 0x00000006 +] + +Weapon(buf::AbstractVector{UInt8}) = FlatBuffers.read(Weapon, buf) +Weapon(io::IO) = FlatBuffers.deserialize(io, Weapon) diff --git a/samples/julia_sample.sh b/samples/julia_sample.sh new file mode 100755 index 00000000000..7ff98b6ac80 --- /dev/null +++ b/samples/julia_sample.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Note: This script runs on Mac and Linux. It requires `julia` to be installed +# and `flatc` to be built (using `cmake` in the root directory). + +sampledir=$(cd $(dirname $BASH_SOURCE) && pwd) +rootdir=$(cd $sampledir/.. && pwd) +currentdir=$(pwd) + +if [[ "$sampledir" != "$currentdir" ]]; then + echo Error: This script must be run from inside the $sampledir directory. + echo You executed it from the $currentdir directory. + exit 1 +fi + +# Run `flatc`. Note: This requires you to compile using `cmake` from the +# root `/flatbuffers` directory. +if [ -e ../flatc ]; then + ../flatc --julia monster.fbs +elif [ -e ../Debug/flatc ]; then + ../Debug/flatc --julia monster.fbs +else + echo 'flatc' could not be found. Make sure to build FlatBuffers from the \ + $rootdir directory. + exit 1 +fi + +echo Running the Julia sample. + +# Execute the sample. +julia sample_binary.jl + +# Clean up the temporary files. +rm -rf MyGame diff --git a/samples/sample_binary.jl b/samples/sample_binary.jl new file mode 100644 index 00000000000..788b2ddbbb1 --- /dev/null +++ b/samples/sample_binary.jl @@ -0,0 +1,49 @@ +import FlatBuffers + +include("MyGame/MyGame.jl") + +# require the files generated from the schema +import .MyGame.Sample.Weapon +import .MyGame.Sample.Monster +import .MyGame.Sample.Vec3 +import .MyGame.Sample.Color +import .MyGame.Sample.Equipment + +sword = Weapon(;name="Sword", damage=3) +axe = Weapon(;name="Axe", damage=5) +orc = Monster(; + name="Orc", + pos=Vec3(1.0, 2.0, 3.0), + hp=300, + inventory=collect(1:10), + color=MyGame.Sample.ColorRed, + weapons=[sword, axe], + equipped=axe +) + +# Get the flatbuffer as a string containing the binary data +io = IOBuffer() +FlatBuffers.serialize(io, orc) +bytes = take!(io) + +# Build the Monster from the raw bytes +mon = Monster(bytes) + +@assert mon.mana == 150 +@assert mon.hp == 300 +@assert mon.name == "Orc" +@assert mon.color == MyGame.Sample.ColorRed +@assert mon.pos.x == 1.0 +@assert mon.pos.y == 2.0 +@assert mon.pos.z == 3.0 +@assert mon.inventory == collect(1:10) + +@assert mon.equipped_type == FlatBuffers.typeorder(Equipment, Weapon) +@assert mon.equipped.name == "Axe" +@assert mon.equipped.damage == 5 + +@assert mon.weapons[1].name == "Sword" +@assert mon.weapons[1].damage == 3 + +@assert mon.weapons[2].name == "Axe" +@assert mon.weapons[2].damage == 5 diff --git a/src/flatc_main.cpp b/src/flatc_main.cpp index 78e66a6f637..ef92a24f052 100644 --- a/src/flatc_main.cpp +++ b/src/flatc_main.cpp @@ -58,6 +58,10 @@ int main(int argc, const char *argv[]) { { flatbuffers::GenerateJSTS, "-s", "--js", "JavaScript", true, nullptr, flatbuffers::IDLOptions::kJs, "Generate JavaScript code for tables/structs", flatbuffers::JSTSMakeRule }, + { flatbuffers::GenerateJulia, nullptr, "--julia", "Julia", true, nullptr, + flatbuffers::IDLOptions::kJulia, + "Generate Julia modules for tables/structs", + flatbuffers::GeneralMakeRule }, { flatbuffers::GenerateDart, "-d", "--dart", "Dart", true, nullptr, flatbuffers::IDLOptions::kDart, "Generate Dart classes for tables/structs", flatbuffers::DartMakeRule }, diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index f91f14e1f82..3823bdfb0f3 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -494,7 +494,7 @@ class CppGenerator : public BaseGenerator { static const char * const ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #CTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 66984da0c6d..b2eb017dec9 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -253,7 +253,7 @@ class GeneralGenerator : public BaseGenerator { // clang-format off static const char * const java_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #JTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -261,7 +261,7 @@ class GeneralGenerator : public BaseGenerator { static const char * const csharp_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #NTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index fe4939edf64..8f32b95ec46 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -693,7 +693,7 @@ static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #GTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp new file mode 100644 index 00000000000..fb1b4782e42 --- /dev/null +++ b/src/idl_gen_julia.cpp @@ -0,0 +1,815 @@ +/* + * Copyright 2018 Dolby Laboratories. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// loosely based on idl_gen_python.cpp + +#include +#include +#include +#include + +#include "flatbuffers/code_generators.h" +#include "flatbuffers/flatbuffers.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { +namespace julia { + +const std::string Indent = " "; +const std::string JuliaFileExtension = ".jl"; +const std::string JuliaPackageName = "FlatBuffers"; + +// Documentation comments +const CommentConfig JuliaCommentConfig = { "#=", "# ", "=#" }; + +// Class to represent a dependency graph +// with a topological sort operation +class DepGraph { + public: + // adjacency list + std::vector *> adj; + // node names + std::vector nodes; + unsigned int GetOrCreateNodeID(std::string name) { + auto it = std::find(nodes.begin(), nodes.end(), name); + if (it == nodes.end()) { + nodes.push_back(name); + return nodes.size() - 1; + } + return std::distance(nodes.begin(), it); + } + DepGraph() {} + ~DepGraph() { + for (unsigned int i = 0; i < adj.size(); i++) delete adj[i]; + } + void AddDep(std::string parent, std::string child) { + unsigned int p = GetOrCreateNodeID(parent); + unsigned int c = GetOrCreateNodeID(child); + // n = max(p, c) + unsigned int n = p > c ? p : c; + // if we just created new node ids, + // grow the adjacency list to the new size + if (n >= adj.size()) { + for (unsigned int i = adj.size(); i <= n; i++) + adj.push_back(new std::vector()); + } + adj[p]->push_back(c); + } + // topological sort + bool TopSort(unsigned int v, bool visited[], bool visiting[], + std::list &stack) { + bool has_cycle = false; + visited[v] = true; + visiting[v] = true; + for (auto it = adj[v]->begin(); it != adj[v]->end(); ++it) { + if (!visited[*it]) has_cycle = TopSort(*it, visited, visiting, stack); + if (visiting[*it] && nodes[v] != nodes[*it]) { + printf( + "warning: %s\n", + ("Circular reference detected (" + nodes[v] + " -> " + nodes[*it] + + "). " + "Julia does not currently suport such definitions.") + .c_str()); + has_cycle = true; + } + } + visiting[v] = false; + stack.push_back(v); + return has_cycle; + } + // Topological sort of dependency graph, so we can include + // julia files in the right order + std::vector TopSort() { + std::vector sorted_nodes; + std::list stack; + unsigned int n = adj.size(); + bool *visited = new bool[n]; + bool *visiting = new bool[n]; + for (unsigned int i = 0; i < n; i++) { + visited[i] = false; + visiting[i] = false; + } + + bool has_cycle = false; + for (unsigned int i = 0; i < n; i++) + if (!visited[i]) has_cycle |= TopSort(i, visited, visiting, stack); + + while (!stack.empty()) { + unsigned int id = stack.back(); + // don't return nodes on which nothing depends + if (!adj[id]->empty()) sorted_nodes.push_back(nodes[id]); + stack.pop_back(); + } + return sorted_nodes; + } +}; + +// Singleton class which keeps track of generated modules +// and their dependencies. A singleton class is necessary +// since our generator is run multiple times with +// multiple different .fbs files, and we want to preserve +// the set of module dependencies across these runs. +// We need to keep track of dependencies since Julia doesn't +// support forward declarations of types. +class ModuleTable { + public: + static ModuleTable &GetInstance() { + static ModuleTable instance; + return instance; + } + bool IsModule(const std::string &m) { + return modules_.find(m) != modules_.end(); + } + void AddFile(const std::string &f) { files_.insert(f); } + bool IsFile(const std::string &f) { return files_.find(f) != files_.end(); } + void AddDependency(std::string mod, std::string parent, std::string child) { + if (!IsModule(mod)) modules_[mod] = new DepGraph(); + modules_[mod]->AddDep(parent, child); + } + std::vector SortedModuleNames() { + std::set module_names; + for (auto it = modules_.begin(); it != modules_.end(); ++it) + module_names.insert(it->first); + // Make sure parent modules come before child modules + std::vector sorted_modules(module_names.begin(), + module_names.end()); + std::sort(sorted_modules.begin(), sorted_modules.end()); + return sorted_modules; + } + DepGraph *GetDependencies(std::string module) { return modules_[module]; } + + private: + std::map modules_; + std::set files_; + ModuleTable() {} + ~ModuleTable() { + for (auto it = modules_.begin(); it != modules_.end(); ++it) + delete it->second; + } + + public: + ModuleTable(ModuleTable const &) = delete; + void operator=(ModuleTable const &) = delete; +}; + +class JuliaGenerator : public BaseGenerator { + public: + JuliaGenerator(const Parser &parser, const std::string &path, + const std::string &file_name) + : BaseGenerator(parser, path, file_name, "" /* not used */, + "" /* not used */) {} + + ~JuliaGenerator() {} + bool generate() { + if (!GenEnums()) return false; + if (!GenStructs()) return false; + if (!GenModules()) return false; + return true; + } + + private: + // the root module is the name of the .fbs file which + // we are compiling, in camel case + std::string root_module_ = MakeCamel(file_name_); + ModuleTable &module_table_ = ModuleTable::GetInstance(); + static const std::unordered_set keywords_; + + bool GenEnums(void) { + for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); + ++it) { + auto &enum_def = **it; + std::string enumcode; + if (enum_def.is_union) + GenUnion(enum_def, &enumcode); + else + GenEnum(enum_def, &enumcode); + if (!SaveType(enum_def, enumcode)) return false; + } + return true; + } + + bool GenStructs(void) { + for (auto it = parser_.structs_.vec.begin(); + it != parser_.structs_.vec.end(); ++it) { + auto &struct_def = **it; + std::string declcode; + GenObject(struct_def, &declcode); + if (!SaveType(struct_def, declcode)) return false; + } + return true; + } + + bool GenModules(void) { + for (auto it = parser_.namespaces_.begin(); it != parser_.namespaces_.end(); + ++it) { + std::string parent; + std::string child; + // Gather all parent namespaces for this namespace + // TODO: this is similar to idl_gen_js.cpp, maybe + // move into common place? + for (auto component = (*it)->components.begin(); + component != (*it)->components.end(); ++component) { + if (parent.empty()) { + parent = *component; + child = *component; + module_table_.AddDependency(root_module_, parent, child); + } else { + child = parent + kPathSeparator + *component; + // Add component to parent's list of children + module_table_.AddDependency(parent, child, *component); + } + parent = child; + } + } + auto sorted_modules = module_table_.SortedModuleNames(); + bool save_root = false; + // iterate through child modules first, then parents + for (auto m = sorted_modules.rbegin(); m != sorted_modules.rend(); ++m) { + if (*m == root_module_) { + save_root = true; + continue; + } + SaveModule(false, *m, module_table_.GetDependencies(*m)); + } + // save the root module last + if (save_root) + SaveModule(true, root_module_, + module_table_.GetDependencies(root_module_)); + return true; + } + + // Begin an object declaration. + static void BeginObject(const StructDef &struct_def, std::string *code_ptr, + bool has_defaults) { + std::string &code = *code_ptr; + if (has_defaults) code += JuliaPackageName + ".@with_kw "; + if (!struct_def.fixed) + code += "mutable struct "; + else + code += JuliaPackageName + ".@STRUCT struct "; + code += NormalizedName(struct_def) + "\n"; + } + + void EndObject(const StructDef &struct_def, + const std::vector &offsets, + std::string *code_ptr) const { + std::string &code = *code_ptr; + std::string name = NormalizedName(struct_def); + code += "end\n"; + code += JuliaPackageName + ".@ALIGN(" + name + +", " + + NumToString(struct_def.minalign) + ")\n"; + std::string method_signature = "(::Type{T}) where {T<:" + name + "}"; + if (!offsets.empty() && !struct_def.fixed) { + bool first = true; + int i = 0; + code += JuliaPackageName + ".slot_offsets" + method_signature; + code += " = ["; + for (auto it = offsets.begin(); it != offsets.end(); ++it) { + if (!first) code += ", "; + if (i == 0) code += "\n" + Indent; + code += *it; + i++; + i %= 4; // print four offsets per line + first = false; + } + code += "\n]\n"; + } + + // emit file identifier and extension for the root type + if (parser_.root_struct_def_ == &struct_def) { + code += JuliaPackageName + ".root_type" + method_signature + " = true\n"; + if (!parser_.file_identifier_.empty()) { + code += JuliaPackageName + ".file_identifier" + method_signature; + code += " = \"" + parser_.file_identifier_ + "\"\n"; + } + if (!parser_.file_extension_.empty()) { + code += JuliaPackageName + ".file_extension" + method_signature; + code += " = \"" + parser_.file_extension_ + "\"\n"; + } + } + code += "\n"; + } + + static std::string EscapeKeyword(const std::string &name) { + return keywords_.find(name) == keywords_.end() ? name : name + "_"; + } + + static std::string const NormalizedName(const Definition &child, + const Definition *parent = NULL) { + std::string prefix = ""; + if (parent != NULL) { + std::string relname = GetRelativeName(*parent, &child, false); + if (!relname.empty()) prefix = relname + "."; + } + return prefix + EscapeKeyword(child.name); + } + + static std::string NormalizedName(const EnumVal &ev) { + return EscapeKeyword(ev.name); + } + + static void BeginEnum(const std::string enum_name, + const std::string enum_type, std::string *code_ptr) { + *code_ptr += "@enum " + enum_name + "::" + enum_type + " begin\n"; + } + + static void EnumMember(const std::string enum_name, const EnumVal ev, + std::string *code_ptr) { + *code_ptr += Indent + enum_name + NormalizedName(ev); + *code_ptr += " = " + NumToString(ev.value) + "\n"; + } + + static void EndEnum(std::string *code_ptr) { *code_ptr += "end\n\n"; } + + static void BeginUnion(const std::string union_name, std::string *code_ptr) { + *code_ptr += JuliaPackageName + ".@UNION(" + union_name + ", (\n"; + } + + static void UnionMember(const std::string type_name, std::string *code_ptr) { + *code_ptr += Indent + type_name + ",\n"; + } + + static void EndUnion(std::string *code_ptr) { *code_ptr += "))\n\n"; } + + static void NewObjectFromBuffer(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += NormalizedName(struct_def) + "(buf::AbstractVector{UInt8})"; + code += " = " + JuliaPackageName + ".read(" + NormalizedName(struct_def) + + ", buf)\n"; + code += NormalizedName(struct_def) + "(io::IO) = " + JuliaPackageName; + code += ".deserialize(io, " + NormalizedName(struct_def) + ")\n"; + } + + void GenScalarField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, bool *has_defaults, + std::set *imports_ptr) { + std::string &code = *code_ptr; + std::string field_name = NormalizedName(field); + code += Indent + field_name; + code += "::"; + code += GenTypeGet(field.value.type); + if (!field.value.constant.empty() && !struct_def.fixed) { + *has_defaults = true; + std::string c = field.value.constant; + if (field.value.type.base_type == BASE_TYPE_BOOL) { + c = c == "0" ? "false" : "true"; + } + code += " = " + c; + } + if (IsScalarEnum(field.value.type)) + AddDependency(struct_def, field.value.type, imports_ptr); + code += "\n"; + } + + // whether two namespaces have the same prefix up to "prefix_size" + // used for figuring out relative import paths + static bool SameNamespacePrefix(Namespace n1, Namespace n2, + size_t prefix_size) { + if (n1.components.size() < prefix_size) return false; + if (n2.components.size() < prefix_size) return false; + size_t i; + for (i = 0; i < prefix_size; i++) { + if (n1.components[i] != n2.components[i]) return false; + } + return true; + } + + static void GetRelativeNamespaces(const Definition &parent_def, + const Definition *child_def, + Namespace *parent, Namespace *child) { + Namespace p, c; + if (parent_def.defined_namespace != NULL) p = *parent_def.defined_namespace; + if (child_def != NULL && child_def->defined_namespace != NULL) + c = *child_def->defined_namespace; + *parent = p; + *child = c; + } + + static void GetRelativeNamespaces(const Definition &def, const Type &type, + Namespace *parent, Namespace *child) { + Namespace p, c; + Definition *child_def = GetBaseDefinition(type); + GetRelativeNamespaces(def, child_def, parent, child); + } + + static std::string GetRelativeName(const Namespace &parent, + const Namespace &child, + bool dotprefix = true) { + // here we are accounting for modules which are at + // different levels of the tree. + std::string relname = ""; + // go up to common level (don't add dots to path if we are + // annotating the type of a field - julia doesn't like that) + unsigned int i = 0; + unsigned int n = parent.components.size(); + while (i <= n && !SameNamespacePrefix(parent, child, n - i)) { + if (dotprefix) relname += "."; + i++; + } + // traverse down to the place we need to be + unsigned int m = parent.components.size() - i; + unsigned int j; + for (j = m; j < child.components.size(); j++) { + if (dotprefix || !relname.empty()) relname += "."; + relname += child.components[j]; + } + return relname; + } + + static std::string GetRelativeName(const Definition &parent_def, + const Definition *child_def, + bool dotprefix = false) { + Namespace parent, child; + GetRelativeNamespaces(parent_def, child_def, &parent, &child); + return GetRelativeName(parent, child, dotprefix); + } + + static std::string GetRelativeName(const Definition &def, const Type &type, + bool dotprefix = false) { + Namespace parent, child; + GetRelativeNamespaces(def, type, &parent, &child); + return GetRelativeName(parent, child, dotprefix); + } + + // This definition depends on this type. Add the necessary relative imports. + void AddDependency(const Definition &def, const Type &type, + std::set *imports_ptr) { + Namespace parent, child; + Definition *child_def = GetBaseDefinition(type); + if (child_def == NULL) return; + + GetRelativeNamespaces(def, child_def, &parent, &child); + std::string relname = GetRelativeName(parent, child); + + // add relative import + if (!relname.empty()) { + // special case: we are importing something from a direct parent + if (relname.find_first_not_of('.') == std::string::npos) + imports_ptr->insert(relname + "." + NormalizedName(*child_def)); + else + imports_ptr->insert(relname); + } + + std::string module = GetCanonicalName(child); + std::string child_name = + GetCanonicalName(child) + kPathSeparator + NormalizedName(*child_def); + std::string parent_name = + GetCanonicalName(parent) + kPathSeparator + NormalizedName(def); + // self-reference - this is fine, but don't add it to the dependency table + if (child_name == parent_name) return; + module_table_.AddDependency(module, parent_name, child_name); + } + + // generate a field which depends upon generated types + void GenDependentField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, + bool *has_defaults, + std::set *imports_ptr) { + std::string type_name = GenTypeGet(field.value.type, &struct_def); + + BaseType bt = field.value.type.base_type; + if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_VECTOR) && + !struct_def.fixed) { + type_name = "Union{" + type_name + ", Nothing}"; + } + // initialise nullable fields to nothing by default + if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_UNION || + bt == BASE_TYPE_VECTOR || bt == BASE_TYPE_STRING) && + !struct_def.fixed) { + type_name += " = nothing"; + *has_defaults = true; + } + *code_ptr += Indent + NormalizedName(field) + "::" + type_name + "\n"; + AddDependency(struct_def, field.value.type, imports_ptr); + } + + static void GenStringField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, bool *has_defaults) { + *code_ptr += Indent + NormalizedName(field) + "::"; + // initialise strings to be empty by default + *code_ptr += "Union{" + GenTypeGet(field.value.type) + ", Nothing}"; + if (!struct_def.fixed) { + *code_ptr += " = nothing"; + *has_defaults = true; + } + *code_ptr += "\n"; + } + + // Generate a field, conditioned on its child type(s). + void GenField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, std::set *imports_ptr, + bool *has_defaults) { + GenComment(field.doc_comment, code_ptr, &JuliaCommentConfig); + if (IsScalar(field.value.type.base_type)) { + GenScalarField(struct_def, field, code_ptr, has_defaults, imports_ptr); + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRING: + GenStringField(struct_def, field, code_ptr, has_defaults); + break; + case BASE_TYPE_STRUCT: + case BASE_TYPE_VECTOR: + case BASE_TYPE_UNION: + GenDependentField(struct_def, field, code_ptr, has_defaults, imports_ptr); + break; + default: FLATBUFFERS_ASSERT(0); + } + } + } + + static std::string GenImports(const std::set &imports) { + std::string impstr = ""; + for (auto it = imports.begin(); it != imports.end(); ++it) + impstr += "import " + *it + "\n"; + return impstr; + } + + // Generate structs/tables + void GenObject(const StructDef &struct_def, std::string *code_ptr) { + if (struct_def.generated) return; + + // always need FlatBuffers package for structs + std::set imports = { JuliaPackageName }; + bool has_defaults = false; + + // generate all the fields + std::vector offsets; + + GenComment(struct_def.doc_comment, code_ptr, &JuliaCommentConfig); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + GenField(struct_def, field, code_ptr, &imports, &has_defaults); + offsets.push_back("0x" + IntToStringHex(field.value.offset, 8)); + } + EndObject(struct_def, offsets, code_ptr); + + // need to call BeginObject after EndObject because we don't know + // the defaults until we've looked at all the fields. + std::string struct_begin; + BeginObject(struct_def, &struct_begin, has_defaults); + + *code_ptr = GenImports(imports) + "\n" + struct_begin + *code_ptr; + + // Generate a functions for constructing the object from a buffer + NewObjectFromBuffer(struct_def, code_ptr); + } + + void GenUnion(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + + // always need FlatBuffers package for unions + std::set imports = { JuliaPackageName }; + std::string union_name = NormalizedName(enum_def); + BeginUnion(union_name, code_ptr); + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + std::string type_name = GenTypeGet(ev.union_type, &enum_def); + GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig); + UnionMember(type_name, code_ptr); + // special case, instead make every Union a union with Nothing + if (ev.name == "NONE") continue; + AddDependency(enum_def, ev.union_type, &imports); + } + EndUnion(code_ptr); + + *code_ptr = GenImports(imports) + "\n" + *code_ptr; + } + + void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + GenComment(enum_def.doc_comment, code_ptr, &JuliaCommentConfig); + std::string enum_name = NormalizedName(enum_def); + BeginEnum(enum_name, GenTypeBasic(enum_def.underlying_type), code_ptr); + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig); + EnumMember(enum_name, ev, code_ptr); + } + EndEnum(code_ptr); + } + + static std::string GenTypeBasic(const Type &type) { + static const char *ctypename[] = { +// clang-format off + #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ + #JLTYPE, + FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + // clang-format on + }; + return ctypename[type.base_type]; + } + + static std::string GenTypePointer(const Type &type, + const Definition *parent) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "String"; + case BASE_TYPE_VECTOR: + return "Vector{" + GenTypeGet(type.VectorType(), parent) + "}"; + case BASE_TYPE_STRUCT: return NormalizedName(*type.struct_def, parent); + case BASE_TYPE_UNION: return NormalizedName(*type.enum_def, parent); + case BASE_TYPE_NONE: return "Nothing"; + default: return "Any"; + } + } + + static Definition *GetBaseDefinition(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_VECTOR: return GetBaseDefinition(type.VectorType()); + case BASE_TYPE_STRUCT: return type.struct_def; + case BASE_TYPE_UNION: return type.enum_def; + default: return NULL; + } + } + + static bool IsScalarEnum(const Type &type) { + return (type.enum_def != nullptr && !type.enum_def->is_union && + IsInteger(type.enum_def->underlying_type.base_type)); + } + + static std::string GenTypeGet(const Type &type, + const Definition *parent = NULL) { + if (IsScalar(type.base_type)) + return IsScalarEnum(type) ? NormalizedName(*type.enum_def, parent) + : GenTypeBasic(type); + return GenTypePointer(type, parent); + } + + void BeginFile(const std::string submodule_name, + std::string *code_ptr) const { + std::string &code = *code_ptr; + code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; + std::string module = submodule_name; + if (module.empty()) module = root_module_; + code += "# module: " + module + "\n\n"; + } + + std::string GetDirname(const Definition &def) const { + std::string d = path_; + if (def.defined_namespace != NULL) + d += GetCanonicalName(*def.defined_namespace); + return d; + } + + std::string GetFilename(const Definition &def) const { + return ConCatPathFileName(GetDirname(def), NormalizedName(def)) + + JuliaFileExtension; + } + + std::string GetModule(const Definition &def) const { + if (def.defined_namespace != NULL) + return GetCanonicalName(*def.defined_namespace); + return root_module_; + } + + static std::string GetSubModule(const Definition &def) { + if (def.defined_namespace != NULL) + return LastNamespacePart(*def.defined_namespace); + return ""; + } + + // Save out the generated code for a Julia module + bool SaveModule(bool is_root, std::string full_module_name, + DepGraph *children) { + std::string module_name = full_module_name; + auto start = full_module_name.rfind(kPathSeparator); + if (start != std::string::npos) + module_name = full_module_name.substr(start + 1); + std::string module_dir = ConCatPathFileName(path_, full_module_name); + if (is_root) { + module_dir = path_; + module_name = root_module_; + } + std::string module_jl = + ConCatPathFileName(module_dir, module_name) + JuliaFileExtension; + std::string code = ""; + bool need_module_file = false; + BeginFile(module_name, &code); + code += "module " + module_name + "\n"; + + // Include all the dependencies of this module in the right order + std::vector sorted_children = children->TopSort(); + for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); + ++it) { + std::string child = *it; + bool is_module = false; + std::string submodule_name = + child.substr(child.rfind(kPathSeparator) + 1); + + // Don't include the root module from the root + if (is_root && submodule_name == root_module_) continue; + + // Modules live in a subdirectory named after themselves + if (module_table_.IsModule(child) && + !(is_root && submodule_name == root_module_)) { + is_module = true; + child = ConCatPathFileName(child, submodule_name); + } + std::string dir = is_root ? path_ : module_dir; + std::string relname; + if (is_root) + relname = child; + else if (is_module) + relname = child.substr(child.find(kPathSeparator) + 1); + else + relname = child.substr(child.rfind(kPathSeparator) + 1); + + // If the file doesn't exist, don't include it + // TODO: this doesn't allow types which reference each other, + // but Julia doesn't support this yet anyway + std::string toinclude = relname + JuliaFileExtension; + std::string fullpath = ConCatPathFileName(dir, toinclude); + if (!module_table_.IsFile(fullpath.c_str())) continue; + code += Indent + "include(\"" + toinclude + "\")\n"; + need_module_file = true; + } + code += "end\n"; + + // only write the module if it has some things to include + if (!need_module_file) return true; + + EnsureDirExists(module_dir); + if (!SaveFile(module_jl.c_str(), code, false)) return false; + + module_table_.AddFile(module_jl); + return true; + } + + // Canonical julia name of a namespace (Foo.Bar.Baz) + std::string GetCanonicalName(const Namespace &ns) const { + std::string name; + for (size_t i = 0; i < ns.components.size(); i++) { + if (i) name += kPathSeparator; + name += std::string(ns.components[i]); + } + if (name.empty()) name = root_module_; + return name; + } + + // Add a dependency between two definitions + void AddDependency(const Definition *parent, const Definition *child) { + FLATBUFFERS_ASSERT(parent != NULL && child != NULL); + std::string parent_name = NormalizedName(*parent); + std::string module = parent_name; + if (parent->defined_namespace != NULL) + module = GetCanonicalName(*parent->defined_namespace); + module_table_.AddDependency(module, parent_name, NormalizedName(*child)); + } + + // Add a definition as a dependency to its own module + void AddToOwnModule(const Definition &def) { + std::string m = GetModule(def); + module_table_.AddDependency(m, m + kPathSeparator + NormalizedName(def), m); + } + + // Save out the generated code for a Julia struct + bool SaveType(const Definition &def, const std::string &declcode) { + if (!declcode.length()) return true; + std::string code = ""; + BeginFile(GetSubModule(def), &code); + code += declcode; + std::string filename = GetFilename(def); + EnsureDirExists(GetDirname(def)); + if (!SaveFile(filename.c_str(), code, false)) return false; + module_table_.AddFile(filename); + AddToOwnModule(def); + return true; + } +}; + +const std::unordered_set JuliaGenerator::keywords_{ + { "begin", "while", "if", "for", "try", "return", + "break", "continue", "function", "macro", "quote", "let", + "local", "global", "const", "do", "struct", "module", + "baremodule", "using", "import", "export", "end", "else", + "catch", "finally", "true", "false", "Any" } +}; + +} // namespace julia + +bool GenerateJulia(const Parser &parser, const std::string &path, + const std::string &file_name) { + julia::JuliaGenerator generator(parser, path, file_name); + return generator.generate(); +} + +} // namespace flatbuffers diff --git a/src/idl_gen_lobster.cpp b/src/idl_gen_lobster.cpp index 5f199e3a1c9..48e752308f6 100644 --- a/src/idl_gen_lobster.cpp +++ b/src/idl_gen_lobster.cpp @@ -81,7 +81,7 @@ class LobsterGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_lua.cpp b/src/idl_gen_lua.cpp index e8ae32802c7..8cdf5b24dd3 100644 --- a/src/idl_gen_lua.cpp +++ b/src/idl_gen_lua.cpp @@ -604,7 +604,7 @@ namespace lua { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_php.cpp b/src/idl_gen_php.cpp index 91ce01367a6..7bb2e04b647 100644 --- a/src/idl_gen_php.cpp +++ b/src/idl_gen_php.cpp @@ -864,7 +864,7 @@ class PhpGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #NTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 11491c00d59..d2edef77bdc 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -615,7 +615,7 @@ class PythonGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_rust.cpp b/src/idl_gen_rust.cpp index b530137b29b..910a7d765d7 100644 --- a/src/idl_gen_rust.cpp +++ b/src/idl_gen_rust.cpp @@ -510,7 +510,7 @@ class RustGenerator : public BaseGenerator { // clang-format off static const char * const ctypename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #RTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -532,7 +532,7 @@ class RustGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #RTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index c129488650f..ed6646b28df 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -131,7 +131,7 @@ bool Print(const void *val, Type type, int indent, switch (type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (!PrintVector( \ *reinterpret_cast *>(val), \ @@ -227,7 +227,7 @@ static bool GenStruct(const StructDef &struct_def, const Table *table, switch (fd.value.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (!GenField(fd, table, struct_def.fixed, \ opts, indent + Indent(opts), _text)) { \ @@ -238,7 +238,7 @@ static bool GenStruct(const StructDef &struct_def, const Table *table, #undef FLATBUFFERS_TD // Generate drop-thru case statements for all pointer types: #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 357337a74f7..89049badeb0 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -30,7 +30,7 @@ const double kPi = 3.14159265358979323846; const char *const kTypeNames[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -41,7 +41,7 @@ const char *const kTypeNames[] = { const char kTypeSizes[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ sizeof(CTYPE), FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -192,7 +192,7 @@ static std::string TokenToString(int t) { FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -1077,7 +1077,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, switch (field_value.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (struct_def.fixed) { \ @@ -1094,7 +1094,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); #undef FLATBUFFERS_TD #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (IsStruct(field->value.type)) { \ @@ -1166,7 +1166,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { switch (val.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ else { \ @@ -1506,7 +1506,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e, switch (e.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: {\ CTYPE val; \ ECHECK(atot(e.constant.c_str(), *this, &val)); \ @@ -1665,7 +1665,7 @@ CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) { switch (enum_def->underlying_type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ - PTYPE, RTYPE) \ + PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_##ENUM: { \ int64_t min_value = static_cast( \ flatbuffers::numeric_limits::lowest()); \ @@ -1749,7 +1749,8 @@ bool Parser::SupportsVectorOfUnions() const { return opts.lang_to_generate != 0 && (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs | IDLOptions::kTs | IDLOptions::kPhp | - IDLOptions::kJava | IDLOptions::kCSharp)) == 0; + IDLOptions::kJava | IDLOptions::kCSharp | + IDLOptions::kJulia)) == 0; } Namespace *Parser::UniqueNamespace(Namespace *ns) { diff --git a/tests/MonsterTest.jl b/tests/MonsterTest.jl new file mode 100644 index 00000000000..b3ad477a2a0 --- /dev/null +++ b/tests/MonsterTest.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MonsterTest + +module MonsterTest + include("MyGame/MyGame.jl") +end diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl new file mode 100644 index 00000000000..e32f1ca69b8 --- /dev/null +++ b/tests/MyGame/Example/Ability.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Ability + id::UInt32 + distance::UInt32 +end +FlatBuffers.@ALIGN(Ability, 4) + +Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) +Ability(io::IO) = FlatBuffers.deserialize(io, Ability) diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl new file mode 100644 index 00000000000..e106020db0f --- /dev/null +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@UNION(AnyAmbiguousAliases, ( + Nothing, + Monster, + Monster, + Monster, +)) + diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl new file mode 100644 index 00000000000..27abea3f997 --- /dev/null +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(AnyUniqueAliases, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl new file mode 100644 index 00000000000..9bdc2a646e1 --- /dev/null +++ b/tests/MyGame/Example/Any_.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(Any_, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/tests/MyGame/Example/Color.jl b/tests/MyGame/Example/Color.jl new file mode 100644 index 00000000000..804144f0ac7 --- /dev/null +++ b/tests/MyGame/Example/Color.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +@enum Color::Int8 begin + ColorRed = 1 + ColorGreen = 2 + ColorBlue = 8 +end + diff --git a/tests/MyGame/Example/Example.jl b/tests/MyGame/Example/Example.jl new file mode 100644 index 00000000000..c8fd32b3cd4 --- /dev/null +++ b/tests/MyGame/Example/Example.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +module Example + include("Color.jl") + include("Test.jl") + include("Vec3.jl") + include("Stat.jl") + include("Ability.jl") + include("Referrable.jl") + include("TestSimpleTableWithEnum.jl") + include("AnyUniqueAliases.jl") + include("AnyAmbiguousAliases.jl") + include("Monster.jl") + include("Any_.jl") + include("TypeAliases.jl") +end diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl new file mode 100644 index 00000000000..15586f65c70 --- /dev/null +++ b/tests/MyGame/Example/Monster.jl @@ -0,0 +1,84 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..InParentNamespace +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Monster +#= +# an example documentation comment: monster object +=# + pos::Union{Vec3, Nothing} = nothing + mana::Int16 = 150 + hp::Int16 = 100 + name::Union{String, Nothing} = nothing + inventory::Union{Vector{UInt8}, Nothing} = nothing + color::Color = 8 + test_type::UInt8 = 0 + test::Any_ = nothing + test4::Union{Vector{Test}, Nothing} = nothing + testarrayofstring::Union{Vector{String}, Nothing} = nothing +#= +# an example documentation comment: this will end up in the generated code +# multiline too +=# + testarrayoftables::Union{Vector{Monster}, Nothing} = nothing + enemy::Union{Monster, Nothing} = nothing + testnestedflatbuffer::Union{Vector{UInt8}, Nothing} = nothing + testempty::Union{Stat, Nothing} = nothing + testbool::Bool = false + testhashs32_fnv1::Int32 = 0 + testhashu32_fnv1::UInt32 = 0 + testhashs64_fnv1::Int64 = 0 + testhashu64_fnv1::UInt64 = 0 + testhashs32_fnv1a::Int32 = 0 + testhashu32_fnv1a::UInt32 = 0 + testhashs64_fnv1a::Int64 = 0 + testhashu64_fnv1a::UInt64 = 0 + testarrayofbools::Union{Vector{Bool}, Nothing} = nothing + testf::Float32 = 3.14159 + testf2::Float32 = 3.0 + testf3::Float32 = 0.0 + testarrayofstring2::Union{Vector{String}, Nothing} = nothing + testarrayofsortedstruct::Union{Vector{Ability}, Nothing} = nothing + flex::Union{Vector{UInt8}, Nothing} = nothing + test5::Union{Vector{Test}, Nothing} = nothing + vector_of_longs::Union{Vector{Int64}, Nothing} = nothing + vector_of_doubles::Union{Vector{Float64}, Nothing} = nothing + parent_namespace_test::Union{InParentNamespace, Nothing} = nothing + vector_of_referrables::Union{Vector{Referrable}, Nothing} = nothing + single_weak_reference::UInt64 = 0 + vector_of_weak_references::Union{Vector{UInt64}, Nothing} = nothing + vector_of_strong_referrables::Union{Vector{Referrable}, Nothing} = nothing + co_owning_reference::UInt64 = 0 + vector_of_co_owning_references::Union{Vector{UInt64}, Nothing} = nothing + non_owning_reference::UInt64 = 0 + vector_of_non_owning_references::Union{Vector{UInt64}, Nothing} = nothing + any_unique_type::UInt8 = 0 + any_unique::AnyUniqueAliases = nothing + any_ambiguous_type::UInt8 = 0 + any_ambiguous::AnyAmbiguousAliases = nothing + vector_of_enums::Union{Vector{Color}, Nothing} = nothing +end +FlatBuffers.@ALIGN(Monster, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000E, 0x00000010, 0x00000012, 0x00000014, + 0x00000016, 0x00000018, 0x0000001A, 0x0000001C, + 0x0000001E, 0x00000020, 0x00000022, 0x00000024, + 0x00000026, 0x00000028, 0x0000002A, 0x0000002C, + 0x0000002E, 0x00000030, 0x00000032, 0x00000034, + 0x00000036, 0x00000038, 0x0000003A, 0x0000003C, + 0x0000003E, 0x00000040, 0x00000042, 0x00000044, + 0x00000046, 0x00000048, 0x0000004A, 0x0000004C, + 0x0000004E, 0x00000050, 0x00000052, 0x00000054, + 0x00000056, 0x00000058, 0x0000005A, 0x0000005C, + 0x0000005E, 0x00000060, 0x00000062 +] +FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true +FlatBuffers.file_identifier(::Type{T}) where {T<:Monster} = "MONS" +FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl new file mode 100644 index 00000000000..7fe024e210e --- /dev/null +++ b/tests/MyGame/Example/Referrable.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Referrable + id::UInt64 = 0 +end +FlatBuffers.@ALIGN(Referrable, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ + 0x00000004 +] + +Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) +Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl new file mode 100644 index 00000000000..00001c46935 --- /dev/null +++ b/tests/MyGame/Example/Stat.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Stat + id::Union{String, Nothing} = nothing + val::Int64 = 0 + count::UInt16 = 0 +end +FlatBuffers.@ALIGN(Stat, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ + 0x00000004, 0x00000006, 0x00000008 +] + +Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) +Stat(io::IO) = FlatBuffers.deserialize(io, Stat) diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl new file mode 100644 index 00000000000..0087689c7eb --- /dev/null +++ b/tests/MyGame/Example/Test.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Test + a::Int16 + b::Int8 +end +FlatBuffers.@ALIGN(Test, 2) + +Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) +Test(io::IO) = FlatBuffers.deserialize(io, Test) diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl new file mode 100644 index 00000000000..41c0e33f3ee --- /dev/null +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum + color::Color = 2 +end +FlatBuffers.@ALIGN(TestSimpleTableWithEnum, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ + 0x00000004 +] + +TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) +TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl new file mode 100644 index 00000000000..a563b74ecbd --- /dev/null +++ b/tests/MyGame/Example/TypeAliases.jl @@ -0,0 +1,29 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TypeAliases + i8::Int8 = 0 + u8::UInt8 = 0 + i16::Int16 = 0 + u16::UInt16 = 0 + i32::Int32 = 0 + u32::UInt32 = 0 + i64::Int64 = 0 + u64::UInt64 = 0 + f32::Float32 = 0.0 + f64::Float64 = 0.0 + v8::Union{Vector{Int8}, Nothing} = nothing + vf64::Union{Vector{Float64}, Nothing} = nothing +end +FlatBuffers.@ALIGN(TypeAliases, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000C, 0x0000000E, 0x00000010, 0x00000012, + 0x00000014, 0x00000016, 0x00000018, 0x0000001A +] + +TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) +TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl new file mode 100644 index 00000000000..8f5442af51f --- /dev/null +++ b/tests/MyGame/Example/Vec3.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 + test1::Float64 + test2::Color + test3::Test +end +FlatBuffers.@ALIGN(Vec3, 16) + +Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) +Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/tests/MyGame/Example2/Example2.jl b/tests/MyGame/Example2/Example2.jl new file mode 100644 index 00000000000..eaef8178155 --- /dev/null +++ b/tests/MyGame/Example2/Example2.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +module Example2 + include("Monster.jl") +end diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl new file mode 100644 index 00000000000..91b98e333fb --- /dev/null +++ b/tests/MyGame/Example2/Monster.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +import FlatBuffers + +mutable struct Monster +end +FlatBuffers.@ALIGN(Monster, 1) + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl new file mode 100644 index 00000000000..518cb89e3f0 --- /dev/null +++ b/tests/MyGame/InParentNamespace.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +import FlatBuffers + +mutable struct InParentNamespace +end +FlatBuffers.@ALIGN(InParentNamespace, 1) + +InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) +InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) diff --git a/tests/MyGame/MyGame.jl b/tests/MyGame/MyGame.jl new file mode 100644 index 00000000000..075ee85e35a --- /dev/null +++ b/tests/MyGame/MyGame.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +module MyGame + include("InParentNamespace.jl") + include("Example2/Example2.jl") + include("Example/Example.jl") +end diff --git a/tests/generate_code.sh b/tests/generate_code.sh index 89b868fd131..840016ef0dd 100755 --- a/tests/generate_code.sh +++ b/tests/generate_code.sh @@ -15,9 +15,9 @@ # limitations under the License. set -e -../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json -../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs -../flatc --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs +../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --julia --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json +../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --julia --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs +../flatc --cpp --js --ts --php --julia --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs ../flatc -b --schema --bfbs-comments -I include_test monster_test.fbs ../flatc --jsonschema --schema -I include_test monster_test.fbs cd ../samples diff --git a/tests/namespace_test/NamespaceA/NamespaceA.jl b/tests/namespace_test/NamespaceA/NamespaceA.jl new file mode 100644 index 00000000000..5f8e0d98900 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceA.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +module NamespaceA + include("NamespaceB/NamespaceB.jl") + include("TableInFirstNS.jl") + include("SecondTableInA.jl") +end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl new file mode 100644 index 00000000000..9fa9777b485 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +@enum EnumInNestedNS::Int8 begin + EnumInNestedNSA = 0 + EnumInNestedNSB = 1 + EnumInNestedNSC = 2 +end + diff --git a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl new file mode 100644 index 00000000000..5dbfb19cf18 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +module NamespaceB + include("EnumInNestedNS.jl") + include("TableInNestedNS.jl") + include("StructInNestedNS.jl") +end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl new file mode 100644 index 00000000000..77aa0b205c1 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +import FlatBuffers + +FlatBuffers.@STRUCT struct StructInNestedNS + a::Int32 + b::Int32 +end +FlatBuffers.@ALIGN(StructInNestedNS, 4) + +StructInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(StructInNestedNS, buf) +StructInNestedNS(io::IO) = FlatBuffers.deserialize(io, StructInNestedNS) diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl new file mode 100644 index 00000000000..06125901142 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInNestedNS + foo::Int32 = 0 +end +FlatBuffers.@ALIGN(TableInNestedNS, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInNestedNS} = [ + 0x00000004 +] + +TableInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInNestedNS, buf) +TableInNestedNS(io::IO) = FlatBuffers.deserialize(io, TableInNestedNS) diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl new file mode 100644 index 00000000000..6cdebe37fe7 --- /dev/null +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -0,0 +1,17 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +import ..NamespaceC +import FlatBuffers + +FlatBuffers.@with_kw mutable struct SecondTableInA + refer_to_c::Union{NamespaceC.TableInC, Nothing} = nothing +end +FlatBuffers.@ALIGN(SecondTableInA, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:SecondTableInA} = [ + 0x00000004 +] + +SecondTableInA(buf::AbstractVector{UInt8}) = FlatBuffers.read(SecondTableInA, buf) +SecondTableInA(io::IO) = FlatBuffers.deserialize(io, SecondTableInA) diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl new file mode 100644 index 00000000000..23eb1b4df4d --- /dev/null +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -0,0 +1,19 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +import .NamespaceB +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInFirstNS + foo_table::Union{NamespaceB.TableInNestedNS, Nothing} = nothing + foo_enum::EnumInNestedNS = 0 + foo_struct::Union{NamespaceB.StructInNestedNS, Nothing} = nothing +end +FlatBuffers.@ALIGN(TableInFirstNS, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInFirstNS} = [ + 0x00000004, 0x00000006, 0x00000008 +] + +TableInFirstNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInFirstNS, buf) +TableInFirstNS(io::IO) = FlatBuffers.deserialize(io, TableInFirstNS) diff --git a/tests/namespace_test/NamespaceC/NamespaceC.jl b/tests/namespace_test/NamespaceC/NamespaceC.jl new file mode 100644 index 00000000000..bc2c2417e76 --- /dev/null +++ b/tests/namespace_test/NamespaceC/NamespaceC.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceC + +module NamespaceC + include("TableInC.jl") +end diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl new file mode 100644 index 00000000000..c5f58206d48 --- /dev/null +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceC + +import ..NamespaceA +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInC + refer_to_a1::Union{NamespaceA.TableInFirstNS, Nothing} = nothing + refer_to_a2::Union{NamespaceA.SecondTableInA, Nothing} = nothing +end +FlatBuffers.@ALIGN(TableInC, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInC} = [ + 0x00000004, 0x00000006 +] + +TableInC(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInC, buf) +TableInC(io::IO) = FlatBuffers.deserialize(io, TableInC) diff --git a/tests/namespace_test/NamespaceTest1.jl b/tests/namespace_test/NamespaceTest1.jl new file mode 100644 index 00000000000..2f158d945f1 --- /dev/null +++ b/tests/namespace_test/NamespaceTest1.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceTest1 + +module NamespaceTest1 + include("NamespaceA/NamespaceA.jl") +end diff --git a/tests/namespace_test/NamespaceTest2.jl b/tests/namespace_test/NamespaceTest2.jl new file mode 100644 index 00000000000..960f5a3a27f --- /dev/null +++ b/tests/namespace_test/NamespaceTest2.jl @@ -0,0 +1,8 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceTest2 + +module NamespaceTest2 + include("NamespaceA/NamespaceA.jl") + include("NamespaceC/NamespaceC.jl") +end diff --git a/tests/union_vector/UnionVector.jl b/tests/union_vector/UnionVector.jl new file mode 100644 index 00000000000..f65c7a3310b --- /dev/null +++ b/tests/union_vector/UnionVector.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +module UnionVector + include("UnionVector/Attacker.jl") + include("UnionVector/Rapunzel.jl") + include("UnionVector/BookReader.jl") + include("UnionVector/Character.jl") + include("UnionVector/Movie.jl") +end diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl new file mode 100644 index 00000000000..e7553a68942 --- /dev/null +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Attacker + sword_attack_damage::Int32 = 0 +end +FlatBuffers.@ALIGN(Attacker, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Attacker} = [ + 0x00000004 +] + +Attacker(buf::AbstractVector{UInt8}) = FlatBuffers.read(Attacker, buf) +Attacker(io::IO) = FlatBuffers.deserialize(io, Attacker) diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl new file mode 100644 index 00000000000..253ea64c972 --- /dev/null +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@STRUCT struct BookReader + books_read::Int32 +end +FlatBuffers.@ALIGN(BookReader, 4) + +BookReader(buf::AbstractVector{UInt8}) = FlatBuffers.read(BookReader, buf) +BookReader(io::IO) = FlatBuffers.deserialize(io, BookReader) diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl new file mode 100644 index 00000000000..603b1055be8 --- /dev/null +++ b/tests/union_vector/UnionVector/Character.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@UNION(Character, ( + Nothing, + Attacker, + Rapunzel, + BookReader, + BookReader, + String, + String, +)) + diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl new file mode 100644 index 00000000000..0544e8b7c2b --- /dev/null +++ b/tests/union_vector/UnionVector/Movie.jl @@ -0,0 +1,21 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Movie + main_character_type::UInt8 = 0 + main_character::Character = nothing + characters_type::Union{Vector{UInt8}, Nothing} = nothing + characters::Union{Vector{Character}, Nothing} = nothing +end +FlatBuffers.@ALIGN(Movie, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Movie} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A +] +FlatBuffers.root_type(::Type{T}) where {T<:Movie} = true +FlatBuffers.file_identifier(::Type{T}) where {T<:Movie} = "MOVI" + +Movie(buf::AbstractVector{UInt8}) = FlatBuffers.read(Movie, buf) +Movie(io::IO) = FlatBuffers.deserialize(io, Movie) diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl new file mode 100644 index 00000000000..b524a9fb0bd --- /dev/null +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@STRUCT struct Rapunzel + hair_length::Int32 +end +FlatBuffers.@ALIGN(Rapunzel, 4) + +Rapunzel(buf::AbstractVector{UInt8}) = FlatBuffers.read(Rapunzel, buf) +Rapunzel(io::IO) = FlatBuffers.deserialize(io, Rapunzel) From 73181da271d39c2db5f3508f25f8e2d4e3f9b787 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Fri, 31 Aug 2018 15:05:27 +1000 Subject: [PATCH 02/19] add Julia support to flatbuffers --- CMakeLists.txt | 1 + docs/source/Compiler.md | 2 + docs/source/FlatBuffers.md | 2 + docs/source/JuliaUsage.md | 57 ++ docs/source/Support.md | 36 +- docs/source/Tutorial.md | 134 ++- docs/source/doxyfile | 1 + docs/source/doxygen_layout.xml | 2 + grpc/README.md | 11 + include/flatbuffers/idl.h | 48 +- samples/Monster.jl | 7 + samples/MyGame/MyGame.jl | 7 + samples/MyGame/Sample/Color.jl | 10 + samples/MyGame/Sample/Equipment.jl | 11 + samples/MyGame/Sample/Monster.jl | 27 + samples/MyGame/Sample/Sample.jl | 11 + samples/MyGame/Sample/Vec3.jl | 15 + samples/MyGame/Sample/Weapon.jl | 17 + samples/julia_sample.sh | 48 ++ samples/sample_binary.jl | 49 ++ src/flatc_main.cpp | 4 + src/idl_gen_cpp.cpp | 2 +- src/idl_gen_general.cpp | 4 +- src/idl_gen_go.cpp | 2 +- src/idl_gen_julia.cpp | 815 ++++++++++++++++++ src/idl_gen_lobster.cpp | 2 +- src/idl_gen_lua.cpp | 2 +- src/idl_gen_php.cpp | 2 +- src/idl_gen_python.cpp | 2 +- src/idl_gen_rust.cpp | 4 +- src/idl_gen_text.cpp | 6 +- src/idl_parser.cpp | 19 +- tests/MonsterTest.jl | 7 + tests/MyGame/Example/Ability.jl | 14 + tests/MyGame/Example/AnyAmbiguousAliases.jl | 13 + tests/MyGame/Example/AnyUniqueAliases.jl | 14 + tests/MyGame/Example/Any_.jl | 14 + tests/MyGame/Example/Color.jl | 10 + tests/MyGame/Example/Example.jl | 18 + tests/MyGame/Example/Monster.jl | 84 ++ tests/MyGame/Example/Referrable.jl | 16 + tests/MyGame/Example/Stat.jl | 18 + tests/MyGame/Example/Test.jl | 14 + .../MyGame/Example/TestSimpleTableWithEnum.jl | 16 + tests/MyGame/Example/TypeAliases.jl | 29 + tests/MyGame/Example/Vec3.jl | 18 + tests/MyGame/Example2/Example2.jl | 7 + tests/MyGame/Example2/Monster.jl | 12 + tests/MyGame/InParentNamespace.jl | 12 + tests/MyGame/MyGame.jl | 9 + tests/generate_code.sh | 6 +- tests/namespace_test/NamespaceA/NamespaceA.jl | 9 + .../NamespaceA/NamespaceB/EnumInNestedNS.jl | 10 + .../NamespaceA/NamespaceB/NamespaceB.jl | 9 + .../NamespaceA/NamespaceB/StructInNestedNS.jl | 14 + .../NamespaceA/NamespaceB/TableInNestedNS.jl | 16 + .../NamespaceA/SecondTableInA.jl | 17 + .../NamespaceA/TableInFirstNS.jl | 19 + tests/namespace_test/NamespaceC/NamespaceC.jl | 7 + tests/namespace_test/NamespaceC/TableInC.jl | 18 + tests/namespace_test/NamespaceTest1.jl | 7 + tests/namespace_test/NamespaceTest2.jl | 8 + tests/union_vector/UnionVector.jl | 11 + tests/union_vector/UnionVector/Attacker.jl | 16 + tests/union_vector/UnionVector/BookReader.jl | 13 + tests/union_vector/UnionVector/Character.jl | 16 + tests/union_vector/UnionVector/Movie.jl | 21 + tests/union_vector/UnionVector/Rapunzel.jl | 13 + 68 files changed, 1851 insertions(+), 64 deletions(-) create mode 100644 docs/source/JuliaUsage.md create mode 100644 samples/Monster.jl create mode 100644 samples/MyGame/MyGame.jl create mode 100644 samples/MyGame/Sample/Color.jl create mode 100644 samples/MyGame/Sample/Equipment.jl create mode 100644 samples/MyGame/Sample/Monster.jl create mode 100644 samples/MyGame/Sample/Sample.jl create mode 100644 samples/MyGame/Sample/Vec3.jl create mode 100644 samples/MyGame/Sample/Weapon.jl create mode 100755 samples/julia_sample.sh create mode 100644 samples/sample_binary.jl create mode 100644 src/idl_gen_julia.cpp create mode 100644 tests/MonsterTest.jl create mode 100644 tests/MyGame/Example/Ability.jl create mode 100644 tests/MyGame/Example/AnyAmbiguousAliases.jl create mode 100644 tests/MyGame/Example/AnyUniqueAliases.jl create mode 100644 tests/MyGame/Example/Any_.jl create mode 100644 tests/MyGame/Example/Color.jl create mode 100644 tests/MyGame/Example/Example.jl create mode 100644 tests/MyGame/Example/Monster.jl create mode 100644 tests/MyGame/Example/Referrable.jl create mode 100644 tests/MyGame/Example/Stat.jl create mode 100644 tests/MyGame/Example/Test.jl create mode 100644 tests/MyGame/Example/TestSimpleTableWithEnum.jl create mode 100644 tests/MyGame/Example/TypeAliases.jl create mode 100644 tests/MyGame/Example/Vec3.jl create mode 100644 tests/MyGame/Example2/Example2.jl create mode 100644 tests/MyGame/Example2/Monster.jl create mode 100644 tests/MyGame/InParentNamespace.jl create mode 100644 tests/MyGame/MyGame.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceA.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl create mode 100644 tests/namespace_test/NamespaceA/SecondTableInA.jl create mode 100644 tests/namespace_test/NamespaceA/TableInFirstNS.jl create mode 100644 tests/namespace_test/NamespaceC/NamespaceC.jl create mode 100644 tests/namespace_test/NamespaceC/TableInC.jl create mode 100644 tests/namespace_test/NamespaceTest1.jl create mode 100644 tests/namespace_test/NamespaceTest2.jl create mode 100644 tests/union_vector/UnionVector.jl create mode 100644 tests/union_vector/UnionVector/Attacker.jl create mode 100644 tests/union_vector/UnionVector/BookReader.jl create mode 100644 tests/union_vector/UnionVector/Character.jl create mode 100644 tests/union_vector/UnionVector/Movie.jl create mode 100644 tests/union_vector/UnionVector/Rapunzel.jl diff --git a/CMakeLists.txt b/CMakeLists.txt index 389596e4f3f..9aa2356678a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,7 @@ set(FlatBuffers_Compiler_SRCS src/idl_gen_general.cpp src/idl_gen_go.cpp src/idl_gen_js_ts.cpp + src/idl_gen_julia.cpp src/idl_gen_php.cpp src/idl_gen_python.cpp src/idl_gen_lobster.cpp diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md index 420019cdce2..f42663570fa 100644 --- a/docs/source/Compiler.md +++ b/docs/source/Compiler.md @@ -45,6 +45,8 @@ For any schema input files, one or more generators can be specified: - `--rust`, `-r` : Generate Rust code. +- `--julia`: Generate Julia code. + For any data input files: - `--binary`, `-b` : If data is contained in this file, generate a diff --git a/docs/source/FlatBuffers.md b/docs/source/FlatBuffers.md index 7cc93b92c44..967ee3f3328 100644 --- a/docs/source/FlatBuffers.md +++ b/docs/source/FlatBuffers.md @@ -146,6 +146,8 @@ sections provide a more in-depth usage guide. own programs. - How to [use the generated Rust code](@ref flatbuffers_guide_use_rust) in your own programs. +- How to [use the generated Julia code](@ref flatbuffers_guide_use_julia) in your + own programs. - [Support matrix](@ref flatbuffers_support) for platforms/languages/features. - Some [benchmarks](@ref flatbuffers_benchmarks) showing the advantage of using FlatBuffers. diff --git a/docs/source/JuliaUsage.md b/docs/source/JuliaUsage.md new file mode 100644 index 00000000000..cd2a80c7905 --- /dev/null +++ b/docs/source/JuliaUsage.md @@ -0,0 +1,57 @@ +Use in Julia {#flatbuffers_guide_use_julia} +============== +## Before you get started +Before diving into the FlatBuffers usage in Julia, it should be noted that the +[Tutorial](@ref flatbuffers_guide_tutorial) page has a complete guide to general +FlatBuffers usage in all of the supported languages (including Julia). This +page is designed to cover the nuances of FlatBuffers usage, specific to +Julia. + +You should also have read the [Building](@ref flatbuffers_guide_building) +documentation to build `flatc` and should be familiar with +[Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) and +[Writing a schema](@ref flatbuffers_guide_writing_schema). + +## FlatBuffers Julia package +The `flatc` backend for Julia makes heavy use of the [FlatBuffers.jl package](https://github.com/JuliaData/FlatBuffers.jl). +Documentation for this package may be found [here](http://juliadata.github.io/FlatBuffers.jl/stable). +You can also browse the source code on the [FlatBuffers.jl GitHub page](https://github.com/JuliaData/FlatBuffers.jl). + +## Testing the FlatBuffers Julia library +The code to test the Julia library can be found at [FlatBuffers.jl/test/flatc.jl](https://github.com/JuliaData/FlatBuffers.jl/tree/master/test/flatc.jl). To run the tests, run `julia test/flatc.jl` in the top-level directory. +To obtain Julia itself, go to the [Julia homepage](http://julialang.org) +for binaries or source code for your platform. + +## Using the FlatBuffers Julia library +*Note: See [Tutorial](@ref flatbuffers_guide_tutorial) for a more in-depth +example of how to use FlatBuffers in Julia.* + +There is support for both reading and writing FlatBuffers in Julia. +To use FlatBuffers in your own code, first generate Julia modules from your +schema with the `--julia` option to `flatc`. Then you can include both +FlatBuffers and the generated code to read or write a FlatBuffer. + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.julia} + import FlatBuffers + + include("MyGame/MyGame.jl") + import .MyGame.Example.Monster + + monster = open("monsterdata_test.mon", "r") do f Monster(f) end +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Now you can access values like this: + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.julia} + hp = monster.hp + pos = monster.pos +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +More detailed documentation for using the FlatBuffers Julia package +may be found [here](http://juliadata.github.io/FlatBuffers.jl/stable). + +## Speed +The FlatBuffers Julia package has not been thoroughly optimized for speed. +For example, instead of referring to memory in-place, values are instead +loaded with `unsafe_load`. It does however support deduplication of vtables, which +saves roughly 30% of time spent writing, when compared to writing objects +with duplicate vtables. + +
\ No newline at end of file diff --git a/docs/source/Support.md b/docs/source/Support.md index c8ac7f7e89d..fd2f01f1e72 100644 --- a/docs/source/Support.md +++ b/docs/source/Support.md @@ -18,23 +18,23 @@ In general: NOTE: this table is a start, it needs to be extended. -Feature | C++ | Java | C# | Go | Python | JS | TS | C | PHP | Dart | Lobster | Rust ------------------------------- | ------ | ------ | ------ | ------ | ------ | --------- | --------- | ------ | --- | ------- | ------- | ---- -Codegen for all basic features | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | WiP | Yes | Yes | Yes -JSON parsing | Yes | No | No | No | No | No | No | Yes | No | No | Yes | No -Simple mutation | Yes | Yes | Yes | Yes | No | No | No | No | No | No | No | No -Reflection | Yes | No | No | No | No | No | No | Basic | No | No | No | No -Buffer verifier | Yes | No | No | No | No | No | No | Yes | No | No | No | No -Testing: basic | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | ? | Yes | Yes | Yes -Testing: fuzz | Yes | No | No | Yes | Yes | No | No | No | ? | No | No | Yes -Performance: | Superb | Great | Great | Great | Ok | ? | ? | Superb | ? | ? | Great | Superb -Platform: Windows | VS2010 | Yes | Yes | ? | ? | ? | Yes | VS2010 | ? | Yes | Yes | Yes -Platform: Linux | GCC282 | Yes | ? | Yes | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes -Platform: OS X | Xcode4 | ? | ? | ? | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes -Platform: Android | NDK10d | Yes | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? -Platform: iOS | ? | ? | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? -Engine: Unity | ? | ? | Yes | ? | ? | ? | ? | ? | ? | ? | No | ? -Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | evanw/ev* | kr* | mik* | ch* | dnfield | aard* | rw +Feature | C++ | Java | C# | Go | Python | JS | TS | C | PHP | Dart | Lobster | Rust | Julia | +------------------------------ | ------ | ------ | ------ | ------ | ------ | --------- | --------- | ------ | --- | ------- | ------- | ---- | ---- | +Codegen for all basic features | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | WiP | Yes | Yes | Yes | Yes | +JSON parsing | Yes | No | No | No | No | No | No | Yes | No | No | Yes | No | No | +Simple mutation | Yes | Yes | Yes | Yes | No | No | No | No | No | No | No | No | No | +Reflection | Yes | No | No | No | No | No | No | Basic | No | No | No | No | No | +Buffer verifier | Yes | No | No | No | No | No | No | Yes | No | No | No | No | No | +Testing: basic | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Testing: fuzz | Yes | No | No | Yes | Yes | No | No | No | ? | No | No | Yes | Yes | +Performance: | Superb | Great | Great | Great | Ok | ? | ? | Superb | ? | ? | Great | Superb | ? | +Platform: Windows | VS2010 | Yes | Yes | ? | ? | ? | Yes | VS2010 | ? | Yes | Yes | Yes | Yes | +Platform: Linux | GCC282 | Yes | ? | Yes | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Platform: OS X | Xcode4 | ? | ? | ? | Yes | ? | Yes | Yes | ? | Yes | Yes | Yes | Yes | +Platform: Android | NDK10d | Yes | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | ? | +Platform: iOS | ? | ? | ? | ? | ? | ? | ? | ? | ? | Flutter | Yes | ? | ? | +Engine: Unity | ? | ? | Yes | ? | ? | ? | ? | ? | ? | ? | No | ? | ? | +Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | evanw/ev* | kr* | mik* | ch* | dnfield | aard* | rw | jq*/rk* | * aard = aardappel (previously: gwvo) * ev = evolutional @@ -42,5 +42,7 @@ Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | ev * mik = mikkelfj * ch = chobie * kr = krojew + * jq = quinnj + * rk = rkat
diff --git a/docs/source/Tutorial.md b/docs/source/Tutorial.md index f2eaa2ea72d..2344a9c99f0 100644 --- a/docs/source/Tutorial.md +++ b/docs/source/Tutorial.md @@ -34,6 +34,7 @@ Please select your desired language for our quest: Lua Lobster Rust + Julia \endhtmlonly @@ -148,7 +149,9 @@ For your chosen language, please cross-reference with:
[sample_binary.rs](https://github.com/google/flatbuffers/blob/master/samples/sample_binary.rs)
- +
+[sample_binary.jl](https://github.com/google/flatbuffers/blob/master/samples/sample_binary.jl) +
## Writing the Monsters' FlatBuffer Schema @@ -353,6 +356,12 @@ Please be aware of the difference between `flatc` and `flatcc` tools. ./../flatc --rust monster.fbs ~~~ +
+~~~{.sh} + cd flatbuffers/sample + ./../flatc --julia monster.fbs +~~~ +
For a more complete guide to using the `flatc` compiler, please read the [Using the schema compiler](@ref flatbuffers_guide_using_schema_compiler) @@ -504,6 +513,18 @@ The first step is to import/include the library, generated files, etc. Weapon, WeaponArgs}; ~~~ +
+~~~{.jl} + import FlatBuffers + + # Generated by `flatc`. + import .MyGame.Sample.Color + import .MyGame.Sample.Equipment + import .MyGame.Sample.Monster + import .MyGame.Sample.Vec3 + import .MyGame.Sample.Weapon +~~~ +
Now we are ready to start building some buffers. In order to start, we need to create an instance of the `FlatBufferBuilder`, which will contain the buffer @@ -602,6 +623,13 @@ which will grow automatically if needed: let mut builder = flatbuffers::FlatBufferBuilder::new_with_capacity(1024); ~~~ +
+~~~{julia} + # types are not built up gradually in Julia, + # instead they are written to the buffer all at once + buf = IOBuffer() +~~~ +
After creating the `builder`, we can start serializing our data. Before we make our `orc` Monster, lets create some `Weapon`s: a `Sword` and an `Axe`. @@ -838,6 +866,12 @@ our `orc` Monster, lets create some `Weapon`s: a `Sword` and an `Axe`. }); ~~~ +
+~~~{.jl} + sword = Weapon(;name="Sword", damage=3) + axe = Weapon(;name="Axe", damage=5) +~~~ +
Now let's create our monster, the `orc`. For this `orc`, lets make him `red` with rage, positioned at `(1.0, 2.0, 3.0)`, and give him @@ -1018,6 +1052,20 @@ traversal. This is generally easy to do on any tree structures. let inventory = builder.create_vector(&[0u8, 1, 2, 3, 4, 5, 6, 7, 8, 9]); ~~~ +
+~~~{.jl} + # we don't need to incrementally build things up here, just make the monster + orc = Monster(; + name="Orc", + pos=Vec3(1.0, 2.0, 3.0), + hp=300, + inventory=collect(1:10), + color=MyGame.Sample.ColorRed, + weapons=[sword, axe], + equipped=axe + ) +~~~ +
We serialized two built-in data types (`string` and `vector`) and captured their return values. These values are offsets into the serialized data, @@ -1280,6 +1328,11 @@ for the `path` field above: // let path = builder.create_vector(&[&x, &y]); ~~~ +
+~~~{.jl} + path = Vec3[Vec3(1.0, 2.0, 3.0), Vec3(4.0, 5.0, 6.0)] +~~~ +
We have now serialized the non-scalar components of the orc, so we can serialize the monster itself: @@ -1536,6 +1589,11 @@ can serialize the monster itself: }); ~~~ +
+~~~{.rs} + FlatBuffers.serialize(io, orc) +~~~ +
Note how we create `Vec3` struct in-line in the table. Unlike tables, structs are simple combinations of scalars that are always stored inline, just like @@ -1698,6 +1756,11 @@ Here is a repetition these lines, to help highlight them more clearly: monster_builder.add_equipped(axe.as_union_value()); // Union data ~~~ +
+ ~~~{.jl} + # this section is unnecessary in Julia + ~~~ +
After you have created your buffer, you will have the offset to the root of the data in the `orc` variable, so you can finish the buffer by calling the @@ -1787,6 +1850,11 @@ appropriate `finish` method. builder.finish(orc, None); ~~~ +
+~~~{.rs} + # this section is unnecessary in Julia +~~~ +
The buffer is now ready to be stored somewhere, sent over the network, be compressed, or whatever you'd like to do with it. You can access the buffer @@ -1903,6 +1971,11 @@ like so: let buf = builder.finished_data(); // Of type `&[u8]` ~~~ +
+~~~{.jl} + buf = take!(io) +~~~ +
Now you can write the bytes to a file, send them over the network.. @@ -2051,6 +2124,17 @@ import './monster_my_game.sample_generated.dart' as myGame; Weapon, WeaponArgs}; ~~~ +
+~~~{.py} + import FlatBuffers + + # Generated by `flatc`. + import .MyGame.Sample.Any + import .MyGame.Sample.Color + import .MyGame.Sample.Monster + import .MyGame.Sample.Vec3 +~~~ +
Then, assuming you have a buffer of bytes received from disk, network, etc., you can create start accessing the buffer like so: @@ -2186,6 +2270,13 @@ myGame.Monster monster = new myGame.Monster(data); let monster = get_root_as_monster(buf); ~~~ +
+~~~{.jl} + # where x is either an IO or an AbstractVector{UInt8} containing the data + monster = Monster(x) +~~~ +
+ If you look in the generated files from the schema compiler, you will see it generated accessors for all non-`deprecated` fields. For example: @@ -2286,6 +2377,13 @@ accessors for all non-`deprecated` fields. For example: let name = monster.name(); ~~~ +
+~~~{.rs} + hp = monster.hp + mana = monster.mana + name = monster.name +~~~ +
These should hold `300`, `150`, and `"Orc"` respectively. @@ -2403,6 +2501,14 @@ To access sub-objects, in the case of our `pos`, which is a `Vec3`: let z = pos.z(); ~~~ +
+~~~{.jl} + pos = monster.pos + x = pos.x + y = pos.y + z = pos.z +~~~ +
`x`, `y`, and `z` will contain `1.0`, `2.0`, and `3.0`, respectively. @@ -2497,6 +2603,11 @@ FlatBuffers `vector`. let third_item = inv[2]; ~~~ +
+~~~{.jl} + third_item = monster.inventory[3] +~~~ +
For `vector`s of `table`s, you can access the elements like any other vector, except your need to handle the result as a FlatBuffer `table`: @@ -2603,6 +2714,13 @@ except your need to handle the result as a FlatBuffer `table`: let second_weapon_damage = wep2.damage(); ~~~ +
+~~~{.jl} + # no tables are needed in Julia + second_weapon_name = monster.weapons[2].name + second_weapon_damage = monster.weapons[2].damage +~~~ +
Last, we can access our `Equipped` FlatBuffer `union`. Just like when we created the `union`, we need to get both parts of the `union`: the type and the data. @@ -2776,6 +2894,11 @@ We can access the type to dynamically cast the data as needed (since the let weapon_damage = equipped.damage(); ~~~ +
+~~~{.jl} + equipped = monster.equipped +~~~ +
## Mutating FlatBuffers @@ -2871,6 +2994,11 @@ mutators like so: ~~~ +
+~~~{.rs} + +~~~ +
We use the somewhat verbose term `mutate` instead of `set` to indicate that this is a special use case, not to be confused with the default way of constructing @@ -2997,5 +3125,7 @@ For your chosen language, see:
[Use in Rust](@ref flatbuffers_guide_use_rust)
- +
+[Use in Julia](@ref flatbuffers_guide_use_julia) +

diff --git a/docs/source/doxyfile b/docs/source/doxyfile index 6ba3c108cc9..9b1d6163eca 100644 --- a/docs/source/doxyfile +++ b/docs/source/doxyfile @@ -761,6 +761,7 @@ INPUT = "FlatBuffers.md" \ "LuaUsage.md" \ "LobsterUsage.md" \ "RustUsage.md" \ + "JuliaUsage.md" \ "Support.md" \ "Benchmarks.md" \ "WhitePaper.md" \ diff --git a/docs/source/doxygen_layout.xml b/docs/source/doxygen_layout.xml index a2325075f2d..5f9dc73aa51 100644 --- a/docs/source/doxygen_layout.xml +++ b/docs/source/doxygen_layout.xml @@ -47,6 +47,8 @@ title="Use in Lobster"/> + diff --git a/grpc/README.md b/grpc/README.md index 544651d6024..b9ffd47a60e 100644 --- a/grpc/README.md +++ b/grpc/README.md @@ -28,4 +28,15 @@ the GRPC libraries for this to compile. This test will build using the 1. `ln -s ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.6 ${GRPC_INSTALL_PATH}/lib/libgrpc++_unsecure.so.1` 2. `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${GRPC_INSTALL_PATH}/lib` +3. `make test ARGS=-V` + +### macOS + +1. Fix the dynamic library paths in grpc ``` +for l in ${GRPC_INSTALL_PATH}/lib/*.dylib; do + for dep in ${GRPC_INSTALL_PATH}/lib/*.dylib; do + install_name_tool -change ${dep##*/} $dep $l; done + done +done``` +2. `for l in libgrpc_unsecure libgrpc++_unsecure libgpr; do install_name_tool -change $l.dylib ${GRPC_INSTALL_PATH}/lib/$l.dylib grpctest; done` 3. `make test ARGS=-V` diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 18e1db9d54b..5fc30cbf81e 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -47,24 +47,24 @@ namespace flatbuffers { // of type tokens. // clang-format off #define FLATBUFFERS_GEN_TYPES_SCALAR(TD) \ - TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8) \ - TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8) /* begin scalar/int */ \ - TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool) \ - TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8) \ - TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8) \ - TD(SHORT, "short", int16_t, short, int16, short, int16, i16) \ - TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16) \ - TD(INT, "int", int32_t, int, int32, int, int32, i32) \ - TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32) \ - TD(LONG, "long", int64_t, long, int64, long, int64, i64) \ - TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64) /* end int */ \ - TD(FLOAT, "float", float, float, float32, float, float32, f32) /* begin float */ \ - TD(DOUBLE, "double", double, double, float64, double, float64, f64) /* end float/scalar */ + TD(NONE, "", uint8_t, byte, byte, byte, uint8, u8, UInt8) \ + TD(UTYPE, "", uint8_t, byte, byte, byte, uint8, u8, UInt8) /* begin scalar/int */ \ + TD(BOOL, "bool", uint8_t, boolean,bool, bool, bool, bool, Bool) \ + TD(CHAR, "byte", int8_t, byte, int8, sbyte, int8, i8, Int8) \ + TD(UCHAR, "ubyte", uint8_t, byte, byte, byte, uint8, u8, UInt8) \ + TD(SHORT, "short", int16_t, short, int16, short, int16, i16, Int16) \ + TD(USHORT, "ushort", uint16_t, short, uint16, ushort, uint16, u16, UInt16) \ + TD(INT, "int", int32_t, int, int32, int, int32, i32, Int32) \ + TD(UINT, "uint", uint32_t, int, uint32, uint, uint32, u32, UInt32) \ + TD(LONG, "long", int64_t, long, int64, long, int64, i64, Int64) \ + TD(ULONG, "ulong", uint64_t, long, uint64, ulong, uint64, u64, UInt64) /* end int */ \ + TD(FLOAT, "float", float, float, float32, float, float32, f32, Float32) /* begin float */ \ + TD(DOUBLE, "double", double, double, float64, double, float64, f64, Float64) /* end float/scalar */ #define FLATBUFFERS_GEN_TYPES_POINTER(TD) \ - TD(STRING, "string", Offset, int, int, StringOffset, int, unused) \ - TD(VECTOR, "", Offset, int, int, VectorOffset, int, unused) \ - TD(STRUCT, "", Offset, int, int, int, int, unused) \ - TD(UNION, "", Offset, int, int, int, int, unused) + TD(STRING, "string", Offset, int, int, StringOffset, int, unused, Int) \ + TD(VECTOR, "", Offset, int, int, VectorOffset, int, unused, Int) \ + TD(STRUCT, "", Offset, int, int, int, int, unused, Int) \ + TD(UNION, "", Offset, int, int, int, int, unused, Int) // The fields are: // - enum @@ -75,13 +75,14 @@ namespace flatbuffers { // - C# / .Net type. // - Python type. // - Rust type. +// - Julia type. // using these macros, we can now write code dealing with types just once, e.g. /* switch (type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ // do something specific to CTYPE here FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -99,14 +100,14 @@ __extension__ // Stop GCC complaining about trailing comma with -Wpendantic. #endif enum BaseType { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ BASE_TYPE_ ## ENUM, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPEL) \ static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \ "define largest_scalar_t as " #CTYPE); FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) @@ -427,6 +428,7 @@ struct IDLOptions { kLua = 1 << 12, kLobster = 1 << 13, kRust = 1 << 14, + kJulia = 1 << 15, kMAX }; @@ -842,6 +844,12 @@ extern bool GeneratePython(const Parser &parser, const std::string &path, const std::string &file_name); +// Generate Julia files from the definitions in the Parser object. +// See idl_gen_julia.cpp. +extern bool GenerateJulia(const Parser &parser, + const std::string &path, + const std::string &file_name); + // Generate Lobster files from the definitions in the Parser object. // See idl_gen_lobster.cpp. extern bool GenerateLobster(const Parser &parser, diff --git a/samples/Monster.jl b/samples/Monster.jl new file mode 100644 index 00000000000..aac20e5f01b --- /dev/null +++ b/samples/Monster.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Monster + +module Monster + include("MyGame/MyGame.jl") +end diff --git a/samples/MyGame/MyGame.jl b/samples/MyGame/MyGame.jl new file mode 100644 index 00000000000..6379ae45d93 --- /dev/null +++ b/samples/MyGame/MyGame.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +module MyGame + include("Sample/Sample.jl") +end diff --git a/samples/MyGame/Sample/Color.jl b/samples/MyGame/Sample/Color.jl new file mode 100644 index 00000000000..e6108b38686 --- /dev/null +++ b/samples/MyGame/Sample/Color.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +@enum Color::Int8 begin + ColorRed = 0 + ColorGreen = 1 + ColorBlue = 2 +end + diff --git a/samples/MyGame/Sample/Equipment.jl b/samples/MyGame/Sample/Equipment.jl new file mode 100644 index 00000000000..dfeb2e26d37 --- /dev/null +++ b/samples/MyGame/Sample/Equipment.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@UNION(Equipment, ( + Nothing, + Weapon, +)) + diff --git a/samples/MyGame/Sample/Monster.jl b/samples/MyGame/Sample/Monster.jl new file mode 100644 index 00000000000..b5904138ac7 --- /dev/null +++ b/samples/MyGame/Sample/Monster.jl @@ -0,0 +1,27 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Monster + pos::Union{Vec3, Nothing} = nothing + mana::Int16 = 150 + hp::Int16 = 100 + name::Union{String, Nothing} = nothing + inventory::Union{Vector{UInt8}, Nothing} = nothing + color::Color = 2 + weapons::Union{Vector{Weapon}, Nothing} = nothing + equipped_type::UInt8 = 0 + equipped::Equipment = nothing +end +FlatBuffers.@ALIGN(Monster, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000E, 0x00000010, 0x00000012, 0x00000014, + 0x00000016 +] +FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/samples/MyGame/Sample/Sample.jl b/samples/MyGame/Sample/Sample.jl new file mode 100644 index 00000000000..07e66c9dfd2 --- /dev/null +++ b/samples/MyGame/Sample/Sample.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +module Sample + include("Color.jl") + include("Weapon.jl") + include("Equipment.jl") + include("Vec3.jl") + include("Monster.jl") +end diff --git a/samples/MyGame/Sample/Vec3.jl b/samples/MyGame/Sample/Vec3.jl new file mode 100644 index 00000000000..57e72494a7a --- /dev/null +++ b/samples/MyGame/Sample/Vec3.jl @@ -0,0 +1,15 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 +end +FlatBuffers.@ALIGN(Vec3, 4) + +Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) +Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/samples/MyGame/Sample/Weapon.jl b/samples/MyGame/Sample/Weapon.jl new file mode 100644 index 00000000000..c94e9b14ca0 --- /dev/null +++ b/samples/MyGame/Sample/Weapon.jl @@ -0,0 +1,17 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Sample + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Weapon + name::Union{String, Nothing} = nothing + damage::Int16 = 0 +end +FlatBuffers.@ALIGN(Weapon, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Weapon} = [ + 0x00000004, 0x00000006 +] + +Weapon(buf::AbstractVector{UInt8}) = FlatBuffers.read(Weapon, buf) +Weapon(io::IO) = FlatBuffers.deserialize(io, Weapon) diff --git a/samples/julia_sample.sh b/samples/julia_sample.sh new file mode 100755 index 00000000000..7ff98b6ac80 --- /dev/null +++ b/samples/julia_sample.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Note: This script runs on Mac and Linux. It requires `julia` to be installed +# and `flatc` to be built (using `cmake` in the root directory). + +sampledir=$(cd $(dirname $BASH_SOURCE) && pwd) +rootdir=$(cd $sampledir/.. && pwd) +currentdir=$(pwd) + +if [[ "$sampledir" != "$currentdir" ]]; then + echo Error: This script must be run from inside the $sampledir directory. + echo You executed it from the $currentdir directory. + exit 1 +fi + +# Run `flatc`. Note: This requires you to compile using `cmake` from the +# root `/flatbuffers` directory. +if [ -e ../flatc ]; then + ../flatc --julia monster.fbs +elif [ -e ../Debug/flatc ]; then + ../Debug/flatc --julia monster.fbs +else + echo 'flatc' could not be found. Make sure to build FlatBuffers from the \ + $rootdir directory. + exit 1 +fi + +echo Running the Julia sample. + +# Execute the sample. +julia sample_binary.jl + +# Clean up the temporary files. +rm -rf MyGame diff --git a/samples/sample_binary.jl b/samples/sample_binary.jl new file mode 100644 index 00000000000..788b2ddbbb1 --- /dev/null +++ b/samples/sample_binary.jl @@ -0,0 +1,49 @@ +import FlatBuffers + +include("MyGame/MyGame.jl") + +# require the files generated from the schema +import .MyGame.Sample.Weapon +import .MyGame.Sample.Monster +import .MyGame.Sample.Vec3 +import .MyGame.Sample.Color +import .MyGame.Sample.Equipment + +sword = Weapon(;name="Sword", damage=3) +axe = Weapon(;name="Axe", damage=5) +orc = Monster(; + name="Orc", + pos=Vec3(1.0, 2.0, 3.0), + hp=300, + inventory=collect(1:10), + color=MyGame.Sample.ColorRed, + weapons=[sword, axe], + equipped=axe +) + +# Get the flatbuffer as a string containing the binary data +io = IOBuffer() +FlatBuffers.serialize(io, orc) +bytes = take!(io) + +# Build the Monster from the raw bytes +mon = Monster(bytes) + +@assert mon.mana == 150 +@assert mon.hp == 300 +@assert mon.name == "Orc" +@assert mon.color == MyGame.Sample.ColorRed +@assert mon.pos.x == 1.0 +@assert mon.pos.y == 2.0 +@assert mon.pos.z == 3.0 +@assert mon.inventory == collect(1:10) + +@assert mon.equipped_type == FlatBuffers.typeorder(Equipment, Weapon) +@assert mon.equipped.name == "Axe" +@assert mon.equipped.damage == 5 + +@assert mon.weapons[1].name == "Sword" +@assert mon.weapons[1].damage == 3 + +@assert mon.weapons[2].name == "Axe" +@assert mon.weapons[2].damage == 5 diff --git a/src/flatc_main.cpp b/src/flatc_main.cpp index 78e66a6f637..ef92a24f052 100644 --- a/src/flatc_main.cpp +++ b/src/flatc_main.cpp @@ -58,6 +58,10 @@ int main(int argc, const char *argv[]) { { flatbuffers::GenerateJSTS, "-s", "--js", "JavaScript", true, nullptr, flatbuffers::IDLOptions::kJs, "Generate JavaScript code for tables/structs", flatbuffers::JSTSMakeRule }, + { flatbuffers::GenerateJulia, nullptr, "--julia", "Julia", true, nullptr, + flatbuffers::IDLOptions::kJulia, + "Generate Julia modules for tables/structs", + flatbuffers::GeneralMakeRule }, { flatbuffers::GenerateDart, "-d", "--dart", "Dart", true, nullptr, flatbuffers::IDLOptions::kDart, "Generate Dart classes for tables/structs", flatbuffers::DartMakeRule }, diff --git a/src/idl_gen_cpp.cpp b/src/idl_gen_cpp.cpp index f91f14e1f82..3823bdfb0f3 100644 --- a/src/idl_gen_cpp.cpp +++ b/src/idl_gen_cpp.cpp @@ -494,7 +494,7 @@ class CppGenerator : public BaseGenerator { static const char * const ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #CTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_general.cpp b/src/idl_gen_general.cpp index 66984da0c6d..b2eb017dec9 100644 --- a/src/idl_gen_general.cpp +++ b/src/idl_gen_general.cpp @@ -253,7 +253,7 @@ class GeneralGenerator : public BaseGenerator { // clang-format off static const char * const java_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #JTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -261,7 +261,7 @@ class GeneralGenerator : public BaseGenerator { static const char * const csharp_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #NTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index fe4939edf64..8f32b95ec46 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -693,7 +693,7 @@ static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #GTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp new file mode 100644 index 00000000000..fb1b4782e42 --- /dev/null +++ b/src/idl_gen_julia.cpp @@ -0,0 +1,815 @@ +/* + * Copyright 2018 Dolby Laboratories. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// loosely based on idl_gen_python.cpp + +#include +#include +#include +#include + +#include "flatbuffers/code_generators.h" +#include "flatbuffers/flatbuffers.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { +namespace julia { + +const std::string Indent = " "; +const std::string JuliaFileExtension = ".jl"; +const std::string JuliaPackageName = "FlatBuffers"; + +// Documentation comments +const CommentConfig JuliaCommentConfig = { "#=", "# ", "=#" }; + +// Class to represent a dependency graph +// with a topological sort operation +class DepGraph { + public: + // adjacency list + std::vector *> adj; + // node names + std::vector nodes; + unsigned int GetOrCreateNodeID(std::string name) { + auto it = std::find(nodes.begin(), nodes.end(), name); + if (it == nodes.end()) { + nodes.push_back(name); + return nodes.size() - 1; + } + return std::distance(nodes.begin(), it); + } + DepGraph() {} + ~DepGraph() { + for (unsigned int i = 0; i < adj.size(); i++) delete adj[i]; + } + void AddDep(std::string parent, std::string child) { + unsigned int p = GetOrCreateNodeID(parent); + unsigned int c = GetOrCreateNodeID(child); + // n = max(p, c) + unsigned int n = p > c ? p : c; + // if we just created new node ids, + // grow the adjacency list to the new size + if (n >= adj.size()) { + for (unsigned int i = adj.size(); i <= n; i++) + adj.push_back(new std::vector()); + } + adj[p]->push_back(c); + } + // topological sort + bool TopSort(unsigned int v, bool visited[], bool visiting[], + std::list &stack) { + bool has_cycle = false; + visited[v] = true; + visiting[v] = true; + for (auto it = adj[v]->begin(); it != adj[v]->end(); ++it) { + if (!visited[*it]) has_cycle = TopSort(*it, visited, visiting, stack); + if (visiting[*it] && nodes[v] != nodes[*it]) { + printf( + "warning: %s\n", + ("Circular reference detected (" + nodes[v] + " -> " + nodes[*it] + + "). " + "Julia does not currently suport such definitions.") + .c_str()); + has_cycle = true; + } + } + visiting[v] = false; + stack.push_back(v); + return has_cycle; + } + // Topological sort of dependency graph, so we can include + // julia files in the right order + std::vector TopSort() { + std::vector sorted_nodes; + std::list stack; + unsigned int n = adj.size(); + bool *visited = new bool[n]; + bool *visiting = new bool[n]; + for (unsigned int i = 0; i < n; i++) { + visited[i] = false; + visiting[i] = false; + } + + bool has_cycle = false; + for (unsigned int i = 0; i < n; i++) + if (!visited[i]) has_cycle |= TopSort(i, visited, visiting, stack); + + while (!stack.empty()) { + unsigned int id = stack.back(); + // don't return nodes on which nothing depends + if (!adj[id]->empty()) sorted_nodes.push_back(nodes[id]); + stack.pop_back(); + } + return sorted_nodes; + } +}; + +// Singleton class which keeps track of generated modules +// and their dependencies. A singleton class is necessary +// since our generator is run multiple times with +// multiple different .fbs files, and we want to preserve +// the set of module dependencies across these runs. +// We need to keep track of dependencies since Julia doesn't +// support forward declarations of types. +class ModuleTable { + public: + static ModuleTable &GetInstance() { + static ModuleTable instance; + return instance; + } + bool IsModule(const std::string &m) { + return modules_.find(m) != modules_.end(); + } + void AddFile(const std::string &f) { files_.insert(f); } + bool IsFile(const std::string &f) { return files_.find(f) != files_.end(); } + void AddDependency(std::string mod, std::string parent, std::string child) { + if (!IsModule(mod)) modules_[mod] = new DepGraph(); + modules_[mod]->AddDep(parent, child); + } + std::vector SortedModuleNames() { + std::set module_names; + for (auto it = modules_.begin(); it != modules_.end(); ++it) + module_names.insert(it->first); + // Make sure parent modules come before child modules + std::vector sorted_modules(module_names.begin(), + module_names.end()); + std::sort(sorted_modules.begin(), sorted_modules.end()); + return sorted_modules; + } + DepGraph *GetDependencies(std::string module) { return modules_[module]; } + + private: + std::map modules_; + std::set files_; + ModuleTable() {} + ~ModuleTable() { + for (auto it = modules_.begin(); it != modules_.end(); ++it) + delete it->second; + } + + public: + ModuleTable(ModuleTable const &) = delete; + void operator=(ModuleTable const &) = delete; +}; + +class JuliaGenerator : public BaseGenerator { + public: + JuliaGenerator(const Parser &parser, const std::string &path, + const std::string &file_name) + : BaseGenerator(parser, path, file_name, "" /* not used */, + "" /* not used */) {} + + ~JuliaGenerator() {} + bool generate() { + if (!GenEnums()) return false; + if (!GenStructs()) return false; + if (!GenModules()) return false; + return true; + } + + private: + // the root module is the name of the .fbs file which + // we are compiling, in camel case + std::string root_module_ = MakeCamel(file_name_); + ModuleTable &module_table_ = ModuleTable::GetInstance(); + static const std::unordered_set keywords_; + + bool GenEnums(void) { + for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); + ++it) { + auto &enum_def = **it; + std::string enumcode; + if (enum_def.is_union) + GenUnion(enum_def, &enumcode); + else + GenEnum(enum_def, &enumcode); + if (!SaveType(enum_def, enumcode)) return false; + } + return true; + } + + bool GenStructs(void) { + for (auto it = parser_.structs_.vec.begin(); + it != parser_.structs_.vec.end(); ++it) { + auto &struct_def = **it; + std::string declcode; + GenObject(struct_def, &declcode); + if (!SaveType(struct_def, declcode)) return false; + } + return true; + } + + bool GenModules(void) { + for (auto it = parser_.namespaces_.begin(); it != parser_.namespaces_.end(); + ++it) { + std::string parent; + std::string child; + // Gather all parent namespaces for this namespace + // TODO: this is similar to idl_gen_js.cpp, maybe + // move into common place? + for (auto component = (*it)->components.begin(); + component != (*it)->components.end(); ++component) { + if (parent.empty()) { + parent = *component; + child = *component; + module_table_.AddDependency(root_module_, parent, child); + } else { + child = parent + kPathSeparator + *component; + // Add component to parent's list of children + module_table_.AddDependency(parent, child, *component); + } + parent = child; + } + } + auto sorted_modules = module_table_.SortedModuleNames(); + bool save_root = false; + // iterate through child modules first, then parents + for (auto m = sorted_modules.rbegin(); m != sorted_modules.rend(); ++m) { + if (*m == root_module_) { + save_root = true; + continue; + } + SaveModule(false, *m, module_table_.GetDependencies(*m)); + } + // save the root module last + if (save_root) + SaveModule(true, root_module_, + module_table_.GetDependencies(root_module_)); + return true; + } + + // Begin an object declaration. + static void BeginObject(const StructDef &struct_def, std::string *code_ptr, + bool has_defaults) { + std::string &code = *code_ptr; + if (has_defaults) code += JuliaPackageName + ".@with_kw "; + if (!struct_def.fixed) + code += "mutable struct "; + else + code += JuliaPackageName + ".@STRUCT struct "; + code += NormalizedName(struct_def) + "\n"; + } + + void EndObject(const StructDef &struct_def, + const std::vector &offsets, + std::string *code_ptr) const { + std::string &code = *code_ptr; + std::string name = NormalizedName(struct_def); + code += "end\n"; + code += JuliaPackageName + ".@ALIGN(" + name + +", " + + NumToString(struct_def.minalign) + ")\n"; + std::string method_signature = "(::Type{T}) where {T<:" + name + "}"; + if (!offsets.empty() && !struct_def.fixed) { + bool first = true; + int i = 0; + code += JuliaPackageName + ".slot_offsets" + method_signature; + code += " = ["; + for (auto it = offsets.begin(); it != offsets.end(); ++it) { + if (!first) code += ", "; + if (i == 0) code += "\n" + Indent; + code += *it; + i++; + i %= 4; // print four offsets per line + first = false; + } + code += "\n]\n"; + } + + // emit file identifier and extension for the root type + if (parser_.root_struct_def_ == &struct_def) { + code += JuliaPackageName + ".root_type" + method_signature + " = true\n"; + if (!parser_.file_identifier_.empty()) { + code += JuliaPackageName + ".file_identifier" + method_signature; + code += " = \"" + parser_.file_identifier_ + "\"\n"; + } + if (!parser_.file_extension_.empty()) { + code += JuliaPackageName + ".file_extension" + method_signature; + code += " = \"" + parser_.file_extension_ + "\"\n"; + } + } + code += "\n"; + } + + static std::string EscapeKeyword(const std::string &name) { + return keywords_.find(name) == keywords_.end() ? name : name + "_"; + } + + static std::string const NormalizedName(const Definition &child, + const Definition *parent = NULL) { + std::string prefix = ""; + if (parent != NULL) { + std::string relname = GetRelativeName(*parent, &child, false); + if (!relname.empty()) prefix = relname + "."; + } + return prefix + EscapeKeyword(child.name); + } + + static std::string NormalizedName(const EnumVal &ev) { + return EscapeKeyword(ev.name); + } + + static void BeginEnum(const std::string enum_name, + const std::string enum_type, std::string *code_ptr) { + *code_ptr += "@enum " + enum_name + "::" + enum_type + " begin\n"; + } + + static void EnumMember(const std::string enum_name, const EnumVal ev, + std::string *code_ptr) { + *code_ptr += Indent + enum_name + NormalizedName(ev); + *code_ptr += " = " + NumToString(ev.value) + "\n"; + } + + static void EndEnum(std::string *code_ptr) { *code_ptr += "end\n\n"; } + + static void BeginUnion(const std::string union_name, std::string *code_ptr) { + *code_ptr += JuliaPackageName + ".@UNION(" + union_name + ", (\n"; + } + + static void UnionMember(const std::string type_name, std::string *code_ptr) { + *code_ptr += Indent + type_name + ",\n"; + } + + static void EndUnion(std::string *code_ptr) { *code_ptr += "))\n\n"; } + + static void NewObjectFromBuffer(const StructDef &struct_def, + std::string *code_ptr) { + std::string &code = *code_ptr; + code += NormalizedName(struct_def) + "(buf::AbstractVector{UInt8})"; + code += " = " + JuliaPackageName + ".read(" + NormalizedName(struct_def) + + ", buf)\n"; + code += NormalizedName(struct_def) + "(io::IO) = " + JuliaPackageName; + code += ".deserialize(io, " + NormalizedName(struct_def) + ")\n"; + } + + void GenScalarField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, bool *has_defaults, + std::set *imports_ptr) { + std::string &code = *code_ptr; + std::string field_name = NormalizedName(field); + code += Indent + field_name; + code += "::"; + code += GenTypeGet(field.value.type); + if (!field.value.constant.empty() && !struct_def.fixed) { + *has_defaults = true; + std::string c = field.value.constant; + if (field.value.type.base_type == BASE_TYPE_BOOL) { + c = c == "0" ? "false" : "true"; + } + code += " = " + c; + } + if (IsScalarEnum(field.value.type)) + AddDependency(struct_def, field.value.type, imports_ptr); + code += "\n"; + } + + // whether two namespaces have the same prefix up to "prefix_size" + // used for figuring out relative import paths + static bool SameNamespacePrefix(Namespace n1, Namespace n2, + size_t prefix_size) { + if (n1.components.size() < prefix_size) return false; + if (n2.components.size() < prefix_size) return false; + size_t i; + for (i = 0; i < prefix_size; i++) { + if (n1.components[i] != n2.components[i]) return false; + } + return true; + } + + static void GetRelativeNamespaces(const Definition &parent_def, + const Definition *child_def, + Namespace *parent, Namespace *child) { + Namespace p, c; + if (parent_def.defined_namespace != NULL) p = *parent_def.defined_namespace; + if (child_def != NULL && child_def->defined_namespace != NULL) + c = *child_def->defined_namespace; + *parent = p; + *child = c; + } + + static void GetRelativeNamespaces(const Definition &def, const Type &type, + Namespace *parent, Namespace *child) { + Namespace p, c; + Definition *child_def = GetBaseDefinition(type); + GetRelativeNamespaces(def, child_def, parent, child); + } + + static std::string GetRelativeName(const Namespace &parent, + const Namespace &child, + bool dotprefix = true) { + // here we are accounting for modules which are at + // different levels of the tree. + std::string relname = ""; + // go up to common level (don't add dots to path if we are + // annotating the type of a field - julia doesn't like that) + unsigned int i = 0; + unsigned int n = parent.components.size(); + while (i <= n && !SameNamespacePrefix(parent, child, n - i)) { + if (dotprefix) relname += "."; + i++; + } + // traverse down to the place we need to be + unsigned int m = parent.components.size() - i; + unsigned int j; + for (j = m; j < child.components.size(); j++) { + if (dotprefix || !relname.empty()) relname += "."; + relname += child.components[j]; + } + return relname; + } + + static std::string GetRelativeName(const Definition &parent_def, + const Definition *child_def, + bool dotprefix = false) { + Namespace parent, child; + GetRelativeNamespaces(parent_def, child_def, &parent, &child); + return GetRelativeName(parent, child, dotprefix); + } + + static std::string GetRelativeName(const Definition &def, const Type &type, + bool dotprefix = false) { + Namespace parent, child; + GetRelativeNamespaces(def, type, &parent, &child); + return GetRelativeName(parent, child, dotprefix); + } + + // This definition depends on this type. Add the necessary relative imports. + void AddDependency(const Definition &def, const Type &type, + std::set *imports_ptr) { + Namespace parent, child; + Definition *child_def = GetBaseDefinition(type); + if (child_def == NULL) return; + + GetRelativeNamespaces(def, child_def, &parent, &child); + std::string relname = GetRelativeName(parent, child); + + // add relative import + if (!relname.empty()) { + // special case: we are importing something from a direct parent + if (relname.find_first_not_of('.') == std::string::npos) + imports_ptr->insert(relname + "." + NormalizedName(*child_def)); + else + imports_ptr->insert(relname); + } + + std::string module = GetCanonicalName(child); + std::string child_name = + GetCanonicalName(child) + kPathSeparator + NormalizedName(*child_def); + std::string parent_name = + GetCanonicalName(parent) + kPathSeparator + NormalizedName(def); + // self-reference - this is fine, but don't add it to the dependency table + if (child_name == parent_name) return; + module_table_.AddDependency(module, parent_name, child_name); + } + + // generate a field which depends upon generated types + void GenDependentField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, + bool *has_defaults, + std::set *imports_ptr) { + std::string type_name = GenTypeGet(field.value.type, &struct_def); + + BaseType bt = field.value.type.base_type; + if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_VECTOR) && + !struct_def.fixed) { + type_name = "Union{" + type_name + ", Nothing}"; + } + // initialise nullable fields to nothing by default + if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_UNION || + bt == BASE_TYPE_VECTOR || bt == BASE_TYPE_STRING) && + !struct_def.fixed) { + type_name += " = nothing"; + *has_defaults = true; + } + *code_ptr += Indent + NormalizedName(field) + "::" + type_name + "\n"; + AddDependency(struct_def, field.value.type, imports_ptr); + } + + static void GenStringField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, bool *has_defaults) { + *code_ptr += Indent + NormalizedName(field) + "::"; + // initialise strings to be empty by default + *code_ptr += "Union{" + GenTypeGet(field.value.type) + ", Nothing}"; + if (!struct_def.fixed) { + *code_ptr += " = nothing"; + *has_defaults = true; + } + *code_ptr += "\n"; + } + + // Generate a field, conditioned on its child type(s). + void GenField(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr, std::set *imports_ptr, + bool *has_defaults) { + GenComment(field.doc_comment, code_ptr, &JuliaCommentConfig); + if (IsScalar(field.value.type.base_type)) { + GenScalarField(struct_def, field, code_ptr, has_defaults, imports_ptr); + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRING: + GenStringField(struct_def, field, code_ptr, has_defaults); + break; + case BASE_TYPE_STRUCT: + case BASE_TYPE_VECTOR: + case BASE_TYPE_UNION: + GenDependentField(struct_def, field, code_ptr, has_defaults, imports_ptr); + break; + default: FLATBUFFERS_ASSERT(0); + } + } + } + + static std::string GenImports(const std::set &imports) { + std::string impstr = ""; + for (auto it = imports.begin(); it != imports.end(); ++it) + impstr += "import " + *it + "\n"; + return impstr; + } + + // Generate structs/tables + void GenObject(const StructDef &struct_def, std::string *code_ptr) { + if (struct_def.generated) return; + + // always need FlatBuffers package for structs + std::set imports = { JuliaPackageName }; + bool has_defaults = false; + + // generate all the fields + std::vector offsets; + + GenComment(struct_def.doc_comment, code_ptr, &JuliaCommentConfig); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + GenField(struct_def, field, code_ptr, &imports, &has_defaults); + offsets.push_back("0x" + IntToStringHex(field.value.offset, 8)); + } + EndObject(struct_def, offsets, code_ptr); + + // need to call BeginObject after EndObject because we don't know + // the defaults until we've looked at all the fields. + std::string struct_begin; + BeginObject(struct_def, &struct_begin, has_defaults); + + *code_ptr = GenImports(imports) + "\n" + struct_begin + *code_ptr; + + // Generate a functions for constructing the object from a buffer + NewObjectFromBuffer(struct_def, code_ptr); + } + + void GenUnion(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + + // always need FlatBuffers package for unions + std::set imports = { JuliaPackageName }; + std::string union_name = NormalizedName(enum_def); + BeginUnion(union_name, code_ptr); + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + std::string type_name = GenTypeGet(ev.union_type, &enum_def); + GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig); + UnionMember(type_name, code_ptr); + // special case, instead make every Union a union with Nothing + if (ev.name == "NONE") continue; + AddDependency(enum_def, ev.union_type, &imports); + } + EndUnion(code_ptr); + + *code_ptr = GenImports(imports) + "\n" + *code_ptr; + } + + void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + if (enum_def.generated) return; + GenComment(enum_def.doc_comment, code_ptr, &JuliaCommentConfig); + std::string enum_name = NormalizedName(enum_def); + BeginEnum(enum_name, GenTypeBasic(enum_def.underlying_type), code_ptr); + for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + ++it) { + auto &ev = **it; + GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig); + EnumMember(enum_name, ev, code_ptr); + } + EndEnum(code_ptr); + } + + static std::string GenTypeBasic(const Type &type) { + static const char *ctypename[] = { +// clang-format off + #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ + #JLTYPE, + FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + // clang-format on + }; + return ctypename[type.base_type]; + } + + static std::string GenTypePointer(const Type &type, + const Definition *parent) { + switch (type.base_type) { + case BASE_TYPE_STRING: return "String"; + case BASE_TYPE_VECTOR: + return "Vector{" + GenTypeGet(type.VectorType(), parent) + "}"; + case BASE_TYPE_STRUCT: return NormalizedName(*type.struct_def, parent); + case BASE_TYPE_UNION: return NormalizedName(*type.enum_def, parent); + case BASE_TYPE_NONE: return "Nothing"; + default: return "Any"; + } + } + + static Definition *GetBaseDefinition(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_VECTOR: return GetBaseDefinition(type.VectorType()); + case BASE_TYPE_STRUCT: return type.struct_def; + case BASE_TYPE_UNION: return type.enum_def; + default: return NULL; + } + } + + static bool IsScalarEnum(const Type &type) { + return (type.enum_def != nullptr && !type.enum_def->is_union && + IsInteger(type.enum_def->underlying_type.base_type)); + } + + static std::string GenTypeGet(const Type &type, + const Definition *parent = NULL) { + if (IsScalar(type.base_type)) + return IsScalarEnum(type) ? NormalizedName(*type.enum_def, parent) + : GenTypeBasic(type); + return GenTypePointer(type, parent); + } + + void BeginFile(const std::string submodule_name, + std::string *code_ptr) const { + std::string &code = *code_ptr; + code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; + std::string module = submodule_name; + if (module.empty()) module = root_module_; + code += "# module: " + module + "\n\n"; + } + + std::string GetDirname(const Definition &def) const { + std::string d = path_; + if (def.defined_namespace != NULL) + d += GetCanonicalName(*def.defined_namespace); + return d; + } + + std::string GetFilename(const Definition &def) const { + return ConCatPathFileName(GetDirname(def), NormalizedName(def)) + + JuliaFileExtension; + } + + std::string GetModule(const Definition &def) const { + if (def.defined_namespace != NULL) + return GetCanonicalName(*def.defined_namespace); + return root_module_; + } + + static std::string GetSubModule(const Definition &def) { + if (def.defined_namespace != NULL) + return LastNamespacePart(*def.defined_namespace); + return ""; + } + + // Save out the generated code for a Julia module + bool SaveModule(bool is_root, std::string full_module_name, + DepGraph *children) { + std::string module_name = full_module_name; + auto start = full_module_name.rfind(kPathSeparator); + if (start != std::string::npos) + module_name = full_module_name.substr(start + 1); + std::string module_dir = ConCatPathFileName(path_, full_module_name); + if (is_root) { + module_dir = path_; + module_name = root_module_; + } + std::string module_jl = + ConCatPathFileName(module_dir, module_name) + JuliaFileExtension; + std::string code = ""; + bool need_module_file = false; + BeginFile(module_name, &code); + code += "module " + module_name + "\n"; + + // Include all the dependencies of this module in the right order + std::vector sorted_children = children->TopSort(); + for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); + ++it) { + std::string child = *it; + bool is_module = false; + std::string submodule_name = + child.substr(child.rfind(kPathSeparator) + 1); + + // Don't include the root module from the root + if (is_root && submodule_name == root_module_) continue; + + // Modules live in a subdirectory named after themselves + if (module_table_.IsModule(child) && + !(is_root && submodule_name == root_module_)) { + is_module = true; + child = ConCatPathFileName(child, submodule_name); + } + std::string dir = is_root ? path_ : module_dir; + std::string relname; + if (is_root) + relname = child; + else if (is_module) + relname = child.substr(child.find(kPathSeparator) + 1); + else + relname = child.substr(child.rfind(kPathSeparator) + 1); + + // If the file doesn't exist, don't include it + // TODO: this doesn't allow types which reference each other, + // but Julia doesn't support this yet anyway + std::string toinclude = relname + JuliaFileExtension; + std::string fullpath = ConCatPathFileName(dir, toinclude); + if (!module_table_.IsFile(fullpath.c_str())) continue; + code += Indent + "include(\"" + toinclude + "\")\n"; + need_module_file = true; + } + code += "end\n"; + + // only write the module if it has some things to include + if (!need_module_file) return true; + + EnsureDirExists(module_dir); + if (!SaveFile(module_jl.c_str(), code, false)) return false; + + module_table_.AddFile(module_jl); + return true; + } + + // Canonical julia name of a namespace (Foo.Bar.Baz) + std::string GetCanonicalName(const Namespace &ns) const { + std::string name; + for (size_t i = 0; i < ns.components.size(); i++) { + if (i) name += kPathSeparator; + name += std::string(ns.components[i]); + } + if (name.empty()) name = root_module_; + return name; + } + + // Add a dependency between two definitions + void AddDependency(const Definition *parent, const Definition *child) { + FLATBUFFERS_ASSERT(parent != NULL && child != NULL); + std::string parent_name = NormalizedName(*parent); + std::string module = parent_name; + if (parent->defined_namespace != NULL) + module = GetCanonicalName(*parent->defined_namespace); + module_table_.AddDependency(module, parent_name, NormalizedName(*child)); + } + + // Add a definition as a dependency to its own module + void AddToOwnModule(const Definition &def) { + std::string m = GetModule(def); + module_table_.AddDependency(m, m + kPathSeparator + NormalizedName(def), m); + } + + // Save out the generated code for a Julia struct + bool SaveType(const Definition &def, const std::string &declcode) { + if (!declcode.length()) return true; + std::string code = ""; + BeginFile(GetSubModule(def), &code); + code += declcode; + std::string filename = GetFilename(def); + EnsureDirExists(GetDirname(def)); + if (!SaveFile(filename.c_str(), code, false)) return false; + module_table_.AddFile(filename); + AddToOwnModule(def); + return true; + } +}; + +const std::unordered_set JuliaGenerator::keywords_{ + { "begin", "while", "if", "for", "try", "return", + "break", "continue", "function", "macro", "quote", "let", + "local", "global", "const", "do", "struct", "module", + "baremodule", "using", "import", "export", "end", "else", + "catch", "finally", "true", "false", "Any" } +}; + +} // namespace julia + +bool GenerateJulia(const Parser &parser, const std::string &path, + const std::string &file_name) { + julia::JuliaGenerator generator(parser, path, file_name); + return generator.generate(); +} + +} // namespace flatbuffers diff --git a/src/idl_gen_lobster.cpp b/src/idl_gen_lobster.cpp index 5f199e3a1c9..48e752308f6 100644 --- a/src/idl_gen_lobster.cpp +++ b/src/idl_gen_lobster.cpp @@ -81,7 +81,7 @@ class LobsterGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_lua.cpp b/src/idl_gen_lua.cpp index e8ae32802c7..8cdf5b24dd3 100644 --- a/src/idl_gen_lua.cpp +++ b/src/idl_gen_lua.cpp @@ -604,7 +604,7 @@ namespace lua { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_php.cpp b/src/idl_gen_php.cpp index 91ce01367a6..7bb2e04b647 100644 --- a/src/idl_gen_php.cpp +++ b/src/idl_gen_php.cpp @@ -864,7 +864,7 @@ class PhpGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #NTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 11491c00d59..d2edef77bdc 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -615,7 +615,7 @@ class PythonGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #PTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_rust.cpp b/src/idl_gen_rust.cpp index b530137b29b..910a7d765d7 100644 --- a/src/idl_gen_rust.cpp +++ b/src/idl_gen_rust.cpp @@ -510,7 +510,7 @@ class RustGenerator : public BaseGenerator { // clang-format off static const char * const ctypename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #RTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -532,7 +532,7 @@ class RustGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE) \ + RTYPE, JLTYPE) \ #RTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index c129488650f..ed6646b28df 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -131,7 +131,7 @@ bool Print(const void *val, Type type, int indent, switch (type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (!PrintVector( \ *reinterpret_cast *>(val), \ @@ -227,7 +227,7 @@ static bool GenStruct(const StructDef &struct_def, const Table *table, switch (fd.value.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (!GenField(fd, table, struct_def.fixed, \ opts, indent + Indent(opts), _text)) { \ @@ -238,7 +238,7 @@ static bool GenStruct(const StructDef &struct_def, const Table *table, #undef FLATBUFFERS_TD // Generate drop-thru case statements for all pointer types: #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 357337a74f7..89049badeb0 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -30,7 +30,7 @@ const double kPi = 3.14159265358979323846; const char *const kTypeNames[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -41,7 +41,7 @@ const char *const kTypeNames[] = { const char kTypeSizes[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ sizeof(CTYPE), FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -192,7 +192,7 @@ static std::string TokenToString(int t) { FLATBUFFERS_GEN_TOKENS(FLATBUFFERS_TOKEN) #undef FLATBUFFERS_TOKEN #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ IDLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -1077,7 +1077,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, switch (field_value.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (struct_def.fixed) { \ @@ -1094,7 +1094,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD); #undef FLATBUFFERS_TD #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ builder_.Pad(field->padding); \ if (IsStruct(field->value.type)) { \ @@ -1166,7 +1166,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue) { switch (val.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) SerializeStruct(*val.type.struct_def, val); \ else { \ @@ -1506,7 +1506,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e, switch (e.type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: {\ CTYPE val; \ ECHECK(atot(e.constant.c_str(), *this, &val)); \ @@ -1665,7 +1665,7 @@ CheckedError Parser::ParseEnum(bool is_union, EnumDef **dest) { switch (enum_def->underlying_type.base_type) { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ - PTYPE, RTYPE) \ + PTYPE, RTYPE, JLTYPE) \ case BASE_TYPE_##ENUM: { \ int64_t min_value = static_cast( \ flatbuffers::numeric_limits::lowest()); \ @@ -1749,7 +1749,8 @@ bool Parser::SupportsVectorOfUnions() const { return opts.lang_to_generate != 0 && (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kJs | IDLOptions::kTs | IDLOptions::kPhp | - IDLOptions::kJava | IDLOptions::kCSharp)) == 0; + IDLOptions::kJava | IDLOptions::kCSharp | + IDLOptions::kJulia)) == 0; } Namespace *Parser::UniqueNamespace(Namespace *ns) { diff --git a/tests/MonsterTest.jl b/tests/MonsterTest.jl new file mode 100644 index 00000000000..b3ad477a2a0 --- /dev/null +++ b/tests/MonsterTest.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MonsterTest + +module MonsterTest + include("MyGame/MyGame.jl") +end diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl new file mode 100644 index 00000000000..e32f1ca69b8 --- /dev/null +++ b/tests/MyGame/Example/Ability.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Ability + id::UInt32 + distance::UInt32 +end +FlatBuffers.@ALIGN(Ability, 4) + +Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) +Ability(io::IO) = FlatBuffers.deserialize(io, Ability) diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl new file mode 100644 index 00000000000..e106020db0f --- /dev/null +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@UNION(AnyAmbiguousAliases, ( + Nothing, + Monster, + Monster, + Monster, +)) + diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl new file mode 100644 index 00000000000..27abea3f997 --- /dev/null +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(AnyUniqueAliases, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl new file mode 100644 index 00000000000..9bdc2a646e1 --- /dev/null +++ b/tests/MyGame/Example/Any_.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(Any_, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/tests/MyGame/Example/Color.jl b/tests/MyGame/Example/Color.jl new file mode 100644 index 00000000000..804144f0ac7 --- /dev/null +++ b/tests/MyGame/Example/Color.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +@enum Color::Int8 begin + ColorRed = 1 + ColorGreen = 2 + ColorBlue = 8 +end + diff --git a/tests/MyGame/Example/Example.jl b/tests/MyGame/Example/Example.jl new file mode 100644 index 00000000000..c8fd32b3cd4 --- /dev/null +++ b/tests/MyGame/Example/Example.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +module Example + include("Color.jl") + include("Test.jl") + include("Vec3.jl") + include("Stat.jl") + include("Ability.jl") + include("Referrable.jl") + include("TestSimpleTableWithEnum.jl") + include("AnyUniqueAliases.jl") + include("AnyAmbiguousAliases.jl") + include("Monster.jl") + include("Any_.jl") + include("TypeAliases.jl") +end diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl new file mode 100644 index 00000000000..15586f65c70 --- /dev/null +++ b/tests/MyGame/Example/Monster.jl @@ -0,0 +1,84 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..InParentNamespace +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Monster +#= +# an example documentation comment: monster object +=# + pos::Union{Vec3, Nothing} = nothing + mana::Int16 = 150 + hp::Int16 = 100 + name::Union{String, Nothing} = nothing + inventory::Union{Vector{UInt8}, Nothing} = nothing + color::Color = 8 + test_type::UInt8 = 0 + test::Any_ = nothing + test4::Union{Vector{Test}, Nothing} = nothing + testarrayofstring::Union{Vector{String}, Nothing} = nothing +#= +# an example documentation comment: this will end up in the generated code +# multiline too +=# + testarrayoftables::Union{Vector{Monster}, Nothing} = nothing + enemy::Union{Monster, Nothing} = nothing + testnestedflatbuffer::Union{Vector{UInt8}, Nothing} = nothing + testempty::Union{Stat, Nothing} = nothing + testbool::Bool = false + testhashs32_fnv1::Int32 = 0 + testhashu32_fnv1::UInt32 = 0 + testhashs64_fnv1::Int64 = 0 + testhashu64_fnv1::UInt64 = 0 + testhashs32_fnv1a::Int32 = 0 + testhashu32_fnv1a::UInt32 = 0 + testhashs64_fnv1a::Int64 = 0 + testhashu64_fnv1a::UInt64 = 0 + testarrayofbools::Union{Vector{Bool}, Nothing} = nothing + testf::Float32 = 3.14159 + testf2::Float32 = 3.0 + testf3::Float32 = 0.0 + testarrayofstring2::Union{Vector{String}, Nothing} = nothing + testarrayofsortedstruct::Union{Vector{Ability}, Nothing} = nothing + flex::Union{Vector{UInt8}, Nothing} = nothing + test5::Union{Vector{Test}, Nothing} = nothing + vector_of_longs::Union{Vector{Int64}, Nothing} = nothing + vector_of_doubles::Union{Vector{Float64}, Nothing} = nothing + parent_namespace_test::Union{InParentNamespace, Nothing} = nothing + vector_of_referrables::Union{Vector{Referrable}, Nothing} = nothing + single_weak_reference::UInt64 = 0 + vector_of_weak_references::Union{Vector{UInt64}, Nothing} = nothing + vector_of_strong_referrables::Union{Vector{Referrable}, Nothing} = nothing + co_owning_reference::UInt64 = 0 + vector_of_co_owning_references::Union{Vector{UInt64}, Nothing} = nothing + non_owning_reference::UInt64 = 0 + vector_of_non_owning_references::Union{Vector{UInt64}, Nothing} = nothing + any_unique_type::UInt8 = 0 + any_unique::AnyUniqueAliases = nothing + any_ambiguous_type::UInt8 = 0 + any_ambiguous::AnyAmbiguousAliases = nothing + vector_of_enums::Union{Vector{Color}, Nothing} = nothing +end +FlatBuffers.@ALIGN(Monster, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000E, 0x00000010, 0x00000012, 0x00000014, + 0x00000016, 0x00000018, 0x0000001A, 0x0000001C, + 0x0000001E, 0x00000020, 0x00000022, 0x00000024, + 0x00000026, 0x00000028, 0x0000002A, 0x0000002C, + 0x0000002E, 0x00000030, 0x00000032, 0x00000034, + 0x00000036, 0x00000038, 0x0000003A, 0x0000003C, + 0x0000003E, 0x00000040, 0x00000042, 0x00000044, + 0x00000046, 0x00000048, 0x0000004A, 0x0000004C, + 0x0000004E, 0x00000050, 0x00000052, 0x00000054, + 0x00000056, 0x00000058, 0x0000005A, 0x0000005C, + 0x0000005E, 0x00000060, 0x00000062 +] +FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true +FlatBuffers.file_identifier(::Type{T}) where {T<:Monster} = "MONS" +FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl new file mode 100644 index 00000000000..7fe024e210e --- /dev/null +++ b/tests/MyGame/Example/Referrable.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Referrable + id::UInt64 = 0 +end +FlatBuffers.@ALIGN(Referrable, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ + 0x00000004 +] + +Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) +Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl new file mode 100644 index 00000000000..00001c46935 --- /dev/null +++ b/tests/MyGame/Example/Stat.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Stat + id::Union{String, Nothing} = nothing + val::Int64 = 0 + count::UInt16 = 0 +end +FlatBuffers.@ALIGN(Stat, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ + 0x00000004, 0x00000006, 0x00000008 +] + +Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) +Stat(io::IO) = FlatBuffers.deserialize(io, Stat) diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl new file mode 100644 index 00000000000..0087689c7eb --- /dev/null +++ b/tests/MyGame/Example/Test.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Test + a::Int16 + b::Int8 +end +FlatBuffers.@ALIGN(Test, 2) + +Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) +Test(io::IO) = FlatBuffers.deserialize(io, Test) diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl new file mode 100644 index 00000000000..41c0e33f3ee --- /dev/null +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum + color::Color = 2 +end +FlatBuffers.@ALIGN(TestSimpleTableWithEnum, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ + 0x00000004 +] + +TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) +TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl new file mode 100644 index 00000000000..a563b74ecbd --- /dev/null +++ b/tests/MyGame/Example/TypeAliases.jl @@ -0,0 +1,29 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TypeAliases + i8::Int8 = 0 + u8::UInt8 = 0 + i16::Int16 = 0 + u16::UInt16 = 0 + i32::Int32 = 0 + u32::UInt32 = 0 + i64::Int64 = 0 + u64::UInt64 = 0 + f32::Float32 = 0.0 + f64::Float64 = 0.0 + v8::Union{Vector{Int8}, Nothing} = nothing + vf64::Union{Vector{Float64}, Nothing} = nothing +end +FlatBuffers.@ALIGN(TypeAliases, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000C, 0x0000000E, 0x00000010, 0x00000012, + 0x00000014, 0x00000016, 0x00000018, 0x0000001A +] + +TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) +TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl new file mode 100644 index 00000000000..8f5442af51f --- /dev/null +++ b/tests/MyGame/Example/Vec3.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 + test1::Float64 + test2::Color + test3::Test +end +FlatBuffers.@ALIGN(Vec3, 16) + +Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) +Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/tests/MyGame/Example2/Example2.jl b/tests/MyGame/Example2/Example2.jl new file mode 100644 index 00000000000..eaef8178155 --- /dev/null +++ b/tests/MyGame/Example2/Example2.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +module Example2 + include("Monster.jl") +end diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl new file mode 100644 index 00000000000..91b98e333fb --- /dev/null +++ b/tests/MyGame/Example2/Monster.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +import FlatBuffers + +mutable struct Monster +end +FlatBuffers.@ALIGN(Monster, 1) + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl new file mode 100644 index 00000000000..518cb89e3f0 --- /dev/null +++ b/tests/MyGame/InParentNamespace.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +import FlatBuffers + +mutable struct InParentNamespace +end +FlatBuffers.@ALIGN(InParentNamespace, 1) + +InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) +InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) diff --git a/tests/MyGame/MyGame.jl b/tests/MyGame/MyGame.jl new file mode 100644 index 00000000000..075ee85e35a --- /dev/null +++ b/tests/MyGame/MyGame.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +module MyGame + include("InParentNamespace.jl") + include("Example2/Example2.jl") + include("Example/Example.jl") +end diff --git a/tests/generate_code.sh b/tests/generate_code.sh index 89b868fd131..840016ef0dd 100755 --- a/tests/generate_code.sh +++ b/tests/generate_code.sh @@ -15,9 +15,9 @@ # limitations under the License. set -e -../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json -../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs -../flatc --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs +../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --julia --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json +../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --julia --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs +../flatc --cpp --js --ts --php --julia --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs ../flatc -b --schema --bfbs-comments -I include_test monster_test.fbs ../flatc --jsonschema --schema -I include_test monster_test.fbs cd ../samples diff --git a/tests/namespace_test/NamespaceA/NamespaceA.jl b/tests/namespace_test/NamespaceA/NamespaceA.jl new file mode 100644 index 00000000000..5f8e0d98900 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceA.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +module NamespaceA + include("NamespaceB/NamespaceB.jl") + include("TableInFirstNS.jl") + include("SecondTableInA.jl") +end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl new file mode 100644 index 00000000000..9fa9777b485 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +@enum EnumInNestedNS::Int8 begin + EnumInNestedNSA = 0 + EnumInNestedNSB = 1 + EnumInNestedNSC = 2 +end + diff --git a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl new file mode 100644 index 00000000000..5dbfb19cf18 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +module NamespaceB + include("EnumInNestedNS.jl") + include("TableInNestedNS.jl") + include("StructInNestedNS.jl") +end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl new file mode 100644 index 00000000000..77aa0b205c1 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +import FlatBuffers + +FlatBuffers.@STRUCT struct StructInNestedNS + a::Int32 + b::Int32 +end +FlatBuffers.@ALIGN(StructInNestedNS, 4) + +StructInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(StructInNestedNS, buf) +StructInNestedNS(io::IO) = FlatBuffers.deserialize(io, StructInNestedNS) diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl new file mode 100644 index 00000000000..06125901142 --- /dev/null +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceB + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInNestedNS + foo::Int32 = 0 +end +FlatBuffers.@ALIGN(TableInNestedNS, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInNestedNS} = [ + 0x00000004 +] + +TableInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInNestedNS, buf) +TableInNestedNS(io::IO) = FlatBuffers.deserialize(io, TableInNestedNS) diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl new file mode 100644 index 00000000000..6cdebe37fe7 --- /dev/null +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -0,0 +1,17 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +import ..NamespaceC +import FlatBuffers + +FlatBuffers.@with_kw mutable struct SecondTableInA + refer_to_c::Union{NamespaceC.TableInC, Nothing} = nothing +end +FlatBuffers.@ALIGN(SecondTableInA, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:SecondTableInA} = [ + 0x00000004 +] + +SecondTableInA(buf::AbstractVector{UInt8}) = FlatBuffers.read(SecondTableInA, buf) +SecondTableInA(io::IO) = FlatBuffers.deserialize(io, SecondTableInA) diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl new file mode 100644 index 00000000000..23eb1b4df4d --- /dev/null +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -0,0 +1,19 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceA + +import .NamespaceB +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInFirstNS + foo_table::Union{NamespaceB.TableInNestedNS, Nothing} = nothing + foo_enum::EnumInNestedNS = 0 + foo_struct::Union{NamespaceB.StructInNestedNS, Nothing} = nothing +end +FlatBuffers.@ALIGN(TableInFirstNS, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInFirstNS} = [ + 0x00000004, 0x00000006, 0x00000008 +] + +TableInFirstNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInFirstNS, buf) +TableInFirstNS(io::IO) = FlatBuffers.deserialize(io, TableInFirstNS) diff --git a/tests/namespace_test/NamespaceC/NamespaceC.jl b/tests/namespace_test/NamespaceC/NamespaceC.jl new file mode 100644 index 00000000000..bc2c2417e76 --- /dev/null +++ b/tests/namespace_test/NamespaceC/NamespaceC.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceC + +module NamespaceC + include("TableInC.jl") +end diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl new file mode 100644 index 00000000000..c5f58206d48 --- /dev/null +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceC + +import ..NamespaceA +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TableInC + refer_to_a1::Union{NamespaceA.TableInFirstNS, Nothing} = nothing + refer_to_a2::Union{NamespaceA.SecondTableInA, Nothing} = nothing +end +FlatBuffers.@ALIGN(TableInC, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInC} = [ + 0x00000004, 0x00000006 +] + +TableInC(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInC, buf) +TableInC(io::IO) = FlatBuffers.deserialize(io, TableInC) diff --git a/tests/namespace_test/NamespaceTest1.jl b/tests/namespace_test/NamespaceTest1.jl new file mode 100644 index 00000000000..2f158d945f1 --- /dev/null +++ b/tests/namespace_test/NamespaceTest1.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceTest1 + +module NamespaceTest1 + include("NamespaceA/NamespaceA.jl") +end diff --git a/tests/namespace_test/NamespaceTest2.jl b/tests/namespace_test/NamespaceTest2.jl new file mode 100644 index 00000000000..960f5a3a27f --- /dev/null +++ b/tests/namespace_test/NamespaceTest2.jl @@ -0,0 +1,8 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: NamespaceTest2 + +module NamespaceTest2 + include("NamespaceA/NamespaceA.jl") + include("NamespaceC/NamespaceC.jl") +end diff --git a/tests/union_vector/UnionVector.jl b/tests/union_vector/UnionVector.jl new file mode 100644 index 00000000000..f65c7a3310b --- /dev/null +++ b/tests/union_vector/UnionVector.jl @@ -0,0 +1,11 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +module UnionVector + include("UnionVector/Attacker.jl") + include("UnionVector/Rapunzel.jl") + include("UnionVector/BookReader.jl") + include("UnionVector/Character.jl") + include("UnionVector/Movie.jl") +end diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl new file mode 100644 index 00000000000..e7553a68942 --- /dev/null +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Attacker + sword_attack_damage::Int32 = 0 +end +FlatBuffers.@ALIGN(Attacker, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Attacker} = [ + 0x00000004 +] + +Attacker(buf::AbstractVector{UInt8}) = FlatBuffers.read(Attacker, buf) +Attacker(io::IO) = FlatBuffers.deserialize(io, Attacker) diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl new file mode 100644 index 00000000000..253ea64c972 --- /dev/null +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@STRUCT struct BookReader + books_read::Int32 +end +FlatBuffers.@ALIGN(BookReader, 4) + +BookReader(buf::AbstractVector{UInt8}) = FlatBuffers.read(BookReader, buf) +BookReader(io::IO) = FlatBuffers.deserialize(io, BookReader) diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl new file mode 100644 index 00000000000..603b1055be8 --- /dev/null +++ b/tests/union_vector/UnionVector/Character.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@UNION(Character, ( + Nothing, + Attacker, + Rapunzel, + BookReader, + BookReader, + String, + String, +)) + diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl new file mode 100644 index 00000000000..0544e8b7c2b --- /dev/null +++ b/tests/union_vector/UnionVector/Movie.jl @@ -0,0 +1,21 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Movie + main_character_type::UInt8 = 0 + main_character::Character = nothing + characters_type::Union{Vector{UInt8}, Nothing} = nothing + characters::Union{Vector{Character}, Nothing} = nothing +end +FlatBuffers.@ALIGN(Movie, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Movie} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A +] +FlatBuffers.root_type(::Type{T}) where {T<:Movie} = true +FlatBuffers.file_identifier(::Type{T}) where {T<:Movie} = "MOVI" + +Movie(buf::AbstractVector{UInt8}) = FlatBuffers.read(Movie, buf) +Movie(io::IO) = FlatBuffers.deserialize(io, Movie) diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl new file mode 100644 index 00000000000..b524a9fb0bd --- /dev/null +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: UnionVector + +import FlatBuffers + +FlatBuffers.@STRUCT struct Rapunzel + hair_length::Int32 +end +FlatBuffers.@ALIGN(Rapunzel, 4) + +Rapunzel(buf::AbstractVector{UInt8}) = FlatBuffers.read(Rapunzel, buf) +Rapunzel(io::IO) = FlatBuffers.deserialize(io, Rapunzel) From 9056551f6badf3b87b7c55f1bb7d1352e4fae351 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Fri, 14 Dec 2018 10:54:24 +1100 Subject: [PATCH 03/19] add FlatBuffers package --- julia/LICENSE.md | 22 + julia/Manifest.toml | 54 ++ julia/Project.toml | 6 + julia/README.md | 56 ++ julia/REQUIRE | 2 + julia/appveyor.yml | 42 + julia/docs/Manifest.toml | 79 ++ julia/docs/Project.toml | 2 + julia/docs/make.jl | 17 + julia/docs/mkdocs.yml | 29 + julia/docs/src/index.md | 97 +++ julia/src/FlatBuffers.jl | 610 +++++++++++++ julia/src/internals.jl | 453 ++++++++++ julia/src/macros.jl | 225 +++++ julia/test/MyGame/Example/Ability.jl | 14 + .../MyGame/Example/AnyAmbiguousAliases.jl | 13 + julia/test/MyGame/Example/AnyUniqueAliases.jl | 14 + julia/test/MyGame/Example/Any_.jl | 14 + julia/test/MyGame/Example/Color.jl | 10 + julia/test/MyGame/Example/Example.jl | 18 + julia/test/MyGame/Example/Monster.jl | 83 ++ julia/test/MyGame/Example/Referrable.jl | 16 + julia/test/MyGame/Example/Stat.jl | 18 + julia/test/MyGame/Example/Test.jl | 14 + .../MyGame/Example/TestSimpleTableWithEnum.jl | 16 + julia/test/MyGame/Example/TypeAliases.jl | 29 + julia/test/MyGame/Example/Vec3.jl | 18 + julia/test/MyGame/Example2/Example2.jl | 7 + julia/test/MyGame/Example2/Monster.jl | 12 + julia/test/MyGame/InParentNamespace.jl | 12 + julia/test/MyGame/MyGame.jl | 9 + julia/test/defaults.jl | 40 + julia/test/flatc.jl | 91 ++ julia/test/internals.jl | 811 ++++++++++++++++++ julia/test/monster.jl | 77 ++ julia/test/monsterdata_python_wire.mon | Bin 0 -> 352 bytes julia/test/monsterdata_test.mon | Bin 0 -> 448 bytes julia/test/runtests.jl | 313 +++++++ 38 files changed, 3343 insertions(+) create mode 100644 julia/LICENSE.md create mode 100644 julia/Manifest.toml create mode 100644 julia/Project.toml create mode 100644 julia/README.md create mode 100644 julia/REQUIRE create mode 100644 julia/appveyor.yml create mode 100644 julia/docs/Manifest.toml create mode 100644 julia/docs/Project.toml create mode 100644 julia/docs/make.jl create mode 100644 julia/docs/mkdocs.yml create mode 100644 julia/docs/src/index.md create mode 100644 julia/src/FlatBuffers.jl create mode 100644 julia/src/internals.jl create mode 100644 julia/src/macros.jl create mode 100644 julia/test/MyGame/Example/Ability.jl create mode 100644 julia/test/MyGame/Example/AnyAmbiguousAliases.jl create mode 100644 julia/test/MyGame/Example/AnyUniqueAliases.jl create mode 100644 julia/test/MyGame/Example/Any_.jl create mode 100644 julia/test/MyGame/Example/Color.jl create mode 100644 julia/test/MyGame/Example/Example.jl create mode 100644 julia/test/MyGame/Example/Monster.jl create mode 100644 julia/test/MyGame/Example/Referrable.jl create mode 100644 julia/test/MyGame/Example/Stat.jl create mode 100644 julia/test/MyGame/Example/Test.jl create mode 100644 julia/test/MyGame/Example/TestSimpleTableWithEnum.jl create mode 100644 julia/test/MyGame/Example/TypeAliases.jl create mode 100644 julia/test/MyGame/Example/Vec3.jl create mode 100644 julia/test/MyGame/Example2/Example2.jl create mode 100644 julia/test/MyGame/Example2/Monster.jl create mode 100644 julia/test/MyGame/InParentNamespace.jl create mode 100644 julia/test/MyGame/MyGame.jl create mode 100644 julia/test/defaults.jl create mode 100644 julia/test/flatc.jl create mode 100644 julia/test/internals.jl create mode 100644 julia/test/monster.jl create mode 100644 julia/test/monsterdata_python_wire.mon create mode 100644 julia/test/monsterdata_test.mon create mode 100644 julia/test/runtests.jl diff --git a/julia/LICENSE.md b/julia/LICENSE.md new file mode 100644 index 00000000000..80c8616b40c --- /dev/null +++ b/julia/LICENSE.md @@ -0,0 +1,22 @@ +The FlatBuffers.jl package is licensed under the MIT "Expat" License: + +> Copyright (c) 2016: Douglas Bates. +> +> Permission is hereby granted, free of charge, to any person obtaining +> a copy of this software and associated documentation files (the +> "Software"), to deal in the Software without restriction, including +> without limitation the rights to use, copy, modify, merge, publish, +> distribute, sublicense, and/or sell copies of the Software, and to +> permit persons to whom the Software is furnished to do so, subject to +> the following conditions: +> +> The above copyright notice and this permission notice shall be +> included in all copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/julia/Manifest.toml b/julia/Manifest.toml new file mode 100644 index 00000000000..57aea68c746 --- /dev/null +++ b/julia/Manifest.toml @@ -0,0 +1,54 @@ +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Distributed]] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[InteractiveUtils]] +deps = ["LinearAlgebra", "Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.0.2" + +[[Parameters]] +deps = ["Markdown", "OrderedCollections", "REPL", "Test"] +git-tree-sha1 = "40f540ec96e50c0b2b9efdb11b5e4d0c63f90923" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.10.1" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/julia/Project.toml b/julia/Project.toml new file mode 100644 index 00000000000..3df258589e9 --- /dev/null +++ b/julia/Project.toml @@ -0,0 +1,6 @@ +name = "FlatBuffers" +uuid = "6a5b833c-db2f-11e8-18d6-cf74c1981d29" + +[deps] +Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/julia/README.md b/julia/README.md new file mode 100644 index 00000000000..c0ff8f66781 --- /dev/null +++ b/julia/README.md @@ -0,0 +1,56 @@ + +# FlatBuffers + +*A Julia implementation of google flatbuffers* + + +| **Documentation** | **PackageEvaluator** | **Build Status** | +|:-------------------------------------------------------------------------------:|:---------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| +| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.6-img]][pkg-0.6-url] [![][pkg-0.7-img]][pkg-0.7-url] | [![][travis-img]][travis-url] [![][appveyor-img]][appveyor-url] [![][codecov-img]][codecov-url] | + + +## Installation + +The package is registered in `METADATA.jl` and so can be installed with `Pkg.add`. + +```julia +julia> Pkg.add("FlatBuffers") +``` + +## Documentation + +- [**STABLE**][docs-stable-url] — **most recently tagged version of the documentation.** +- [**LATEST**][docs-latest-url] — *in-development version of the documentation.* + +## Project Status + +The package is tested against Julia `1.0` and nightly on Linux, OS X, and Windows. + +## Contributing and Questions + +Contributions are very welcome, as are feature requests and suggestions. Please open an +[issue][issues-url] if you encounter any problems or would just like to ask a question. + + + +[docs-latest-img]: https://img.shields.io/badge/docs-latest-blue.svg +[docs-latest-url]: https://JuliaData.github.io/FlatBuffers.jl/latest + +[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-stable-url]: https://JuliaData.github.io/FlatBuffers.jl/stable + +[travis-img]: https://travis-ci.org/JuliaData/FlatBuffers.jl.svg?branch=master +[travis-url]: https://travis-ci.org/JuliaData/FlatBuffers.jl + +[appveyor-img]: https://ci.appveyor.com/api/projects/status/h227adt6ovd1u3sx/branch/master?svg=true +[appveyor-url]: https://ci.appveyor.com/project/JuliaData/documenter-jl/branch/master + +[codecov-img]: https://codecov.io/gh/JuliaData/FlatBuffers.jl/branch/master/graph/badge.svg +[codecov-url]: https://codecov.io/gh/JuliaData/FlatBuffers.jl + +[issues-url]: https://github.com/JuliaData/FlatBuffers.jl/issues + +[pkg-0.6-img]: https://pkg.julialang.org/badges/FlatBuffers_0.6.svg +[pkg-0.6-url]: https://pkg.julialang.org/?pkg=FlatBuffers +[pkg-0.7-img]: https://pkg.julialang.org/badges/FlatBuffers_0.7.svg +[pkg-0.7-url]: https://pkg.julialang.org/?pkg=FlatBuffers diff --git a/julia/REQUIRE b/julia/REQUIRE new file mode 100644 index 00000000000..3b034579702 --- /dev/null +++ b/julia/REQUIRE @@ -0,0 +1,2 @@ +julia 0.7 1.1- +Parameters 0.10.2 0.11- diff --git a/julia/appveyor.yml b/julia/appveyor.yml new file mode 100644 index 00000000000..9aa56ab8dee --- /dev/null +++ b/julia/appveyor.yml @@ -0,0 +1,42 @@ +environment: + matrix: + - julia_version: 1.0 + - julia_version: nightly + +platform: + - x86 # 32-bit + - x64 # 64-bit + +# # Uncomment the following lines to allow failures on nightly julia +# # (tests will run but not make your overall status red) +# matrix: +# allow_failures: +# - julia_version: nightly + +branches: + only: + - master + - /release-.*/ + +notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + +install: + - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) + +build_script: + - echo "%JL_BUILD_SCRIPT%" + - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" + +test_script: + - echo "%JL_TEST_SCRIPT%" + - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" + +# # Uncomment to support code coverage upload. Should only be enabled for packages +# # which would have coverage gaps without running on Windows +# on_success: +# - echo "%JL_CODECOV_SCRIPT%" +# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/julia/docs/Manifest.toml b/julia/docs/Manifest.toml new file mode 100644 index 00000000000..b341c92ea83 --- /dev/null +++ b/julia/docs/Manifest.toml @@ -0,0 +1,79 @@ +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.6.0" + +[[Documenter]] +deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] +git-tree-sha1 = "9f2135e0e7ecb63f9c3ef73ea15a31d8cdb79bb7" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.20.0" + +[[InteractiveUtils]] +deps = ["LinearAlgebra", "Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[UUIDs]] +deps = ["Random"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/julia/docs/Project.toml b/julia/docs/Project.toml new file mode 100644 index 00000000000..dfa65cd107d --- /dev/null +++ b/julia/docs/Project.toml @@ -0,0 +1,2 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/julia/docs/make.jl b/julia/docs/make.jl new file mode 100644 index 00000000000..2384c6a66d3 --- /dev/null +++ b/julia/docs/make.jl @@ -0,0 +1,17 @@ +import Pkg +Pkg.instantiate() +using Documenter, FlatBuffers + +makedocs( + modules = [FlatBuffers], + format = :html, + sitename = "FlatBuffers.jl", + pages = ["Home" => "index.md"] +) + +deploydocs( + repo = "github.com/JuliaData/FlatBuffers.jl.git", + target = "build", + deps = nothing, + make = nothing +) diff --git a/julia/docs/mkdocs.yml b/julia/docs/mkdocs.yml new file mode 100644 index 00000000000..da901f9e2d0 --- /dev/null +++ b/julia/docs/mkdocs.yml @@ -0,0 +1,29 @@ +site_name: FlatBuffers.jl +repo_url: https://github.com/dmbates/FlatBuffers.jl +site_description: Julia implementation of google flatbuffers +site_author: Jacob Quinn + +theme: material + +extra: + palette: + primary: 'indigo' + accent: 'blue' + +extra_css: + - assets/Documenter.css + +extra_javascript: + - https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML + - assets/mathjaxhelper.js + +markdown_extensions: + - extra + - tables + - fenced_code + - mdx_math + +docs_dir: 'build' + +pages: + - Home: index.md diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md new file mode 100644 index 00000000000..6e06b214606 --- /dev/null +++ b/julia/docs/src/index.md @@ -0,0 +1,97 @@ +# FlatBuffers.jl Documentation + +#### Overview +FlatBuffers.jl provides native Julia support for reading and writing binary structures following the google flatbuffer schema (see [here](https://google.github.io/flatbuffers/flatbuffers_internals.html) for a more in-depth review of the binary format). + +The typical language support for flatbuffers involves utilizing the `flatc` compiler to translate a flatbuffer schema file (.fbs) into a langugage-specific set of types/classes and methods. See [here](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) for the official guide on writing schemas. + +This Julia package provides the serialization primitives used by code that has been generated by `flatc`. Since it was originally built without `flatc` support, it can also be used as a minimal set of macros to provide flatbuffer-compatible serialization of existing Julia types. This has led to the Julia code generated by `flatc` appearing somewhat more readable than for other languages. + +For example, for this schema: +``` +namespace example; + +table SimpleType { + x: int = 1; +} + +root_type SimpleType; +``` +the code generated by `flatc` looks like this: +```julia +module Example + +using FlatBuffers +@with_kw mutable struct SimpleType + x::Int32 = 1 +end + +# ... other generated stuff +end +``` +If you don't want to write a schema, you can pepper your existing Julia types +with these macros and then call the functions below to produce flatbuffer-compatible +binaries. + +#### Usage +`FlatBuffers` provides the following functions for reading and writing flatbuffers: +``` +FlatBuffers.serialize(stream::IO, value::T) +FlatBuffers.deserialize(stream::IO, ::Type{T}) +``` +These methods are not exported to avoid naming clashes with the `Serialization` module. +For convenience, there are also two additional constructors defined for each generated type: +* `T(buf::AbstractVector{UInt8}, pos::Integer=0)` +* `T(io::IO)` + +Here is an example showing how to use them to serialize the example type above. +```julia +import FlatBuffers, Example + +# create an instance of our type +val = Example.SimpleType(2) + +# serialize it to example.bin +open("example.bin", "w") do f FlatBuffers.serialize(f, val) end + +# read the value back again from file +val2 = open("example.bin", "r") do f Example.SimpleType(f) end +``` +In addition, this package provides the following types and methods, which are useful +when inspecting and constructing flatbuffers: +* `FlatBuffers.Table{T}` - type for deserializing a Julia type `T` from a flatbuffer +* `FlatBuffers.Builder{T}` - type for serializing a Julia type `T` to a flatbuffer +* `FlatBuffers.read` - performs the actual deserializing on a `FlatBuffer.Table` +* `FlatBuffers.build!` - performs the actual serializing on a `FlatBuffer.Builder` + +#### Methods for Generated Types +For a generated type `T`, in addition to the constructors mentioned above: +* if `T` has default values, constructors will be defined as per the `@with_kw` macro in [Parameters.jl](https://github.com/mauro3/Parameters.jl) +* `FlatBuffers.file_extension(T)` - returns the `file_extension` specified in the schema (if any) +* `FlatBuffers.file_identifier(T)` - returns the `file_identifier` specified in the schema (if any) +* `FlatBuffers.has_identifier(T, bytes)` - returns whether the given bytes contain the identifier for `T` at the offset designated by the flatbuffers specification +* `FlatBuffers.slot_offsets(T)` - an array containing the positions of the slots in the vtable for type `T`, accounting for gaps caused by deprecated fields +* `FlatBuffers.root_type(T)` - returns whether the type is designated as the root type by the schema. Also note however that no `root_type` definition is necessary in Julia; any of the generated `mutable struct`s can be a valid root table type. + +#### Circular References +It's a bit unfortunate that the flatbuffers example uses mutually referential types, something which Julia doesn't have support for yet. +However, there is a [workaround](https://github.com/JuliaLang/julia/issues/269#issuecomment-68421745) - by modifying the +code generated by `flatc` slightly to add a type parameter, we can refer to a type that hasn't yet been defined. +```julia +FlatBuffers.@with_kw mutable struct Monster{T} + # ... + test::T = nothing + # ... +end +``` +In general though, try to avoid schemas which introduce these kinds of circular references. +For the full `Monster` example see the test suite [here](https://github.com/JuliaData/FlatBuffers.jl/blob/master/test/MyGame/Example/Monster.jl). + +#### Internal Utilities +These functions are used by the code generated by `flatc`. Documentation is also included for many +internal methods and may be queried using `?` at the REPL. +* `@ALIGN T size_in_bytes` - convenience macro for forcing a flatbuffer alignment on the Julia type `T` to `size_in_bytes` +* `@with_kw mutable struct T fields...` - convenience macro for defining default field values for Julia type `T` +* `@UNION T Union{T1,T2,...}` - convenience macro for defining a flatbuffer union type `T` +* `@STRUCT struct T fields... end` - convenience macro for defining flatbuffer struct types, ensuring any necessary padding gets added to the type definition + diff --git a/julia/src/FlatBuffers.jl b/julia/src/FlatBuffers.jl new file mode 100644 index 00000000000..1c9804be5f6 --- /dev/null +++ b/julia/src/FlatBuffers.jl @@ -0,0 +1,610 @@ +module FlatBuffers + +# utils +""" +serialize(stream::IO, value::T) where {T} +Serialize `value` to `stream` using the `FlatBuffer` format. +""" +function serialize(stream::IO, value::T) where {T} + write(stream, bytes(build!(value))) +end + +""" +deserialize(stream::IO, ::Type{T}) where {T} +Read a `T` from the flatbuffer-formatted `stream`. +""" +function deserialize(stream::IO, ::Type{T}) where {T} + read(T, read(stream)) +end + +struct UndefinedType end +const Undefined = UndefinedType() +getfieldvalue(obj::T, i) where {T} = isdefined(obj, i) ? getfield(obj, i) : Undefined +getprevfieldvalue(obj::T, i) where {T} = i == 1 ? missing : getfieldvalue(obj, i - 1) + +""" +Scalar +A Union of the Julia types `T <: Number` that are allowed in FlatBuffers schema +""" +const Scalar = Union{Bool, +Int8, Int16, Int32, Int64, +UInt8, UInt16, UInt32, UInt64, +Float32, Float64} + +isstruct(T) = isconcretetype(T) && !T.mutable +isbitstype(T) = fieldcount(T) == 0 +isunionwithnothing(T) = T isa Union && T.a == Nothing && !(isa(T.b, Union)) + +file_identifier(T) = "" +file_extension(T) = "" +slot_offsets(T) = [4 + ((i - 1) * 2) for i = 1:length(T.types)] + +default(T, TT, sym) = default(TT) +default(::Type{T}) where {T <: Scalar} = zero(T) +default(::Type{T}) where {T <: AbstractString} = nothing +default(::Type{T}) where {T <: Enum} = enumtype(T)(T(0)) +default(::Type{Vector{T}}) where {T} = nothing + +# attempt to call default constructors for the type, +# use above methods as fallback +function default(::Type{T}, i::Integer) where {T} + TT = T.types[i] + try + return FlatBuffers.default(T, TT, fieldnames(T)[i]) + # catch because Parameters throws an error if there is no + # default value defined... + catch + end + return default(TT) +end + +# fallback that recursively builds a default; for structs/tables +function default(::Type{T}) where {T} + if isa(T, Union) || isa(T, UnionAll) + return nothing + else + return T([default(T, i) for i = 1:length(T.types)]...) + end +end + +function typeorder end + +enumtype(::Type{<:Enum}) = UInt8 + +# Types +""" +Table + +The object containing the flatbuffer and positional information specific to the table. +The `vtable` containing the offsets for specific members precedes `pos`. +The actual values in the table follow `pos` offset and size of the vtable. + +- `bytes::AbstractVector{UInt8}`: the flatbuffer itself +- `pos::Integer`: the base position in `bytes` of the table +""" +mutable struct Table{T} + bytes::AbstractVector{UInt8} + pos::Integer +end + +""" +Builder is a state machine for creating FlatBuffer objects. +Use a Builder to construct object(s) starting from leaf nodes. + +A Builder constructs byte buffers in a last-first manner for simplicity and +performance. +""" +mutable struct Builder{T} + bytes::AbstractVector{UInt8} + minalign::Int + vtable::Vector{Int} + objectend::Int + vtables::Vector{Int} + head::Int + nested::Bool + finished::Bool +end + +function hexloc(x) + "0x" * lpad("$(string(x-1, base=16)) ", 6, '0') +end + +function hexbyte(io, z) + printstyled(io, lpad("$(string(z, base=16)) ", 3, '0'), color=Int(z)) +end + +function hexoffset(x) + "0x$(lpad(string(x, base=16), 4, '0'))" +end + +function stringify(io, buf, offset, x, y, msg="", msgcolor=:blue) + y = min(y, length(buf)) + printstyled(io, hexloc(x + offset), color=:blue) + for i = x:y + hexbyte(io, buf[i]) + end + if length(msg) > 0 + printstyled(io, " " * msg, color=msgcolor) + end + println(io) +end + +function showvtable(io::IO, T, buffer, vtabstart, vtabsize) + syms = T.name.names + printstyled(io, "vtable start pos: $(hexoffset(vtabstart))\n", color=:green) + printstyled(io, "vtable size: $vtabsize\n", color=:green) + i = vtabstart + 4 + soff = slot_offsets(T) + numslots = div(soff[end] - 4, 2) + 1 + field = 1 + slot = 1 + numfields = length(T.types) + while slot <= numslots + # leave holes for deprecated fields + j = 2 + start = field == 1 ? soff[1] : soff[field - 1] + while (start + j) < soff[field] + # empty slot + stringify(io, buffer, 1, i, i+1, "[deprecated field]", :red) + slot += 1 + j += 2 + i += 2 + if (i - vtabstart) > vtabsize + break + end + end + if (i - vtabstart) > vtabsize + break + end + stringify(io, buffer, 1, i, i+1, "[$(fieldnames(T)[field])]") + slot += 1 + field += 1 + i += 2 + if (i - vtabstart) > vtabsize + break + end + end + # now we're pointing at data + printstyled(io, "payload:\n", color=:green) + while i < length(buffer) + stringify(io, buffer, 1, i, i+7, "") + i += 8 + end +end + +function Base.show(io::IO, x::Union{Builder{T}, Table{T}}) where {T} + printstyled(io, "FlatBuffers.$(typeof(x)):\n", color=:green) + buffer = x isa Builder ? x.bytes[x.head+1:end] : x.bytes + if isempty(buffer) + printstyled(io, " (empty flatbuffer)", color=:red) + else + pos = Int(typeof(x) <: Table ? x.pos : readbuffer(buffer, 0, Int32)) + printstyled(io, "root offset: $(hexoffset(pos))\n", color=:green) + vtaboff = readbuffer(buffer, pos, Int32) + vtabstart = pos - vtaboff + vtabsize = readbuffer(buffer, vtabstart, Int16) + showvtable(io, T, buffer, vtabstart, vtabsize) + end +end + +include("internals.jl") + +function Table(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T} + return Table{T}(buffer, pos) +end + +Table(b::Builder{T}) where {T} = Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32)) + +getvalue(t, o, ::Type{Nothing}) = nothing +getvalue(t, o, ::Type{T}) where {T <: Scalar} = get(t, t.pos + o, T) +getvalue(t, o, ::Type{T}) where {T <: Enum} = T(get(t, t.pos + o, enumtype(T))) +function getvalue(t, o, ::Type{T}) where {T <: AbstractString} + o += get(t, t.pos + o, Int32) + strlen = get(t, t.pos + o, Int32) + o += t.pos + sizeof(Int32) + return String(t.bytes[o + 1:o + strlen]) +end +function getvalue(t, o, ::Type{Vector{UInt8}}) + o += get(t, t.pos + o, Int32) + len = get(t, t.pos + o, Int32) + o += t.pos + sizeof(Int32) + return t.bytes[o + 1:o + len] #TODO: maybe not make copy here? +end + +getarray(t, vp, len, ::Type{T}) where {T <: Scalar} = (ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len]) +getarray(t, vp, len, ::Type{T}) where {T <: Enum} = (ptr = convert(Ptr{enumtype(T)}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len]) +function getarray(t, vp, len, ::Type{T}) where {T <: Union{AbstractString, Vector{UInt8}}} + A = Vector{T}(undef, len) + for i = 1:len + A[i] = getvalue(t, vp - t.pos, T) + vp += sizeof(Int32) + end + return A +end +function getarray(t, vp, len, ::Type{T}) where {T} + if isstruct(T) + ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)) + return [unsafe_load(ptr, i) for i = 1:len] + else + A = Vector{T}(undef, len) + for i = 1:len + A[i] = getvalue(t, vp - t.pos, T) + vp += sizeof(Int32) + end + return A + end +end + +function getvalue(t, o, ::Type{Vector{T}}) where {T} + vl = vectorlen(t, o) + vp = vector(t, o) + return getarray(t, vp, vl, T) +end + +Base.convert(::Type{T}, e::Integer) where {T <: Enum} = T(e) + +# fallback which recursively calls read +function getvalue(t, o, ::Type{T}) where {T} + if isstruct(T) + if any(x-> x <: Enum, T.types) + args = [] + o = t.pos + o + 1 + for typ in T.types + val = unsafe_load(convert(Ptr{typ <: Enum ? enumtype(typ) : typ}, pointer(view(t.bytes, o:length(t.bytes))))) + push!(args, val) + o += sizeof(typ <: Enum ? enumtype(typ) : typ) + end + return T(args...) + else + return unsafe_load(convert(Ptr{T}, pointer(view(t.bytes, (t.pos + o + 1):length(t.bytes))))) + end + else + o += t.pos + newt = Table{T}(t.bytes, indirect(t, o)) + return FlatBuffers.read(newt, T) + end +end + +function typetoread(prevfield, ::Type{T}, ::Type{V}) where {T <:Any, V <: AbstractVector} + # hacks! if it's a union all, assume it's because we're working around circular dependencies + if isa(eltype(V), UnionAll) + return Vector{T}, false + # if it's a vector of Unions, use the previous field to figure out the types of all the elements + elseif isa(eltype(V), Union) + types = typeorder.(eltype(V), prevfield) + R = definestruct(types) + return R, true + end + return V, false +end + +function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT} + # if it's a Union type, use the previous arg to figure out the true type that was serialized + if !isunionwithnothing(TT) && TT isa Union + R = typeorder(TT, prevfield) + else + R = TT + end + R, false +end + +""" +`FlatBuffers.read` parses a `T` at `t.pos` in Table `t`. +Will recurse as necessary for nested types (Arrays, Tables, etc.) +""" +function FlatBuffers.read(t::Table{T1}, ::Type{T}=T1) where {T1, T} + args = [] + numfields = length(T.types) + soff = slot_offsets(T) + for i = 1:numfields + TT = T.types[i] + o = offset(t, soff[i]) + R, isunionvector = typetoread(i == 1 ? nothing : args[end], T, TT) + nullable = false + if isunionwithnothing(R) + nullable = true + R = R.b + end + if o == 0 + push!(args, nullable ? nothing : default(T, TT, T.name.names[i])) + else + if isunionvector + eval(:(newr = getvalue($t, $o, $R))) + eval(:(n = length($R.types))) + push!(args, [getfieldvalue(newr, j) for j = 1:n]) + else + push!(args, getvalue(t, o, R)) + end + end + end + + return T(args...) +end + +FlatBuffers.read(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T} = FlatBuffers.read(Table(T, buffer, pos)) +FlatBuffers.read(b::Builder{T}) where {T} = FlatBuffers.read(Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32))) +# assume `bytes` is a pure flatbuffer buffer where we can read the root position at the beginning +FlatBuffers.read(::Type{T}, bytes) where {T} = FlatBuffers.read(T, bytes, read(IOBuffer(bytes), Int32)) + +has_identifier(::Type{T}, bytes) where {T} = length(bytes) >= 8 && String(bytes[5:8]) == FlatBuffers.file_identifier(T) +root_type(::Type{T}) where {T} = false + +""" +flat_bytes = bytes(b) + +`flat_bytes` are the serialized bytes for the FlatBuffer. This discards the Julia specific `head`. +""" +bytes(b::Builder) = unsafe_wrap(Array{UInt8,1}, pointer(b.bytes, b.head+1), (length(b.bytes)-b.head)) + +function Builder(::Type{T}=Any, size=0) where {T} + objectend = 0 + vtables = zeros(Int, 0) + head = size + nested = false + bytes = zeros(UInt8, size) + minalign = 1 + vtable = zeros(Int, 0) + finished = false + b = Builder{T}(bytes, minalign, vtable, objectend, + vtables, head, nested, finished) + return b +end + +# build! +"`alignment` looks for the largest scalar member of `T` that represents a flatbuffer Struct" +function alignment(::Type{T}) where {T} + largest = 0 + for typ in T.types + largest = isbitstype(typ) ? max(largest,sizeof(typ)) : alignment(typ) + end + return largest +end + +""" +`buildvector!` is for building vectors with all kinds of element types, +even building its elements recursively if needed (Array of Arrays, Array of tables, etc.). +""" +function buildvector! end + +# empty vector +function buildvector!(b, A::Vector{Nothing}, len, prev) + startvector(b, 1, 0, 1) + return endvector(b, 0) +end +# scalar type vector +function buildvector!(b, A::Vector{T}, len, prev) where {T <: Scalar} + startvector(b, sizeof(T), len, sizeof(T)) + foreach(x->prepend!(b, A[x]), len:-1:1) + return endvector(b, len) +end +function buildvector!(b, A::Vector{T}, len, prev) where {T <: Enum} + startvector(b, sizeof(enumtype(T)), len, sizeof(enumtype(T))) + foreach(x->prepend!(b, enumtype(T)(A[x])), len:-1:1) + return endvector(b, len) +end + +function putoffsetvector!(b, offsets, len) + startvector(b, 4, len, 4) #TODO: elsize/alignment correct here? + foreach(x->prependoffset!(b, offsets[x]), len:-1:1) + return endvector(b, len) +end +# byte vector vector +function buildvector!(b, A::Vector{Vector{UInt8}}, len, prev) + offsets = map(x->createbytevector(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) +end +# string vector +function buildvector!(b, A::Vector{T}, len, prev) where {T <: AbstractString} + offsets = map(x->createstring(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) +end +# array vector +function buildvector!(b, A::Vector{Vector{T}}, len, prev) where {T} + offsets = map(x->buildbuffer!(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) +end + +# make a new struct which has fields of the given type +function definestruct(types::Vector{DataType}) + fields = [:($(gensym())::$(TT)) for TT in types] + T1 = gensym() + eval(:(mutable struct $T1 + $(fields...) + end)) + return T1 +end + +# make a new struct which has fields of the given type +# and populate them with values from the vector +function createstruct(types::Vector{DataType}, A::Vector{T}) where {T} + T1 = definestruct(types) + eval(:(newt = $T1($(A...)))) + return newt +end + +# struct or table/object vector +function buildvector!(b, A::Vector{T}, len, prev) where {T} + if isstruct(T) + # struct + startvector(b, sizeof(T), len, alignment(T)) #TODO: forced elsize/alignment correct here? + foreach(x->buildbuffer!(b, A[x]), len:-1:1) + return endvector(b, len) + elseif isa(T, Union) + types = typeorder.(T, prev) + + # define a new type, construct one, and pack it into the buffer + newt = createstruct(types, A) + buildbuffer!(b, newt) + else + # table/object + offsets = map(x->buildbuffer!(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) + end +end + +""" +`getoffset` checks if a given field argument needs to be built +offset (Arrays, Strings, other tables) or can be inlined (Scalar or Struct types). +`getoffset` has a recursive nature in that it will build offset types +down to their last leaf scalar types before returning the highest-level offset. +""" +function getoffset end + +getoffset(b, arg::Nothing, prev=nothing) = 0 +getoffset(b, arg::T, prev=nothing) where {T <: Scalar} = 0 +getoffset(b, arg::T, prev=nothing) where {T <: Enum} = 0 +getoffset(b, arg::AbstractString, prev=nothing) = createstring(b, arg) +getoffset(b, arg::Vector{UInt8}, prev) = createbytevector(b, arg) +getoffset(b, arg::Vector{T}, prev) where {T} = buildbuffer!(b, arg, prev) + +# structs or table/object +getoffset(b, arg::T, prev=nothing) where {T} = isstruct(T) ? 0 : buildbuffer!(b, arg, prev) + +""" +`putslot!` is one of the final steps in building a flatbuffer. +It puts the final value in the "data" section of the flatbuffer, +whether that be an actual value (for `Scalar`, Struct types) or an offset +to the actual data (Arrays, Strings, other tables) +""" +function putslot! end + +putslot!(b, i, arg::T, off, default, prev) where {T <: Scalar} = prependslot!(b, i, arg, default) +putslot!(b, i, arg::T, off, default, prev) where {T <: Enum} = prependslot!(b, i, enumtype(T)(arg), default) +putslot!(b, i, arg::AbstractString, off, default, prev) = prependoffsetslot!(b, i, off, 0) +putslot!(b, i, arg::Vector{T}, off, default, prev) where {T} = prependoffsetslot!(b, i, off, 0) +# structs or table/object +function putslot!(b, i, arg::T, off, default, prev) where {T} + if isstruct(T) + prependstructslot!(b, i, buildbuffer!(b, arg, prev), 0) + else + prependoffsetslot!(b, i, off, 0) + end +end + +function needreconstruct(T) + for TT in T.types + if TT <: Vector && eltype(TT) isa Union && !(eltype(TT) isa UnionAll) + return true + elseif TT isa Union && !isunionwithnothing(TT) + return true + end + end + return false +end + +function reconstructkwargs(arg::T) where {T} + kwargs = Dict{Symbol, Any}() + numfields = length(T.types) + fnames = fieldnames(T) + for i = 2:numfields + field = getfield(arg, i) + prevname = fnames[i - 1] + # hack to make the example work + TT = field isa Vector ? eltype(field) : typeof(field) + if :parameters in propertynames(TT) && length(TT.parameters) > 0 + TT = TT.name.wrapper + end + if field isa Vector && eltype(field) isa Union && !(eltype(field) isa UnionAll) + kwargs[prevname] = [FlatBuffers.typeorder(TT, typeof(x)) for x in field] + elseif (T.types[i] isa Union && !isunionwithnothing(T.types[i])) + kwargs[prevname] = FlatBuffers.typeorder(T.types[i], TT) + end + end + return kwargs +end + +function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Array} + # array of things + buildvector!(b, arg, length(arg), prev) +end + +function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Any} + # populate the _type field before unions/vectors of unions + if needreconstruct(T) + # reconstruct it so the types before the fields + # are populated correctly + kwargs = reconstructkwargs(arg) + arg = Parameters.reconstruct(arg; kwargs...) + end + if isstruct(T) + # build a struct type with provided `arg` + all(isstruct, T.types) || throw(ArgumentError("can't seralize flatbuffer, $T is not a pure struct")) + align = alignment(T) + prep!(b, align, 2align) + for i = length(T.types):-1:1 + typ = T.types[i] + if typ <: Enum + prepend!(b, enumtype(typ)(getfield(arg,i))) + elseif isbitstype(typ) + prepend!(b, getfield(arg,i)) + else + buildbuffer!(b, getfield(arg, i), getprevfieldvalue(arg, i)) + end + end + n = offset(b) + else + # build a table type + # check for string/array/table types + numfields = length(T.types) + # early exit for empty objects + if numfields == 0 + startobject(b, 0) + return endobject(b) + end + os = Int[] + isdefault = falses(numfields) + for i = 1:numfields + push!(os, getoffset(b, getfieldvalue(arg, i), getprevfieldvalue(arg, i))) + end + # all nested have been written, with offsets in `os[]` + # don't use slots for the last N members if they are all default + # also leave slots for deprecated fields + i = numfields + isdefault = getfieldvalue(arg, i) == default(T, i) + while isdefault && i > 0 + i -= 1 + isdefault = getfieldvalue(arg, i) == default(T, i) + end + soff = slot_offsets(T) + numslots = div(soff[i] - 4, 2) + 1 + startobject(b, numslots) + i = 1 + field = 1 + while i <= numslots + # leave holes for deprecated fields + j = 2 + start = field == 1 ? soff[1] : soff[field - 1] + while (start + j) < soff[field] + # empty slot + i += 1 + j += 2 + end + val = getfieldvalue(arg, field) + d = default(T, field) + if !(isunionwithnothing(T.types[field]) && val == nothing) + putslot!(b, i, + val, + os[field], + d, + getprevfieldvalue(arg, field) + ) + end + field += 1 + i += 1 + end + n = endobject(b) + end + return n +end + +function build!(b, arg) + n = buildbuffer!(b, arg) + finish!(b, n) + return b +end + +build!(arg::T) where {T} = build!(Builder(T), arg) + +include("macros.jl") + +end # module diff --git a/julia/src/internals.jl b/julia/src/internals.jl new file mode 100644 index 00000000000..054b573d29e --- /dev/null +++ b/julia/src/internals.jl @@ -0,0 +1,453 @@ +# table.go +const VtableMetadataFields = 2 + +const TableOrBuilder = Union{Table,Builder} + +const Bytes2Type = Dict{Int, DataType}(1=>UInt8, 2=>UInt16, 4=>UInt32, 8=>UInt64) + +Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T} = read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), T) +readbuffer(t::AbstractVector{UInt8}, pos::Int, ::Type{T}) where {T} = read(IOBuffer(view(t, (pos+1):length(t))), T) +Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), Bytes2Type[sizeof(T)])) + +""" +`offset` provides access into the Table's vtable. + +Deprecated fields are ignored by checking against the vtable's length. +""" +function offset(t::Table, vtableoffset) + vtable = t.pos - get(t, t.pos, Int32) + return vtableoffset < get(t, vtable, Int16) ? get(t, vtable + vtableoffset, Int16) : 0 +end + +"`indirect` retrieves the relative offset stored at `offset`." +indirect(t::Table, off) = off + get(t, off, Int32) + +""" +`vectorlen` retrieves the length of the vector whose offset is stored at +`off` in this object. +""" +function vectorlen(t::Table, off) + off += t.pos + off += get(t, off, Int32) + return get(t, off, Int32) +end + +""" +`vector` retrieves the start of data of the vector whose offset is stored +at `off` in this object. +""" +function vector(t::Table, off) + off += t.pos + off += get(t, off, Int32) + # data starts after metadata containing the vector length + return off + sizeof(Int32) +end + +""" +GetVOffsetTSlot retrieves the VOffsetT that the given vtable location +points to. If the vtable value is zero, the default value `d` +will be returned. +""" +function getoffsetslot(t::Table, slot, d) + off = offset(t, slot) + if off == 0 + return d + end + return off +end + +""" +`getslot` retrieves the `T` that the given vtable location +points to. If the vtable value is zero, the default value `d` +will be returned. +""" +function getslot(t::Table, slot, d::T) where {T} + off = offset(t, slot) + if off == 0 + return d + end + + return get(t, t.pos + off, T) +end + +# builder.go +value(x::T) where {T <: Enum} = length(T.types) == 0 ? Int(x) : getfield(x,1) + +Base.write(sink::Builder, o, x::Union{Bool,UInt8}) = sink.bytes[o+1] = UInt8(x) +function Base.write(sink::Builder, off, x::T) where {T} + off += 1 + for (i,ind) = enumerate(off:(off + sizeof(T) - 1)) + sink.bytes[ind] = (x >> ((i-1) * 8)) % UInt8 + end +end +Base.write(b::Builder, o, x::Float32) = write(b, o, reinterpret(UInt32, x)) +Base.write(b::Builder, o, x::Float64) = write(b, o, reinterpret(UInt64, x)) +Base.write(b::Builder, o, x::Enum) = write(b, o, reinterpret(Bytes2Type[sizeof(x)], value(x))) + +"Offset relative to the end of the buffer." +offset(b::Builder) = length(b.bytes) - b.head + +pad!(b::Builder, n) = foreach(x->place!(b, 0x00), 1:n) + +""" +`finishedbytes` returns a pointer to the written data in the byte buffer. +Panics if the builder is not in a finished state (which is caused by calling +`finish!()`). +""" +function finishedbytes(b::Builder) + assertfinished(b) + return b.bytes[b.head+1:end] +end + +function startobject(b::Builder, numslots) + assertnotnested(b) + b.nested = true + b.vtable = zeros(Int, numslots) + b.objectend = offset(b) + b.minalign = 1 + return b +end + +""" +`endobject` writes data necessary to finish object construction. +""" +function endobject(b::Builder{T}) where {T} + assertnested(b) + n = writevtable!(b) + b.nested = false + return n +end + +""" +`prep!` prepares to write an element of `size` after `additionalbytes` +have been written, e.g. if you write a string, you need to align such +the int length field is aligned to sizeof(Int32), and the string data follows it +directly. +If all you need to do is align, `additionalbytes` will be 0. +""" +function prep!(b::Builder, size, additionalbytes) + # Track the biggest thing we've ever aligned to. + if size > b.minalign + b.minalign = size + end + # Find the amount of alignment needed such that `size` is properly + # aligned after `additionalBytes`: + alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1 + alignsize &= (size - 1) + + # Reallocate the buffer if needed: + totalsize = alignsize + size + additionalbytes + if b.head <= totalsize + len = length(b.bytes) + prepend!(b.bytes, zeros(UInt8, totalsize)) + b.head += length(b.bytes) - len + end + pad!(b, alignsize) + return +end + +""" +`prepend!` prepends a `T` to the Builder buffer. +Aligns and checks for space. +""" +function Base.prepend!(b::Builder, x::T) where {T} + prep!(b, sizeof(T), 0) + place!(b, x) + return +end + +""" +`place!` prepends a `T` to the Builder, without checking for space. +""" +function place!(b::Builder, x::T) where {T} + b.head -= sizeof(T) + write(b, b.head, x) + return +end + +""" +`startvector` initializes bookkeeping for writing a new vector. + +A vector has the following format: + ++, where T is the type of elements of this vector. +""" +function startvector(b::Builder, elemSize, numElems, alignment) + assertnotnested(b) + b.nested = true + prep!(b, sizeof(UInt32), elemSize * numElems) + prep!(b, alignment, elemSize * numElems) + return offset(b) +end + +""" +`endvector` writes data necessary to finish vector construction. +""" +function endvector(b::Builder, vectorNumElems) + assertnested(b) + place!(b, UInt32(vectorNumElems)) + b.nested = false + return offset(b) +end + +""" +`createstring` writes a null-terminated string as a vector. +""" +function createstring(b::Builder, s::AbstractString) + assertnotnested(b) + b.nested = true + s = codeunits(s) + prep!(b, sizeof(UInt32), length(s) + 1) + place!(b, UInt8(0)) + + l = length(s) + + b.head -= l + copyto!(b.bytes, b.head+1, s, 1, l) + return endvector(b, length(s)) +end + +""" +`createbytevector` writes a byte vector +""" +function createbytevector(b::Builder, v::AbstractVector{UInt8}) + assertnotnested(b) + b.nested = true + + prep!(b, sizeof(UInt32), length(v)) + + l = length(v) + + b.head -= l + copyto!(b.bytes, b.head+1, v, 1, l) + + return endvector(b, length(v)) +end + +""" +`prependoffset!` prepends an Int32, relative to where it will be written. +""" +function prependoffset!(b::Builder, off) + prep!(b, sizeof(Int32), 0) # Ensure alignment is already done. + if !(off <= offset(b)) + throw(ArgumentError("unreachable: $off <= $(offset(b))")) + end + off2 = offset(b) - off + sizeof(Int32) + place!(b, Int32(off2)) + return +end + +function prependoffsetslot!(b::Builder, o::Int, x::T, d) where {T} + if x != T(d) + prependoffset!(b, x) + slot!(b, o) + end + return +end + +""" +`prependslot!` prepends a `T` onto the object at vtable slot `o`. +If value `x` equals default `d`, then the slot will be set to zero and no +other data will be written. +""" +function prependslot!(b::Builder, o::Int, x::T, d) where {T} + if x != T(d) + prepend!(b, x) + slot!(b, o) + end + return +end + +""" +`prependstructslot!` prepends a struct onto the object at vtable slot `o`. +Structs are stored inline, so nothing additional is being added. +In generated code, `d` is always 0. +""" +function prependstructslot!(b::Builder, voffset, x, d) + if x != d + assertnested(b) + if x != offset(b) + throw(ArgumentError("inline data write outside of object")) + end + slot!(b, voffset) + end + return +end + +""" +`slot!` sets the vtable key `voffset` to the current location in the buffer. +""" +function slot!(b::Builder, slotnum) + b.vtable[slotnum] = offset(b) +end + +""" +`finish!` finalizes a buffer, pointing to the given `rootTable`. +""" +function finish!(b::Builder{T}, rootTable) where {T} + assertnotnested(b) + identifier = file_identifier(T) + n = length(identifier) + prep!(b, b.minalign, sizeof(UInt32)) + for i = 0:(n-1) + prepend!(b, UInt8(identifier[n - i])) + end + prependoffset!(b, Int32(rootTable)) + b.finished = true + return +end + +function assertnested(b::Builder) + # If you get this assert, you're in an object while trying to write + # data that belongs outside of an object. + # To fix this, write non-inline data (like vectors) before creating + # objects. + if !b.nested + throw(ArgumentError("Incorrect creation order: must be inside object.")) + end + return +end + +function assertnotnested(b::Builder) + # If you hit this, you're trying to construct a Table/Vector/String + # during the construction of its parent table (between the MyTableBuilder + # and builder.Finish()). + # Move the creation of these view-objects to above the MyTableBuilder to + # not get this assert. + # Ignoring this assert may appear to work in simple cases, but the reason + # it is here is that storing objects in-line may cause vtable offsets + # to not fit anymore. It also leads to vtable duplication. + if b.nested + throw(ArgumentError("Incorrect creation order: object must not be nested.")) + end + return +end + +function assertfinished(b::Builder) + # If you get this assert, you're attempting to get access a buffer + # which hasn't been finished yet. Be sure to call builder.Finish() + # with your root table. + # If you really need to access an unfinished buffer, use the bytes + # buffer directly. + if !b.finished + throw(ArgumentError("Incorrect use of FinishedBytes(): must call 'Finish' first.")) + end +end + +""" +WriteVtable serializes the vtable for the current object, if applicable. + +Before writing out the vtable, this checks pre-existing vtables for equality +to this one. If an equal vtable is found, point the object to the existing +vtable and return. + +Because vtable values are sensitive to alignment of object data, not all +logically-equal vtables will be deduplicated. + +A vtable has the following format: + + + * N, where N is the number of fields in +the schema for this type. Includes deprecated fields. +Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. + +An object has the following format: + ++ +""" +function writevtable!(b::Builder{T}) where {T} + # Prepend a zero scalar to the object. Later in this function we'll + # write an offset here that points to the object's vtable: + prepend!(b, Int32(0)) + + objectOffset = offset(b) + existingVtable = 0 + + # Search backwards through existing vtables, because similar vtables + # are likely to have been recently appended. See + # BenchmarkVtableDeduplication for a case in which this heuristic + # saves about 30% of the time used in writing objects with duplicate + # tables. + for i = length(b.vtables):-1:1 + # Find the other vtable, which is associated with `i`: + vt2Offset = b.vtables[i] + vt2Start = length(b.bytes) - vt2Offset + vt2Len = readbuffer(b.bytes, vt2Start, Int16) + + metadata = VtableMetadataFields * sizeof(Int16) + vt2End = vt2Start + vt2Len + vt2 = view(b.bytes, (vt2Start + metadata + 1):vt2End) #TODO: might need a +1 on the start of range here + + # Compare the other vtable to the one under consideration. + # If they are equal, store the offset and break: + if vtableEqual(b.vtable, objectOffset, vt2) + existingVtable = vt2Offset + break + end + end + + if existingVtable == 0 + # Did not find a vtable, so write this one to the buffer. + + # Write out the current vtable in reverse , because + # serialization occurs in last-first order: + for i = length(b.vtable):-1:1 + off::Int16 = 0 + if b.vtable[i] != 0 + # Forward reference to field; + # use 32bit number to assert no overflow: + off = objectOffset - b.vtable[i] + end + prepend!(b, Int16(off)) + end + + # The two metadata fields are written last. + + # First, store the object bytesize: + objectSize::Int16 = objectOffset - b.objectend + prepend!(b, objectSize) + + # Second, store the vtable bytesize: + vbytes::Int16 = (length(b.vtable) + VtableMetadataFields) * sizeof(Int16) + prepend!(b, vbytes) + + # Next, write the offset to the new vtable in the + # already-allocated SOffsetT at the beginning of this object: + objectStart::Int32 = length(b.bytes) - objectOffset + write(b, objectStart, Int32(offset(b) - objectOffset)) + + # Finally, store this vtable in memory for future + # deduplication: + push!(b.vtables, offset(b)) + else + # Found a duplicate vtable. + + objectStart = length(b.bytes) - objectOffset + b.head = objectStart + + # Write the offset to the found vtable in the + # already-allocated SOffsetT at the beginning of this object: + write(b, b.head, Int32(existingVtable - objectOffset)) + end + + empty!(b.vtable) + return objectOffset +end + +"vtableEqual compares an unwritten vtable to a written vtable." +function vtableEqual(a::Vector{Int}, objectStart, b::AbstractVector{UInt8}) + if length(a) * sizeof(Int16) != length(b) + return false + end + + for i = 0:(length(a)-1) + x = read(IOBuffer(view(b, (i * sizeof(Int16) + 1):length(b))), Int16) + + # Skip vtable entries that indicate a default value. + x == 0 && a[i+1] == 0 && continue + + y = objectStart - a[i+1] + x != y && return false + end + return true +end diff --git a/julia/src/macros.jl b/julia/src/macros.jl new file mode 100644 index 00000000000..281fefc2f83 --- /dev/null +++ b/julia/src/macros.jl @@ -0,0 +1,225 @@ +export @UNION, @DEFAULT, @ALIGN, @STRUCT, @with_kw + +const __module__ = 0 + +function indexof(needle, haystack) + for (i, v) in enumerate(haystack) + v == needle && return i-1 + end + return -1 +end + +macro UNION(T, TT) + typeof(T) == Symbol || throw(ArgumentError("1st argument must be a symbol to represent a Union type")) + TT.head == :tuple || throw(ArgumentError("2nd argument must be a tuple of types like `(T1,T2,...)`")) + return esc(quote + const $T = $(Expr(:curly, :Union, TT.args...)) + FlatBuffers.typeorder(::Type{$T}, ::Type{TT}) where {TT} = FlatBuffers.indexof(TT, $TT) + FlatBuffers.typeorder(::Type{$T}, i::Integer) = ($TT)[i+1] + FlatBuffers.isunionwithnothing(::Type{$T}) = false + end) +end + +macro ALIGN(T, sz) + return esc(quote + FlatBuffers.alignment(::Type{$T}) = $sz + end) +end + +macro enumtype(T, typ) + return esc(quote + FlatBuffers.enumtype(::Type{$T}) = $typ + end) +end + +# recursively finds largest field of a STRUCT +fbsizeof(::Type{T}) where {T<:Enum} = sizeof(enumtype(T)) +fbsizeof(::Type{T}) where {T} = sizeof(T) + +maxsizeof(::Type{T}) where {T<:Enum} = sizeof(enumtype(T)) +maxsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : maximum(map(x->maxsizeof(x), T.types)) + +nextsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : nextsizeof(T.types[1]) + +function fieldlayout(mod, typ, exprs...) + fields = Expr[] + values = [] + largest_field = maximum(map(x->maxsizeof(Core.eval(mod, x.args[2])), exprs)) + sz = cur_sz = 0 + x = 0 + for (i,expr) in enumerate(exprs) + T = Core.eval(mod, expr.args[2]) + if !isbitstype(T) + exprs2 = [Expr(:(::), nm, typ) for (nm,typ) in zip(fieldnames(T),T.types)] + fields2, values2 = fieldlayout(mod, T, exprs2...) + append!(fields, map(x->Expr(:(::), Symbol(string(expr.args[1],'_',x.args[1])), x.args[2]), fields2)) + append!(values, map(x->x == 0 ? 0 : Expr(:call, :getfield, expr.args[1], QuoteNode(x)), values2)) + else + push!(fields, expr) + push!(values, expr.args[1]) + end + sz += cur_sz = fbsizeof(T) + if sz % largest_field == 0 + sz = cur_sz = 0 + continue + end + nextsz = i == length(exprs) ? 0 : nextsizeof(Core.eval(mod, exprs[i+1].args[2])) + if i == length(exprs) || cur_sz < nextsz || (sz + nextsz) > largest_field + # this is the last field and we're not `sz % largest_field` + # potential diffs = 7, 6, 5, 4, 3, 2, 1 + sym = expr.args[1] + diff = cur_sz < nextsz ? nextsz - cur_sz : largest_field - sz + if diff == 7 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0); push!(values, 0) + elseif diff == 6 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 5 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 4 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0) + elseif diff == 3 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 2 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(values, 0) + elseif diff == 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(values, 0) + end + sz = (sz + diff) % largest_field == 0 ? 0 : (cur_sz < nextsz ? sz + diff : 0) + cur_sz = 0 + end + end + return fields, values +end + +linefilter = x->typeof(x) != LineNumberNode + +macro STRUCT(expr) + !expr.args[1] || throw(ArgumentError("@STRUCT is only applicable for immutable types")) + exprs = filter(linefilter, expr.args[3].args) + fields, values = FlatBuffers.fieldlayout(__module__, expr.args[2], exprs...) + expr.args[3].args = fields + # generate convenience outer constructors if necessary + # if there are nested structs or padding: + # build an outer constructor that takes all direct, original fields + # recursively flatten/splat all nested structs into one big args tuple + # adding zeros for padded arguments + # pass big, flat, args tuple to inner constructor + T = expr.args[2] + if any(x->!FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])), exprs) || + length(fields) > length(exprs) + exprs2 = map(x->FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])) ? x.args[1] : x, exprs) + sig = Expr(:call, T, exprs2...) + body = Expr(:call, T, values...) + outer = Expr(:function, sig, body) + else + outer = :(nothing) + end + return esc(quote + $expr + $outer + end) +end + +macro DEFAULT(T, kwargs...) + ifblock = quote end + if length(kwargs) > 0 && isa(kwargs[1], Expr) && length(kwargs[1].args) > 1 + for kw in kwargs + push!(ifblock.args, :(if sym == $(QuoteNode(kw.args[1])) + return $(kw.args[2]) + end)) + end + end + esc(quote + if $T <: Enum + FlatBuffers.default(::Type{$T}) = FlatBuffers.enumtype($T)($(kwargs[1])) + else + function FlatBuffers.default(::Type{$T}, TT, sym) + $ifblock + return FlatBuffers.default(TT) + end + end + end) +end + +import Parameters + +# TODO: this is so evil and will fall over in the slightest breeze. +function getdef(typedef::Expr) + typedef.args[end-16].args[end].args[end] +end + +function getfielddefs(typedef::Expr) + [a for a in getdef(typedef).args[1:end-2] if a isa Expr] +end + +function getconstructor(typedef::Expr) + cons = getdef(typedef).args[end-1].args[1] + typevars = [] + if :head in propertynames(cons) && cons.head == :where + typevars = cons.args[2:end] + cons = cons.args[1] + end + cons, typevars +end + +function getkwtype(defs, name) + for d in defs + if :args in propertynames(d) && d.args[1] == name + return d.args[end] + end + end + return nothing +end + +function createdefaultfns(typedef::Expr) + cons, typevars = getconstructor(typedef) + T = cons.args[1] + params = cons.args[2] + + @assert params.head == :parameters + + kwargs = [] + defs = getfielddefs(typedef) + kwdict = Dict{Any, Any}() + for p in params.args + name = p.args[1] + t = getkwtype(defs, name) + if t == nothing + continue + end + value = p.args[end] + ifblock = get(kwdict, t, quote end) + push!(ifblock.args, :(if sym == $(QuoteNode(name)) + return convert($t, $value) + end)) + kwdict[t] = ifblock + end + + [:(function FlatBuffers.default(::Type{$T}, ::Type{$TT}, sym) where {$(typevars...)} + $(kwdict[TT]) + return FlatBuffers.default($TT) + end) for TT in keys(kwdict)] +end + +macro with_kw(typedef) + body = Parameters.with_kw(typedef, __module__, true) + defaults = createdefaultfns(body) + defaultsblock = Expr(:block, body, defaults...) + esc(defaultsblock) +end + +#TODO: +# handle id? +# nested_flatbuffer diff --git a/julia/test/MyGame/Example/Ability.jl b/julia/test/MyGame/Example/Ability.jl new file mode 100644 index 00000000000..e32f1ca69b8 --- /dev/null +++ b/julia/test/MyGame/Example/Ability.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Ability + id::UInt32 + distance::UInt32 +end +FlatBuffers.@ALIGN(Ability, 4) + +Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) +Ability(io::IO) = FlatBuffers.deserialize(io, Ability) diff --git a/julia/test/MyGame/Example/AnyAmbiguousAliases.jl b/julia/test/MyGame/Example/AnyAmbiguousAliases.jl new file mode 100644 index 00000000000..e106020db0f --- /dev/null +++ b/julia/test/MyGame/Example/AnyAmbiguousAliases.jl @@ -0,0 +1,13 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@UNION(AnyAmbiguousAliases, ( + Nothing, + Monster, + Monster, + Monster, +)) + diff --git a/julia/test/MyGame/Example/AnyUniqueAliases.jl b/julia/test/MyGame/Example/AnyUniqueAliases.jl new file mode 100644 index 00000000000..27abea3f997 --- /dev/null +++ b/julia/test/MyGame/Example/AnyUniqueAliases.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(AnyUniqueAliases, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/julia/test/MyGame/Example/Any_.jl b/julia/test/MyGame/Example/Any_.jl new file mode 100644 index 00000000000..9bdc2a646e1 --- /dev/null +++ b/julia/test/MyGame/Example/Any_.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..Example2 +import FlatBuffers + +FlatBuffers.@UNION(Any_, ( + Nothing, + Monster, + TestSimpleTableWithEnum, + Example2.Monster, +)) + diff --git a/julia/test/MyGame/Example/Color.jl b/julia/test/MyGame/Example/Color.jl new file mode 100644 index 00000000000..804144f0ac7 --- /dev/null +++ b/julia/test/MyGame/Example/Color.jl @@ -0,0 +1,10 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +@enum Color::Int8 begin + ColorRed = 1 + ColorGreen = 2 + ColorBlue = 8 +end + diff --git a/julia/test/MyGame/Example/Example.jl b/julia/test/MyGame/Example/Example.jl new file mode 100644 index 00000000000..4c720515ca8 --- /dev/null +++ b/julia/test/MyGame/Example/Example.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +module Example + include("Color.jl") + include("Test.jl") + include("Vec3.jl") + include("Stat.jl") + include("Ability.jl") + include("Referrable.jl") + include("TestSimpleTableWithEnum.jl") + # include("AnyUniqueAliases.jl") + # include("AnyAmbiguousAliases.jl") + include("Monster.jl") + include("Any_.jl") + include("TypeAliases.jl") +end diff --git a/julia/test/MyGame/Example/Monster.jl b/julia/test/MyGame/Example/Monster.jl new file mode 100644 index 00000000000..07fd74c0be7 --- /dev/null +++ b/julia/test/MyGame/Example/Monster.jl @@ -0,0 +1,83 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import ..InParentNamespace +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Monster{T} +#= +# an example documentation comment: monster object +=# + pos::Union{Vec3, Nothing} = nothing + mana::Int16 = 150 + hp::Int16 = 100 + name::Union{String, Nothing} = nothing + inventory::Union{Vector{UInt8}, Nothing} = nothing + color::Color = 8 + test_type::UInt8 = 0 + test::T = nothing + test4::Union{Vector{Test}, Nothing} = nothing + testarrayofstring::Union{Vector{String}, Nothing} = nothing +#= +# an example documentation comment: this will end up in the generated code +# multiline too +=# + testarrayoftables::Union{Vector{Monster{T}}, Nothing} = nothing + enemy::Union{Monster{T}, Nothing} = nothing + testnestedflatbuffer::Union{Vector{UInt8}, Nothing} = nothing + testempty::Union{Stat, Nothing} = nothing + testbool::Bool = false + testhashs32_fnv1::Int32 = 0 + testhashu32_fnv1::UInt32 = 0 + testhashs64_fnv1::Int64 = 0 + testhashu64_fnv1::UInt64 = 0 + testhashs32_fnv1a::Int32 = 0 + testhashu32_fnv1a::UInt32 = 0 + testhashs64_fnv1a::Int64 = 0 + testhashu64_fnv1a::UInt64 = 0 + testarrayofbools::Union{Vector{Bool}, Nothing} = nothing + testf::Float32 = 3.14159 + testf2::Float32 = 3.0 + testf3::Float32 = 0.0 + testarrayofstring2::Union{Vector{String}, Nothing} = nothing + testarrayofsortedstruct::Union{Vector{Ability}, Nothing} = nothing + flex::Union{Vector{UInt8}, Nothing} = nothing + test5::Union{Vector{Test}, Nothing} = nothing + vector_of_longs::Union{Vector{Int64}, Nothing} = nothing + vector_of_doubles::Union{Vector{Float64}, Nothing} = nothing + parent_namespace_test::Union{InParentNamespace, Nothing} = nothing + vector_of_referrables::Union{Vector{Referrable}, Nothing} = nothing + single_weak_reference::UInt64 = 0 + vector_of_weak_references::Union{Vector{UInt64}, Nothing} = nothing + vector_of_strong_referrables::Union{Vector{Referrable}, Nothing} = nothing + co_owning_reference::UInt64 = 0 + vector_of_co_owning_references::Union{Vector{UInt64}, Nothing} = nothing + non_owning_reference::UInt64 = 0 + vector_of_non_owning_references::Union{Vector{UInt64}, Nothing} = nothing + # any_unique_type::UInt8 = 0 + # any_unique::AnyUniqueAliases = nothing + # any_ambiguous_type::UInt8 = 0 + # any_ambiguous::AnyAmbiguousAliases = nothing +end +FlatBuffers.@ALIGN(Monster, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000E, 0x00000010, 0x00000012, 0x00000014, + 0x00000016, 0x00000018, 0x0000001A, 0x0000001C, + 0x0000001E, 0x00000020, 0x00000022, 0x00000024, + 0x00000026, 0x00000028, 0x0000002A, 0x0000002C, + 0x0000002E, 0x00000030, 0x00000032, 0x00000034, + 0x00000036, 0x00000038, 0x0000003A, 0x0000003C, + 0x0000003E, 0x00000040, 0x00000042, 0x00000044, + 0x00000046, 0x00000048, 0x0000004A, 0x0000004C, + 0x0000004E, 0x00000050, 0x00000052, 0x00000054, + 0x00000056, 0x00000058, 0x0000005A, 0x0000005C, + 0x0000005E, 0x00000060 +] +FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true +FlatBuffers.file_identifier(::Type{T}) where {T<:Monster} = "MONS" +FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" + +Monster{T}(buf::AbstractVector{UInt8}) where {T} = FlatBuffers.read(Monster{T}, buf) +Monster{T}(io::IO) where {T} = FlatBuffers.deserialize(io, Monster{T}) diff --git a/julia/test/MyGame/Example/Referrable.jl b/julia/test/MyGame/Example/Referrable.jl new file mode 100644 index 00000000000..7fe024e210e --- /dev/null +++ b/julia/test/MyGame/Example/Referrable.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Referrable + id::UInt64 = 0 +end +FlatBuffers.@ALIGN(Referrable, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ + 0x00000004 +] + +Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) +Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) diff --git a/julia/test/MyGame/Example/Stat.jl b/julia/test/MyGame/Example/Stat.jl new file mode 100644 index 00000000000..00001c46935 --- /dev/null +++ b/julia/test/MyGame/Example/Stat.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct Stat + id::Union{String, Nothing} = nothing + val::Int64 = 0 + count::UInt16 = 0 +end +FlatBuffers.@ALIGN(Stat, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ + 0x00000004, 0x00000006, 0x00000008 +] + +Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) +Stat(io::IO) = FlatBuffers.deserialize(io, Stat) diff --git a/julia/test/MyGame/Example/Test.jl b/julia/test/MyGame/Example/Test.jl new file mode 100644 index 00000000000..0087689c7eb --- /dev/null +++ b/julia/test/MyGame/Example/Test.jl @@ -0,0 +1,14 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Test + a::Int16 + b::Int8 +end +FlatBuffers.@ALIGN(Test, 2) + +Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) +Test(io::IO) = FlatBuffers.deserialize(io, Test) diff --git a/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl b/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl new file mode 100644 index 00000000000..41c0e33f3ee --- /dev/null +++ b/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl @@ -0,0 +1,16 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum + color::Color = 2 +end +FlatBuffers.@ALIGN(TestSimpleTableWithEnum, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ + 0x00000004 +] + +TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) +TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) diff --git a/julia/test/MyGame/Example/TypeAliases.jl b/julia/test/MyGame/Example/TypeAliases.jl new file mode 100644 index 00000000000..a563b74ecbd --- /dev/null +++ b/julia/test/MyGame/Example/TypeAliases.jl @@ -0,0 +1,29 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@with_kw mutable struct TypeAliases + i8::Int8 = 0 + u8::UInt8 = 0 + i16::Int16 = 0 + u16::UInt16 = 0 + i32::Int32 = 0 + u32::UInt32 = 0 + i64::Int64 = 0 + u64::UInt64 = 0 + f32::Float32 = 0.0 + f64::Float64 = 0.0 + v8::Union{Vector{Int8}, Nothing} = nothing + vf64::Union{Vector{Float64}, Nothing} = nothing +end +FlatBuffers.@ALIGN(TypeAliases, 1) +FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ + 0x00000004, 0x00000006, 0x00000008, 0x0000000A, + 0x0000000C, 0x0000000E, 0x00000010, 0x00000012, + 0x00000014, 0x00000016, 0x00000018, 0x0000001A +] + +TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) +TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) diff --git a/julia/test/MyGame/Example/Vec3.jl b/julia/test/MyGame/Example/Vec3.jl new file mode 100644 index 00000000000..8f5442af51f --- /dev/null +++ b/julia/test/MyGame/Example/Vec3.jl @@ -0,0 +1,18 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example + +import FlatBuffers + +FlatBuffers.@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 + test1::Float64 + test2::Color + test3::Test +end +FlatBuffers.@ALIGN(Vec3, 16) + +Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) +Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/julia/test/MyGame/Example2/Example2.jl b/julia/test/MyGame/Example2/Example2.jl new file mode 100644 index 00000000000..eaef8178155 --- /dev/null +++ b/julia/test/MyGame/Example2/Example2.jl @@ -0,0 +1,7 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +module Example2 + include("Monster.jl") +end diff --git a/julia/test/MyGame/Example2/Monster.jl b/julia/test/MyGame/Example2/Monster.jl new file mode 100644 index 00000000000..91b98e333fb --- /dev/null +++ b/julia/test/MyGame/Example2/Monster.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: Example2 + +import FlatBuffers + +mutable struct Monster +end +FlatBuffers.@ALIGN(Monster, 1) + +Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) +Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/julia/test/MyGame/InParentNamespace.jl b/julia/test/MyGame/InParentNamespace.jl new file mode 100644 index 00000000000..518cb89e3f0 --- /dev/null +++ b/julia/test/MyGame/InParentNamespace.jl @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +import FlatBuffers + +mutable struct InParentNamespace +end +FlatBuffers.@ALIGN(InParentNamespace, 1) + +InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) +InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) diff --git a/julia/test/MyGame/MyGame.jl b/julia/test/MyGame/MyGame.jl new file mode 100644 index 00000000000..075ee85e35a --- /dev/null +++ b/julia/test/MyGame/MyGame.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# module: MyGame + +module MyGame + include("InParentNamespace.jl") + include("Example2/Example2.jl") + include("Example/Example.jl") +end diff --git a/julia/test/defaults.jl b/julia/test/defaults.jl new file mode 100644 index 00000000000..ee978f8da3d --- /dev/null +++ b/julia/test/defaults.jl @@ -0,0 +1,40 @@ +using FlatBuffers +using Test +import Parameters + +# test default fields +@with_kw mutable struct UltimateAnswer + answer::Int32 = 42 + question::String + highwaysbuilt::Int32 = 7 +end + +x = UltimateAnswer(;question="How many roads must a man walk down?") +@test x.answer == 42 +@test FlatBuffers.default(UltimateAnswer, Int32, :answer) == 42 +@test FlatBuffers.default(UltimateAnswer, Int32, :highwaysbuilt) == 7 +b = FlatBuffers.Builder(UltimateAnswer) +FlatBuffers.build!(b, x) +xbytes = FlatBuffers.bytes(b) +y = FlatBuffers.read(UltimateAnswer, xbytes) + +@test y.answer == x.answer +@test y.question == x.question +@test y.highwaysbuilt == x.highwaysbuilt +@test x.highwaysbuilt == 7 + +y = Parameters.reconstruct(x, highwaysbuilt = 0) +b = FlatBuffers.Builder(UltimateAnswer) +FlatBuffers.build!(b, y) +ybytes = FlatBuffers.bytes(b) + +# check that we save bytes with default integer values +@test length(ybytes) > length(xbytes) + +@test y.answer == x.answer +@test y.question == x.question +@test y.highwaysbuilt == 0 + +y = FlatBuffers.read(UltimateAnswer, ybytes) +@test y.highwaysbuilt == 0 + diff --git a/julia/test/flatc.jl b/julia/test/flatc.jl new file mode 100644 index 00000000000..d3996c7a3af --- /dev/null +++ b/julia/test/flatc.jl @@ -0,0 +1,91 @@ +using Test +import FlatBuffers + +# generated code +include("MyGame/MyGame.jl") +import .MyGame +import .MyGame.Example +import .MyGame.Example2 +import .MyGame.Example.Any_ +import .MyGame.Example.Monster +import .MyGame.Example.TestSimpleTableWithEnum + +# override the typeorder function, since we are working around +# circular type definitions using type parameters +FlatBuffers.typeorder(::Type{Any_}, i::Integer) = [Nothing, Monster{Any_}, TestSimpleTableWithEnum, Example2.Monster][i+1] + +function loadmonsterfile(filename) + open(joinpath(@__DIR__, filename), "r") do f Monster{Any_}(f) end +end + +function checkmonster(monster) + @test monster.hp == 80 + @test monster.mana == 150 + @test monster.name == "MyMonster" + + vec = monster.pos + + @test vec.x == 1.0 + @test vec.y == 2.0 + @test vec.z == 3.0 + @test vec.test1 == 3.0 + @test vec.test2 == MyGame.Example.ColorGreen + @test vec.test3_a == 5 + @test vec.test3_b == 6 + + monster2 = monster.test + @test monster2.name == "Fred" + + @test length(monster.inventory) == 5 + @test sum(monster.inventory) == 10 + + @test monster.vector_of_longs == [10 ^ (2*i) for i = 0:4] + @test monster.vector_of_doubles == [-1.7976931348623157e+308, 0, 1.7976931348623157e+308] + + @test length(monster.test4) == 2 + + (test0, test1) = monster.test4 + @test sum([test0.a, test0.b, test1.a, test1.b]) == 100 + + @test monster.testarrayofstring == ["test1", "test2"] + @test monster.testarrayoftables == nothing + @test monster.testf == 3.14159f0 +end + +function checkpassthrough(monster) + b = FlatBuffers.Builder(Monster{Any_}) + FlatBuffers.build!(b, monster) + bytes = FlatBuffers.bytes(b) + @test FlatBuffers.has_identifier(Monster{Any_}, bytes) + newmonster = FlatBuffers.read(Monster{Any_}, bytes) + checkmonster(newmonster) +end + +function checkserialize(monster) + io = IOBuffer() + FlatBuffers.serialize(io, monster) + bytes = take!(io) + newmonster = FlatBuffers.deserialize(IOBuffer(bytes), Monster{Any_}) + checkmonster(newmonster) +end + +@test FlatBuffers.root_type(Monster) == true +@test FlatBuffers.file_identifier(Monster) == "MONS" +@test FlatBuffers.file_extension(Monster) == "mon" + +for testcase in ["test", "python_wire"] + mon = loadmonsterfile("monsterdata_$testcase.mon") + checkmonster(mon) + checkpassthrough(mon) + checkserialize(mon) +end + +# test printing +mon = loadmonsterfile("monsterdata_test.mon") +b = FlatBuffers.Builder(Monster{Any_}) +FlatBuffers.build!(b, mon) +io = IOBuffer() +show(io, b) +output = String(take!(io)) +@test occursin("deprecated field", split(output, "\n")[9]) + diff --git a/julia/test/internals.jl b/julia/test/internals.jl new file mode 100644 index 00000000000..392f7ca7e93 --- /dev/null +++ b/julia/test/internals.jl @@ -0,0 +1,811 @@ +# Store specific byte patterns in these variables for the fuzzer. These +# values are taken verbatim from the C++ function FuzzTest1. +const overflowingInt32Val = read(IOBuffer(UInt8[0x83, 0x33, 0x33, 0x33]), Int32) +const overflowingInt64Val = read(IOBuffer(UInt8[0x84, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44]), Int64) + +# CheckByteLayout verifies the bytes of a Builder in various scenarios. +function CheckByteLayout() + check = want-> begin + got = b.bytes[b.head+1:end] + @test want == got + return +end + +# test 1: numbers + +b = FlatBuffers.Builder() +check(UInt8[]) +FlatBuffers.prepend!(b, true) +check(UInt8[1]) +FlatBuffers.prepend!(b, Int8(-127)) +check(UInt8[129, 1]) +FlatBuffers.prepend!(b, UInt8(255)) +check(UInt8[255, 129, 1]) +FlatBuffers.prepend!(b, Int16(-32222)) +check(UInt8[0x22, 0x82, 0, 255, 129, 1]) # first pad +FlatBuffers.prepend!(b, UInt16(0xFEEE)) +check(UInt8[0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) # no pad this time +FlatBuffers.prepend!(b, Int32(-53687092)) +check(UInt8[204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) +FlatBuffers.prepend!(b, UInt32(0x98765432)) +check(UInt8[0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) + +# test 1b: numbers 2 + +b = FlatBuffers.Builder() +prepend!(b, 0x1122334455667788) +check(UInt8[0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]) + +# test 2: 1xbyte vector + +b = FlatBuffers.Builder() +check(UInt8[]) +FlatBuffers.startvector(b, sizeof(Bool), 1, 1) +check(UInt8[0, 0, 0]) # align to 4bytes +FlatBuffers.prepend!(b, UInt8(1)) +check(UInt8[1, 0, 0, 0]) +FlatBuffers.endvector(b, 1) +check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding + +# test 3: 2xbyte vector + +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(UInt8), 2, 1) +check(UInt8[0, 0]) # align to 4bytes +FlatBuffers.prepend!(b, UInt8(1)) +check(UInt8[1, 0, 0]) +FlatBuffers.prepend!(b, UInt8(2)) +check(UInt8[2, 1, 0, 0]) +FlatBuffers.endvector(b, 2) +check(UInt8[2, 0, 0, 0, 2, 1, 0, 0]) # padding + +# test 3b: 11xbyte vector matches builder size + +b = FlatBuffers.Builder(Any, 12) +FlatBuffers.startvector(b, sizeof(UInt8), 8, 1) +start = UInt8[] +check(start) +for i = 1:11 + FlatBuffers.prepend!(b, UInt8(i)) + start = append!(UInt8[i], start) + check(start) +end +FlatBuffers.endvector(b, 8) +check(append!(UInt8[8, 0, 0, 0], start)) + +# test 4: 1xuint16 vector + +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(UInt16), 1, 1) +check(UInt8[0, 0]) # align to 4bytes +FlatBuffers.prepend!(b, UInt16(1)) +check(UInt8[1, 0, 0, 0]) +FlatBuffers.endvector(b, 1) +check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding + +# test 5: 2xuint16 vector + +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(UInt16), 2, 1) +check(UInt8[]) # align to 4bytes +FlatBuffers.prepend!(b, UInt16(0xABCD)) +check(UInt8[0xCD, 0xAB]) +FlatBuffers.prepend!(b, UInt16(0xDCBA)) +check(UInt8[0xBA, 0xDC, 0xCD, 0xAB]) +FlatBuffers.endvector(b, 2) +check(UInt8[2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB]) + +# test 6: CreateString + +b = FlatBuffers.Builder() +FlatBuffers.createstring(b, "foo") +check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad +FlatBuffers.createstring(b, "moop") +check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0]) + +# test 6b: CreateString unicode + +b = FlatBuffers.Builder() +# These characters are chinese from blog.golang.org/strings +# We use escape codes here so that editors without unicode support +# aren't bothered: +uni_str = "\u65e5\u672c\u8a9e" +FlatBuffers.createstring(b, uni_str) +check(UInt8[9, 0, 0, 0, 230, 151, 165, 230, 156, 172, 232, 170, 158, 0, # null-terminated, 2-byte pad + 0, 0]) + +# test 6c: CreateUInt8String + +b = FlatBuffers.Builder() +FlatBuffers.createstring(b, "foo") +check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad +FlatBuffers.createstring(b, "moop") +check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0]) + +# test 7: empty vtable +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 0) +check(UInt8[]) +FlatBuffers.endobject(b) +check(UInt8[4, 0, 4, 0, 4, 0, 0, 0]) + +# test 8: vtable with one true bool +b = FlatBuffers.Builder() +check(UInt8[]) +FlatBuffers.startobject(b, 1) +check(UInt8[]) +FlatBuffers.prependslot!(b, 1, true, false) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # length of object including vtable offset + 7, 0, # start of bool value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, 0, # padded to 4 bytes + 1, # bool value + ]) + +# test 9: vtable with one default bool +b = FlatBuffers.Builder() +check(UInt8[]) +FlatBuffers.startobject(b, 1) +check(UInt8[]) +FlatBuffers.prependslot!(b, 1, false, false) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 4, 0, # end of object from here + 0, 0, # entry 1 is zero + 6, 0, 0, 0, # offset for start of vtable (int32) + ]) + +# test 10: vtable with one int16 +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 1) +FlatBuffers.prependslot!(b, 1, Int16(0x789A), Int16(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding to 4 bytes + 0x9A, 0x78, + ]) + +# test 11: vtable with two int16 +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 2) +FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) +FlatBuffers.prependslot!(b, 2, Int16(0x789A), Int16(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 4, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0x9A, 0x78, # value 1 + 0x56, 0x34, # value 0 + ]) + +# test 12: vtable with int16 and bool +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 2) +FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) +FlatBuffers.prependslot!(b, 2, true, false) +FlatBuffers.endobject(b) +check(UInt8[ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 5, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 1, # value 1 + 0x56, 0x34, # value 0 + ]) + +# test 12: vtable with empty vector +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) +vecend = FlatBuffers.endvector(b, 0) +FlatBuffers.startobject(b, 1) +FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset to vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, + 0, 0, 0, 0, # length of vector (not in struct) + ]) + +# test 12b: vtable with empty vector of byte and some scalars +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) +vecend = FlatBuffers.endvector(b, 0) +FlatBuffers.startobject(b, 2) +FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) +FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 8, 0, # vtable bytes + 12, 0, + 10, 0, # offset to value 0 + 4, 0, # offset to vector offset + 8, 0, 0, 0, # vtable loc + 8, 0, 0, 0, # value 1 + 0, 0, 55, 0, # value 0 + + 0, 0, 0, 0, # length of vector (not in struct) + ]) + +# test 13: vtable with 1 int16 and 2-vector of int16 +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(Int16), 2, 1) +FlatBuffers.prepend!(b, Int16(0x1234)) +FlatBuffers.prepend!(b, Int16(0x5678)) +vecend = FlatBuffers.endvector(b, 2) +FlatBuffers.startobject(b, 2) +FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) +FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 8, 0, # vtable bytes + 12, 0, # length of object + 6, 0, # start of value 0 from end of vtable + 8, 0, # start of value 1 from end of buffer + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 55, 0, # value 0 + 4, 0, 0, 0, # vector position from here + 2, 0, 0, 0, # length of vector (uint32) + 0x78, 0x56, # vector value 1 + 0x34, 0x12, # vector value 0 + ]) + +# test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32 +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 1) +FlatBuffers.prep!(b, 4+4+4, 0) +FlatBuffers.prepend!(b, Int8(55)) +FlatBuffers.pad!(b, 3) +FlatBuffers.prepend!(b, Int16(0x1234)) +FlatBuffers.pad!(b, 2) +FlatBuffers.prepend!(b, Int32(0x12345678)) +structStart = FlatBuffers.offset(b) +FlatBuffers.prependstructslot!(b, 1, structStart, 0) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 16, 0, # end of object from here + 4, 0, # start of struct from here + 6, 0, 0, 0, # offset for start of vtable (int32) + 0x78, 0x56, 0x34, 0x12, # value 2 + 0, 0, # padding + 0x34, 0x12, # value 1 + 0, 0, 0, # padding + 55, # value 0 + ]) + +# test 15: vtable with 1 vector of 2 struct of 2 int8 +b = FlatBuffers.Builder() +FlatBuffers.startvector(b, sizeof(Int8)*2, 2, 1) +FlatBuffers.prepend!(b, Int8(33)) +FlatBuffers.prepend!(b, Int8(44)) +FlatBuffers.prepend!(b, Int8(55)) +FlatBuffers.prepend!(b, Int8(66)) +vecend = FlatBuffers.endvector(b, 2) +FlatBuffers.startobject(b, 1) +FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) +FlatBuffers.endobject(b) +check(UInt8[ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset of vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, # vector start offset + + 2, 0, 0, 0, # vector length + 66, # vector value 1,1 + 55, # vector value 1,0 + 44, # vector value 0,1 + 33, # vector value 0,0 + ]) + +# test 16: table with some elements +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 2) +FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) +FlatBuffers.prependslot!(b, 2, Int16(66), Int16(0)) +off = FlatBuffers.endobject(b) +FlatBuffers.finish!(b, off) #TODO + +check(UInt8[ + 12, 0, 0, 0, # root of table: points to vtable offset + + 8, 0, # vtable bytes + 8, 0, # end of object from here + 7, 0, # start of value 0 + 4, 0, # start of value 1 + + 8, 0, 0, 0, # offset for start of vtable (int32) + + 66, 0, # value 1 + 0, # padding + 33, # value 0 + ]) + +# test 17: one unfinished table and one finished table +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 2) +FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) +FlatBuffers.prependslot!(b, 2, Int8(44), Int8(0)) +off = FlatBuffers.endobject(b) +FlatBuffers.finish!(b, off) + +FlatBuffers.startobject(b, 3) +FlatBuffers.prependslot!(b, 1, Int8(55), Int8(0)) +FlatBuffers.prependslot!(b, 2, Int8(66), Int8(0)) +FlatBuffers.prependslot!(b, 3, Int8(77), Int8(0)) +off = FlatBuffers.endobject(b) +FlatBuffers.finish!(b, off) + +check(UInt8[ + 16, 0, 0, 0, # root of table: points to object + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 77, # value 2 + 66, # value 1 + 55, # value 0 + + 12, 0, 0, 0, # root of table: points to object + + 8, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 44, # value 1 + 33, # value 0 + ]) + +# test 18: a bunch of bools +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 8) +FlatBuffers.prependslot!(b, 1, true, false) +FlatBuffers.prependslot!(b, 2, true, false) +FlatBuffers.prependslot!(b, 3, true, false) +FlatBuffers.prependslot!(b, 4, true, false) +FlatBuffers.prependslot!(b, 5, true, false) +FlatBuffers.prependslot!(b, 6, true, false) +FlatBuffers.prependslot!(b, 7, true, false) +FlatBuffers.prependslot!(b, 8, true, false) +off = FlatBuffers.endobject(b) +FlatBuffers.finish!(b, off) + +check(UInt8[ + 24, 0, 0, 0, # root of table: points to vtable offset + + 20, 0, # vtable bytes + 12, 0, # size of object + 11, 0, # start of value 0 + 10, 0, # start of value 1 + 9, 0, # start of value 2 + 8, 0, # start of value 3 + 7, 0, # start of value 4 + 6, 0, # start of value 5 + 5, 0, # start of value 6 + 4, 0, # start of value 7 + 20, 0, 0, 0, # vtable offset + + 1, # value 7 + 1, # value 6 + 1, # value 5 + 1, # value 4 + 1, # value 3 + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + +# test 19: three bools +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 3) +FlatBuffers.prependslot!(b, 1, true, false) +FlatBuffers.prependslot!(b, 2, true, false) +FlatBuffers.prependslot!(b, 3, true, false) +off = FlatBuffers.endobject(b) +FlatBuffers.finish!(b, off) + +check(UInt8[ + 16, 0, 0, 0, # root of table: points to vtable offset + + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # vtable offset from here + + 0, # padding + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + +# test 20: some floats +b = FlatBuffers.Builder() +FlatBuffers.startobject(b, 1) +FlatBuffers.prependslot!(b, 1, Float32(1.0), Float32(0.0)) +off = FlatBuffers.endobject(b) + +check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # size of object + 4, 0, # start of value 0 + 6, 0, 0, 0, # vtable offset + + 0, 0, 128, 63, # value 0 + ]) +end + +# CheckManualBuild builds a Monster manually. +function CheckManualBuild() + b = FlatBuffers.Builder() + str = FlatBuffers.createstring(b, "MyMonster") + + FlatBuffers.startvector(b, 1, 5, 1) + FlatBuffers.prepend!(b, UInt8(4)) + FlatBuffers.prepend!(b, UInt8(3)) + FlatBuffers.prepend!(b, UInt8(2)) + FlatBuffers.prepend!(b, UInt8(1)) + FlatBuffers.prepend!(b, UInt8(0)) + inv = FlatBuffers.endvector(b, 5) + + FlatBuffers.startobject(b, 13) + FlatBuffers.prependslot!(b, 2, Int16(20), Int16(100)) + mon2 = FlatBuffers.endobject(b) + + # Test4Vector + FlatBuffers.startvector(b, 4, 2, 1) + + # Test 0 + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, Int8(20)) + FlatBuffers.place!(b, Int16(10)) + + # Test 1 + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, Int8(40)) + FlatBuffers.place!(b, Int16(30)) + + # end testvector + test4 = FlatBuffers.endvector(b, 2) + + FlatBuffers.startobject(b, 13) + + # a vec3 + FlatBuffers.prep!(b, 16, 32) + FlatBuffers.pad!(b, 2) + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, UInt8(6)) + FlatBuffers.place!(b, Int16(5)) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, UInt8(4)) + FlatBuffers.place!(b, Float64(3.0)) + FlatBuffers.pad!(b, 4) + FlatBuffers.place!(b, Float32(3.0)) + FlatBuffers.place!(b, Float32(2.0)) + FlatBuffers.place!(b, Float32(1.0)) + vec3Loc = FlatBuffers.offset(b) + # end vec3 + + FlatBuffers.prependstructslot!(b, 1, vec3Loc, 0) # vec3. noop + FlatBuffers.prependslot!(b, 3, Int16(80), Int16(100)) # hp + FlatBuffers.prependoffsetslot!(b, 4, Int32(str), Int32(0)) + FlatBuffers.prependoffsetslot!(b, 6, Int32(inv), Int32(0)) # inventory + FlatBuffers.prependslot!(b, 8, UInt8(1), UInt8(0)) + FlatBuffers.prependoffsetslot!(b, 9, Int32(mon2), Int32(0)) + FlatBuffers.prependoffsetslot!(b, 10, Int32(test4), Int32(0)) + mon = FlatBuffers.endobject(b) + + FlatBuffers.finish!(b, mon) + + return b.bytes, b.head +end + +# CheckVtableDeduplication verifies that vtables are deduplicated. +function CheckVtableDeduplication() + b = FlatBuffers.Builder() + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(11), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(22), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(33), Int16(0)) + obj0 = FlatBuffers.endobject(b) + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(44), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(55), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(66), Int16(0)) + obj1 = FlatBuffers.endobject(b) + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(77), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(88), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(99), Int16(0)) + obj2 = FlatBuffers.endobject(b) + + got = b.bytes[b.head+1:end] + + want = UInt8[ + 240, 255, 255, 255, # == -12. offset to dedupped vtable. + 99, 0, + 88, + 77, + 248, 255, 255, 255, # == -8. offset to dedupped vtable. + 66, 0, + 55, + 44, + 12, 0, # start of vtable + 8, 0, + 0, 0, + 7, 0, + 6, 0, + 4, 0, + 12, 0, 0, 0, # table0 + 33, 0, + 22, + 11 + ] + + @test got == want + + table0 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj0) + table1 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj1) + table2 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj2) + + function testTable(tab, a, b, c, d) + # vtable size + got = FlatBuffers.getoffsetslot(tab, 0, Int16(0)) + @test 12 == got + # object size + got = FlatBuffers.getoffsetslot(tab, 2, Int16(0)) + @test 8 == got + # default value + got = FlatBuffers.getoffsetslot(tab, 4, Int16(0)) + @test a == got + got = FlatBuffers.getslot(tab, 6, UInt8(0)) + @test b == got + val = FlatBuffers.getslot(tab, 8, UInt8(0)) + c != val && throw(ArgumentError("failed 8, 0: $got")) + got = FlatBuffers.getslot(tab, 10, UInt8(0)) + @test d == got + return + end + + testTable(table0, UInt16(0), UInt8(11), UInt8(22), UInt8(33)) + testTable(table1, UInt16(0), UInt8(44), UInt8(55), UInt8(66)) + testTable(table2, UInt16(0), UInt8(77), UInt8(88), UInt8(99)) +end + +# CheckNotInObjectError verifies that `endobject` fails if not inside an +# object. +function CheckNotInObjectError() + b = FlatBuffers.Builder() + + @test_throws ArgumentError FlatBuffers.endobject(b) +end + +# CheckStringIsNestedError verifies that a string can not be created inside +# another object. +function CheckStringIsNestedError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.createstring(b, "foo") +end + +# CheckByteStringIsNestedError verifies that a bytestring can not be created +# inside another object. +function CheckByteStringIsNestedError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.createstring(b, "foo") +end + +# CheckStructIsNotInlineError verifies that writing a struct in a location +# away from where it is used will cause a panic. +function CheckStructIsNotInlineError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.prependstructslot!(b, 0, 1, 0) +end + +# CheckFinishedBytesError verifies that `FinishedBytes` panics if the table +# is not finished. +function CheckFinishedBytesError() + b = FlatBuffers.Builder() + + @test_throws ArgumentError FlatBuffers.finishedbytes(b) +end + +function CheckCreateByteVector() + raw = UInt8(0):UInt8(29) + + for size = 1:30 + b1 = FlatBuffers.Builder() + b2 = FlatBuffers.Builder() + FlatBuffers.startvector(b1, 1, size, 1) + for i = size:-1:1 + FlatBuffers.prepend!(b1, raw[i]) + end + FlatBuffers.endvector(b1, size) + FlatBuffers.createbytevector(b2, raw[1:size]) + @test b1.bytes == b2.bytes + end +end + +const InitialLCGSeed = 48271 +mutable struct LCG + val::UInt32 + LCG() = new(UInt32(InitialLCGSeed)) +end +reset!(lcg::LCG) = lcg.val = UInt32(InitialLCGSeed) +function next(lcg::LCG) + n = UInt32((UInt64(lcg.val) * UInt64(279470273)) % UInt64(4294967291)) + lcg.val = n + return n +end + +# Low level stress/fuzz test: serialize/deserialize a variety of +# different kinds of data in different combinations +function checkFuzz(fuzzFields, fuzzObjects, verbose=true) + + # Values we're testing against: chosen to ensure no bits get chopped + # off anywhere, and also be different from eachother. + boolVal = true + int8Val = Int8(-127) # 0x81 + uint8Val = UInt8(0xFF) + int16Val = Int16(-32222) # 0x8222 + uint16Val = UInt16(0xFEEE) + int32Val = Int32(overflowingInt32Val) + uint32Val = UInt32(0xFDDDDDDD) + int64Val = Int64(overflowingInt64Val) + uint64Val = UInt64(0xFCCCCCCCCCCCCCCC) + float32Val = Float32(3.14159) + float64Val = Float64(3.14159265359) + + testValuesMax = 11 # hardcoded to the number of scalar types + + b = FlatBuffers.Builder() + l = LCG() + + objects = fill(0, fuzzObjects) + + # Generate fuzzObjects random objects each consisting of + # fuzzFields fields, each of a random type. + for i = 1:fuzzObjects + FlatBuffers.startobject(b, fuzzFields) + + for f = 1:fuzzFields + choice = next(l) % UInt32(testValuesMax) + if choice == 0 + FlatBuffers.prependslot!(b, f, boolVal, false) + elseif choice == 1 + FlatBuffers.prependslot!(b, f, int8Val, Int8(0)) + elseif choice == 2 + FlatBuffers.prependslot!(b, f, uint8Val, UInt8(0)) + elseif choice == 3 + FlatBuffers.prependslot!(b, f, int16Val, Int16(0)) + elseif choice == 4 + FlatBuffers.prependslot!(b, f, uint16Val, UInt16(0)) + elseif choice == 5 + FlatBuffers.prependslot!(b, f, int32Val, Int32(0)) + elseif choice == 6 + FlatBuffers.prependslot!(b, f, uint32Val, UInt32(0)) + elseif choice == 7 + FlatBuffers.prependslot!(b, f, int64Val, Int64(0)) + elseif choice == 8 + FlatBuffers.prependslot!(b, f, uint64Val, UInt64(0)) + elseif choice == 9 + FlatBuffers.prependslot!(b, f, float32Val, Float32(0)) + elseif choice == 10 + FlatBuffers.prependslot!(b, f, float64Val, Float64(0)) + end + end + + off = FlatBuffers.endobject(b) + + # store the offset from the end of the builder buffer, + # since it will keep growing: + objects[i] = off + end + + # Do some bookkeeping to generate stats on fuzzes: + stats = Dict{String,Int}() + function check(desc, want, got) + v = get!(stats, desc, 0) + stats[desc] = v + 1 + @test want == got + end + + l = LCG() # Reset. + + # Test that all objects we generated are readable and return the + # expected values. We generate random objects in the same order + # so this is deterministic. + for i = 1:fuzzObjects + + table = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - objects[i]) + + for j = 0:(fuzzFields - 1) + f = (FlatBuffers.VtableMetadataFields + j) * sizeof(Int16) + choice = next(l) % UInt32(testValuesMax) + + if choice == 0 + check("bool", boolVal, FlatBuffers.getslot(table, f, false)) + elseif choice == 1 + check("int8", int8Val, FlatBuffers.getslot(table, f, Int8(0))) + elseif choice == 2 + check("uint8", uint8Val, FlatBuffers.getslot(table, f, UInt8(0))) + elseif choice == 3 + check("int16", int16Val, FlatBuffers.getslot(table, f, Int16(0))) + elseif choice == 4 + check("uint16", uint16Val, FlatBuffers.getslot(table, f, UInt16(0))) + elseif choice == 5 + check("int32", int32Val, FlatBuffers.getslot(table, f, Int32(0))) + elseif choice == 6 + check("uint32", uint32Val, FlatBuffers.getslot(table, f, UInt32(0))) + elseif choice == 7 + check("int64", int64Val, FlatBuffers.getslot(table, f, Int64(0))) + elseif choice == 8 + check("uint64", uint64Val, FlatBuffers.getslot(table, f, UInt64(0))) + elseif choice == 9 + check("float32", float32Val, FlatBuffers.getslot(table, f, Float32(0))) + elseif choice == 10 + check("float64", float64Val, FlatBuffers.getslot(table, f, Float64(0))) + end + end + end + + # If enough checks were made, verify that all scalar types were used: + if fuzzFields*fuzzObjects >= testValuesMax + if length(stats) != testValuesMax + throw(ArgumentError("fuzzing failed to test all scalar types")) + end + end + + # Print some counts, if needed: + if verbose + if fuzzFields == 0 || fuzzObjects == 0 + println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t[none]\t 0") + else + ks = sort!(collect(keys(stats))) + for k in ks + println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t$(k): $(stats[k])") + end + end + end + return +end diff --git a/julia/test/monster.jl b/julia/test/monster.jl new file mode 100644 index 00000000000..b37a5685f9f --- /dev/null +++ b/julia/test/monster.jl @@ -0,0 +1,77 @@ +module SmallExample + +using FlatBuffers + +@enum(Color, Red = 1, Green = 2, Blue = 8) +@DEFAULT Color Red + +@STRUCT struct Test + a::Int16 + b::UInt8 + # _1::UInt8 # padding +end + +mutable struct TestSimpleTableWithEnum + color::Color +end + +@DEFAULT TestSimpleTableWithEnum color=Green + +@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 + # _::UInt32 # padding + test1::Float64 + test2::Color + # __::UInt8 # padding + test3::Test + # ___::UInt16 # padding +end + +@ALIGN Vec3 16 + +mutable struct Stat + id::Union{Nothing, String} + val::Int64 + count::UInt16 +end + +# Julia doesn't support forward referencing of types +# @union Any_ Union{Monster, TestSimpleTableWithEnum} + +mutable struct Monster + pos::Vec3 + mana::Int16 + hp::Int16 + name::Union{Nothing, String} + friendly::Bool # deprecated + inventory::Union{Nothing, Vector{UInt8}} + color::Color + # test_type::Any_ + # test::Union{Nothing, Vector{UInt8}} + test4::Union{Nothing, Vector{Test}} + testarrayofstring::Union{Nothing, Vector{String}} + testarrayoftables::Union{Nothing, Vector{Monster}} + # don't support nested circulr reference objects yet + # enemy::Monster + testnestedflatbuffer::Union{Nothing, Vector{UInt8}} + testempty::Union{Nothing, Stat} + testbool::Bool + testhashs32_fnv1::Int32 + testhashu32_fnv1::UInt32 + testhashs64_fnv1::Int64 + testhashu64_fnv1::UInt64 + testhashs32_fnv1a::Int32 + testhashu32_fnv1a::UInt32 + testhashs64_fnv1a::Int64 + testhashu64_fnv1a::UInt64 + testarrayofbools::Union{Nothing, Vector{Bool}} + testf::Float32 + testf2::Float32 + testf3::Float32 +end + +@DEFAULT Monster hp=Int16(100) mana=Int16(150) color=convert(UInt8, Blue) friendly=false testf=Float32(3.14159) testf2=Float32(3) + +end # module diff --git a/julia/test/monsterdata_python_wire.mon b/julia/test/monsterdata_python_wire.mon new file mode 100644 index 0000000000000000000000000000000000000000..55e37bf03961f9afdc7bd84220dcde2c8a5e8264 GIT binary patch literal 352 zcma!GKm{HQJ`5T_AvFdCASnaH(hMRD0t`Gr7O{YXfrY^Xs2GG(fOrBB&j8{zKnw(o zAo>QF82}_3?16*>5CJg^a5ykAurjbgxy(TNKN!6K4^si<*Ms=1K+Fi00NDou0_qUj z!HFNjc=#2h3=Einqz@3=05OOy2lO!)(3d<696%ljuz>w50AztRF)}f;fJGS?+=^1c gR)EAyQj1HBplU#DLm-f!oUG!u`y@>NgW_&VK87YVXy$QY=D@7ftA65!G$4#A%nq# z!H1!Mp@Ja*EDj?=7$O*AfN~5BjEo?e274gk07O6x0~`)aKy%oDTmwc11_dDA0c6hs z;tU`L>2f?-c^8BWmo)Z)unADC0EmwO@d6-joLK$oYy*EDjQ)5x{rDY^pUyBEsDyzT zh#7%!!~g&PS%7R1b}LFv0a6T1Kr8~pATd@TE=eseF+yS+0@)xnTtHQFz_37O^MFGL zC<)Zf113?;2H6X=5(wV^2dM%9D8C-W2H6MnG8ljz17r!PL--C({1C>&uOMY$0A?^U VF@xO73Bx.x, inst7.x) .== map(x->x.x, inst7_2.x)) + +# self-referential type test (type has subtype of itself) +# type TestCircT +# x::Int8 +# y::TestCircT +# end +# +# struct TestCircI +# x::Int8 +# y::TestCircI +# end + +# simple Union (Any_) + +# fbs +# table TestUnionT { +# x::TestUnionI +# } + +@UNION TestUnionU (Nothing,TestInt8T,TestInt8A) + +mutable struct TestUnionT + x_type::Int8 + x::TestUnionU +end + +TestUnionT(x::TestUnionU) = TestUnionT(FlatBuffers.typeorder(TestUnionU, typeof(x)), x) + +inst8 = TestUnionT(inst1) + +b = FlatBuffers.Builder(TestUnionT) +FlatBuffers.build!(b, inst8) +t = FlatBuffers.Table(b) +inst8_2 = FlatBuffers.read(t) + +@test inst8.x_type == inst8_2.x_type && inst8.x.x == inst8_2.x.x + +inst9 = TestUnionT(inst3) + +b = FlatBuffers.Builder(TestUnionT) +FlatBuffers.build!(b, inst9) +t = FlatBuffers.Table(b) +inst9_2 = FlatBuffers.read(t) + +@test inst9.x_type == inst9_2.x_type && inst9.x.x == inst9_2.x.x + +const Nachos = String +const Burgers = Int +FlatBuffers.@UNION(Delicious, + (Burgers, Nachos) +) + +FlatBuffers.@with_kw mutable struct TestVecUnionT + xs_type::Vector{UInt8} + xs::Vector{TestUnionU} + ys_type::Vector{UInt8} + ys::Vector{Delicious} +end + +function TestVecUnionT(xs::Vector{A}, ys::Vector{B}) where {A<:TestUnionU, B<:Delicious} + xs_type = [FlatBuffers.typeorder(TestUnionU, typeof(x)) for x in xs] + ys_type = [FlatBuffers.typeorder(Delicious, typeof(y)) for y in ys] + TestVecUnionT(xs_type, xs, ys_type, ys) +end + +dinner = "beef" +inst10 = TestVecUnionT(TestUnionU[inst1, inst1, inst1], [dinner]) + +b = FlatBuffers.Builder(TestVecUnionT) +FlatBuffers.build!(b, inst10) + +t = FlatBuffers.Table(b) +inst10_2 = FlatBuffers.read(t) + +@test inst10.xs_type == inst10_2.xs_type +@test [x.x for x in inst10.xs] == [x.x for x in inst10_2.xs] + +inst11 = TestVecUnionT(TestUnionU[inst3, inst3], [dinner for _ = 1:9]) + +b = FlatBuffers.Builder(TestVecUnionT) +FlatBuffers.build!(b, inst11) +t = FlatBuffers.Table(b) +inst11_2 = FlatBuffers.read(t) + +@test inst11.xs_type == inst11_2.xs_type +@test [x.x for x in inst11.xs] == [x.x for x in inst11_2.xs] +@test inst11.ys_type == inst11_2.ys_type +@test [y for y in inst11.ys] == [y for y in inst11_2.ys] + +# test @STRUCT macro +@STRUCT struct A + a::Int32 +end +@test sizeof(A) == 4 +@test all(fieldnames(A) .== [:a]) +@test A(1) == A(1) + +@STRUCT struct B + a::Int8 + b::Int32 +end +@test sizeof(B) == 8 +@test all(fieldnames(B) .== [:a, :_pad_a_B_0, :_pad_a_B_1, :b]) +@test B(1,2) == B(1,2) + +@STRUCT struct C + a::Int16 + b::Int32 + c::Int16 +end +@test sizeof(C) == 12 +@test all(fieldnames(C) .== [:a, :_pad_a_C_0, :b, :c, :_pad_c_C_1]) +@test C(1,2,3) == C(1,2,3) + +@STRUCT struct D + a::Int8 + b::Int64 +end +@test sizeof(D) == 16 +@test all(fieldnames(D) .== [:a, :_pad_a_D_0, :_pad_a_D_1, :_pad_a_D_2, :b]) +@test D(1,2) == D(1,2) + +@STRUCT struct E + a::Int64 + b::Int32 +end +@test sizeof(E) == 16 +@test all(fieldnames(E) .== [:a, :b, :_pad_b_E_0]) +@test E(1,2) == E(1,2) + +@STRUCT struct F + a::Int32 + b::Int16 + c::Int32 + d::Int32 + e::Int64 +end +@test sizeof(F) == 24 +@test all(fieldnames(F) .== [:a, :b, :_pad_b_F_0, :c, :d, :e]) +@test F(1,2,3,4,5) == F(1,2,3,4,5) + +@STRUCT struct G + a::Float64 + b::Int8 + c::Int16 + d::Int32 +end +@test sizeof(G) == 24 +@test all(fieldnames(G) .== [:a, :b, :_pad_b_G_0, :c, :_pad_c_G_1, :d]) +@test G(1,2,3,4) == G(1,2,3,4) + +@STRUCT struct H + a::Float32 + b::Int8 + c::Int16 +end +@test sizeof(H) == 8 +@test all(fieldnames(H) .== [:a, :b, :_pad_b_H_0, :c]) +@test H(1,2,3) == H(1,2,3) + +@STRUCT struct I + a::Float64 + b::Int8 + c::Int32 +end +@test sizeof(I) == 16 +@test all(fieldnames(I) .== [:a, :b, :_pad_b_I_0, :_pad_b_I_1, :c]) +@test I(1,2,3) == I(1,2,3) + +@STRUCT struct J + a::Int8 + b::A +end +@test sizeof(J) == 8 +@test all(fieldnames(J) .== [:a, :_pad_a_J_0, :_pad_a_J_1, :b_a]) +@test J(1,A(2)) == J(1,A(2)) + +@STRUCT struct K + a::J + b::I + c::J +end +@test sizeof(K) == 48 +@test all(fieldnames(K) .== [:a_a, :a__pad_a_J_0, :a__pad__pad_a_J_0_J_0, :a__pad_a_J_1, :a__pad__pad_a_J_1_J_1, :a_b_a, :b_a, :b_b, :b__pad_b_I_0, :b__pad__pad_b_I_0_I_0, :b__pad_b_I_1, :b__pad__pad_b_I_1_I_1, :b_c, :c_a, :c__pad_a_J_0, :c__pad__pad_a_J_0_J_0, :c__pad_a_J_1, :c__pad__pad_a_J_1_J_1, :c_b_a]) +@test K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) == K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) From 33b20f600951d4d3bd6d73d6ed951cfbd2891a66 Mon Sep 17 00:00:00 2001 From: Rowan Date: Thu, 20 Dec 2018 10:57:03 +1100 Subject: [PATCH 04/19] Apply suggestions from code review --- docs/source/Support.md | 2 +- docs/source/Tutorial.md | 8 ++++---- include/flatbuffers/idl.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/Support.md b/docs/source/Support.md index fd2f01f1e72..7b4b20efdac 100644 --- a/docs/source/Support.md +++ b/docs/source/Support.md @@ -43,6 +43,6 @@ Primary authors (github) | aard* | aard* | ev*/js*| rw | rw | ev * ch = chobie * kr = krojew * jq = quinnj - * rk = rkat + * rk = rjkat
diff --git a/docs/source/Tutorial.md b/docs/source/Tutorial.md index 2344a9c99f0..77a5f7219c0 100644 --- a/docs/source/Tutorial.md +++ b/docs/source/Tutorial.md @@ -1851,7 +1851,7 @@ appropriate `finish` method. ~~~
-~~~{.rs} +~~~{.jl} # this section is unnecessary in Julia ~~~
@@ -2125,7 +2125,7 @@ import './monster_my_game.sample_generated.dart' as myGame; ~~~
-~~~{.py} +~~~{.jl} import FlatBuffers # Generated by `flatc`. @@ -2378,7 +2378,7 @@ accessors for all non-`deprecated` fields. For example: ~~~
-~~~{.rs} +~~~{.jl} hp = monster.hp mana = monster.mana name = monster.name @@ -2995,7 +2995,7 @@ mutators like so: ~~~
-~~~{.rs} +~~~{.jl} ~~~
diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 8255a942121..18e2c756d35 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -107,7 +107,7 @@ enum BaseType { }; #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE, JLTYPEL) \ + RTYPE, JLTYPE) \ static_assert(sizeof(CTYPE) <= sizeof(largest_scalar_t), \ "define largest_scalar_t as " #CTYPE); FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) From f2726ec0bf9102141e9bf377f33074d761008d62 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Thu, 20 Dec 2018 15:54:59 +1100 Subject: [PATCH 05/19] code review changes --- julia/README.md | 2 +- julia/docs/Manifest.toml | 79 -- julia/docs/Project.toml | 2 - julia/docs/make.jl | 17 - julia/docs/mkdocs.yml | 29 - julia/docs/src/index.md | 97 --- julia/test/MyGame/Example/Ability.jl | 14 - .../MyGame/Example/AnyAmbiguousAliases.jl | 13 - julia/test/MyGame/Example/AnyUniqueAliases.jl | 14 - julia/test/MyGame/Example/Any_.jl | 14 - julia/test/MyGame/Example/Color.jl | 10 - julia/test/MyGame/Example/Example.jl | 18 - julia/test/MyGame/Example/Monster.jl | 83 -- julia/test/MyGame/Example/Referrable.jl | 16 - julia/test/MyGame/Example/Stat.jl | 18 - julia/test/MyGame/Example/Test.jl | 14 - .../MyGame/Example/TestSimpleTableWithEnum.jl | 16 - julia/test/MyGame/Example/TypeAliases.jl | 29 - julia/test/MyGame/Example/Vec3.jl | 18 - julia/test/MyGame/Example2/Example2.jl | 7 - julia/test/MyGame/Example2/Monster.jl | 12 - julia/test/MyGame/InParentNamespace.jl | 12 - julia/test/MyGame/MyGame.jl | 9 - julia/test/defaults.jl | 40 - julia/test/flatc.jl | 91 -- julia/test/internals.jl | 811 ------------------ julia/test/monster.jl | 77 -- julia/test/monsterdata_python_wire.mon | Bin 352 -> 0 bytes julia/test/monsterdata_test.mon | Bin 448 -> 0 bytes julia/test/runtests.jl | 313 ------- src/idl_gen_julia.cpp | 75 +- 31 files changed, 37 insertions(+), 1913 deletions(-) delete mode 100644 julia/docs/Manifest.toml delete mode 100644 julia/docs/Project.toml delete mode 100644 julia/docs/make.jl delete mode 100644 julia/docs/mkdocs.yml delete mode 100644 julia/docs/src/index.md delete mode 100644 julia/test/MyGame/Example/Ability.jl delete mode 100644 julia/test/MyGame/Example/AnyAmbiguousAliases.jl delete mode 100644 julia/test/MyGame/Example/AnyUniqueAliases.jl delete mode 100644 julia/test/MyGame/Example/Any_.jl delete mode 100644 julia/test/MyGame/Example/Color.jl delete mode 100644 julia/test/MyGame/Example/Example.jl delete mode 100644 julia/test/MyGame/Example/Monster.jl delete mode 100644 julia/test/MyGame/Example/Referrable.jl delete mode 100644 julia/test/MyGame/Example/Stat.jl delete mode 100644 julia/test/MyGame/Example/Test.jl delete mode 100644 julia/test/MyGame/Example/TestSimpleTableWithEnum.jl delete mode 100644 julia/test/MyGame/Example/TypeAliases.jl delete mode 100644 julia/test/MyGame/Example/Vec3.jl delete mode 100644 julia/test/MyGame/Example2/Example2.jl delete mode 100644 julia/test/MyGame/Example2/Monster.jl delete mode 100644 julia/test/MyGame/InParentNamespace.jl delete mode 100644 julia/test/MyGame/MyGame.jl delete mode 100644 julia/test/defaults.jl delete mode 100644 julia/test/flatc.jl delete mode 100644 julia/test/internals.jl delete mode 100644 julia/test/monster.jl delete mode 100644 julia/test/monsterdata_python_wire.mon delete mode 100644 julia/test/monsterdata_test.mon delete mode 100644 julia/test/runtests.jl diff --git a/julia/README.md b/julia/README.md index c0ff8f66781..d3677956a48 100644 --- a/julia/README.md +++ b/julia/README.md @@ -1,7 +1,7 @@ # FlatBuffers -*A Julia implementation of google flatbuffers* +*A Julia implementation of [google flatbuffers](https://google.github.io/flatbuffers/)* | **Documentation** | **PackageEvaluator** | **Build Status** | diff --git a/julia/docs/Manifest.toml b/julia/docs/Manifest.toml deleted file mode 100644 index b341c92ea83..00000000000 --- a/julia/docs/Manifest.toml +++ /dev/null @@ -1,79 +0,0 @@ -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[Distributed]] -deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[DocStringExtensions]] -deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.6.0" - -[[Documenter]] -deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] -git-tree-sha1 = "9f2135e0e7ecb63f9c3ef73ea15a31d8cdb79bb7" -uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.20.0" - -[[InteractiveUtils]] -deps = ["LinearAlgebra", "Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[LibGit2]] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[UUIDs]] -deps = ["Random"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/julia/docs/Project.toml b/julia/docs/Project.toml deleted file mode 100644 index dfa65cd107d..00000000000 --- a/julia/docs/Project.toml +++ /dev/null @@ -1,2 +0,0 @@ -[deps] -Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/julia/docs/make.jl b/julia/docs/make.jl deleted file mode 100644 index 2384c6a66d3..00000000000 --- a/julia/docs/make.jl +++ /dev/null @@ -1,17 +0,0 @@ -import Pkg -Pkg.instantiate() -using Documenter, FlatBuffers - -makedocs( - modules = [FlatBuffers], - format = :html, - sitename = "FlatBuffers.jl", - pages = ["Home" => "index.md"] -) - -deploydocs( - repo = "github.com/JuliaData/FlatBuffers.jl.git", - target = "build", - deps = nothing, - make = nothing -) diff --git a/julia/docs/mkdocs.yml b/julia/docs/mkdocs.yml deleted file mode 100644 index da901f9e2d0..00000000000 --- a/julia/docs/mkdocs.yml +++ /dev/null @@ -1,29 +0,0 @@ -site_name: FlatBuffers.jl -repo_url: https://github.com/dmbates/FlatBuffers.jl -site_description: Julia implementation of google flatbuffers -site_author: Jacob Quinn - -theme: material - -extra: - palette: - primary: 'indigo' - accent: 'blue' - -extra_css: - - assets/Documenter.css - -extra_javascript: - - https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML - - assets/mathjaxhelper.js - -markdown_extensions: - - extra - - tables - - fenced_code - - mdx_math - -docs_dir: 'build' - -pages: - - Home: index.md diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md deleted file mode 100644 index 6e06b214606..00000000000 --- a/julia/docs/src/index.md +++ /dev/null @@ -1,97 +0,0 @@ -# FlatBuffers.jl Documentation - -#### Overview -FlatBuffers.jl provides native Julia support for reading and writing binary structures following the google flatbuffer schema (see [here](https://google.github.io/flatbuffers/flatbuffers_internals.html) for a more in-depth review of the binary format). - -The typical language support for flatbuffers involves utilizing the `flatc` compiler to translate a flatbuffer schema file (.fbs) into a langugage-specific set of types/classes and methods. See [here](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) for the official guide on writing schemas. - -This Julia package provides the serialization primitives used by code that has been generated by `flatc`. Since it was originally built without `flatc` support, it can also be used as a minimal set of macros to provide flatbuffer-compatible serialization of existing Julia types. This has led to the Julia code generated by `flatc` appearing somewhat more readable than for other languages. - -For example, for this schema: -``` -namespace example; - -table SimpleType { - x: int = 1; -} - -root_type SimpleType; -``` -the code generated by `flatc` looks like this: -```julia -module Example - -using FlatBuffers -@with_kw mutable struct SimpleType - x::Int32 = 1 -end - -# ... other generated stuff -end -``` -If you don't want to write a schema, you can pepper your existing Julia types -with these macros and then call the functions below to produce flatbuffer-compatible -binaries. - -#### Usage -`FlatBuffers` provides the following functions for reading and writing flatbuffers: -``` -FlatBuffers.serialize(stream::IO, value::T) -FlatBuffers.deserialize(stream::IO, ::Type{T}) -``` -These methods are not exported to avoid naming clashes with the `Serialization` module. -For convenience, there are also two additional constructors defined for each generated type: -* `T(buf::AbstractVector{UInt8}, pos::Integer=0)` -* `T(io::IO)` - -Here is an example showing how to use them to serialize the example type above. -```julia -import FlatBuffers, Example - -# create an instance of our type -val = Example.SimpleType(2) - -# serialize it to example.bin -open("example.bin", "w") do f FlatBuffers.serialize(f, val) end - -# read the value back again from file -val2 = open("example.bin", "r") do f Example.SimpleType(f) end -``` -In addition, this package provides the following types and methods, which are useful -when inspecting and constructing flatbuffers: -* `FlatBuffers.Table{T}` - type for deserializing a Julia type `T` from a flatbuffer -* `FlatBuffers.Builder{T}` - type for serializing a Julia type `T` to a flatbuffer -* `FlatBuffers.read` - performs the actual deserializing on a `FlatBuffer.Table` -* `FlatBuffers.build!` - performs the actual serializing on a `FlatBuffer.Builder` - -#### Methods for Generated Types -For a generated type `T`, in addition to the constructors mentioned above: -* if `T` has default values, constructors will be defined as per the `@with_kw` macro in [Parameters.jl](https://github.com/mauro3/Parameters.jl) -* `FlatBuffers.file_extension(T)` - returns the `file_extension` specified in the schema (if any) -* `FlatBuffers.file_identifier(T)` - returns the `file_identifier` specified in the schema (if any) -* `FlatBuffers.has_identifier(T, bytes)` - returns whether the given bytes contain the identifier for `T` at the offset designated by the flatbuffers specification -* `FlatBuffers.slot_offsets(T)` - an array containing the positions of the slots in the vtable for type `T`, accounting for gaps caused by deprecated fields -* `FlatBuffers.root_type(T)` - returns whether the type is designated as the root type by the schema. Also note however that no `root_type` definition is necessary in Julia; any of the generated `mutable struct`s can be a valid root table type. - -#### Circular References -It's a bit unfortunate that the flatbuffers example uses mutually referential types, something which Julia doesn't have support for yet. -However, there is a [workaround](https://github.com/JuliaLang/julia/issues/269#issuecomment-68421745) - by modifying the -code generated by `flatc` slightly to add a type parameter, we can refer to a type that hasn't yet been defined. -```julia -FlatBuffers.@with_kw mutable struct Monster{T} - # ... - test::T = nothing - # ... -end -``` -In general though, try to avoid schemas which introduce these kinds of circular references. -For the full `Monster` example see the test suite [here](https://github.com/JuliaData/FlatBuffers.jl/blob/master/test/MyGame/Example/Monster.jl). - -#### Internal Utilities -These functions are used by the code generated by `flatc`. Documentation is also included for many -internal methods and may be queried using `?` at the REPL. -* `@ALIGN T size_in_bytes` - convenience macro for forcing a flatbuffer alignment on the Julia type `T` to `size_in_bytes` -* `@with_kw mutable struct T fields...` - convenience macro for defining default field values for Julia type `T` -* `@UNION T Union{T1,T2,...}` - convenience macro for defining a flatbuffer union type `T` -* `@STRUCT struct T fields... end` - convenience macro for defining flatbuffer struct types, ensuring any necessary padding gets added to the type definition - diff --git a/julia/test/MyGame/Example/Ability.jl b/julia/test/MyGame/Example/Ability.jl deleted file mode 100644 index e32f1ca69b8..00000000000 --- a/julia/test/MyGame/Example/Ability.jl +++ /dev/null @@ -1,14 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@STRUCT struct Ability - id::UInt32 - distance::UInt32 -end -FlatBuffers.@ALIGN(Ability, 4) - -Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) -Ability(io::IO) = FlatBuffers.deserialize(io, Ability) diff --git a/julia/test/MyGame/Example/AnyAmbiguousAliases.jl b/julia/test/MyGame/Example/AnyAmbiguousAliases.jl deleted file mode 100644 index e106020db0f..00000000000 --- a/julia/test/MyGame/Example/AnyAmbiguousAliases.jl +++ /dev/null @@ -1,13 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@UNION(AnyAmbiguousAliases, ( - Nothing, - Monster, - Monster, - Monster, -)) - diff --git a/julia/test/MyGame/Example/AnyUniqueAliases.jl b/julia/test/MyGame/Example/AnyUniqueAliases.jl deleted file mode 100644 index 27abea3f997..00000000000 --- a/julia/test/MyGame/Example/AnyUniqueAliases.jl +++ /dev/null @@ -1,14 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import ..Example2 -import FlatBuffers - -FlatBuffers.@UNION(AnyUniqueAliases, ( - Nothing, - Monster, - TestSimpleTableWithEnum, - Example2.Monster, -)) - diff --git a/julia/test/MyGame/Example/Any_.jl b/julia/test/MyGame/Example/Any_.jl deleted file mode 100644 index 9bdc2a646e1..00000000000 --- a/julia/test/MyGame/Example/Any_.jl +++ /dev/null @@ -1,14 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import ..Example2 -import FlatBuffers - -FlatBuffers.@UNION(Any_, ( - Nothing, - Monster, - TestSimpleTableWithEnum, - Example2.Monster, -)) - diff --git a/julia/test/MyGame/Example/Color.jl b/julia/test/MyGame/Example/Color.jl deleted file mode 100644 index 804144f0ac7..00000000000 --- a/julia/test/MyGame/Example/Color.jl +++ /dev/null @@ -1,10 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -@enum Color::Int8 begin - ColorRed = 1 - ColorGreen = 2 - ColorBlue = 8 -end - diff --git a/julia/test/MyGame/Example/Example.jl b/julia/test/MyGame/Example/Example.jl deleted file mode 100644 index 4c720515ca8..00000000000 --- a/julia/test/MyGame/Example/Example.jl +++ /dev/null @@ -1,18 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -module Example - include("Color.jl") - include("Test.jl") - include("Vec3.jl") - include("Stat.jl") - include("Ability.jl") - include("Referrable.jl") - include("TestSimpleTableWithEnum.jl") - # include("AnyUniqueAliases.jl") - # include("AnyAmbiguousAliases.jl") - include("Monster.jl") - include("Any_.jl") - include("TypeAliases.jl") -end diff --git a/julia/test/MyGame/Example/Monster.jl b/julia/test/MyGame/Example/Monster.jl deleted file mode 100644 index 07fd74c0be7..00000000000 --- a/julia/test/MyGame/Example/Monster.jl +++ /dev/null @@ -1,83 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import ..InParentNamespace -import FlatBuffers - -FlatBuffers.@with_kw mutable struct Monster{T} -#= -# an example documentation comment: monster object -=# - pos::Union{Vec3, Nothing} = nothing - mana::Int16 = 150 - hp::Int16 = 100 - name::Union{String, Nothing} = nothing - inventory::Union{Vector{UInt8}, Nothing} = nothing - color::Color = 8 - test_type::UInt8 = 0 - test::T = nothing - test4::Union{Vector{Test}, Nothing} = nothing - testarrayofstring::Union{Vector{String}, Nothing} = nothing -#= -# an example documentation comment: this will end up in the generated code -# multiline too -=# - testarrayoftables::Union{Vector{Monster{T}}, Nothing} = nothing - enemy::Union{Monster{T}, Nothing} = nothing - testnestedflatbuffer::Union{Vector{UInt8}, Nothing} = nothing - testempty::Union{Stat, Nothing} = nothing - testbool::Bool = false - testhashs32_fnv1::Int32 = 0 - testhashu32_fnv1::UInt32 = 0 - testhashs64_fnv1::Int64 = 0 - testhashu64_fnv1::UInt64 = 0 - testhashs32_fnv1a::Int32 = 0 - testhashu32_fnv1a::UInt32 = 0 - testhashs64_fnv1a::Int64 = 0 - testhashu64_fnv1a::UInt64 = 0 - testarrayofbools::Union{Vector{Bool}, Nothing} = nothing - testf::Float32 = 3.14159 - testf2::Float32 = 3.0 - testf3::Float32 = 0.0 - testarrayofstring2::Union{Vector{String}, Nothing} = nothing - testarrayofsortedstruct::Union{Vector{Ability}, Nothing} = nothing - flex::Union{Vector{UInt8}, Nothing} = nothing - test5::Union{Vector{Test}, Nothing} = nothing - vector_of_longs::Union{Vector{Int64}, Nothing} = nothing - vector_of_doubles::Union{Vector{Float64}, Nothing} = nothing - parent_namespace_test::Union{InParentNamespace, Nothing} = nothing - vector_of_referrables::Union{Vector{Referrable}, Nothing} = nothing - single_weak_reference::UInt64 = 0 - vector_of_weak_references::Union{Vector{UInt64}, Nothing} = nothing - vector_of_strong_referrables::Union{Vector{Referrable}, Nothing} = nothing - co_owning_reference::UInt64 = 0 - vector_of_co_owning_references::Union{Vector{UInt64}, Nothing} = nothing - non_owning_reference::UInt64 = 0 - vector_of_non_owning_references::Union{Vector{UInt64}, Nothing} = nothing - # any_unique_type::UInt8 = 0 - # any_unique::AnyUniqueAliases = nothing - # any_ambiguous_type::UInt8 = 0 - # any_ambiguous::AnyAmbiguousAliases = nothing -end -FlatBuffers.@ALIGN(Monster, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ - 0x00000004, 0x00000006, 0x00000008, 0x0000000A, - 0x0000000E, 0x00000010, 0x00000012, 0x00000014, - 0x00000016, 0x00000018, 0x0000001A, 0x0000001C, - 0x0000001E, 0x00000020, 0x00000022, 0x00000024, - 0x00000026, 0x00000028, 0x0000002A, 0x0000002C, - 0x0000002E, 0x00000030, 0x00000032, 0x00000034, - 0x00000036, 0x00000038, 0x0000003A, 0x0000003C, - 0x0000003E, 0x00000040, 0x00000042, 0x00000044, - 0x00000046, 0x00000048, 0x0000004A, 0x0000004C, - 0x0000004E, 0x00000050, 0x00000052, 0x00000054, - 0x00000056, 0x00000058, 0x0000005A, 0x0000005C, - 0x0000005E, 0x00000060 -] -FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true -FlatBuffers.file_identifier(::Type{T}) where {T<:Monster} = "MONS" -FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" - -Monster{T}(buf::AbstractVector{UInt8}) where {T} = FlatBuffers.read(Monster{T}, buf) -Monster{T}(io::IO) where {T} = FlatBuffers.deserialize(io, Monster{T}) diff --git a/julia/test/MyGame/Example/Referrable.jl b/julia/test/MyGame/Example/Referrable.jl deleted file mode 100644 index 7fe024e210e..00000000000 --- a/julia/test/MyGame/Example/Referrable.jl +++ /dev/null @@ -1,16 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct Referrable - id::UInt64 = 0 -end -FlatBuffers.@ALIGN(Referrable, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ - 0x00000004 -] - -Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) -Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) diff --git a/julia/test/MyGame/Example/Stat.jl b/julia/test/MyGame/Example/Stat.jl deleted file mode 100644 index 00001c46935..00000000000 --- a/julia/test/MyGame/Example/Stat.jl +++ /dev/null @@ -1,18 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct Stat - id::Union{String, Nothing} = nothing - val::Int64 = 0 - count::UInt16 = 0 -end -FlatBuffers.@ALIGN(Stat, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ - 0x00000004, 0x00000006, 0x00000008 -] - -Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) -Stat(io::IO) = FlatBuffers.deserialize(io, Stat) diff --git a/julia/test/MyGame/Example/Test.jl b/julia/test/MyGame/Example/Test.jl deleted file mode 100644 index 0087689c7eb..00000000000 --- a/julia/test/MyGame/Example/Test.jl +++ /dev/null @@ -1,14 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@STRUCT struct Test - a::Int16 - b::Int8 -end -FlatBuffers.@ALIGN(Test, 2) - -Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) -Test(io::IO) = FlatBuffers.deserialize(io, Test) diff --git a/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl b/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl deleted file mode 100644 index 41c0e33f3ee..00000000000 --- a/julia/test/MyGame/Example/TestSimpleTableWithEnum.jl +++ /dev/null @@ -1,16 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum - color::Color = 2 -end -FlatBuffers.@ALIGN(TestSimpleTableWithEnum, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ - 0x00000004 -] - -TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) -TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) diff --git a/julia/test/MyGame/Example/TypeAliases.jl b/julia/test/MyGame/Example/TypeAliases.jl deleted file mode 100644 index a563b74ecbd..00000000000 --- a/julia/test/MyGame/Example/TypeAliases.jl +++ /dev/null @@ -1,29 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct TypeAliases - i8::Int8 = 0 - u8::UInt8 = 0 - i16::Int16 = 0 - u16::UInt16 = 0 - i32::Int32 = 0 - u32::UInt32 = 0 - i64::Int64 = 0 - u64::UInt64 = 0 - f32::Float32 = 0.0 - f64::Float64 = 0.0 - v8::Union{Vector{Int8}, Nothing} = nothing - vf64::Union{Vector{Float64}, Nothing} = nothing -end -FlatBuffers.@ALIGN(TypeAliases, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ - 0x00000004, 0x00000006, 0x00000008, 0x0000000A, - 0x0000000C, 0x0000000E, 0x00000010, 0x00000012, - 0x00000014, 0x00000016, 0x00000018, 0x0000001A -] - -TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) -TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) diff --git a/julia/test/MyGame/Example/Vec3.jl b/julia/test/MyGame/Example/Vec3.jl deleted file mode 100644 index 8f5442af51f..00000000000 --- a/julia/test/MyGame/Example/Vec3.jl +++ /dev/null @@ -1,18 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - -import FlatBuffers - -FlatBuffers.@STRUCT struct Vec3 - x::Float32 - y::Float32 - z::Float32 - test1::Float64 - test2::Color - test3::Test -end -FlatBuffers.@ALIGN(Vec3, 16) - -Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) -Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/julia/test/MyGame/Example2/Example2.jl b/julia/test/MyGame/Example2/Example2.jl deleted file mode 100644 index eaef8178155..00000000000 --- a/julia/test/MyGame/Example2/Example2.jl +++ /dev/null @@ -1,7 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example2 - -module Example2 - include("Monster.jl") -end diff --git a/julia/test/MyGame/Example2/Monster.jl b/julia/test/MyGame/Example2/Monster.jl deleted file mode 100644 index 91b98e333fb..00000000000 --- a/julia/test/MyGame/Example2/Monster.jl +++ /dev/null @@ -1,12 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example2 - -import FlatBuffers - -mutable struct Monster -end -FlatBuffers.@ALIGN(Monster, 1) - -Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) -Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/julia/test/MyGame/InParentNamespace.jl b/julia/test/MyGame/InParentNamespace.jl deleted file mode 100644 index 518cb89e3f0..00000000000 --- a/julia/test/MyGame/InParentNamespace.jl +++ /dev/null @@ -1,12 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: MyGame - -import FlatBuffers - -mutable struct InParentNamespace -end -FlatBuffers.@ALIGN(InParentNamespace, 1) - -InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) -InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) diff --git a/julia/test/MyGame/MyGame.jl b/julia/test/MyGame/MyGame.jl deleted file mode 100644 index 075ee85e35a..00000000000 --- a/julia/test/MyGame/MyGame.jl +++ /dev/null @@ -1,9 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: MyGame - -module MyGame - include("InParentNamespace.jl") - include("Example2/Example2.jl") - include("Example/Example.jl") -end diff --git a/julia/test/defaults.jl b/julia/test/defaults.jl deleted file mode 100644 index ee978f8da3d..00000000000 --- a/julia/test/defaults.jl +++ /dev/null @@ -1,40 +0,0 @@ -using FlatBuffers -using Test -import Parameters - -# test default fields -@with_kw mutable struct UltimateAnswer - answer::Int32 = 42 - question::String - highwaysbuilt::Int32 = 7 -end - -x = UltimateAnswer(;question="How many roads must a man walk down?") -@test x.answer == 42 -@test FlatBuffers.default(UltimateAnswer, Int32, :answer) == 42 -@test FlatBuffers.default(UltimateAnswer, Int32, :highwaysbuilt) == 7 -b = FlatBuffers.Builder(UltimateAnswer) -FlatBuffers.build!(b, x) -xbytes = FlatBuffers.bytes(b) -y = FlatBuffers.read(UltimateAnswer, xbytes) - -@test y.answer == x.answer -@test y.question == x.question -@test y.highwaysbuilt == x.highwaysbuilt -@test x.highwaysbuilt == 7 - -y = Parameters.reconstruct(x, highwaysbuilt = 0) -b = FlatBuffers.Builder(UltimateAnswer) -FlatBuffers.build!(b, y) -ybytes = FlatBuffers.bytes(b) - -# check that we save bytes with default integer values -@test length(ybytes) > length(xbytes) - -@test y.answer == x.answer -@test y.question == x.question -@test y.highwaysbuilt == 0 - -y = FlatBuffers.read(UltimateAnswer, ybytes) -@test y.highwaysbuilt == 0 - diff --git a/julia/test/flatc.jl b/julia/test/flatc.jl deleted file mode 100644 index d3996c7a3af..00000000000 --- a/julia/test/flatc.jl +++ /dev/null @@ -1,91 +0,0 @@ -using Test -import FlatBuffers - -# generated code -include("MyGame/MyGame.jl") -import .MyGame -import .MyGame.Example -import .MyGame.Example2 -import .MyGame.Example.Any_ -import .MyGame.Example.Monster -import .MyGame.Example.TestSimpleTableWithEnum - -# override the typeorder function, since we are working around -# circular type definitions using type parameters -FlatBuffers.typeorder(::Type{Any_}, i::Integer) = [Nothing, Monster{Any_}, TestSimpleTableWithEnum, Example2.Monster][i+1] - -function loadmonsterfile(filename) - open(joinpath(@__DIR__, filename), "r") do f Monster{Any_}(f) end -end - -function checkmonster(monster) - @test monster.hp == 80 - @test monster.mana == 150 - @test monster.name == "MyMonster" - - vec = monster.pos - - @test vec.x == 1.0 - @test vec.y == 2.0 - @test vec.z == 3.0 - @test vec.test1 == 3.0 - @test vec.test2 == MyGame.Example.ColorGreen - @test vec.test3_a == 5 - @test vec.test3_b == 6 - - monster2 = monster.test - @test monster2.name == "Fred" - - @test length(monster.inventory) == 5 - @test sum(monster.inventory) == 10 - - @test monster.vector_of_longs == [10 ^ (2*i) for i = 0:4] - @test monster.vector_of_doubles == [-1.7976931348623157e+308, 0, 1.7976931348623157e+308] - - @test length(monster.test4) == 2 - - (test0, test1) = monster.test4 - @test sum([test0.a, test0.b, test1.a, test1.b]) == 100 - - @test monster.testarrayofstring == ["test1", "test2"] - @test monster.testarrayoftables == nothing - @test monster.testf == 3.14159f0 -end - -function checkpassthrough(monster) - b = FlatBuffers.Builder(Monster{Any_}) - FlatBuffers.build!(b, monster) - bytes = FlatBuffers.bytes(b) - @test FlatBuffers.has_identifier(Monster{Any_}, bytes) - newmonster = FlatBuffers.read(Monster{Any_}, bytes) - checkmonster(newmonster) -end - -function checkserialize(monster) - io = IOBuffer() - FlatBuffers.serialize(io, monster) - bytes = take!(io) - newmonster = FlatBuffers.deserialize(IOBuffer(bytes), Monster{Any_}) - checkmonster(newmonster) -end - -@test FlatBuffers.root_type(Monster) == true -@test FlatBuffers.file_identifier(Monster) == "MONS" -@test FlatBuffers.file_extension(Monster) == "mon" - -for testcase in ["test", "python_wire"] - mon = loadmonsterfile("monsterdata_$testcase.mon") - checkmonster(mon) - checkpassthrough(mon) - checkserialize(mon) -end - -# test printing -mon = loadmonsterfile("monsterdata_test.mon") -b = FlatBuffers.Builder(Monster{Any_}) -FlatBuffers.build!(b, mon) -io = IOBuffer() -show(io, b) -output = String(take!(io)) -@test occursin("deprecated field", split(output, "\n")[9]) - diff --git a/julia/test/internals.jl b/julia/test/internals.jl deleted file mode 100644 index 392f7ca7e93..00000000000 --- a/julia/test/internals.jl +++ /dev/null @@ -1,811 +0,0 @@ -# Store specific byte patterns in these variables for the fuzzer. These -# values are taken verbatim from the C++ function FuzzTest1. -const overflowingInt32Val = read(IOBuffer(UInt8[0x83, 0x33, 0x33, 0x33]), Int32) -const overflowingInt64Val = read(IOBuffer(UInt8[0x84, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44]), Int64) - -# CheckByteLayout verifies the bytes of a Builder in various scenarios. -function CheckByteLayout() - check = want-> begin - got = b.bytes[b.head+1:end] - @test want == got - return -end - -# test 1: numbers - -b = FlatBuffers.Builder() -check(UInt8[]) -FlatBuffers.prepend!(b, true) -check(UInt8[1]) -FlatBuffers.prepend!(b, Int8(-127)) -check(UInt8[129, 1]) -FlatBuffers.prepend!(b, UInt8(255)) -check(UInt8[255, 129, 1]) -FlatBuffers.prepend!(b, Int16(-32222)) -check(UInt8[0x22, 0x82, 0, 255, 129, 1]) # first pad -FlatBuffers.prepend!(b, UInt16(0xFEEE)) -check(UInt8[0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) # no pad this time -FlatBuffers.prepend!(b, Int32(-53687092)) -check(UInt8[204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) -FlatBuffers.prepend!(b, UInt32(0x98765432)) -check(UInt8[0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) - -# test 1b: numbers 2 - -b = FlatBuffers.Builder() -prepend!(b, 0x1122334455667788) -check(UInt8[0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]) - -# test 2: 1xbyte vector - -b = FlatBuffers.Builder() -check(UInt8[]) -FlatBuffers.startvector(b, sizeof(Bool), 1, 1) -check(UInt8[0, 0, 0]) # align to 4bytes -FlatBuffers.prepend!(b, UInt8(1)) -check(UInt8[1, 0, 0, 0]) -FlatBuffers.endvector(b, 1) -check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding - -# test 3: 2xbyte vector - -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(UInt8), 2, 1) -check(UInt8[0, 0]) # align to 4bytes -FlatBuffers.prepend!(b, UInt8(1)) -check(UInt8[1, 0, 0]) -FlatBuffers.prepend!(b, UInt8(2)) -check(UInt8[2, 1, 0, 0]) -FlatBuffers.endvector(b, 2) -check(UInt8[2, 0, 0, 0, 2, 1, 0, 0]) # padding - -# test 3b: 11xbyte vector matches builder size - -b = FlatBuffers.Builder(Any, 12) -FlatBuffers.startvector(b, sizeof(UInt8), 8, 1) -start = UInt8[] -check(start) -for i = 1:11 - FlatBuffers.prepend!(b, UInt8(i)) - start = append!(UInt8[i], start) - check(start) -end -FlatBuffers.endvector(b, 8) -check(append!(UInt8[8, 0, 0, 0], start)) - -# test 4: 1xuint16 vector - -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(UInt16), 1, 1) -check(UInt8[0, 0]) # align to 4bytes -FlatBuffers.prepend!(b, UInt16(1)) -check(UInt8[1, 0, 0, 0]) -FlatBuffers.endvector(b, 1) -check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding - -# test 5: 2xuint16 vector - -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(UInt16), 2, 1) -check(UInt8[]) # align to 4bytes -FlatBuffers.prepend!(b, UInt16(0xABCD)) -check(UInt8[0xCD, 0xAB]) -FlatBuffers.prepend!(b, UInt16(0xDCBA)) -check(UInt8[0xBA, 0xDC, 0xCD, 0xAB]) -FlatBuffers.endvector(b, 2) -check(UInt8[2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB]) - -# test 6: CreateString - -b = FlatBuffers.Builder() -FlatBuffers.createstring(b, "foo") -check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad -FlatBuffers.createstring(b, "moop") -check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad - 3, 0, 0, 0, 'f', 'o', 'o', 0]) - -# test 6b: CreateString unicode - -b = FlatBuffers.Builder() -# These characters are chinese from blog.golang.org/strings -# We use escape codes here so that editors without unicode support -# aren't bothered: -uni_str = "\u65e5\u672c\u8a9e" -FlatBuffers.createstring(b, uni_str) -check(UInt8[9, 0, 0, 0, 230, 151, 165, 230, 156, 172, 232, 170, 158, 0, # null-terminated, 2-byte pad - 0, 0]) - -# test 6c: CreateUInt8String - -b = FlatBuffers.Builder() -FlatBuffers.createstring(b, "foo") -check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad -FlatBuffers.createstring(b, "moop") -check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad - 3, 0, 0, 0, 'f', 'o', 'o', 0]) - -# test 7: empty vtable -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 0) -check(UInt8[]) -FlatBuffers.endobject(b) -check(UInt8[4, 0, 4, 0, 4, 0, 0, 0]) - -# test 8: vtable with one true bool -b = FlatBuffers.Builder() -check(UInt8[]) -FlatBuffers.startobject(b, 1) -check(UInt8[]) -FlatBuffers.prependslot!(b, 1, true, false) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 8, 0, # length of object including vtable offset - 7, 0, # start of bool value - 6, 0, 0, 0, # offset for start of vtable (int32) - 0, 0, 0, # padded to 4 bytes - 1, # bool value - ]) - -# test 9: vtable with one default bool -b = FlatBuffers.Builder() -check(UInt8[]) -FlatBuffers.startobject(b, 1) -check(UInt8[]) -FlatBuffers.prependslot!(b, 1, false, false) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 4, 0, # end of object from here - 0, 0, # entry 1 is zero - 6, 0, 0, 0, # offset for start of vtable (int32) - ]) - -# test 10: vtable with one int16 -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 1) -FlatBuffers.prependslot!(b, 1, Int16(0x789A), Int16(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 8, 0, # end of object from here - 6, 0, # offset to value - 6, 0, 0, 0, # offset for start of vtable (int32) - 0, 0, # padding to 4 bytes - 0x9A, 0x78, - ]) - -# test 11: vtable with two int16 -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 2) -FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) -FlatBuffers.prependslot!(b, 2, Int16(0x789A), Int16(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 8, 0, # vtable bytes - 8, 0, # end of object from here - 6, 0, # offset to value 0 - 4, 0, # offset to value 1 - 8, 0, 0, 0, # offset for start of vtable (int32) - 0x9A, 0x78, # value 1 - 0x56, 0x34, # value 0 - ]) - -# test 12: vtable with int16 and bool -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 2) -FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) -FlatBuffers.prependslot!(b, 2, true, false) -FlatBuffers.endobject(b) -check(UInt8[ - 8, 0, # vtable bytes - 8, 0, # end of object from here - 6, 0, # offset to value 0 - 5, 0, # offset to value 1 - 8, 0, 0, 0, # offset for start of vtable (int32) - 0, # padding - 1, # value 1 - 0x56, 0x34, # value 0 - ]) - -# test 12: vtable with empty vector -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) -vecend = FlatBuffers.endvector(b, 0) -FlatBuffers.startobject(b, 1) -FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 8, 0, - 4, 0, # offset to vector offset - 6, 0, 0, 0, # offset for start of vtable (int32) - 4, 0, 0, 0, - 0, 0, 0, 0, # length of vector (not in struct) - ]) - -# test 12b: vtable with empty vector of byte and some scalars -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) -vecend = FlatBuffers.endvector(b, 0) -FlatBuffers.startobject(b, 2) -FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) -FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 8, 0, # vtable bytes - 12, 0, - 10, 0, # offset to value 0 - 4, 0, # offset to vector offset - 8, 0, 0, 0, # vtable loc - 8, 0, 0, 0, # value 1 - 0, 0, 55, 0, # value 0 - - 0, 0, 0, 0, # length of vector (not in struct) - ]) - -# test 13: vtable with 1 int16 and 2-vector of int16 -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(Int16), 2, 1) -FlatBuffers.prepend!(b, Int16(0x1234)) -FlatBuffers.prepend!(b, Int16(0x5678)) -vecend = FlatBuffers.endvector(b, 2) -FlatBuffers.startobject(b, 2) -FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) -FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 8, 0, # vtable bytes - 12, 0, # length of object - 6, 0, # start of value 0 from end of vtable - 8, 0, # start of value 1 from end of buffer - 8, 0, 0, 0, # offset for start of vtable (int32) - 0, 0, # padding - 55, 0, # value 0 - 4, 0, 0, 0, # vector position from here - 2, 0, 0, 0, # length of vector (uint32) - 0x78, 0x56, # vector value 1 - 0x34, 0x12, # vector value 0 - ]) - -# test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32 -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 1) -FlatBuffers.prep!(b, 4+4+4, 0) -FlatBuffers.prepend!(b, Int8(55)) -FlatBuffers.pad!(b, 3) -FlatBuffers.prepend!(b, Int16(0x1234)) -FlatBuffers.pad!(b, 2) -FlatBuffers.prepend!(b, Int32(0x12345678)) -structStart = FlatBuffers.offset(b) -FlatBuffers.prependstructslot!(b, 1, structStart, 0) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 16, 0, # end of object from here - 4, 0, # start of struct from here - 6, 0, 0, 0, # offset for start of vtable (int32) - 0x78, 0x56, 0x34, 0x12, # value 2 - 0, 0, # padding - 0x34, 0x12, # value 1 - 0, 0, 0, # padding - 55, # value 0 - ]) - -# test 15: vtable with 1 vector of 2 struct of 2 int8 -b = FlatBuffers.Builder() -FlatBuffers.startvector(b, sizeof(Int8)*2, 2, 1) -FlatBuffers.prepend!(b, Int8(33)) -FlatBuffers.prepend!(b, Int8(44)) -FlatBuffers.prepend!(b, Int8(55)) -FlatBuffers.prepend!(b, Int8(66)) -vecend = FlatBuffers.endvector(b, 2) -FlatBuffers.startobject(b, 1) -FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) -FlatBuffers.endobject(b) -check(UInt8[ - 6, 0, # vtable bytes - 8, 0, - 4, 0, # offset of vector offset - 6, 0, 0, 0, # offset for start of vtable (int32) - 4, 0, 0, 0, # vector start offset - - 2, 0, 0, 0, # vector length - 66, # vector value 1,1 - 55, # vector value 1,0 - 44, # vector value 0,1 - 33, # vector value 0,0 - ]) - -# test 16: table with some elements -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 2) -FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) -FlatBuffers.prependslot!(b, 2, Int16(66), Int16(0)) -off = FlatBuffers.endobject(b) -FlatBuffers.finish!(b, off) #TODO - -check(UInt8[ - 12, 0, 0, 0, # root of table: points to vtable offset - - 8, 0, # vtable bytes - 8, 0, # end of object from here - 7, 0, # start of value 0 - 4, 0, # start of value 1 - - 8, 0, 0, 0, # offset for start of vtable (int32) - - 66, 0, # value 1 - 0, # padding - 33, # value 0 - ]) - -# test 17: one unfinished table and one finished table -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 2) -FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) -FlatBuffers.prependslot!(b, 2, Int8(44), Int8(0)) -off = FlatBuffers.endobject(b) -FlatBuffers.finish!(b, off) - -FlatBuffers.startobject(b, 3) -FlatBuffers.prependslot!(b, 1, Int8(55), Int8(0)) -FlatBuffers.prependslot!(b, 2, Int8(66), Int8(0)) -FlatBuffers.prependslot!(b, 3, Int8(77), Int8(0)) -off = FlatBuffers.endobject(b) -FlatBuffers.finish!(b, off) - -check(UInt8[ - 16, 0, 0, 0, # root of table: points to object - 0, 0, # padding - - 10, 0, # vtable bytes - 8, 0, # size of object - 7, 0, # start of value 0 - 6, 0, # start of value 1 - 5, 0, # start of value 2 - 10, 0, 0, 0, # offset for start of vtable (int32) - 0, # padding - 77, # value 2 - 66, # value 1 - 55, # value 0 - - 12, 0, 0, 0, # root of table: points to object - - 8, 0, # vtable bytes - 8, 0, # size of object - 7, 0, # start of value 0 - 6, 0, # start of value 1 - 8, 0, 0, 0, # offset for start of vtable (int32) - 0, 0, # padding - 44, # value 1 - 33, # value 0 - ]) - -# test 18: a bunch of bools -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 8) -FlatBuffers.prependslot!(b, 1, true, false) -FlatBuffers.prependslot!(b, 2, true, false) -FlatBuffers.prependslot!(b, 3, true, false) -FlatBuffers.prependslot!(b, 4, true, false) -FlatBuffers.prependslot!(b, 5, true, false) -FlatBuffers.prependslot!(b, 6, true, false) -FlatBuffers.prependslot!(b, 7, true, false) -FlatBuffers.prependslot!(b, 8, true, false) -off = FlatBuffers.endobject(b) -FlatBuffers.finish!(b, off) - -check(UInt8[ - 24, 0, 0, 0, # root of table: points to vtable offset - - 20, 0, # vtable bytes - 12, 0, # size of object - 11, 0, # start of value 0 - 10, 0, # start of value 1 - 9, 0, # start of value 2 - 8, 0, # start of value 3 - 7, 0, # start of value 4 - 6, 0, # start of value 5 - 5, 0, # start of value 6 - 4, 0, # start of value 7 - 20, 0, 0, 0, # vtable offset - - 1, # value 7 - 1, # value 6 - 1, # value 5 - 1, # value 4 - 1, # value 3 - 1, # value 2 - 1, # value 1 - 1, # value 0 - ]) - -# test 19: three bools -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 3) -FlatBuffers.prependslot!(b, 1, true, false) -FlatBuffers.prependslot!(b, 2, true, false) -FlatBuffers.prependslot!(b, 3, true, false) -off = FlatBuffers.endobject(b) -FlatBuffers.finish!(b, off) - -check(UInt8[ - 16, 0, 0, 0, # root of table: points to vtable offset - - 0, 0, # padding - - 10, 0, # vtable bytes - 8, 0, # size of object - 7, 0, # start of value 0 - 6, 0, # start of value 1 - 5, 0, # start of value 2 - 10, 0, 0, 0, # vtable offset from here - - 0, # padding - 1, # value 2 - 1, # value 1 - 1, # value 0 - ]) - -# test 20: some floats -b = FlatBuffers.Builder() -FlatBuffers.startobject(b, 1) -FlatBuffers.prependslot!(b, 1, Float32(1.0), Float32(0.0)) -off = FlatBuffers.endobject(b) - -check(UInt8[ - 6, 0, # vtable bytes - 8, 0, # size of object - 4, 0, # start of value 0 - 6, 0, 0, 0, # vtable offset - - 0, 0, 128, 63, # value 0 - ]) -end - -# CheckManualBuild builds a Monster manually. -function CheckManualBuild() - b = FlatBuffers.Builder() - str = FlatBuffers.createstring(b, "MyMonster") - - FlatBuffers.startvector(b, 1, 5, 1) - FlatBuffers.prepend!(b, UInt8(4)) - FlatBuffers.prepend!(b, UInt8(3)) - FlatBuffers.prepend!(b, UInt8(2)) - FlatBuffers.prepend!(b, UInt8(1)) - FlatBuffers.prepend!(b, UInt8(0)) - inv = FlatBuffers.endvector(b, 5) - - FlatBuffers.startobject(b, 13) - FlatBuffers.prependslot!(b, 2, Int16(20), Int16(100)) - mon2 = FlatBuffers.endobject(b) - - # Test4Vector - FlatBuffers.startvector(b, 4, 2, 1) - - # Test 0 - FlatBuffers.prep!(b, 2, 4) - FlatBuffers.pad!(b, 1) - FlatBuffers.place!(b, Int8(20)) - FlatBuffers.place!(b, Int16(10)) - - # Test 1 - FlatBuffers.prep!(b, 2, 4) - FlatBuffers.pad!(b, 1) - FlatBuffers.place!(b, Int8(40)) - FlatBuffers.place!(b, Int16(30)) - - # end testvector - test4 = FlatBuffers.endvector(b, 2) - - FlatBuffers.startobject(b, 13) - - # a vec3 - FlatBuffers.prep!(b, 16, 32) - FlatBuffers.pad!(b, 2) - FlatBuffers.prep!(b, 2, 4) - FlatBuffers.pad!(b, 1) - FlatBuffers.place!(b, UInt8(6)) - FlatBuffers.place!(b, Int16(5)) - FlatBuffers.pad!(b, 1) - FlatBuffers.place!(b, UInt8(4)) - FlatBuffers.place!(b, Float64(3.0)) - FlatBuffers.pad!(b, 4) - FlatBuffers.place!(b, Float32(3.0)) - FlatBuffers.place!(b, Float32(2.0)) - FlatBuffers.place!(b, Float32(1.0)) - vec3Loc = FlatBuffers.offset(b) - # end vec3 - - FlatBuffers.prependstructslot!(b, 1, vec3Loc, 0) # vec3. noop - FlatBuffers.prependslot!(b, 3, Int16(80), Int16(100)) # hp - FlatBuffers.prependoffsetslot!(b, 4, Int32(str), Int32(0)) - FlatBuffers.prependoffsetslot!(b, 6, Int32(inv), Int32(0)) # inventory - FlatBuffers.prependslot!(b, 8, UInt8(1), UInt8(0)) - FlatBuffers.prependoffsetslot!(b, 9, Int32(mon2), Int32(0)) - FlatBuffers.prependoffsetslot!(b, 10, Int32(test4), Int32(0)) - mon = FlatBuffers.endobject(b) - - FlatBuffers.finish!(b, mon) - - return b.bytes, b.head -end - -# CheckVtableDeduplication verifies that vtables are deduplicated. -function CheckVtableDeduplication() - b = FlatBuffers.Builder() - - FlatBuffers.startobject(b, 4) - FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) - FlatBuffers.prependslot!(b, 2, UInt8(11), UInt8(0)) - FlatBuffers.prependslot!(b, 3, UInt8(22), UInt8(0)) - FlatBuffers.prependslot!(b, 4, Int16(33), Int16(0)) - obj0 = FlatBuffers.endobject(b) - - FlatBuffers.startobject(b, 4) - FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) - FlatBuffers.prependslot!(b, 2, UInt8(44), UInt8(0)) - FlatBuffers.prependslot!(b, 3, UInt8(55), UInt8(0)) - FlatBuffers.prependslot!(b, 4, Int16(66), Int16(0)) - obj1 = FlatBuffers.endobject(b) - - FlatBuffers.startobject(b, 4) - FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) - FlatBuffers.prependslot!(b, 2, UInt8(77), UInt8(0)) - FlatBuffers.prependslot!(b, 3, UInt8(88), UInt8(0)) - FlatBuffers.prependslot!(b, 4, Int16(99), Int16(0)) - obj2 = FlatBuffers.endobject(b) - - got = b.bytes[b.head+1:end] - - want = UInt8[ - 240, 255, 255, 255, # == -12. offset to dedupped vtable. - 99, 0, - 88, - 77, - 248, 255, 255, 255, # == -8. offset to dedupped vtable. - 66, 0, - 55, - 44, - 12, 0, # start of vtable - 8, 0, - 0, 0, - 7, 0, - 6, 0, - 4, 0, - 12, 0, 0, 0, # table0 - 33, 0, - 22, - 11 - ] - - @test got == want - - table0 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj0) - table1 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj1) - table2 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj2) - - function testTable(tab, a, b, c, d) - # vtable size - got = FlatBuffers.getoffsetslot(tab, 0, Int16(0)) - @test 12 == got - # object size - got = FlatBuffers.getoffsetslot(tab, 2, Int16(0)) - @test 8 == got - # default value - got = FlatBuffers.getoffsetslot(tab, 4, Int16(0)) - @test a == got - got = FlatBuffers.getslot(tab, 6, UInt8(0)) - @test b == got - val = FlatBuffers.getslot(tab, 8, UInt8(0)) - c != val && throw(ArgumentError("failed 8, 0: $got")) - got = FlatBuffers.getslot(tab, 10, UInt8(0)) - @test d == got - return - end - - testTable(table0, UInt16(0), UInt8(11), UInt8(22), UInt8(33)) - testTable(table1, UInt16(0), UInt8(44), UInt8(55), UInt8(66)) - testTable(table2, UInt16(0), UInt8(77), UInt8(88), UInt8(99)) -end - -# CheckNotInObjectError verifies that `endobject` fails if not inside an -# object. -function CheckNotInObjectError() - b = FlatBuffers.Builder() - - @test_throws ArgumentError FlatBuffers.endobject(b) -end - -# CheckStringIsNestedError verifies that a string can not be created inside -# another object. -function CheckStringIsNestedError() - b = FlatBuffers.Builder() - FlatBuffers.startobject(b, 0) - @test_throws ArgumentError FlatBuffers.createstring(b, "foo") -end - -# CheckByteStringIsNestedError verifies that a bytestring can not be created -# inside another object. -function CheckByteStringIsNestedError() - b = FlatBuffers.Builder() - FlatBuffers.startobject(b, 0) - @test_throws ArgumentError FlatBuffers.createstring(b, "foo") -end - -# CheckStructIsNotInlineError verifies that writing a struct in a location -# away from where it is used will cause a panic. -function CheckStructIsNotInlineError() - b = FlatBuffers.Builder() - FlatBuffers.startobject(b, 0) - @test_throws ArgumentError FlatBuffers.prependstructslot!(b, 0, 1, 0) -end - -# CheckFinishedBytesError verifies that `FinishedBytes` panics if the table -# is not finished. -function CheckFinishedBytesError() - b = FlatBuffers.Builder() - - @test_throws ArgumentError FlatBuffers.finishedbytes(b) -end - -function CheckCreateByteVector() - raw = UInt8(0):UInt8(29) - - for size = 1:30 - b1 = FlatBuffers.Builder() - b2 = FlatBuffers.Builder() - FlatBuffers.startvector(b1, 1, size, 1) - for i = size:-1:1 - FlatBuffers.prepend!(b1, raw[i]) - end - FlatBuffers.endvector(b1, size) - FlatBuffers.createbytevector(b2, raw[1:size]) - @test b1.bytes == b2.bytes - end -end - -const InitialLCGSeed = 48271 -mutable struct LCG - val::UInt32 - LCG() = new(UInt32(InitialLCGSeed)) -end -reset!(lcg::LCG) = lcg.val = UInt32(InitialLCGSeed) -function next(lcg::LCG) - n = UInt32((UInt64(lcg.val) * UInt64(279470273)) % UInt64(4294967291)) - lcg.val = n - return n -end - -# Low level stress/fuzz test: serialize/deserialize a variety of -# different kinds of data in different combinations -function checkFuzz(fuzzFields, fuzzObjects, verbose=true) - - # Values we're testing against: chosen to ensure no bits get chopped - # off anywhere, and also be different from eachother. - boolVal = true - int8Val = Int8(-127) # 0x81 - uint8Val = UInt8(0xFF) - int16Val = Int16(-32222) # 0x8222 - uint16Val = UInt16(0xFEEE) - int32Val = Int32(overflowingInt32Val) - uint32Val = UInt32(0xFDDDDDDD) - int64Val = Int64(overflowingInt64Val) - uint64Val = UInt64(0xFCCCCCCCCCCCCCCC) - float32Val = Float32(3.14159) - float64Val = Float64(3.14159265359) - - testValuesMax = 11 # hardcoded to the number of scalar types - - b = FlatBuffers.Builder() - l = LCG() - - objects = fill(0, fuzzObjects) - - # Generate fuzzObjects random objects each consisting of - # fuzzFields fields, each of a random type. - for i = 1:fuzzObjects - FlatBuffers.startobject(b, fuzzFields) - - for f = 1:fuzzFields - choice = next(l) % UInt32(testValuesMax) - if choice == 0 - FlatBuffers.prependslot!(b, f, boolVal, false) - elseif choice == 1 - FlatBuffers.prependslot!(b, f, int8Val, Int8(0)) - elseif choice == 2 - FlatBuffers.prependslot!(b, f, uint8Val, UInt8(0)) - elseif choice == 3 - FlatBuffers.prependslot!(b, f, int16Val, Int16(0)) - elseif choice == 4 - FlatBuffers.prependslot!(b, f, uint16Val, UInt16(0)) - elseif choice == 5 - FlatBuffers.prependslot!(b, f, int32Val, Int32(0)) - elseif choice == 6 - FlatBuffers.prependslot!(b, f, uint32Val, UInt32(0)) - elseif choice == 7 - FlatBuffers.prependslot!(b, f, int64Val, Int64(0)) - elseif choice == 8 - FlatBuffers.prependslot!(b, f, uint64Val, UInt64(0)) - elseif choice == 9 - FlatBuffers.prependslot!(b, f, float32Val, Float32(0)) - elseif choice == 10 - FlatBuffers.prependslot!(b, f, float64Val, Float64(0)) - end - end - - off = FlatBuffers.endobject(b) - - # store the offset from the end of the builder buffer, - # since it will keep growing: - objects[i] = off - end - - # Do some bookkeeping to generate stats on fuzzes: - stats = Dict{String,Int}() - function check(desc, want, got) - v = get!(stats, desc, 0) - stats[desc] = v + 1 - @test want == got - end - - l = LCG() # Reset. - - # Test that all objects we generated are readable and return the - # expected values. We generate random objects in the same order - # so this is deterministic. - for i = 1:fuzzObjects - - table = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - objects[i]) - - for j = 0:(fuzzFields - 1) - f = (FlatBuffers.VtableMetadataFields + j) * sizeof(Int16) - choice = next(l) % UInt32(testValuesMax) - - if choice == 0 - check("bool", boolVal, FlatBuffers.getslot(table, f, false)) - elseif choice == 1 - check("int8", int8Val, FlatBuffers.getslot(table, f, Int8(0))) - elseif choice == 2 - check("uint8", uint8Val, FlatBuffers.getslot(table, f, UInt8(0))) - elseif choice == 3 - check("int16", int16Val, FlatBuffers.getslot(table, f, Int16(0))) - elseif choice == 4 - check("uint16", uint16Val, FlatBuffers.getslot(table, f, UInt16(0))) - elseif choice == 5 - check("int32", int32Val, FlatBuffers.getslot(table, f, Int32(0))) - elseif choice == 6 - check("uint32", uint32Val, FlatBuffers.getslot(table, f, UInt32(0))) - elseif choice == 7 - check("int64", int64Val, FlatBuffers.getslot(table, f, Int64(0))) - elseif choice == 8 - check("uint64", uint64Val, FlatBuffers.getslot(table, f, UInt64(0))) - elseif choice == 9 - check("float32", float32Val, FlatBuffers.getslot(table, f, Float32(0))) - elseif choice == 10 - check("float64", float64Val, FlatBuffers.getslot(table, f, Float64(0))) - end - end - end - - # If enough checks were made, verify that all scalar types were used: - if fuzzFields*fuzzObjects >= testValuesMax - if length(stats) != testValuesMax - throw(ArgumentError("fuzzing failed to test all scalar types")) - end - end - - # Print some counts, if needed: - if verbose - if fuzzFields == 0 || fuzzObjects == 0 - println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t[none]\t 0") - else - ks = sort!(collect(keys(stats))) - for k in ks - println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t$(k): $(stats[k])") - end - end - end - return -end diff --git a/julia/test/monster.jl b/julia/test/monster.jl deleted file mode 100644 index b37a5685f9f..00000000000 --- a/julia/test/monster.jl +++ /dev/null @@ -1,77 +0,0 @@ -module SmallExample - -using FlatBuffers - -@enum(Color, Red = 1, Green = 2, Blue = 8) -@DEFAULT Color Red - -@STRUCT struct Test - a::Int16 - b::UInt8 - # _1::UInt8 # padding -end - -mutable struct TestSimpleTableWithEnum - color::Color -end - -@DEFAULT TestSimpleTableWithEnum color=Green - -@STRUCT struct Vec3 - x::Float32 - y::Float32 - z::Float32 - # _::UInt32 # padding - test1::Float64 - test2::Color - # __::UInt8 # padding - test3::Test - # ___::UInt16 # padding -end - -@ALIGN Vec3 16 - -mutable struct Stat - id::Union{Nothing, String} - val::Int64 - count::UInt16 -end - -# Julia doesn't support forward referencing of types -# @union Any_ Union{Monster, TestSimpleTableWithEnum} - -mutable struct Monster - pos::Vec3 - mana::Int16 - hp::Int16 - name::Union{Nothing, String} - friendly::Bool # deprecated - inventory::Union{Nothing, Vector{UInt8}} - color::Color - # test_type::Any_ - # test::Union{Nothing, Vector{UInt8}} - test4::Union{Nothing, Vector{Test}} - testarrayofstring::Union{Nothing, Vector{String}} - testarrayoftables::Union{Nothing, Vector{Monster}} - # don't support nested circulr reference objects yet - # enemy::Monster - testnestedflatbuffer::Union{Nothing, Vector{UInt8}} - testempty::Union{Nothing, Stat} - testbool::Bool - testhashs32_fnv1::Int32 - testhashu32_fnv1::UInt32 - testhashs64_fnv1::Int64 - testhashu64_fnv1::UInt64 - testhashs32_fnv1a::Int32 - testhashu32_fnv1a::UInt32 - testhashs64_fnv1a::Int64 - testhashu64_fnv1a::UInt64 - testarrayofbools::Union{Nothing, Vector{Bool}} - testf::Float32 - testf2::Float32 - testf3::Float32 -end - -@DEFAULT Monster hp=Int16(100) mana=Int16(150) color=convert(UInt8, Blue) friendly=false testf=Float32(3.14159) testf2=Float32(3) - -end # module diff --git a/julia/test/monsterdata_python_wire.mon b/julia/test/monsterdata_python_wire.mon deleted file mode 100644 index 55e37bf03961f9afdc7bd84220dcde2c8a5e8264..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 352 zcma!GKm{HQJ`5T_AvFdCASnaH(hMRD0t`Gr7O{YXfrY^Xs2GG(fOrBB&j8{zKnw(o zAo>QF82}_3?16*>5CJg^a5ykAurjbgxy(TNKN!6K4^si<*Ms=1K+Fi00NDou0_qUj z!HFNjc=#2h3=Einqz@3=05OOy2lO!)(3d<696%ljuz>w50AztRF)}f;fJGS?+=^1c gR)EAyQj1HBplU#DLm-f!oUG!u`y@>NgW_&VK87YVXy$QY=D@7ftA65!G$4#A%nq# z!H1!Mp@Ja*EDj?=7$O*AfN~5BjEo?e274gk07O6x0~`)aKy%oDTmwc11_dDA0c6hs z;tU`L>2f?-c^8BWmo)Z)unADC0EmwO@d6-joLK$oYy*EDjQ)5x{rDY^pUyBEsDyzT zh#7%!!~g&PS%7R1b}LFv0a6T1Kr8~pATd@TE=eseF+yS+0@)xnTtHQFz_37O^MFGL zC<)Zf113?;2H6X=5(wV^2dM%9D8C-W2H6MnG8ljz17r!PL--C({1C>&uOMY$0A?^U VF@xO73Bx.x, inst7.x) .== map(x->x.x, inst7_2.x)) - -# self-referential type test (type has subtype of itself) -# type TestCircT -# x::Int8 -# y::TestCircT -# end -# -# struct TestCircI -# x::Int8 -# y::TestCircI -# end - -# simple Union (Any_) - -# fbs -# table TestUnionT { -# x::TestUnionI -# } - -@UNION TestUnionU (Nothing,TestInt8T,TestInt8A) - -mutable struct TestUnionT - x_type::Int8 - x::TestUnionU -end - -TestUnionT(x::TestUnionU) = TestUnionT(FlatBuffers.typeorder(TestUnionU, typeof(x)), x) - -inst8 = TestUnionT(inst1) - -b = FlatBuffers.Builder(TestUnionT) -FlatBuffers.build!(b, inst8) -t = FlatBuffers.Table(b) -inst8_2 = FlatBuffers.read(t) - -@test inst8.x_type == inst8_2.x_type && inst8.x.x == inst8_2.x.x - -inst9 = TestUnionT(inst3) - -b = FlatBuffers.Builder(TestUnionT) -FlatBuffers.build!(b, inst9) -t = FlatBuffers.Table(b) -inst9_2 = FlatBuffers.read(t) - -@test inst9.x_type == inst9_2.x_type && inst9.x.x == inst9_2.x.x - -const Nachos = String -const Burgers = Int -FlatBuffers.@UNION(Delicious, - (Burgers, Nachos) -) - -FlatBuffers.@with_kw mutable struct TestVecUnionT - xs_type::Vector{UInt8} - xs::Vector{TestUnionU} - ys_type::Vector{UInt8} - ys::Vector{Delicious} -end - -function TestVecUnionT(xs::Vector{A}, ys::Vector{B}) where {A<:TestUnionU, B<:Delicious} - xs_type = [FlatBuffers.typeorder(TestUnionU, typeof(x)) for x in xs] - ys_type = [FlatBuffers.typeorder(Delicious, typeof(y)) for y in ys] - TestVecUnionT(xs_type, xs, ys_type, ys) -end - -dinner = "beef" -inst10 = TestVecUnionT(TestUnionU[inst1, inst1, inst1], [dinner]) - -b = FlatBuffers.Builder(TestVecUnionT) -FlatBuffers.build!(b, inst10) - -t = FlatBuffers.Table(b) -inst10_2 = FlatBuffers.read(t) - -@test inst10.xs_type == inst10_2.xs_type -@test [x.x for x in inst10.xs] == [x.x for x in inst10_2.xs] - -inst11 = TestVecUnionT(TestUnionU[inst3, inst3], [dinner for _ = 1:9]) - -b = FlatBuffers.Builder(TestVecUnionT) -FlatBuffers.build!(b, inst11) -t = FlatBuffers.Table(b) -inst11_2 = FlatBuffers.read(t) - -@test inst11.xs_type == inst11_2.xs_type -@test [x.x for x in inst11.xs] == [x.x for x in inst11_2.xs] -@test inst11.ys_type == inst11_2.ys_type -@test [y for y in inst11.ys] == [y for y in inst11_2.ys] - -# test @STRUCT macro -@STRUCT struct A - a::Int32 -end -@test sizeof(A) == 4 -@test all(fieldnames(A) .== [:a]) -@test A(1) == A(1) - -@STRUCT struct B - a::Int8 - b::Int32 -end -@test sizeof(B) == 8 -@test all(fieldnames(B) .== [:a, :_pad_a_B_0, :_pad_a_B_1, :b]) -@test B(1,2) == B(1,2) - -@STRUCT struct C - a::Int16 - b::Int32 - c::Int16 -end -@test sizeof(C) == 12 -@test all(fieldnames(C) .== [:a, :_pad_a_C_0, :b, :c, :_pad_c_C_1]) -@test C(1,2,3) == C(1,2,3) - -@STRUCT struct D - a::Int8 - b::Int64 -end -@test sizeof(D) == 16 -@test all(fieldnames(D) .== [:a, :_pad_a_D_0, :_pad_a_D_1, :_pad_a_D_2, :b]) -@test D(1,2) == D(1,2) - -@STRUCT struct E - a::Int64 - b::Int32 -end -@test sizeof(E) == 16 -@test all(fieldnames(E) .== [:a, :b, :_pad_b_E_0]) -@test E(1,2) == E(1,2) - -@STRUCT struct F - a::Int32 - b::Int16 - c::Int32 - d::Int32 - e::Int64 -end -@test sizeof(F) == 24 -@test all(fieldnames(F) .== [:a, :b, :_pad_b_F_0, :c, :d, :e]) -@test F(1,2,3,4,5) == F(1,2,3,4,5) - -@STRUCT struct G - a::Float64 - b::Int8 - c::Int16 - d::Int32 -end -@test sizeof(G) == 24 -@test all(fieldnames(G) .== [:a, :b, :_pad_b_G_0, :c, :_pad_c_G_1, :d]) -@test G(1,2,3,4) == G(1,2,3,4) - -@STRUCT struct H - a::Float32 - b::Int8 - c::Int16 -end -@test sizeof(H) == 8 -@test all(fieldnames(H) .== [:a, :b, :_pad_b_H_0, :c]) -@test H(1,2,3) == H(1,2,3) - -@STRUCT struct I - a::Float64 - b::Int8 - c::Int32 -end -@test sizeof(I) == 16 -@test all(fieldnames(I) .== [:a, :b, :_pad_b_I_0, :_pad_b_I_1, :c]) -@test I(1,2,3) == I(1,2,3) - -@STRUCT struct J - a::Int8 - b::A -end -@test sizeof(J) == 8 -@test all(fieldnames(J) .== [:a, :_pad_a_J_0, :_pad_a_J_1, :b_a]) -@test J(1,A(2)) == J(1,A(2)) - -@STRUCT struct K - a::J - b::I - c::J -end -@test sizeof(K) == 48 -@test all(fieldnames(K) .== [:a_a, :a__pad_a_J_0, :a__pad__pad_a_J_0_J_0, :a__pad_a_J_1, :a__pad__pad_a_J_1_J_1, :a_b_a, :b_a, :b_b, :b__pad_b_I_0, :b__pad__pad_b_I_0_I_0, :b__pad_b_I_1, :b__pad__pad_b_I_1_I_1, :b_c, :c_a, :c__pad_a_J_0, :c__pad__pad_a_J_0_J_0, :c__pad_a_J_1, :c__pad__pad_a_J_1_J_1, :c_b_a]) -@test K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) == K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 80449b5537d..89f3e39e240 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2018 Dolby Laboratories. All rights reserved. + * Copyright 2018 Google Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -172,7 +172,7 @@ class JuliaGenerator : public BaseGenerator { "" /* not used */), root_module_(MakeCamel(file_name_)), module_table_(ModuleTable::GetInstance()) { - static const char * const keywords[] = { + static const char *const keywords[] = { "begin", "while", "if", "for", "try", "return", "break", "continue", "function", "macro", "quote", "let", "local", "global", "const", "do", "struct", "module", @@ -264,7 +264,7 @@ class JuliaGenerator : public BaseGenerator { // Begin an object declaration. void BeginObject(const StructDef &struct_def, std::string *code_ptr, bool has_defaults) { - std::string &code = *code_ptr; + auto &code = *code_ptr; if (has_defaults) code += JuliaPackageName + ".@with_kw "; if (!struct_def.fixed) code += "mutable struct "; @@ -276,12 +276,12 @@ class JuliaGenerator : public BaseGenerator { void EndObject(const StructDef &struct_def, const std::vector &offsets, std::string *code_ptr) const { - std::string &code = *code_ptr; - std::string name = NormalizedName(struct_def); + auto &code = *code_ptr; + auto name = NormalizedName(struct_def); code += "end\n"; code += JuliaPackageName + ".@ALIGN(" + name + +", " + NumToString(struct_def.minalign) + ")\n"; - std::string method_signature = "(::Type{T}) where {T<:" + name + "}"; + auto method_signature = "(::Type{T}) where {T<:" + name + "}"; if (!offsets.empty() && !struct_def.fixed) { bool first = true; int i = 0; @@ -354,9 +354,8 @@ class JuliaGenerator : public BaseGenerator { static void EndUnion(std::string *code_ptr) { *code_ptr += "))\n\n"; } - void NewObjectFromBuffer(const StructDef &struct_def, - std::string *code_ptr) { - std::string &code = *code_ptr; + void NewObjectFromBuffer(const StructDef &struct_def, std::string *code_ptr) { + auto &code = *code_ptr; code += NormalizedName(struct_def) + "(buf::AbstractVector{UInt8})"; code += " = " + JuliaPackageName + ".read(" + NormalizedName(struct_def) + ", buf)\n"; @@ -367,8 +366,8 @@ class JuliaGenerator : public BaseGenerator { void GenScalarField(const StructDef &struct_def, const FieldDef &field, std::string *code_ptr, bool *has_defaults, std::set *imports_ptr) { - std::string &code = *code_ptr; - std::string field_name = NormalizedName(field); + auto &code = *code_ptr; + auto field_name = NormalizedName(field); code += Indent + field_name; code += "::"; code += GenTypeGet(field.value.type); @@ -424,15 +423,15 @@ class JuliaGenerator : public BaseGenerator { std::string relname = ""; // go up to common level (don't add dots to path if we are // annotating the type of a field - julia doesn't like that) - unsigned int i = 0; - unsigned int n = parent.components.size(); + size_t i = 0; + size_t n = parent.components.size(); while (i <= n && !SameNamespacePrefix(parent, child, n - i)) { if (dotprefix) relname += "."; i++; } // traverse down to the place we need to be - unsigned int m = parent.components.size() - i; - unsigned int j; + size_t m = parent.components.size() - i; + size_t j; for (j = m; j < child.components.size(); j++) { if (dotprefix || !relname.empty()) relname += "."; relname += child.components[j]; @@ -463,7 +462,7 @@ class JuliaGenerator : public BaseGenerator { if (child_def == NULL) return; GetRelativeNamespaces(def, child_def, &parent, &child); - std::string relname = GetRelativeName(parent, child); + auto relname = GetRelativeName(parent, child); // add relative import if (!relname.empty()) { @@ -474,10 +473,10 @@ class JuliaGenerator : public BaseGenerator { imports_ptr->insert(relname); } - std::string module = GetCanonicalName(child); - std::string child_name = + auto module = GetCanonicalName(child); + auto child_name = GetCanonicalName(child) + kPathSeparator + NormalizedName(*child_def); - std::string parent_name = + auto parent_name = GetCanonicalName(parent) + kPathSeparator + NormalizedName(def); // self-reference - this is fine, but don't add it to the dependency table if (child_name == parent_name) return; @@ -486,8 +485,7 @@ class JuliaGenerator : public BaseGenerator { // generate a field which depends upon generated types void GenDependentField(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr, - bool *has_defaults, + std::string *code_ptr, bool *has_defaults, std::set *imports_ptr) { std::string type_name = GenTypeGet(field.value.type, &struct_def); @@ -534,7 +532,8 @@ class JuliaGenerator : public BaseGenerator { case BASE_TYPE_STRUCT: case BASE_TYPE_VECTOR: case BASE_TYPE_UNION: - GenDependentField(struct_def, field, code_ptr, has_defaults, imports_ptr); + GenDependentField(struct_def, field, code_ptr, has_defaults, + imports_ptr); break; default: FLATBUFFERS_ASSERT(0); } @@ -588,7 +587,7 @@ class JuliaGenerator : public BaseGenerator { // always need FlatBuffers package for unions std::set imports; imports.insert(JuliaPackageName); - std::string union_name = NormalizedName(enum_def); + auto union_name = NormalizedName(enum_def); BeginUnion(union_name, code_ptr); for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); ++it) { @@ -608,7 +607,7 @@ class JuliaGenerator : public BaseGenerator { void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { if (enum_def.generated) return; GenComment(enum_def.doc_comment, code_ptr, &JuliaCommentConfig); - std::string enum_name = NormalizedName(enum_def); + auto enum_name = NormalizedName(enum_def); BeginEnum(enum_name, GenTypeBasic(enum_def.underlying_type), code_ptr); for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); ++it) { @@ -632,8 +631,7 @@ class JuliaGenerator : public BaseGenerator { return ctypename[type.base_type]; } - std::string GenTypePointer(const Type &type, - const Definition *parent) { + std::string GenTypePointer(const Type &type, const Definition *parent) { switch (type.base_type) { case BASE_TYPE_STRING: return "String"; case BASE_TYPE_VECTOR: @@ -659,8 +657,7 @@ class JuliaGenerator : public BaseGenerator { IsInteger(type.enum_def->underlying_type.base_type)); } - std::string GenTypeGet(const Type &type, - const Definition *parent = NULL) { + std::string GenTypeGet(const Type &type, const Definition *parent = NULL) { if (IsScalar(type.base_type)) return IsScalarEnum(type) ? NormalizedName(*type.enum_def, parent) : GenTypeBasic(type); @@ -669,9 +666,9 @@ class JuliaGenerator : public BaseGenerator { void BeginFile(const std::string submodule_name, std::string *code_ptr) const { - std::string &code = *code_ptr; + auto &code = *code_ptr; code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; - std::string module = submodule_name; + auto module = submodule_name; if (module.empty()) module = root_module_; code += "# module: " + module + "\n\n"; } @@ -703,16 +700,16 @@ class JuliaGenerator : public BaseGenerator { // Save out the generated code for a Julia module bool SaveModule(bool is_root, std::string full_module_name, DepGraph *children) { - std::string module_name = full_module_name; + auto module_name = full_module_name; auto start = full_module_name.rfind(kPathSeparator); if (start != std::string::npos) module_name = full_module_name.substr(start + 1); - std::string module_dir = ConCatPathFileName(path_, full_module_name); + auto module_dir = ConCatPathFileName(path_, full_module_name); if (is_root) { module_dir = path_; module_name = root_module_; } - std::string module_jl = + auto module_jl = ConCatPathFileName(module_dir, module_name) + JuliaFileExtension; std::string code = ""; bool need_module_file = false; @@ -720,7 +717,7 @@ class JuliaGenerator : public BaseGenerator { code += "module " + module_name + "\n"; // Include all the dependencies of this module in the right order - std::vector sorted_children = children->TopSort(); + auto sorted_children = children->TopSort(); for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); ++it) { std::string child = *it; @@ -737,7 +734,7 @@ class JuliaGenerator : public BaseGenerator { is_module = true; child = ConCatPathFileName(child, submodule_name); } - std::string dir = is_root ? path_ : module_dir; + auto dir = is_root ? path_ : module_dir; std::string relname; if (is_root) relname = child; @@ -781,8 +778,8 @@ class JuliaGenerator : public BaseGenerator { // Add a dependency between two definitions void AddDependency(const Definition *parent, const Definition *child) { FLATBUFFERS_ASSERT(parent != NULL && child != NULL); - std::string parent_name = NormalizedName(*parent); - std::string module = parent_name; + auto parent_name = NormalizedName(*parent); + auto module = parent_name; if (parent->defined_namespace != NULL) module = GetCanonicalName(*parent->defined_namespace); module_table_.AddDependency(module, parent_name, NormalizedName(*child)); @@ -790,7 +787,7 @@ class JuliaGenerator : public BaseGenerator { // Add a definition as a dependency to its own module void AddToOwnModule(const Definition &def) { - std::string m = GetModule(def); + auto m = GetModule(def); module_table_.AddDependency(m, m + kPathSeparator + NormalizedName(def), m); } @@ -800,7 +797,7 @@ class JuliaGenerator : public BaseGenerator { std::string code = ""; BeginFile(GetSubModule(def), &code); code += declcode; - std::string filename = GetFilename(def); + auto filename = GetFilename(def); EnsureDirExists(GetDirname(def)); if (!SaveFile(filename.c_str(), code, false)) return false; module_table_.AddFile(filename); From 630addd2549d26c077cb8d32e8bc83c27dacd9e6 Mon Sep 17 00:00:00 2001 From: rowan Date: Sat, 22 Dec 2018 11:15:00 +1100 Subject: [PATCH 06/19] fix namespace test --- src/idl_gen_julia.cpp | 2 +- tests/namespace_test/NamespaceA/TableInFirstNS.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 89f3e39e240..28d89204c75 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -370,7 +370,7 @@ class JuliaGenerator : public BaseGenerator { auto field_name = NormalizedName(field); code += Indent + field_name; code += "::"; - code += GenTypeGet(field.value.type); + code += GenTypeGet(field.value.type, &struct_def); if (!field.value.constant.empty() && !struct_def.fixed) { *has_defaults = true; std::string c = field.value.constant; diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl index 23eb1b4df4d..ab26e22b09a 100644 --- a/tests/namespace_test/NamespaceA/TableInFirstNS.jl +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -7,7 +7,7 @@ import FlatBuffers FlatBuffers.@with_kw mutable struct TableInFirstNS foo_table::Union{NamespaceB.TableInNestedNS, Nothing} = nothing - foo_enum::EnumInNestedNS = 0 + foo_enum::NamespaceB.EnumInNestedNS = 0 foo_struct::Union{NamespaceB.StructInNestedNS, Nothing} = nothing end FlatBuffers.@ALIGN(TableInFirstNS, 1) From 258fd5592bda5320d033d9cf385c6d6cc096b279 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Fri, 4 Jan 2019 11:28:51 +1100 Subject: [PATCH 07/19] put defaults back the way they were --- src/idl_gen_julia.cpp | 14 +++++---- tests/MyGame/Example/Monster.jl | 38 ++++++++++++------------- tests/MyGame/Example/Stat.jl | 2 +- tests/MyGame/Example/TypeAliases.jl | 4 +-- tests/union_vector/UnionVector/Movie.jl | 4 +-- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 89f3e39e240..78701cdeb39 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -490,15 +490,19 @@ class JuliaGenerator : public BaseGenerator { std::string type_name = GenTypeGet(field.value.type, &struct_def); BaseType bt = field.value.type.base_type; - if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_VECTOR) && - !struct_def.fixed) { + if (bt == BASE_TYPE_STRUCT && !struct_def.fixed) { type_name = "Union{" + type_name + ", Nothing}"; } // initialise nullable fields to nothing by default if ((bt == BASE_TYPE_STRUCT || bt == BASE_TYPE_UNION || bt == BASE_TYPE_VECTOR || bt == BASE_TYPE_STRING) && !struct_def.fixed) { - type_name += " = nothing"; + if (bt == BASE_TYPE_VECTOR) + type_name += " = []"; + else if (bt == BASE_TYPE_STRING) + type_name += " = \"\""; + else + type_name += " = nothing"; *has_defaults = true; } *code_ptr += Indent + NormalizedName(field) + "::" + type_name + "\n"; @@ -509,9 +513,9 @@ class JuliaGenerator : public BaseGenerator { std::string *code_ptr, bool *has_defaults) { *code_ptr += Indent + NormalizedName(field) + "::"; // initialise strings to be empty by default - *code_ptr += "Union{" + GenTypeGet(field.value.type) + ", Nothing}"; + *code_ptr += GenTypeGet(field.value.type); if (!struct_def.fixed) { - *code_ptr += " = nothing"; + *code_ptr += " = \"\""; *has_defaults = true; } *code_ptr += "\n"; diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl index 15586f65c70..98dcb579aac 100644 --- a/tests/MyGame/Example/Monster.jl +++ b/tests/MyGame/Example/Monster.jl @@ -12,20 +12,20 @@ FlatBuffers.@with_kw mutable struct Monster pos::Union{Vec3, Nothing} = nothing mana::Int16 = 150 hp::Int16 = 100 - name::Union{String, Nothing} = nothing - inventory::Union{Vector{UInt8}, Nothing} = nothing + name::String = "" + inventory::Vector{UInt8} = [] color::Color = 8 test_type::UInt8 = 0 test::Any_ = nothing - test4::Union{Vector{Test}, Nothing} = nothing - testarrayofstring::Union{Vector{String}, Nothing} = nothing + test4::Vector{Test} = [] + testarrayofstring::Vector{String} = [] #= # an example documentation comment: this will end up in the generated code # multiline too =# - testarrayoftables::Union{Vector{Monster}, Nothing} = nothing + testarrayoftables::Vector{Monster} = [] enemy::Union{Monster, Nothing} = nothing - testnestedflatbuffer::Union{Vector{UInt8}, Nothing} = nothing + testnestedflatbuffer::Vector{UInt8} = [] testempty::Union{Stat, Nothing} = nothing testbool::Bool = false testhashs32_fnv1::Int32 = 0 @@ -36,30 +36,30 @@ FlatBuffers.@with_kw mutable struct Monster testhashu32_fnv1a::UInt32 = 0 testhashs64_fnv1a::Int64 = 0 testhashu64_fnv1a::UInt64 = 0 - testarrayofbools::Union{Vector{Bool}, Nothing} = nothing + testarrayofbools::Vector{Bool} = [] testf::Float32 = 3.14159 testf2::Float32 = 3.0 testf3::Float32 = 0.0 - testarrayofstring2::Union{Vector{String}, Nothing} = nothing - testarrayofsortedstruct::Union{Vector{Ability}, Nothing} = nothing - flex::Union{Vector{UInt8}, Nothing} = nothing - test5::Union{Vector{Test}, Nothing} = nothing - vector_of_longs::Union{Vector{Int64}, Nothing} = nothing - vector_of_doubles::Union{Vector{Float64}, Nothing} = nothing + testarrayofstring2::Vector{String} = [] + testarrayofsortedstruct::Vector{Ability} = [] + flex::Vector{UInt8} = [] + test5::Vector{Test} = [] + vector_of_longs::Vector{Int64} = [] + vector_of_doubles::Vector{Float64} = [] parent_namespace_test::Union{InParentNamespace, Nothing} = nothing - vector_of_referrables::Union{Vector{Referrable}, Nothing} = nothing + vector_of_referrables::Vector{Referrable} = [] single_weak_reference::UInt64 = 0 - vector_of_weak_references::Union{Vector{UInt64}, Nothing} = nothing - vector_of_strong_referrables::Union{Vector{Referrable}, Nothing} = nothing + vector_of_weak_references::Vector{UInt64} = [] + vector_of_strong_referrables::Vector{Referrable} = [] co_owning_reference::UInt64 = 0 - vector_of_co_owning_references::Union{Vector{UInt64}, Nothing} = nothing + vector_of_co_owning_references::Vector{UInt64} = [] non_owning_reference::UInt64 = 0 - vector_of_non_owning_references::Union{Vector{UInt64}, Nothing} = nothing + vector_of_non_owning_references::Vector{UInt64} = [] any_unique_type::UInt8 = 0 any_unique::AnyUniqueAliases = nothing any_ambiguous_type::UInt8 = 0 any_ambiguous::AnyAmbiguousAliases = nothing - vector_of_enums::Union{Vector{Color}, Nothing} = nothing + vector_of_enums::Vector{Color} = [] end FlatBuffers.@ALIGN(Monster, 1) FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl index 00001c46935..d24c0d9d08d 100644 --- a/tests/MyGame/Example/Stat.jl +++ b/tests/MyGame/Example/Stat.jl @@ -5,7 +5,7 @@ import FlatBuffers FlatBuffers.@with_kw mutable struct Stat - id::Union{String, Nothing} = nothing + id::String = "" val::Int64 = 0 count::UInt16 = 0 end diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl index a563b74ecbd..eda9162bea9 100644 --- a/tests/MyGame/Example/TypeAliases.jl +++ b/tests/MyGame/Example/TypeAliases.jl @@ -15,8 +15,8 @@ FlatBuffers.@with_kw mutable struct TypeAliases u64::UInt64 = 0 f32::Float32 = 0.0 f64::Float64 = 0.0 - v8::Union{Vector{Int8}, Nothing} = nothing - vf64::Union{Vector{Float64}, Nothing} = nothing + v8::Vector{Int8} = [] + vf64::Vector{Float64} = [] end FlatBuffers.@ALIGN(TypeAliases, 1) FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl index 0544e8b7c2b..ae8a7ac3932 100644 --- a/tests/union_vector/UnionVector/Movie.jl +++ b/tests/union_vector/UnionVector/Movie.jl @@ -7,8 +7,8 @@ import FlatBuffers FlatBuffers.@with_kw mutable struct Movie main_character_type::UInt8 = 0 main_character::Character = nothing - characters_type::Union{Vector{UInt8}, Nothing} = nothing - characters::Union{Vector{Character}, Nothing} = nothing + characters_type::Vector{UInt8} = [] + characters::Vector{Character} = [] end FlatBuffers.@ALIGN(Movie, 1) FlatBuffers.slot_offsets(::Type{T}) where {T<:Movie} = [ From a466f5da8f02cb755283a8601512a87cf099c8c9 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Fri, 4 Jan 2019 12:01:20 +1100 Subject: [PATCH 08/19] formatting --- src/idl_gen_julia.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index efe85de1259..17389862e5a 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -624,7 +624,7 @@ class JuliaGenerator : public BaseGenerator { static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { -// clang-format off + // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #JLTYPE, From bb70036121e9f393e87c17eea8d509f5f4cfb6c0 Mon Sep 17 00:00:00 2001 From: rowan Date: Sun, 6 Jan 2019 10:57:30 +1100 Subject: [PATCH 09/19] fix last commit --- src/idl_gen_go.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/idl_gen_go.cpp b/src/idl_gen_go.cpp index 4a40293609f..779d9ff2c4c 100644 --- a/src/idl_gen_go.cpp +++ b/src/idl_gen_go.cpp @@ -738,7 +738,7 @@ class GoGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #GTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD From b7b78ed2ef9f574f0af07ff623c3eedb01f42a33 Mon Sep 17 00:00:00 2001 From: rowan Date: Wed, 9 Jan 2019 15:42:04 +1100 Subject: [PATCH 10/19] work in progress on stateless compilation --- src/idl_gen_julia.cpp | 139 ++++++------------ tests/MonsterTest.jl | 19 ++- tests/MyGame/Example/Ability.jl | 10 +- tests/MyGame/Example/AnyAmbiguousAliases.jl | 10 +- tests/MyGame/Example/AnyUniqueAliases.jl | 10 +- tests/MyGame/Example/Any_.jl | 10 +- tests/MyGame/Example/Color.jl | 10 +- tests/MyGame/Example/Example.jl | 28 ++-- tests/MyGame/Example/Monster.jl | 10 +- tests/MyGame/Example/Referrable.jl | 10 +- tests/MyGame/Example/Stat.jl | 10 +- tests/MyGame/Example/Test.jl | 10 +- .../MyGame/Example/TestSimpleTableWithEnum.jl | 10 +- tests/MyGame/Example/TypeAliases.jl | 10 +- tests/MyGame/Example/Vec3.jl | 10 +- tests/MyGame/Example2/Example2.jl | 6 +- tests/MyGame/Example2/Monster.jl | 10 +- tests/MyGame/InParentNamespace.jl | 5 +- tests/MyGame/MyGame.jl | 10 +- tests/namespace_test/NamespaceA/NamespaceA.jl | 10 +- .../NamespaceA/NamespaceB/EnumInNestedNS.jl | 10 +- .../NamespaceA/NamespaceB/NamespaceB.jl | 10 +- .../NamespaceA/NamespaceB/StructInNestedNS.jl | 10 +- .../NamespaceA/NamespaceB/TableInNestedNS.jl | 10 +- .../NamespaceA/SecondTableInA.jl | 5 +- .../NamespaceA/TableInFirstNS.jl | 5 +- tests/namespace_test/NamespaceC/NamespaceC.jl | 6 +- tests/namespace_test/NamespaceC/TableInC.jl | 5 +- tests/namespace_test/NamespaceTest1.jl | 8 +- tests/namespace_test/NamespaceTest2.jl | 10 +- tests/union_vector/UnionVector.jl | 14 +- tests/union_vector/UnionVector/Attacker.jl | 2 - tests/union_vector/UnionVector/BookReader.jl | 2 - tests/union_vector/UnionVector/Character.jl | 2 - tests/union_vector/UnionVector/Movie.jl | 2 - tests/union_vector/UnionVector/Rapunzel.jl | 2 - 36 files changed, 254 insertions(+), 196 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 17389862e5a..14fb1bc4e36 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -117,19 +117,8 @@ class DepGraph { } }; -// Singleton class which keeps track of generated modules -// and their dependencies. A singleton class is necessary -// since our generator is run multiple times with -// multiple different .fbs files, and we want to preserve -// the set of module dependencies across these runs. -// We need to keep track of dependencies since Julia doesn't -// support forward declarations of types. class ModuleTable { public: - static ModuleTable &GetInstance() { - static ModuleTable instance; - return instance; - } bool IsModule(const std::string &m) { return modules_.find(m) != modules_.end(); } @@ -154,14 +143,6 @@ class ModuleTable { private: std::map modules_; std::set files_; - ModuleTable() {} - ~ModuleTable() { - for (auto it = modules_.begin(); it != modules_.end(); ++it) - delete it->second; - } - - ModuleTable(ModuleTable const &); - void operator=(ModuleTable const &); }; class JuliaGenerator : public BaseGenerator { @@ -170,8 +151,7 @@ class JuliaGenerator : public BaseGenerator { const std::string &file_name) : BaseGenerator(parser, path, file_name, "" /* not used */, "" /* not used */), - root_module_(MakeCamel(file_name_)), - module_table_(ModuleTable::GetInstance()) { + root_module_(MakeCamel(file_name_)) { static const char *const keywords[] = { "begin", "while", "if", "for", "try", "return", "break", "continue", "function", "macro", "quote", "let", @@ -186,7 +166,7 @@ class JuliaGenerator : public BaseGenerator { bool generate() { if (!GenEnums()) return false; if (!GenStructs()) return false; - if (!GenModules()) return false; + if (!GenTopLevel()) return false; return true; } @@ -194,7 +174,7 @@ class JuliaGenerator : public BaseGenerator { // the root module is the name of the .fbs file which // we are compiling, in camel case const std::string root_module_; - ModuleTable &module_table_; + ModuleTable module_table_; std::unordered_set keywords_; bool GenEnums(void) { @@ -222,7 +202,23 @@ class JuliaGenerator : public BaseGenerator { return true; } - bool GenModules(void) { + bool GenTopLevel(void) { + std::string code = "# "; + std::set included; + code += FlatBuffersGeneratedWarning(); + code += "\n\n"; + // include other included .fbs files first + for (auto it = parser_.included_files_.begin(); + it != parser_.included_files_.end(); ++it) { + if (it->second.empty()) continue; + auto dir = flatbuffers::StripFileName(it->second); + auto basename = MakeCamel(flatbuffers::StripPath(flatbuffers::StripExtension(it->second))); + auto toinclude = (dir + kPathSeparator + basename) + JuliaFileExtension; + auto fullpath = path_ + kPathSeparator + toinclude; + if (!FileExists(fullpath.c_str())) + continue; + code += "include(\"" + toinclude + "\")\n"; + } for (auto it = parser_.namespaces_.begin(); it != parser_.namespaces_.end(); ++it) { std::string parent; @@ -245,19 +241,12 @@ class JuliaGenerator : public BaseGenerator { } } auto sorted_modules = module_table_.SortedModuleNames(); - bool save_root = false; // iterate through child modules first, then parents for (auto m = sorted_modules.rbegin(); m != sorted_modules.rend(); ++m) { - if (*m == root_module_) { - save_root = true; - continue; - } - SaveModule(false, *m, module_table_.GetDependencies(*m)); + GenIncludes(module_table_.GetDependencies(*m), included, &code); } - // save the root module last - if (save_root) - SaveModule(true, root_module_, - module_table_.GetDependencies(root_module_)); + auto filename = ConCatPathFileName(path_, root_module_) + JuliaFileExtension; + if (!SaveFile(filename.c_str(), code, false)) return false; return true; } @@ -668,13 +657,19 @@ class JuliaGenerator : public BaseGenerator { return GenTypePointer(type, parent); } - void BeginFile(const std::string submodule_name, - std::string *code_ptr) const { + void BeginFile(const Namespace &ns, std::string *code_ptr) const { auto &code = *code_ptr; code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; - auto module = submodule_name; - if (module.empty()) module = root_module_; - code += "# module: " + module + "\n\n"; + for (size_t i = 0; i < ns.components.size(); i++) { + code += "module " + std::string(ns.components[i]) + "\n\n"; + } + } + + void EndFile(const Namespace &ns, std::string *code_ptr) const { + auto &code = *code_ptr; + for (size_t i = 0; i < ns.components.size(); i++) { + code += "\nend\n\n"; + } } std::string GetDirname(const Definition &def) const { @@ -701,70 +696,25 @@ class JuliaGenerator : public BaseGenerator { return ""; } - // Save out the generated code for a Julia module - bool SaveModule(bool is_root, std::string full_module_name, - DepGraph *children) { - auto module_name = full_module_name; - auto start = full_module_name.rfind(kPathSeparator); - if (start != std::string::npos) - module_name = full_module_name.substr(start + 1); - auto module_dir = ConCatPathFileName(path_, full_module_name); - if (is_root) { - module_dir = path_; - module_name = root_module_; - } - auto module_jl = - ConCatPathFileName(module_dir, module_name) + JuliaFileExtension; - std::string code = ""; - bool need_module_file = false; - BeginFile(module_name, &code); - code += "module " + module_name + "\n"; - - // Include all the dependencies of this module in the right order + bool GenIncludes(DepGraph *children, std::set &included, std::string *code_ptr) { + auto &code = *code_ptr; + // Include all the contents of this module in the right order auto sorted_children = children->TopSort(); for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); ++it) { std::string child = *it; - bool is_module = false; - std::string submodule_name = - child.substr(child.rfind(kPathSeparator) + 1); - - // Don't include the root module from the root - if (is_root && submodule_name == root_module_) continue; - - // Modules live in a subdirectory named after themselves - if (module_table_.IsModule(child) && - !(is_root && submodule_name == root_module_)) { - is_module = true; - child = ConCatPathFileName(child, submodule_name); - } - auto dir = is_root ? path_ : module_dir; - std::string relname; - if (is_root) - relname = child; - else if (is_module) - relname = child.substr(child.find(kPathSeparator) + 1); - else - relname = child.substr(child.rfind(kPathSeparator) + 1); + if (module_table_.IsModule(child) || included.find(child) != included.end()) + continue; // If the file doesn't exist, don't include it // TODO: this doesn't allow types which reference each other, // but Julia doesn't support this yet anyway - std::string toinclude = relname + JuliaFileExtension; - std::string fullpath = ConCatPathFileName(dir, toinclude); + std::string toinclude = child + JuliaFileExtension; + std::string fullpath = ConCatPathFileName(path_, toinclude); if (!module_table_.IsFile(fullpath.c_str())) continue; - code += Indent + "include(\"" + toinclude + "\")\n"; - need_module_file = true; + code += "include(\"" + toinclude + "\")\n"; + included.insert(child); } - code += "end\n"; - - // only write the module if it has some things to include - if (!need_module_file) return true; - - EnsureDirExists(module_dir); - if (!SaveFile(module_jl.c_str(), code, false)) return false; - - module_table_.AddFile(module_jl); return true; } @@ -799,8 +749,9 @@ class JuliaGenerator : public BaseGenerator { bool SaveType(const Definition &def, const std::string &declcode) { if (!declcode.length()) return true; std::string code = ""; - BeginFile(GetSubModule(def), &code); + BeginFile(*def.defined_namespace, &code); code += declcode; + EndFile(*def.defined_namespace, &code); auto filename = GetFilename(def); EnsureDirExists(GetDirname(def)); if (!SaveFile(filename.c_str(), code, false)) return false; diff --git a/tests/MonsterTest.jl b/tests/MonsterTest.jl index b3ad477a2a0..1d49eda47d4 100644 --- a/tests/MonsterTest.jl +++ b/tests/MonsterTest.jl @@ -1,7 +1,16 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: MonsterTest - -module MonsterTest - include("MyGame/MyGame.jl") -end +include("MyGame/Example2/Monster.jl") +include("MyGame/Example/Any_.jl") +include("MyGame/Example/AnyUniqueAliases.jl") +include("MyGame/Example/Color.jl") +include("MyGame/Example/Test.jl") +include("MyGame/Example/Vec3.jl") +include("MyGame/Example/Stat.jl") +include("MyGame/Example/Ability.jl") +include("MyGame/Example/Referrable.jl") +include("MyGame/Example/TestSimpleTableWithEnum.jl") +include("MyGame/Example/AnyAmbiguousAliases.jl") +include("MyGame/Example/Monster.jl") +include("MyGame/Example/TypeAliases.jl") +include("MyGame/InParentNamespace.jl") diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl index e32f1ca69b8..a42406c691f 100644 --- a/tests/MyGame/Example/Ability.jl +++ b/tests/MyGame/Example/Ability.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -12,3 +14,9 @@ FlatBuffers.@ALIGN(Ability, 4) Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) Ability(io::IO) = FlatBuffers.deserialize(io, Ability) + +end + + +end + diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl index e106020db0f..96bc8437f54 100644 --- a/tests/MyGame/Example/AnyAmbiguousAliases.jl +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -11,3 +13,9 @@ FlatBuffers.@UNION(AnyAmbiguousAliases, ( Monster, )) + +end + + +end + diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl index 27abea3f997..7c24aac8236 100644 --- a/tests/MyGame/Example/AnyUniqueAliases.jl +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import ..Example2 import FlatBuffers @@ -12,3 +14,9 @@ FlatBuffers.@UNION(AnyUniqueAliases, ( Example2.Monster, )) + +end + + +end + diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl index 9bdc2a646e1..cfd66ea17d2 100644 --- a/tests/MyGame/Example/Any_.jl +++ b/tests/MyGame/Example/Any_.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import ..Example2 import FlatBuffers @@ -12,3 +14,9 @@ FlatBuffers.@UNION(Any_, ( Example2.Monster, )) + +end + + +end + diff --git a/tests/MyGame/Example/Color.jl b/tests/MyGame/Example/Color.jl index 804144f0ac7..347fcd51f66 100644 --- a/tests/MyGame/Example/Color.jl +++ b/tests/MyGame/Example/Color.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example @enum Color::Int8 begin ColorRed = 1 @@ -8,3 +10,9 @@ ColorBlue = 8 end + +end + + +end + diff --git a/tests/MyGame/Example/Example.jl b/tests/MyGame/Example/Example.jl index c8fd32b3cd4..e35be429849 100644 --- a/tests/MyGame/Example/Example.jl +++ b/tests/MyGame/Example/Example.jl @@ -1,18 +1,14 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example - module Example - include("Color.jl") - include("Test.jl") - include("Vec3.jl") - include("Stat.jl") - include("Ability.jl") - include("Referrable.jl") - include("TestSimpleTableWithEnum.jl") - include("AnyUniqueAliases.jl") - include("AnyAmbiguousAliases.jl") - include("Monster.jl") - include("Any_.jl") - include("TypeAliases.jl") +include("Color.jl") +include("Test.jl") +include("Vec3.jl") +include("Stat.jl") +include("Ability.jl") +include("Referrable.jl") +include("TestSimpleTableWithEnum.jl") +include("AnyUniqueAliases.jl") +include("AnyAmbiguousAliases.jl") +include("Monster.jl") +include("Any_.jl") +include("TypeAliases.jl") end diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl index 98dcb579aac..67765f0f38a 100644 --- a/tests/MyGame/Example/Monster.jl +++ b/tests/MyGame/Example/Monster.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import ..InParentNamespace import FlatBuffers @@ -82,3 +84,9 @@ FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) Monster(io::IO) = FlatBuffers.deserialize(io, Monster) + +end + + +end + diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl index 7fe024e210e..78f96a3bd88 100644 --- a/tests/MyGame/Example/Referrable.jl +++ b/tests/MyGame/Example/Referrable.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -14,3 +16,9 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) + +end + + +end + diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl index d24c0d9d08d..b3d00b8eeba 100644 --- a/tests/MyGame/Example/Stat.jl +++ b/tests/MyGame/Example/Stat.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -16,3 +18,9 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) Stat(io::IO) = FlatBuffers.deserialize(io, Stat) + +end + + +end + diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl index 0087689c7eb..04f8859ab83 100644 --- a/tests/MyGame/Example/Test.jl +++ b/tests/MyGame/Example/Test.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -12,3 +14,9 @@ FlatBuffers.@ALIGN(Test, 2) Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) Test(io::IO) = FlatBuffers.deserialize(io, Test) + +end + + +end + diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl index 41c0e33f3ee..e68fd066f0c 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.jl +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -14,3 +16,9 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) + +end + + +end + diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl index eda9162bea9..4ddb3c19157 100644 --- a/tests/MyGame/Example/TypeAliases.jl +++ b/tests/MyGame/Example/TypeAliases.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -27,3 +29,9 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) + +end + + +end + diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl index 8f5442af51f..1809b4895ce 100644 --- a/tests/MyGame/Example/Vec3.jl +++ b/tests/MyGame/Example/Vec3.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example +module MyGame + +module Example import FlatBuffers @@ -16,3 +18,9 @@ FlatBuffers.@ALIGN(Vec3, 16) Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) + +end + + +end + diff --git a/tests/MyGame/Example2/Example2.jl b/tests/MyGame/Example2/Example2.jl index eaef8178155..520c7e9209a 100644 --- a/tests/MyGame/Example2/Example2.jl +++ b/tests/MyGame/Example2/Example2.jl @@ -1,7 +1,3 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Example2 - module Example2 - include("Monster.jl") +include("Monster.jl") end diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl index 91b98e333fb..440848a546c 100644 --- a/tests/MyGame/Example2/Monster.jl +++ b/tests/MyGame/Example2/Monster.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: Example2 +module MyGame + +module Example2 import FlatBuffers @@ -10,3 +12,9 @@ FlatBuffers.@ALIGN(Monster, 1) Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) Monster(io::IO) = FlatBuffers.deserialize(io, Monster) + +end + + +end + diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl index 518cb89e3f0..9de4b1cba9b 100644 --- a/tests/MyGame/InParentNamespace.jl +++ b/tests/MyGame/InParentNamespace.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: MyGame +module MyGame import FlatBuffers @@ -10,3 +10,6 @@ FlatBuffers.@ALIGN(InParentNamespace, 1) InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) + +end + diff --git a/tests/MyGame/MyGame.jl b/tests/MyGame/MyGame.jl index 075ee85e35a..a4914e69067 100644 --- a/tests/MyGame/MyGame.jl +++ b/tests/MyGame/MyGame.jl @@ -1,9 +1,5 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: MyGame - module MyGame - include("InParentNamespace.jl") - include("Example2/Example2.jl") - include("Example/Example.jl") +include("InParentNamespace.jl") +include("Example2/Example2.jl") +include("Example/Example.jl") end diff --git a/tests/namespace_test/NamespaceA/NamespaceA.jl b/tests/namespace_test/NamespaceA/NamespaceA.jl index 5f8e0d98900..b38c579c542 100644 --- a/tests/namespace_test/NamespaceA/NamespaceA.jl +++ b/tests/namespace_test/NamespaceA/NamespaceA.jl @@ -1,9 +1,5 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: NamespaceA - module NamespaceA - include("NamespaceB/NamespaceB.jl") - include("TableInFirstNS.jl") - include("SecondTableInA.jl") +include("NamespaceB/NamespaceB.jl") +include("TableInFirstNS.jl") +include("SecondTableInA.jl") end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl index 9fa9777b485..461ee707793 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceB +module NamespaceA + +module NamespaceB @enum EnumInNestedNS::Int8 begin EnumInNestedNSA = 0 @@ -8,3 +10,9 @@ EnumInNestedNSC = 2 end + +end + + +end + diff --git a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl index 5dbfb19cf18..d5523138b2d 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl @@ -1,9 +1,5 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: NamespaceB - module NamespaceB - include("EnumInNestedNS.jl") - include("TableInNestedNS.jl") - include("StructInNestedNS.jl") +include("EnumInNestedNS.jl") +include("TableInNestedNS.jl") +include("StructInNestedNS.jl") end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl index 77aa0b205c1..602df5d33d4 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceB +module NamespaceA + +module NamespaceB import FlatBuffers @@ -12,3 +14,9 @@ FlatBuffers.@ALIGN(StructInNestedNS, 4) StructInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(StructInNestedNS, buf) StructInNestedNS(io::IO) = FlatBuffers.deserialize(io, StructInNestedNS) + +end + + +end + diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl index 06125901142..56be2e4f643 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -1,6 +1,8 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceB +module NamespaceA + +module NamespaceB import FlatBuffers @@ -14,3 +16,9 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInNestedNS} = [ TableInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInNestedNS, buf) TableInNestedNS(io::IO) = FlatBuffers.deserialize(io, TableInNestedNS) + +end + + +end + diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl index 6cdebe37fe7..85863907dd0 100644 --- a/tests/namespace_test/NamespaceA/SecondTableInA.jl +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceA +module NamespaceA import ..NamespaceC import FlatBuffers @@ -15,3 +15,6 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:SecondTableInA} = [ SecondTableInA(buf::AbstractVector{UInt8}) = FlatBuffers.read(SecondTableInA, buf) SecondTableInA(io::IO) = FlatBuffers.deserialize(io, SecondTableInA) + +end + diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl index ab26e22b09a..42da210f8e7 100644 --- a/tests/namespace_test/NamespaceA/TableInFirstNS.jl +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceA +module NamespaceA import .NamespaceB import FlatBuffers @@ -17,3 +17,6 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInFirstNS} = [ TableInFirstNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInFirstNS, buf) TableInFirstNS(io::IO) = FlatBuffers.deserialize(io, TableInFirstNS) + +end + diff --git a/tests/namespace_test/NamespaceC/NamespaceC.jl b/tests/namespace_test/NamespaceC/NamespaceC.jl index bc2c2417e76..720ac3fc819 100644 --- a/tests/namespace_test/NamespaceC/NamespaceC.jl +++ b/tests/namespace_test/NamespaceC/NamespaceC.jl @@ -1,7 +1,3 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: NamespaceC - module NamespaceC - include("TableInC.jl") +include("TableInC.jl") end diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl index c5f58206d48..bdeb9e2f90a 100644 --- a/tests/namespace_test/NamespaceC/TableInC.jl +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceC +module NamespaceC import ..NamespaceA import FlatBuffers @@ -16,3 +16,6 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInC} = [ TableInC(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInC, buf) TableInC(io::IO) = FlatBuffers.deserialize(io, TableInC) + +end + diff --git a/tests/namespace_test/NamespaceTest1.jl b/tests/namespace_test/NamespaceTest1.jl index 2f158d945f1..807b650b629 100644 --- a/tests/namespace_test/NamespaceTest1.jl +++ b/tests/namespace_test/NamespaceTest1.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceTest1 - -module NamespaceTest1 - include("NamespaceA/NamespaceA.jl") -end +include("NamespaceA/NamespaceB/EnumInNestedNS.jl") +include("NamespaceA/NamespaceB/TableInNestedNS.jl") +include("NamespaceA/NamespaceB/StructInNestedNS.jl") diff --git a/tests/namespace_test/NamespaceTest2.jl b/tests/namespace_test/NamespaceTest2.jl index 960f5a3a27f..174dd873d95 100644 --- a/tests/namespace_test/NamespaceTest2.jl +++ b/tests/namespace_test/NamespaceTest2.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: NamespaceTest2 - -module NamespaceTest2 - include("NamespaceA/NamespaceA.jl") - include("NamespaceC/NamespaceC.jl") -end +include("/NamespaceTest1.jl") +include("NamespaceC/TableInC.jl") +include("NamespaceA/SecondTableInA.jl") +include("NamespaceA/TableInFirstNS.jl") diff --git a/tests/union_vector/UnionVector.jl b/tests/union_vector/UnionVector.jl index f65c7a3310b..5cb0d7adf75 100644 --- a/tests/union_vector/UnionVector.jl +++ b/tests/union_vector/UnionVector.jl @@ -1,11 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - -module UnionVector - include("UnionVector/Attacker.jl") - include("UnionVector/Rapunzel.jl") - include("UnionVector/BookReader.jl") - include("UnionVector/Character.jl") - include("UnionVector/Movie.jl") -end +include("UnionVector/Attacker.jl") +include("UnionVector/Rapunzel.jl") +include("UnionVector/BookReader.jl") +include("UnionVector/Character.jl") +include("UnionVector/Movie.jl") diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl index e7553a68942..20dc9b1d828 100644 --- a/tests/union_vector/UnionVector/Attacker.jl +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - import FlatBuffers FlatBuffers.@with_kw mutable struct Attacker diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl index 253ea64c972..43c6f6d4631 100644 --- a/tests/union_vector/UnionVector/BookReader.jl +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - import FlatBuffers FlatBuffers.@STRUCT struct BookReader diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl index 603b1055be8..8b517cd30b3 100644 --- a/tests/union_vector/UnionVector/Character.jl +++ b/tests/union_vector/UnionVector/Character.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - import FlatBuffers FlatBuffers.@UNION(Character, ( diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl index ae8a7ac3932..cbf160107be 100644 --- a/tests/union_vector/UnionVector/Movie.jl +++ b/tests/union_vector/UnionVector/Movie.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - import FlatBuffers FlatBuffers.@with_kw mutable struct Movie diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl index b524a9fb0bd..04227dcabde 100644 --- a/tests/union_vector/UnionVector/Rapunzel.jl +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -1,7 +1,5 @@ # automatically generated by the FlatBuffers compiler, do not modify -# module: UnionVector - import FlatBuffers FlatBuffers.@STRUCT struct Rapunzel From bfd1ab404b3005949dcf2b56678b8d883f9c95d5 Mon Sep 17 00:00:00 2001 From: rowan Date: Wed, 9 Jan 2019 16:41:41 +1100 Subject: [PATCH 11/19] work in progress --- src/idl_gen_julia.cpp | 38 +++++++++++-------- tests/MonsterTest.jl | 7 ++++ tests/MyGame/Example/Ability.jl | 9 +---- tests/MyGame/Example/AnyAmbiguousAliases.jl | 9 +---- tests/MyGame/Example/AnyUniqueAliases.jl | 9 +---- tests/MyGame/Example/Any_.jl | 9 +---- tests/MyGame/Example/Color.jl | 9 +---- tests/MyGame/Example/Monster.jl | 9 +---- tests/MyGame/Example/Referrable.jl | 9 +---- tests/MyGame/Example/Stat.jl | 9 +---- tests/MyGame/Example/Test.jl | 9 +---- .../MyGame/Example/TestSimpleTableWithEnum.jl | 9 +---- tests/MyGame/Example/TypeAliases.jl | 9 +---- tests/MyGame/Example/Vec3.jl | 9 +---- tests/MyGame/Example2/Monster.jl | 9 +---- tests/MyGame/InParentNamespace.jl | 4 +- .../NamespaceA/NamespaceB/EnumInNestedNS.jl | 9 +---- .../NamespaceA/NamespaceB/StructInNestedNS.jl | 9 +---- .../NamespaceA/NamespaceB/TableInNestedNS.jl | 9 +---- .../NamespaceA/SecondTableInA.jl | 4 +- .../NamespaceA/TableInFirstNS.jl | 4 +- tests/namespace_test/NamespaceC/TableInC.jl | 4 +- tests/namespace_test/NamespaceTest1.jl | 2 + tests/namespace_test/NamespaceTest2.jl | 6 ++- tests/union_vector/UnionVector/Attacker.jl | 5 +++ tests/union_vector/UnionVector/BookReader.jl | 5 +++ tests/union_vector/UnionVector/Character.jl | 5 +++ tests/union_vector/UnionVector/Movie.jl | 5 +++ tests/union_vector/UnionVector/Rapunzel.jl | 5 +++ 29 files changed, 102 insertions(+), 136 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 14fb1bc4e36..55417693f23 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "flatbuffers/code_generators.h" @@ -115,6 +116,10 @@ class DepGraph { } return sorted_nodes; } + + size_t size() { + return adj.size(); + } }; class ModuleTable { @@ -213,8 +218,8 @@ class JuliaGenerator : public BaseGenerator { if (it->second.empty()) continue; auto dir = flatbuffers::StripFileName(it->second); auto basename = MakeCamel(flatbuffers::StripPath(flatbuffers::StripExtension(it->second))); - auto toinclude = (dir + kPathSeparator + basename) + JuliaFileExtension; - auto fullpath = path_ + kPathSeparator + toinclude; + auto toinclude = ConCatPathFileName(dir, basename + JuliaFileExtension); + auto fullpath = ConCatPathFileName(path_, toinclude); if (!FileExists(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; @@ -224,26 +229,33 @@ class JuliaGenerator : public BaseGenerator { std::string parent; std::string child; // Gather all parent namespaces for this namespace - // TODO: this is similar to idl_gen_js.cpp, maybe - // move into common place? for (auto component = (*it)->components.begin(); component != (*it)->components.end(); ++component) { if (parent.empty()) { parent = *component; child = *component; module_table_.AddDependency(root_module_, parent, child); + + code += "if !isdefined(@__MODULE__(), :" + *component + ") @__MODULE__().eval(:(module " + *component + " import FlatBuffers end)) end\n"; } else { child = parent + kPathSeparator + *component; // Add component to parent's list of children module_table_.AddDependency(parent, child, *component); + + std::string mod = parent; + std::replace(mod.begin(), mod.end(), kPathSeparator, '.'); + // Create module if it doesn't exist + code += "if !isdefined(" + mod + ", :" + *component + ") Core.eval(" + mod + ", :(module " + *component + " import FlatBuffers end)) end\n"; } parent = child; } } + auto sorted_modules = module_table_.SortedModuleNames(); // iterate through child modules first, then parents for (auto m = sorted_modules.rbegin(); m != sorted_modules.rend(); ++m) { - GenIncludes(module_table_.GetDependencies(*m), included, &code); + DepGraph *deps = module_table_.GetDependencies(*m); + GenIncludes(deps, included, &code); } auto filename = ConCatPathFileName(path_, root_module_) + JuliaFileExtension; if (!SaveFile(filename.c_str(), code, false)) return false; @@ -660,16 +672,12 @@ class JuliaGenerator : public BaseGenerator { void BeginFile(const Namespace &ns, std::string *code_ptr) const { auto &code = *code_ptr; code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; - for (size_t i = 0; i < ns.components.size(); i++) { - code += "module " + std::string(ns.components[i]) + "\n\n"; - } + code += "Core.eval(" + GetCanonicalName(ns, '.') + ", quote\n\n"; } - void EndFile(const Namespace &ns, std::string *code_ptr) const { + void EndFile(std::string *code_ptr) const { auto &code = *code_ptr; - for (size_t i = 0; i < ns.components.size(); i++) { - code += "\nend\n\n"; - } + code += "\nend)\n\n"; } std::string GetDirname(const Definition &def) const { @@ -719,10 +727,10 @@ class JuliaGenerator : public BaseGenerator { } // Canonical julia name of a namespace (Foo.Bar.Baz) - std::string GetCanonicalName(const Namespace &ns) const { + std::string GetCanonicalName(const Namespace &ns, char separator = kPathSeparator) const { std::string name; for (size_t i = 0; i < ns.components.size(); i++) { - if (i) name += kPathSeparator; + if (i) name += separator; name += std::string(ns.components[i]); } if (name.empty()) name = root_module_; @@ -751,7 +759,7 @@ class JuliaGenerator : public BaseGenerator { std::string code = ""; BeginFile(*def.defined_namespace, &code); code += declcode; - EndFile(*def.defined_namespace, &code); + EndFile(&code); auto filename = GetFilename(def); EnsureDirExists(GetDirname(def)); if (!SaveFile(filename.c_str(), code, false)) return false; diff --git a/tests/MonsterTest.jl b/tests/MonsterTest.jl index 1d49eda47d4..bbcbfc01a68 100644 --- a/tests/MonsterTest.jl +++ b/tests/MonsterTest.jl @@ -1,5 +1,12 @@ # automatically generated by the FlatBuffers compiler, do not modify +if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end +if !isdefined(MyGame, :OtherNameSpace) Core.eval(MyGame, :(module OtherNameSpace import FlatBuffers end)) end +if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end +if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end +if !isdefined(MyGame, :Example2) Core.eval(MyGame, :(module Example2 import FlatBuffers end)) end +if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end +if !isdefined(MyGame, :Example) Core.eval(MyGame, :(module Example import FlatBuffers end)) end include("MyGame/Example2/Monster.jl") include("MyGame/Example/Any_.jl") include("MyGame/Example/AnyUniqueAliases.jl") diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl index a42406c691f..4f6c8596cca 100644 --- a/tests/MyGame/Example/Ability.jl +++ b/tests/MyGame/Example/Ability.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -15,8 +13,5 @@ FlatBuffers.@ALIGN(Ability, 4) Ability(buf::AbstractVector{UInt8}) = FlatBuffers.read(Ability, buf) Ability(io::IO) = FlatBuffers.deserialize(io, Ability) -end - - -end +end) diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl index 96bc8437f54..780806c3b24 100644 --- a/tests/MyGame/Example/AnyAmbiguousAliases.jl +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -14,8 +12,5 @@ FlatBuffers.@UNION(AnyAmbiguousAliases, ( )) -end - - -end +end) diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl index 7c24aac8236..d4ddc18e3ad 100644 --- a/tests/MyGame/Example/AnyUniqueAliases.jl +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import ..Example2 import FlatBuffers @@ -15,8 +13,5 @@ FlatBuffers.@UNION(AnyUniqueAliases, ( )) -end - - -end +end) diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl index cfd66ea17d2..c2f621bd557 100644 --- a/tests/MyGame/Example/Any_.jl +++ b/tests/MyGame/Example/Any_.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import ..Example2 import FlatBuffers @@ -15,8 +13,5 @@ FlatBuffers.@UNION(Any_, ( )) -end - - -end +end) diff --git a/tests/MyGame/Example/Color.jl b/tests/MyGame/Example/Color.jl index 347fcd51f66..11cbaf58dd6 100644 --- a/tests/MyGame/Example/Color.jl +++ b/tests/MyGame/Example/Color.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote @enum Color::Int8 begin ColorRed = 1 @@ -11,8 +9,5 @@ module Example end -end - - -end +end) diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl index 67765f0f38a..9fb263b2b1d 100644 --- a/tests/MyGame/Example/Monster.jl +++ b/tests/MyGame/Example/Monster.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import ..InParentNamespace import FlatBuffers @@ -85,8 +83,5 @@ FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon" Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) Monster(io::IO) = FlatBuffers.deserialize(io, Monster) -end - - -end +end) diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl index 78f96a3bd88..cd80ede0b58 100644 --- a/tests/MyGame/Example/Referrable.jl +++ b/tests/MyGame/Example/Referrable.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -17,8 +15,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:Referrable} = [ Referrable(buf::AbstractVector{UInt8}) = FlatBuffers.read(Referrable, buf) Referrable(io::IO) = FlatBuffers.deserialize(io, Referrable) -end - - -end +end) diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl index b3d00b8eeba..ef3ce9d4c99 100644 --- a/tests/MyGame/Example/Stat.jl +++ b/tests/MyGame/Example/Stat.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -19,8 +17,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:Stat} = [ Stat(buf::AbstractVector{UInt8}) = FlatBuffers.read(Stat, buf) Stat(io::IO) = FlatBuffers.deserialize(io, Stat) -end - - -end +end) diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl index 04f8859ab83..912fa9e7701 100644 --- a/tests/MyGame/Example/Test.jl +++ b/tests/MyGame/Example/Test.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -15,8 +13,5 @@ FlatBuffers.@ALIGN(Test, 2) Test(buf::AbstractVector{UInt8}) = FlatBuffers.read(Test, buf) Test(io::IO) = FlatBuffers.deserialize(io, Test) -end - - -end +end) diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl index e68fd066f0c..88b8a3974f6 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.jl +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -17,8 +15,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TestSimpleTableWithEnum} = [ TestSimpleTableWithEnum(buf::AbstractVector{UInt8}) = FlatBuffers.read(TestSimpleTableWithEnum, buf) TestSimpleTableWithEnum(io::IO) = FlatBuffers.deserialize(io, TestSimpleTableWithEnum) -end - - -end +end) diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl index 4ddb3c19157..75c940bbf66 100644 --- a/tests/MyGame/Example/TypeAliases.jl +++ b/tests/MyGame/Example/TypeAliases.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -30,8 +28,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TypeAliases} = [ TypeAliases(buf::AbstractVector{UInt8}) = FlatBuffers.read(TypeAliases, buf) TypeAliases(io::IO) = FlatBuffers.deserialize(io, TypeAliases) -end - - -end +end) diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl index 1809b4895ce..07a6bde1bfd 100644 --- a/tests/MyGame/Example/Vec3.jl +++ b/tests/MyGame/Example/Vec3.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example +Core.eval(MyGame.Example, quote import FlatBuffers @@ -19,8 +17,5 @@ FlatBuffers.@ALIGN(Vec3, 16) Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) -end - - -end +end) diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl index 440848a546c..5974b4ca4c8 100644 --- a/tests/MyGame/Example2/Monster.jl +++ b/tests/MyGame/Example2/Monster.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame - -module Example2 +Core.eval(MyGame.Example2, quote import FlatBuffers @@ -13,8 +11,5 @@ FlatBuffers.@ALIGN(Monster, 1) Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) Monster(io::IO) = FlatBuffers.deserialize(io, Monster) -end - - -end +end) diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl index 9de4b1cba9b..3b351709e8a 100644 --- a/tests/MyGame/InParentNamespace.jl +++ b/tests/MyGame/InParentNamespace.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module MyGame +Core.eval(MyGame, quote import FlatBuffers @@ -11,5 +11,5 @@ FlatBuffers.@ALIGN(InParentNamespace, 1) InParentNamespace(buf::AbstractVector{UInt8}) = FlatBuffers.read(InParentNamespace, buf) InParentNamespace(io::IO) = FlatBuffers.deserialize(io, InParentNamespace) -end +end) diff --git a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl index 461ee707793..1701427b546 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceA - -module NamespaceB +Core.eval(NamespaceA.NamespaceB, quote @enum EnumInNestedNS::Int8 begin EnumInNestedNSA = 0 @@ -11,8 +9,5 @@ module NamespaceB end -end - - -end +end) diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl index 602df5d33d4..379b7501fd5 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceA - -module NamespaceB +Core.eval(NamespaceA.NamespaceB, quote import FlatBuffers @@ -15,8 +13,5 @@ FlatBuffers.@ALIGN(StructInNestedNS, 4) StructInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(StructInNestedNS, buf) StructInNestedNS(io::IO) = FlatBuffers.deserialize(io, StructInNestedNS) -end - - -end +end) diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl index 56be2e4f643..8a69098fc75 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -1,8 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceA - -module NamespaceB +Core.eval(NamespaceA.NamespaceB, quote import FlatBuffers @@ -17,8 +15,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInNestedNS} = [ TableInNestedNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInNestedNS, buf) TableInNestedNS(io::IO) = FlatBuffers.deserialize(io, TableInNestedNS) -end - - -end +end) diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl index 85863907dd0..06f52fc41e5 100644 --- a/tests/namespace_test/NamespaceA/SecondTableInA.jl +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceA +Core.eval(NamespaceA, quote import ..NamespaceC import FlatBuffers @@ -16,5 +16,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:SecondTableInA} = [ SecondTableInA(buf::AbstractVector{UInt8}) = FlatBuffers.read(SecondTableInA, buf) SecondTableInA(io::IO) = FlatBuffers.deserialize(io, SecondTableInA) -end +end) diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl index 42da210f8e7..9c908b13c1c 100644 --- a/tests/namespace_test/NamespaceA/TableInFirstNS.jl +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceA +Core.eval(NamespaceA, quote import .NamespaceB import FlatBuffers @@ -18,5 +18,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInFirstNS} = [ TableInFirstNS(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInFirstNS, buf) TableInFirstNS(io::IO) = FlatBuffers.deserialize(io, TableInFirstNS) -end +end) diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl index bdeb9e2f90a..228d2c9cfbf 100644 --- a/tests/namespace_test/NamespaceC/TableInC.jl +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -module NamespaceC +Core.eval(NamespaceC, quote import ..NamespaceA import FlatBuffers @@ -17,5 +17,5 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:TableInC} = [ TableInC(buf::AbstractVector{UInt8}) = FlatBuffers.read(TableInC, buf) TableInC(io::IO) = FlatBuffers.deserialize(io, TableInC) -end +end) diff --git a/tests/namespace_test/NamespaceTest1.jl b/tests/namespace_test/NamespaceTest1.jl index 807b650b629..e169884d83d 100644 --- a/tests/namespace_test/NamespaceTest1.jl +++ b/tests/namespace_test/NamespaceTest1.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end +if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end include("NamespaceA/NamespaceB/EnumInNestedNS.jl") include("NamespaceA/NamespaceB/TableInNestedNS.jl") include("NamespaceA/NamespaceB/StructInNestedNS.jl") diff --git a/tests/namespace_test/NamespaceTest2.jl b/tests/namespace_test/NamespaceTest2.jl index 174dd873d95..9da42a5ace5 100644 --- a/tests/namespace_test/NamespaceTest2.jl +++ b/tests/namespace_test/NamespaceTest2.jl @@ -1,6 +1,10 @@ # automatically generated by the FlatBuffers compiler, do not modify -include("/NamespaceTest1.jl") +include("NamespaceTest1.jl") +if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end +if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end +if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end +if !isdefined(@__MODULE__(), :NamespaceC) @__MODULE__().eval(:(module NamespaceC import FlatBuffers end)) end include("NamespaceC/TableInC.jl") include("NamespaceA/SecondTableInA.jl") include("NamespaceA/TableInFirstNS.jl") diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl index 20dc9b1d828..12f17e7b5a3 100644 --- a/tests/union_vector/UnionVector/Attacker.jl +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +Core.eval(UnionVector, quote + import FlatBuffers FlatBuffers.@with_kw mutable struct Attacker @@ -12,3 +14,6 @@ FlatBuffers.slot_offsets(::Type{T}) where {T<:Attacker} = [ Attacker(buf::AbstractVector{UInt8}) = FlatBuffers.read(Attacker, buf) Attacker(io::IO) = FlatBuffers.deserialize(io, Attacker) + +end) + diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl index 43c6f6d4631..1aeb3df5594 100644 --- a/tests/union_vector/UnionVector/BookReader.jl +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +Core.eval(UnionVector, quote + import FlatBuffers FlatBuffers.@STRUCT struct BookReader @@ -9,3 +11,6 @@ FlatBuffers.@ALIGN(BookReader, 4) BookReader(buf::AbstractVector{UInt8}) = FlatBuffers.read(BookReader, buf) BookReader(io::IO) = FlatBuffers.deserialize(io, BookReader) + +end) + diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl index 8b517cd30b3..c18ddd7c72a 100644 --- a/tests/union_vector/UnionVector/Character.jl +++ b/tests/union_vector/UnionVector/Character.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +Core.eval(UnionVector, quote + import FlatBuffers FlatBuffers.@UNION(Character, ( @@ -12,3 +14,6 @@ FlatBuffers.@UNION(Character, ( String, )) + +end) + diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl index cbf160107be..cc599687bb5 100644 --- a/tests/union_vector/UnionVector/Movie.jl +++ b/tests/union_vector/UnionVector/Movie.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +Core.eval(UnionVector, quote + import FlatBuffers FlatBuffers.@with_kw mutable struct Movie @@ -17,3 +19,6 @@ FlatBuffers.file_identifier(::Type{T}) where {T<:Movie} = "MOVI" Movie(buf::AbstractVector{UInt8}) = FlatBuffers.read(Movie, buf) Movie(io::IO) = FlatBuffers.deserialize(io, Movie) + +end) + diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl index 04227dcabde..2fb632db439 100644 --- a/tests/union_vector/UnionVector/Rapunzel.jl +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -1,5 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify +Core.eval(UnionVector, quote + import FlatBuffers FlatBuffers.@STRUCT struct Rapunzel @@ -9,3 +11,6 @@ FlatBuffers.@ALIGN(Rapunzel, 4) Rapunzel(buf::AbstractVector{UInt8}) = FlatBuffers.read(Rapunzel, buf) Rapunzel(io::IO) = FlatBuffers.deserialize(io, Rapunzel) + +end) + From f6cd1a40a997ba5f0232eb6efa1ee0f4b4f46364 Mon Sep 17 00:00:00 2001 From: rowan Date: Wed, 9 Jan 2019 17:31:19 +1100 Subject: [PATCH 12/19] stateless flatbuffers --- src/idl_gen_julia.cpp | 32 ++++++++++++------- tests/MyGame/Example/Ability.jl | 1 - tests/MyGame/Example/AnyAmbiguousAliases.jl | 1 - tests/MyGame/Example/AnyUniqueAliases.jl | 1 - tests/MyGame/Example/Any_.jl | 1 - tests/MyGame/Example/Example.jl | 14 -------- tests/MyGame/Example/Monster.jl | 1 - tests/MyGame/Example/Referrable.jl | 1 - tests/MyGame/Example/Stat.jl | 1 - tests/MyGame/Example/Test.jl | 1 - .../MyGame/Example/TestSimpleTableWithEnum.jl | 1 - tests/MyGame/Example/TypeAliases.jl | 1 - tests/MyGame/Example/Vec3.jl | 1 - tests/MyGame/Example2/Example2.jl | 3 -- tests/MyGame/Example2/Monster.jl | 1 - tests/MyGame/InParentNamespace.jl | 1 - tests/MyGame/MyGame.jl | 5 --- ...nsterTest.jl => monster_test_generated.jl} | 14 ++++---- tests/namespace_test/NamespaceA/NamespaceA.jl | 5 --- .../NamespaceA/NamespaceB/NamespaceB.jl | 5 --- .../NamespaceA/NamespaceB/StructInNestedNS.jl | 1 - .../NamespaceA/NamespaceB/TableInNestedNS.jl | 1 - .../NamespaceA/SecondTableInA.jl | 1 - .../NamespaceA/TableInFirstNS.jl | 1 - tests/namespace_test/NamespaceC/NamespaceC.jl | 3 -- tests/namespace_test/NamespaceC/TableInC.jl | 1 - ...eTest1.jl => namespace_test1_generated.jl} | 2 ++ ...eTest2.jl => namespace_test2_generated.jl} | 9 +++--- tests/union_vector/UnionVector/Attacker.jl | 1 - tests/union_vector/UnionVector/BookReader.jl | 1 - tests/union_vector/UnionVector/Character.jl | 1 - tests/union_vector/UnionVector/Movie.jl | 1 - tests/union_vector/UnionVector/Rapunzel.jl | 1 - ...ionVector.jl => union_vector_generated.jl} | 0 34 files changed, 34 insertions(+), 81 deletions(-) delete mode 100644 tests/MyGame/Example/Example.jl delete mode 100644 tests/MyGame/Example2/Example2.jl delete mode 100644 tests/MyGame/MyGame.jl rename tests/{MonsterTest.jl => monster_test_generated.jl} (77%) delete mode 100644 tests/namespace_test/NamespaceA/NamespaceA.jl delete mode 100644 tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl delete mode 100644 tests/namespace_test/NamespaceC/NamespaceC.jl rename tests/namespace_test/{NamespaceTest1.jl => namespace_test1_generated.jl} (89%) rename tests/namespace_test/{NamespaceTest2.jl => namespace_test2_generated.jl} (78%) rename tests/union_vector/{UnionVector.jl => union_vector_generated.jl} (100%) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 55417693f23..218498bd63b 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -210,6 +210,7 @@ class JuliaGenerator : public BaseGenerator { bool GenTopLevel(void) { std::string code = "# "; std::set included; + std::set toplevel; code += FlatBuffersGeneratedWarning(); code += "\n\n"; // include other included .fbs files first @@ -235,8 +236,11 @@ class JuliaGenerator : public BaseGenerator { parent = *component; child = *component; module_table_.AddDependency(root_module_, parent, child); - - code += "if !isdefined(@__MODULE__(), :" + *component + ") @__MODULE__().eval(:(module " + *component + " import FlatBuffers end)) end\n"; + // only create toplevel modules once + if (toplevel.find(*component) == toplevel.end()) { + code += "if !isdefined(@__MODULE__(), :" + *component + ") @__MODULE__().eval(:(module " + *component + " import FlatBuffers end)) end\n"; + toplevel.insert(*component); + } } else { child = parent + kPathSeparator + *component; // Add component to parent's list of children @@ -253,11 +257,10 @@ class JuliaGenerator : public BaseGenerator { auto sorted_modules = module_table_.SortedModuleNames(); // iterate through child modules first, then parents - for (auto m = sorted_modules.rbegin(); m != sorted_modules.rend(); ++m) { - DepGraph *deps = module_table_.GetDependencies(*m); - GenIncludes(deps, included, &code); + for (auto m = sorted_modules.begin(); m != sorted_modules.end(); ++m) { + GenIncludes(*m, included, &code); } - auto filename = ConCatPathFileName(path_, root_module_) + JuliaFileExtension; + auto filename = ConCatPathFileName(path_, StripExtension(file_name_) + "_generated") + JuliaFileExtension; if (!SaveFile(filename.c_str(), code, false)) return false; return true; } @@ -556,9 +559,7 @@ class JuliaGenerator : public BaseGenerator { void GenObject(const StructDef &struct_def, std::string *code_ptr) { if (struct_def.generated) return; - // always need FlatBuffers package for structs std::set imports; - imports.insert(JuliaPackageName); bool has_defaults = false; @@ -589,9 +590,7 @@ class JuliaGenerator : public BaseGenerator { void GenUnion(const EnumDef &enum_def, std::string *code_ptr) { if (enum_def.generated) return; - // always need FlatBuffers package for unions std::set imports; - imports.insert(JuliaPackageName); auto union_name = NormalizedName(enum_def); BeginUnion(union_name, code_ptr); for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); @@ -704,14 +703,23 @@ class JuliaGenerator : public BaseGenerator { return ""; } - bool GenIncludes(DepGraph *children, std::set &included, std::string *code_ptr) { + bool GenIncludes(std::string &mod, std::set &included, std::string *code_ptr) { auto &code = *code_ptr; + DepGraph *children = module_table_.GetDependencies(mod); + if (mod != root_module_) + code += "# module: " + mod + "\n"; // Include all the contents of this module in the right order auto sorted_children = children->TopSort(); for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); ++it) { std::string child = *it; + // this is not a direct child of this module, so don't include here + if (child.find(mod) == std::string::npos + || child.length() < (mod.length() + 1) + || child.substr(mod.length() + 1).find(kPathSeparator) != std::string::npos) + continue; + // this is a module or something that's already been included, don't include here if (module_table_.IsModule(child) || included.find(child) != included.end()) continue; // If the file doesn't exist, don't include it @@ -721,7 +729,7 @@ class JuliaGenerator : public BaseGenerator { std::string fullpath = ConCatPathFileName(path_, toinclude); if (!module_table_.IsFile(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; - included.insert(child); + // included.insert(child); } return true; } diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl index 4f6c8596cca..d1e5825b25b 100644 --- a/tests/MyGame/Example/Ability.jl +++ b/tests/MyGame/Example/Ability.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@STRUCT struct Ability id::UInt32 diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl index 780806c3b24..a52455ca228 100644 --- a/tests/MyGame/Example/AnyAmbiguousAliases.jl +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@UNION(AnyAmbiguousAliases, ( Nothing, diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl index d4ddc18e3ad..bf5328ab321 100644 --- a/tests/MyGame/Example/AnyUniqueAliases.jl +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -3,7 +3,6 @@ Core.eval(MyGame.Example, quote import ..Example2 -import FlatBuffers FlatBuffers.@UNION(AnyUniqueAliases, ( Nothing, diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl index c2f621bd557..f22845237a4 100644 --- a/tests/MyGame/Example/Any_.jl +++ b/tests/MyGame/Example/Any_.jl @@ -3,7 +3,6 @@ Core.eval(MyGame.Example, quote import ..Example2 -import FlatBuffers FlatBuffers.@UNION(Any_, ( Nothing, diff --git a/tests/MyGame/Example/Example.jl b/tests/MyGame/Example/Example.jl deleted file mode 100644 index e35be429849..00000000000 --- a/tests/MyGame/Example/Example.jl +++ /dev/null @@ -1,14 +0,0 @@ -module Example -include("Color.jl") -include("Test.jl") -include("Vec3.jl") -include("Stat.jl") -include("Ability.jl") -include("Referrable.jl") -include("TestSimpleTableWithEnum.jl") -include("AnyUniqueAliases.jl") -include("AnyAmbiguousAliases.jl") -include("Monster.jl") -include("Any_.jl") -include("TypeAliases.jl") -end diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl index 9fb263b2b1d..04d52d6fc19 100644 --- a/tests/MyGame/Example/Monster.jl +++ b/tests/MyGame/Example/Monster.jl @@ -3,7 +3,6 @@ Core.eval(MyGame.Example, quote import ..InParentNamespace -import FlatBuffers FlatBuffers.@with_kw mutable struct Monster #= diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl index cd80ede0b58..139b936c19e 100644 --- a/tests/MyGame/Example/Referrable.jl +++ b/tests/MyGame/Example/Referrable.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct Referrable id::UInt64 = 0 diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl index ef3ce9d4c99..9a99a2b377e 100644 --- a/tests/MyGame/Example/Stat.jl +++ b/tests/MyGame/Example/Stat.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct Stat id::String = "" diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl index 912fa9e7701..3a7dd43ab86 100644 --- a/tests/MyGame/Example/Test.jl +++ b/tests/MyGame/Example/Test.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@STRUCT struct Test a::Int16 diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl index 88b8a3974f6..9084d6c2415 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.jl +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum color::Color = 2 diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl index 75c940bbf66..7e3f0583fc0 100644 --- a/tests/MyGame/Example/TypeAliases.jl +++ b/tests/MyGame/Example/TypeAliases.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct TypeAliases i8::Int8 = 0 diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl index 07a6bde1bfd..5e5d7075c3e 100644 --- a/tests/MyGame/Example/Vec3.jl +++ b/tests/MyGame/Example/Vec3.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example, quote -import FlatBuffers FlatBuffers.@STRUCT struct Vec3 x::Float32 diff --git a/tests/MyGame/Example2/Example2.jl b/tests/MyGame/Example2/Example2.jl deleted file mode 100644 index 520c7e9209a..00000000000 --- a/tests/MyGame/Example2/Example2.jl +++ /dev/null @@ -1,3 +0,0 @@ -module Example2 -include("Monster.jl") -end diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl index 5974b4ca4c8..9e5e7e7a313 100644 --- a/tests/MyGame/Example2/Monster.jl +++ b/tests/MyGame/Example2/Monster.jl @@ -2,7 +2,6 @@ Core.eval(MyGame.Example2, quote -import FlatBuffers mutable struct Monster end diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl index 3b351709e8a..f48506b7863 100644 --- a/tests/MyGame/InParentNamespace.jl +++ b/tests/MyGame/InParentNamespace.jl @@ -2,7 +2,6 @@ Core.eval(MyGame, quote -import FlatBuffers mutable struct InParentNamespace end diff --git a/tests/MyGame/MyGame.jl b/tests/MyGame/MyGame.jl deleted file mode 100644 index a4914e69067..00000000000 --- a/tests/MyGame/MyGame.jl +++ /dev/null @@ -1,5 +0,0 @@ -module MyGame -include("InParentNamespace.jl") -include("Example2/Example2.jl") -include("Example/Example.jl") -end diff --git a/tests/MonsterTest.jl b/tests/monster_test_generated.jl similarity index 77% rename from tests/MonsterTest.jl rename to tests/monster_test_generated.jl index bbcbfc01a68..5c705fb9206 100644 --- a/tests/MonsterTest.jl +++ b/tests/monster_test_generated.jl @@ -2,14 +2,11 @@ if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end if !isdefined(MyGame, :OtherNameSpace) Core.eval(MyGame, :(module OtherNameSpace import FlatBuffers end)) end -if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end -if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end if !isdefined(MyGame, :Example2) Core.eval(MyGame, :(module Example2 import FlatBuffers end)) end -if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end if !isdefined(MyGame, :Example) Core.eval(MyGame, :(module Example import FlatBuffers end)) end -include("MyGame/Example2/Monster.jl") -include("MyGame/Example/Any_.jl") -include("MyGame/Example/AnyUniqueAliases.jl") +# module: MyGame +include("MyGame/InParentNamespace.jl") +# module: MyGame/Example include("MyGame/Example/Color.jl") include("MyGame/Example/Test.jl") include("MyGame/Example/Vec3.jl") @@ -17,7 +14,10 @@ include("MyGame/Example/Stat.jl") include("MyGame/Example/Ability.jl") include("MyGame/Example/Referrable.jl") include("MyGame/Example/TestSimpleTableWithEnum.jl") +include("MyGame/Example/AnyUniqueAliases.jl") include("MyGame/Example/AnyAmbiguousAliases.jl") include("MyGame/Example/Monster.jl") +include("MyGame/Example/Any_.jl") include("MyGame/Example/TypeAliases.jl") -include("MyGame/InParentNamespace.jl") +# module: MyGame/Example2 +include("MyGame/Example2/Monster.jl") diff --git a/tests/namespace_test/NamespaceA/NamespaceA.jl b/tests/namespace_test/NamespaceA/NamespaceA.jl deleted file mode 100644 index b38c579c542..00000000000 --- a/tests/namespace_test/NamespaceA/NamespaceA.jl +++ /dev/null @@ -1,5 +0,0 @@ -module NamespaceA -include("NamespaceB/NamespaceB.jl") -include("TableInFirstNS.jl") -include("SecondTableInA.jl") -end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl b/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl deleted file mode 100644 index d5523138b2d..00000000000 --- a/tests/namespace_test/NamespaceA/NamespaceB/NamespaceB.jl +++ /dev/null @@ -1,5 +0,0 @@ -module NamespaceB -include("EnumInNestedNS.jl") -include("TableInNestedNS.jl") -include("StructInNestedNS.jl") -end diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl index 379b7501fd5..9d0ace543de 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -2,7 +2,6 @@ Core.eval(NamespaceA.NamespaceB, quote -import FlatBuffers FlatBuffers.@STRUCT struct StructInNestedNS a::Int32 diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl index 8a69098fc75..4f0bde9cb70 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -2,7 +2,6 @@ Core.eval(NamespaceA.NamespaceB, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct TableInNestedNS foo::Int32 = 0 diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl index 06f52fc41e5..12bc220fa04 100644 --- a/tests/namespace_test/NamespaceA/SecondTableInA.jl +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -3,7 +3,6 @@ Core.eval(NamespaceA, quote import ..NamespaceC -import FlatBuffers FlatBuffers.@with_kw mutable struct SecondTableInA refer_to_c::Union{NamespaceC.TableInC, Nothing} = nothing diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl index 9c908b13c1c..2a88bd366ff 100644 --- a/tests/namespace_test/NamespaceA/TableInFirstNS.jl +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -3,7 +3,6 @@ Core.eval(NamespaceA, quote import .NamespaceB -import FlatBuffers FlatBuffers.@with_kw mutable struct TableInFirstNS foo_table::Union{NamespaceB.TableInNestedNS, Nothing} = nothing diff --git a/tests/namespace_test/NamespaceC/NamespaceC.jl b/tests/namespace_test/NamespaceC/NamespaceC.jl deleted file mode 100644 index 720ac3fc819..00000000000 --- a/tests/namespace_test/NamespaceC/NamespaceC.jl +++ /dev/null @@ -1,3 +0,0 @@ -module NamespaceC -include("TableInC.jl") -end diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl index 228d2c9cfbf..567610c128a 100644 --- a/tests/namespace_test/NamespaceC/TableInC.jl +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -3,7 +3,6 @@ Core.eval(NamespaceC, quote import ..NamespaceA -import FlatBuffers FlatBuffers.@with_kw mutable struct TableInC refer_to_a1::Union{NamespaceA.TableInFirstNS, Nothing} = nothing diff --git a/tests/namespace_test/NamespaceTest1.jl b/tests/namespace_test/namespace_test1_generated.jl similarity index 89% rename from tests/namespace_test/NamespaceTest1.jl rename to tests/namespace_test/namespace_test1_generated.jl index e169884d83d..c607cec73fc 100644 --- a/tests/namespace_test/NamespaceTest1.jl +++ b/tests/namespace_test/namespace_test1_generated.jl @@ -2,6 +2,8 @@ if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end +# module: NamespaceA +# module: NamespaceA/NamespaceB include("NamespaceA/NamespaceB/EnumInNestedNS.jl") include("NamespaceA/NamespaceB/TableInNestedNS.jl") include("NamespaceA/NamespaceB/StructInNestedNS.jl") diff --git a/tests/namespace_test/NamespaceTest2.jl b/tests/namespace_test/namespace_test2_generated.jl similarity index 78% rename from tests/namespace_test/NamespaceTest2.jl rename to tests/namespace_test/namespace_test2_generated.jl index 9da42a5ace5..df2299a8d64 100644 --- a/tests/namespace_test/NamespaceTest2.jl +++ b/tests/namespace_test/namespace_test2_generated.jl @@ -1,10 +1,11 @@ # automatically generated by the FlatBuffers compiler, do not modify -include("NamespaceTest1.jl") if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end -if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end if !isdefined(@__MODULE__(), :NamespaceC) @__MODULE__().eval(:(module NamespaceC import FlatBuffers end)) end -include("NamespaceC/TableInC.jl") -include("NamespaceA/SecondTableInA.jl") +# module: NamespaceA include("NamespaceA/TableInFirstNS.jl") +include("NamespaceA/SecondTableInA.jl") +# module: NamespaceA/NamespaceB +# module: NamespaceC +include("NamespaceC/TableInC.jl") diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl index 12f17e7b5a3..cc025dd14f2 100644 --- a/tests/union_vector/UnionVector/Attacker.jl +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -2,7 +2,6 @@ Core.eval(UnionVector, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct Attacker sword_attack_damage::Int32 = 0 diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl index 1aeb3df5594..8d21d80c61e 100644 --- a/tests/union_vector/UnionVector/BookReader.jl +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -2,7 +2,6 @@ Core.eval(UnionVector, quote -import FlatBuffers FlatBuffers.@STRUCT struct BookReader books_read::Int32 diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl index c18ddd7c72a..34076927d22 100644 --- a/tests/union_vector/UnionVector/Character.jl +++ b/tests/union_vector/UnionVector/Character.jl @@ -2,7 +2,6 @@ Core.eval(UnionVector, quote -import FlatBuffers FlatBuffers.@UNION(Character, ( Nothing, diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl index cc599687bb5..d1ea33b3d88 100644 --- a/tests/union_vector/UnionVector/Movie.jl +++ b/tests/union_vector/UnionVector/Movie.jl @@ -2,7 +2,6 @@ Core.eval(UnionVector, quote -import FlatBuffers FlatBuffers.@with_kw mutable struct Movie main_character_type::UInt8 = 0 diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl index 2fb632db439..6ce331a07f4 100644 --- a/tests/union_vector/UnionVector/Rapunzel.jl +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -2,7 +2,6 @@ Core.eval(UnionVector, quote -import FlatBuffers FlatBuffers.@STRUCT struct Rapunzel hair_length::Int32 diff --git a/tests/union_vector/UnionVector.jl b/tests/union_vector/union_vector_generated.jl similarity index 100% rename from tests/union_vector/UnionVector.jl rename to tests/union_vector/union_vector_generated.jl From c3103a3b5766396ee67d51a6558502a2b1f3b70f Mon Sep 17 00:00:00 2001 From: rowan Date: Wed, 9 Jan 2019 17:33:50 +1100 Subject: [PATCH 13/19] clang format --- src/idl_gen_julia.cpp | 60 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 218498bd63b..abdef31f343 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -16,10 +16,10 @@ // loosely based on idl_gen_python.cpp +#include #include #include #include -#include #include #include "flatbuffers/code_generators.h" @@ -117,9 +117,7 @@ class DepGraph { return sorted_nodes; } - size_t size() { - return adj.size(); - } + size_t size() { return adj.size(); } }; class ModuleTable { @@ -215,14 +213,14 @@ class JuliaGenerator : public BaseGenerator { code += "\n\n"; // include other included .fbs files first for (auto it = parser_.included_files_.begin(); - it != parser_.included_files_.end(); ++it) { + it != parser_.included_files_.end(); ++it) { if (it->second.empty()) continue; auto dir = flatbuffers::StripFileName(it->second); - auto basename = MakeCamel(flatbuffers::StripPath(flatbuffers::StripExtension(it->second))); + auto basename = MakeCamel( + flatbuffers::StripPath(flatbuffers::StripExtension(it->second))); auto toinclude = ConCatPathFileName(dir, basename + JuliaFileExtension); auto fullpath = ConCatPathFileName(path_, toinclude); - if (!FileExists(fullpath.c_str())) - continue; + if (!FileExists(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; } for (auto it = parser_.namespaces_.begin(); it != parser_.namespaces_.end(); @@ -238,29 +236,35 @@ class JuliaGenerator : public BaseGenerator { module_table_.AddDependency(root_module_, parent, child); // only create toplevel modules once if (toplevel.find(*component) == toplevel.end()) { - code += "if !isdefined(@__MODULE__(), :" + *component + ") @__MODULE__().eval(:(module " + *component + " import FlatBuffers end)) end\n"; + code += "if !isdefined(@__MODULE__(), :" + *component + + ") @__MODULE__().eval(:(module " + *component + + " import FlatBuffers end)) end\n"; toplevel.insert(*component); } } else { child = parent + kPathSeparator + *component; // Add component to parent's list of children module_table_.AddDependency(parent, child, *component); - + std::string mod = parent; std::replace(mod.begin(), mod.end(), kPathSeparator, '.'); // Create module if it doesn't exist - code += "if !isdefined(" + mod + ", :" + *component + ") Core.eval(" + mod + ", :(module " + *component + " import FlatBuffers end)) end\n"; + code += "if !isdefined(" + mod + ", :" + *component + ") Core.eval(" + + mod + ", :(module " + *component + + " import FlatBuffers end)) end\n"; } parent = child; } } - + auto sorted_modules = module_table_.SortedModuleNames(); // iterate through child modules first, then parents for (auto m = sorted_modules.begin(); m != sorted_modules.end(); ++m) { GenIncludes(*m, included, &code); } - auto filename = ConCatPathFileName(path_, StripExtension(file_name_) + "_generated") + JuliaFileExtension; + auto filename = + ConCatPathFileName(path_, StripExtension(file_name_) + "_generated") + + JuliaFileExtension; if (!SaveFile(filename.c_str(), code, false)) return false; return true; } @@ -624,7 +628,7 @@ class JuliaGenerator : public BaseGenerator { static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { - // clang-format off +// clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #JLTYPE, @@ -703,11 +707,11 @@ class JuliaGenerator : public BaseGenerator { return ""; } - bool GenIncludes(std::string &mod, std::set &included, std::string *code_ptr) { + bool GenIncludes(std::string &mod, std::set &included, + std::string *code_ptr) { auto &code = *code_ptr; - DepGraph *children = module_table_.GetDependencies(mod); - if (mod != root_module_) - code += "# module: " + mod + "\n"; + DepGraph *children = module_table_.GetDependencies(mod); + if (mod != root_module_) code += "# module: " + mod + "\n"; // Include all the contents of this module in the right order auto sorted_children = children->TopSort(); for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); @@ -715,13 +719,16 @@ class JuliaGenerator : public BaseGenerator { std::string child = *it; // this is not a direct child of this module, so don't include here - if (child.find(mod) == std::string::npos - || child.length() < (mod.length() + 1) - || child.substr(mod.length() + 1).find(kPathSeparator) != std::string::npos) - continue; - // this is a module or something that's already been included, don't include here - if (module_table_.IsModule(child) || included.find(child) != included.end()) - continue; + if (child.find(mod) == std::string::npos || + child.length() < (mod.length() + 1) || + child.substr(mod.length() + 1).find(kPathSeparator) != + std::string::npos) + continue; + // this is a module or something that's already been included, don't + // include here + if (module_table_.IsModule(child) || + included.find(child) != included.end()) + continue; // If the file doesn't exist, don't include it // TODO: this doesn't allow types which reference each other, // but Julia doesn't support this yet anyway @@ -735,7 +742,8 @@ class JuliaGenerator : public BaseGenerator { } // Canonical julia name of a namespace (Foo.Bar.Baz) - std::string GetCanonicalName(const Namespace &ns, char separator = kPathSeparator) const { + std::string GetCanonicalName(const Namespace &ns, + char separator = kPathSeparator) const { std::string name; for (size_t i = 0; i < ns.components.size(); i++) { if (i) name += separator; From d4e9df38845db3d051c5ed7c1b12cf36cd5d84b5 Mon Sep 17 00:00:00 2001 From: rowan Date: Thu, 10 Jan 2019 09:52:17 +1100 Subject: [PATCH 14/19] stateless julia code generation --- samples/Monster.jl | 7 ---- samples/MyGame/MyGame.jl | 7 ---- samples/MyGame/Sample/Color.jl | 10 ----- samples/MyGame/Sample/Equipment.jl | 11 ------ samples/MyGame/Sample/Monster.jl | 27 ------------- samples/MyGame/Sample/Sample.jl | 11 ------ samples/MyGame/Sample/Vec3.jl | 15 -------- samples/MyGame/Sample/Weapon.jl | 17 --------- samples/monster_generated.jl | 9 +++++ samples/sample_binary.jl | 2 +- src/idl_gen_julia.cpp | 38 ++++++++++--------- tests/MyGame/Example/Ability.jl | 2 +- tests/MyGame/Example/AnyAmbiguousAliases.jl | 2 +- tests/MyGame/Example/AnyUniqueAliases.jl | 2 +- tests/MyGame/Example/Any_.jl | 2 +- tests/MyGame/Example/Color.jl | 2 +- tests/MyGame/Example/Monster.jl | 2 +- tests/MyGame/Example/Referrable.jl | 2 +- tests/MyGame/Example/Stat.jl | 2 +- tests/MyGame/Example/Test.jl | 2 +- .../MyGame/Example/TestSimpleTableWithEnum.jl | 2 +- tests/MyGame/Example/TypeAliases.jl | 2 +- tests/MyGame/Example/Vec3.jl | 2 +- tests/MyGame/Example2/Monster.jl | 2 +- tests/MyGame/InParentNamespace.jl | 2 +- tests/monster_test_generated.jl | 11 ++---- .../NamespaceA/NamespaceB/EnumInNestedNS.jl | 2 +- .../NamespaceA/NamespaceB/StructInNestedNS.jl | 2 +- .../NamespaceA/NamespaceB/TableInNestedNS.jl | 2 +- .../NamespaceA/SecondTableInA.jl | 2 +- .../NamespaceA/TableInFirstNS.jl | 2 +- tests/namespace_test/NamespaceC/TableInC.jl | 2 +- .../namespace_test1_generated.jl | 4 +- .../namespace_test2_generated.jl | 6 +-- tests/union_vector/UnionVector/Attacker.jl | 2 +- tests/union_vector/UnionVector/BookReader.jl | 2 +- tests/union_vector/UnionVector/Character.jl | 2 +- tests/union_vector/UnionVector/Movie.jl | 2 +- tests/union_vector/UnionVector/Rapunzel.jl | 2 +- 39 files changed, 62 insertions(+), 163 deletions(-) delete mode 100644 samples/Monster.jl delete mode 100644 samples/MyGame/MyGame.jl delete mode 100644 samples/MyGame/Sample/Color.jl delete mode 100644 samples/MyGame/Sample/Equipment.jl delete mode 100644 samples/MyGame/Sample/Monster.jl delete mode 100644 samples/MyGame/Sample/Sample.jl delete mode 100644 samples/MyGame/Sample/Vec3.jl delete mode 100644 samples/MyGame/Sample/Weapon.jl create mode 100644 samples/monster_generated.jl diff --git a/samples/Monster.jl b/samples/Monster.jl deleted file mode 100644 index aac20e5f01b..00000000000 --- a/samples/Monster.jl +++ /dev/null @@ -1,7 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Monster - -module Monster - include("MyGame/MyGame.jl") -end diff --git a/samples/MyGame/MyGame.jl b/samples/MyGame/MyGame.jl deleted file mode 100644 index 6379ae45d93..00000000000 --- a/samples/MyGame/MyGame.jl +++ /dev/null @@ -1,7 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: MyGame - -module MyGame - include("Sample/Sample.jl") -end diff --git a/samples/MyGame/Sample/Color.jl b/samples/MyGame/Sample/Color.jl deleted file mode 100644 index e6108b38686..00000000000 --- a/samples/MyGame/Sample/Color.jl +++ /dev/null @@ -1,10 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -@enum Color::Int8 begin - ColorRed = 0 - ColorGreen = 1 - ColorBlue = 2 -end - diff --git a/samples/MyGame/Sample/Equipment.jl b/samples/MyGame/Sample/Equipment.jl deleted file mode 100644 index dfeb2e26d37..00000000000 --- a/samples/MyGame/Sample/Equipment.jl +++ /dev/null @@ -1,11 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -import FlatBuffers - -FlatBuffers.@UNION(Equipment, ( - Nothing, - Weapon, -)) - diff --git a/samples/MyGame/Sample/Monster.jl b/samples/MyGame/Sample/Monster.jl deleted file mode 100644 index b5904138ac7..00000000000 --- a/samples/MyGame/Sample/Monster.jl +++ /dev/null @@ -1,27 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct Monster - pos::Union{Vec3, Nothing} = nothing - mana::Int16 = 150 - hp::Int16 = 100 - name::Union{String, Nothing} = nothing - inventory::Union{Vector{UInt8}, Nothing} = nothing - color::Color = 2 - weapons::Union{Vector{Weapon}, Nothing} = nothing - equipped_type::UInt8 = 0 - equipped::Equipment = nothing -end -FlatBuffers.@ALIGN(Monster, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:Monster} = [ - 0x00000004, 0x00000006, 0x00000008, 0x0000000A, - 0x0000000E, 0x00000010, 0x00000012, 0x00000014, - 0x00000016 -] -FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true - -Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf) -Monster(io::IO) = FlatBuffers.deserialize(io, Monster) diff --git a/samples/MyGame/Sample/Sample.jl b/samples/MyGame/Sample/Sample.jl deleted file mode 100644 index 07e66c9dfd2..00000000000 --- a/samples/MyGame/Sample/Sample.jl +++ /dev/null @@ -1,11 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -module Sample - include("Color.jl") - include("Weapon.jl") - include("Equipment.jl") - include("Vec3.jl") - include("Monster.jl") -end diff --git a/samples/MyGame/Sample/Vec3.jl b/samples/MyGame/Sample/Vec3.jl deleted file mode 100644 index 57e72494a7a..00000000000 --- a/samples/MyGame/Sample/Vec3.jl +++ /dev/null @@ -1,15 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -import FlatBuffers - -FlatBuffers.@STRUCT struct Vec3 - x::Float32 - y::Float32 - z::Float32 -end -FlatBuffers.@ALIGN(Vec3, 4) - -Vec3(buf::AbstractVector{UInt8}) = FlatBuffers.read(Vec3, buf) -Vec3(io::IO) = FlatBuffers.deserialize(io, Vec3) diff --git a/samples/MyGame/Sample/Weapon.jl b/samples/MyGame/Sample/Weapon.jl deleted file mode 100644 index c94e9b14ca0..00000000000 --- a/samples/MyGame/Sample/Weapon.jl +++ /dev/null @@ -1,17 +0,0 @@ -# automatically generated by the FlatBuffers compiler, do not modify - -# module: Sample - -import FlatBuffers - -FlatBuffers.@with_kw mutable struct Weapon - name::Union{String, Nothing} = nothing - damage::Int16 = 0 -end -FlatBuffers.@ALIGN(Weapon, 1) -FlatBuffers.slot_offsets(::Type{T}) where {T<:Weapon} = [ - 0x00000004, 0x00000006 -] - -Weapon(buf::AbstractVector{UInt8}) = FlatBuffers.read(Weapon, buf) -Weapon(io::IO) = FlatBuffers.deserialize(io, Weapon) diff --git a/samples/monster_generated.jl b/samples/monster_generated.jl new file mode 100644 index 00000000000..be19e5d8378 --- /dev/null +++ b/samples/monster_generated.jl @@ -0,0 +1,9 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end +if !isdefined(MyGame, :Sample) MyGame.eval(:(module Sample import FlatBuffers end)) end +include("MyGame/Sample/Color.jl") +include("MyGame/Sample/Weapon.jl") +include("MyGame/Sample/Equipment.jl") +include("MyGame/Sample/Vec3.jl") +include("MyGame/Sample/Monster.jl") diff --git a/samples/sample_binary.jl b/samples/sample_binary.jl index 788b2ddbbb1..7cea9c4af44 100644 --- a/samples/sample_binary.jl +++ b/samples/sample_binary.jl @@ -1,6 +1,6 @@ import FlatBuffers -include("MyGame/MyGame.jl") +include("monster_generated.jl") # require the files generated from the schema import .MyGame.Sample.Weapon diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index abdef31f343..fa211a2baf6 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -205,6 +205,11 @@ class JuliaGenerator : public BaseGenerator { return true; } + std::string DefineModule(std::string scope, std::string mod) { + return "if !isdefined(" + scope + ", :" + mod + ") " + scope + + ".eval(:(module " + mod + " import " + JuliaPackageName + " end)) end\n"; + } + bool GenTopLevel(void) { std::string code = "# "; std::set included; @@ -216,9 +221,8 @@ class JuliaGenerator : public BaseGenerator { it != parser_.included_files_.end(); ++it) { if (it->second.empty()) continue; auto dir = flatbuffers::StripFileName(it->second); - auto basename = MakeCamel( - flatbuffers::StripPath(flatbuffers::StripExtension(it->second))); - auto toinclude = ConCatPathFileName(dir, basename + JuliaFileExtension); + auto basename = flatbuffers::StripPath(flatbuffers::StripExtension(it->second)); + auto toinclude = ConCatPathFileName(dir, basename + "_generated" + JuliaFileExtension); auto fullpath = ConCatPathFileName(path_, toinclude); if (!FileExists(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; @@ -230,15 +234,14 @@ class JuliaGenerator : public BaseGenerator { // Gather all parent namespaces for this namespace for (auto component = (*it)->components.begin(); component != (*it)->components.end(); ++component) { + std::string scope; if (parent.empty()) { parent = *component; child = *component; module_table_.AddDependency(root_module_, parent, child); // only create toplevel modules once if (toplevel.find(*component) == toplevel.end()) { - code += "if !isdefined(@__MODULE__(), :" + *component + - ") @__MODULE__().eval(:(module " + *component + - " import FlatBuffers end)) end\n"; + code += DefineModule("@__MODULE__()", *component); toplevel.insert(*component); } } else { @@ -248,10 +251,7 @@ class JuliaGenerator : public BaseGenerator { std::string mod = parent; std::replace(mod.begin(), mod.end(), kPathSeparator, '.'); - // Create module if it doesn't exist - code += "if !isdefined(" + mod + ", :" + *component + ") Core.eval(" + - mod + ", :(module " + *component + - " import FlatBuffers end)) end\n"; + code += DefineModule(mod, *component); } parent = child; } @@ -675,7 +675,7 @@ class JuliaGenerator : public BaseGenerator { void BeginFile(const Namespace &ns, std::string *code_ptr) const { auto &code = *code_ptr; code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; - code += "Core.eval(" + GetCanonicalName(ns, '.') + ", quote\n\n"; + code += GetCanonicalName(ns, '.') + ".eval(quote\n\n"; } void EndFile(std::string *code_ptr) const { @@ -711,24 +711,26 @@ class JuliaGenerator : public BaseGenerator { std::string *code_ptr) { auto &code = *code_ptr; DepGraph *children = module_table_.GetDependencies(mod); - if (mod != root_module_) code += "# module: " + mod + "\n"; // Include all the contents of this module in the right order auto sorted_children = children->TopSort(); for (auto it = sorted_children.rbegin(); it != sorted_children.rend(); ++it) { std::string child = *it; + // if this module depends on another module, go and generate that module + // first + if (module_table_.IsModule(child)) { + GenIncludes(child, included, code_ptr); + continue; + } + // this is not a direct child of this module, so don't include here if (child.find(mod) == std::string::npos || child.length() < (mod.length() + 1) || child.substr(mod.length() + 1).find(kPathSeparator) != std::string::npos) continue; - // this is a module or something that's already been included, don't - // include here - if (module_table_.IsModule(child) || - included.find(child) != included.end()) - continue; + if (included.find(child) != included.end()) continue; // If the file doesn't exist, don't include it // TODO: this doesn't allow types which reference each other, // but Julia doesn't support this yet anyway @@ -736,7 +738,7 @@ class JuliaGenerator : public BaseGenerator { std::string fullpath = ConCatPathFileName(path_, toinclude); if (!module_table_.IsFile(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; - // included.insert(child); + included.insert(child); } return true; } diff --git a/tests/MyGame/Example/Ability.jl b/tests/MyGame/Example/Ability.jl index d1e5825b25b..1c6099b445c 100644 --- a/tests/MyGame/Example/Ability.jl +++ b/tests/MyGame/Example/Ability.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@STRUCT struct Ability diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl index a52455ca228..768d12cdc61 100644 --- a/tests/MyGame/Example/AnyAmbiguousAliases.jl +++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@UNION(AnyAmbiguousAliases, ( diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl index bf5328ab321..2744542c82a 100644 --- a/tests/MyGame/Example/AnyUniqueAliases.jl +++ b/tests/MyGame/Example/AnyUniqueAliases.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote import ..Example2 diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl index f22845237a4..33d7ea894c5 100644 --- a/tests/MyGame/Example/Any_.jl +++ b/tests/MyGame/Example/Any_.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote import ..Example2 diff --git a/tests/MyGame/Example/Color.jl b/tests/MyGame/Example/Color.jl index 11cbaf58dd6..89803313e63 100644 --- a/tests/MyGame/Example/Color.jl +++ b/tests/MyGame/Example/Color.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote @enum Color::Int8 begin ColorRed = 1 diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl index 04d52d6fc19..96a7e850389 100644 --- a/tests/MyGame/Example/Monster.jl +++ b/tests/MyGame/Example/Monster.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote import ..InParentNamespace diff --git a/tests/MyGame/Example/Referrable.jl b/tests/MyGame/Example/Referrable.jl index 139b936c19e..5c51738118f 100644 --- a/tests/MyGame/Example/Referrable.jl +++ b/tests/MyGame/Example/Referrable.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@with_kw mutable struct Referrable diff --git a/tests/MyGame/Example/Stat.jl b/tests/MyGame/Example/Stat.jl index 9a99a2b377e..23fde7b56cd 100644 --- a/tests/MyGame/Example/Stat.jl +++ b/tests/MyGame/Example/Stat.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@with_kw mutable struct Stat diff --git a/tests/MyGame/Example/Test.jl b/tests/MyGame/Example/Test.jl index 3a7dd43ab86..c366a724e77 100644 --- a/tests/MyGame/Example/Test.jl +++ b/tests/MyGame/Example/Test.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@STRUCT struct Test diff --git a/tests/MyGame/Example/TestSimpleTableWithEnum.jl b/tests/MyGame/Example/TestSimpleTableWithEnum.jl index 9084d6c2415..41dc5cf42f9 100644 --- a/tests/MyGame/Example/TestSimpleTableWithEnum.jl +++ b/tests/MyGame/Example/TestSimpleTableWithEnum.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@with_kw mutable struct TestSimpleTableWithEnum diff --git a/tests/MyGame/Example/TypeAliases.jl b/tests/MyGame/Example/TypeAliases.jl index 7e3f0583fc0..e5366108deb 100644 --- a/tests/MyGame/Example/TypeAliases.jl +++ b/tests/MyGame/Example/TypeAliases.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@with_kw mutable struct TypeAliases diff --git a/tests/MyGame/Example/Vec3.jl b/tests/MyGame/Example/Vec3.jl index 5e5d7075c3e..a100986a961 100644 --- a/tests/MyGame/Example/Vec3.jl +++ b/tests/MyGame/Example/Vec3.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example, quote +MyGame.Example.eval(quote FlatBuffers.@STRUCT struct Vec3 diff --git a/tests/MyGame/Example2/Monster.jl b/tests/MyGame/Example2/Monster.jl index 9e5e7e7a313..28474d6b12a 100644 --- a/tests/MyGame/Example2/Monster.jl +++ b/tests/MyGame/Example2/Monster.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame.Example2, quote +MyGame.Example2.eval(quote mutable struct Monster diff --git a/tests/MyGame/InParentNamespace.jl b/tests/MyGame/InParentNamespace.jl index f48506b7863..7e5b662a184 100644 --- a/tests/MyGame/InParentNamespace.jl +++ b/tests/MyGame/InParentNamespace.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(MyGame, quote +MyGame.eval(quote mutable struct InParentNamespace diff --git a/tests/monster_test_generated.jl b/tests/monster_test_generated.jl index 5c705fb9206..9730de0ce90 100644 --- a/tests/monster_test_generated.jl +++ b/tests/monster_test_generated.jl @@ -1,12 +1,11 @@ # automatically generated by the FlatBuffers compiler, do not modify if !isdefined(@__MODULE__(), :MyGame) @__MODULE__().eval(:(module MyGame import FlatBuffers end)) end -if !isdefined(MyGame, :OtherNameSpace) Core.eval(MyGame, :(module OtherNameSpace import FlatBuffers end)) end -if !isdefined(MyGame, :Example2) Core.eval(MyGame, :(module Example2 import FlatBuffers end)) end -if !isdefined(MyGame, :Example) Core.eval(MyGame, :(module Example import FlatBuffers end)) end -# module: MyGame +if !isdefined(MyGame, :OtherNameSpace) MyGame.eval(:(module OtherNameSpace import FlatBuffers end)) end +if !isdefined(MyGame, :Example2) MyGame.eval(:(module Example2 import FlatBuffers end)) end +if !isdefined(MyGame, :Example) MyGame.eval(:(module Example import FlatBuffers end)) end include("MyGame/InParentNamespace.jl") -# module: MyGame/Example +include("MyGame/Example2/Monster.jl") include("MyGame/Example/Color.jl") include("MyGame/Example/Test.jl") include("MyGame/Example/Vec3.jl") @@ -19,5 +18,3 @@ include("MyGame/Example/AnyAmbiguousAliases.jl") include("MyGame/Example/Monster.jl") include("MyGame/Example/Any_.jl") include("MyGame/Example/TypeAliases.jl") -# module: MyGame/Example2 -include("MyGame/Example2/Monster.jl") diff --git a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl index 1701427b546..cfb847c204a 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceA.NamespaceB, quote +NamespaceA.NamespaceB.eval(quote @enum EnumInNestedNS::Int8 begin EnumInNestedNSA = 0 diff --git a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl index 9d0ace543de..bba26bb3756 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceA.NamespaceB, quote +NamespaceA.NamespaceB.eval(quote FlatBuffers.@STRUCT struct StructInNestedNS diff --git a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl index 4f0bde9cb70..2440cf36758 100644 --- a/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl +++ b/tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceA.NamespaceB, quote +NamespaceA.NamespaceB.eval(quote FlatBuffers.@with_kw mutable struct TableInNestedNS diff --git a/tests/namespace_test/NamespaceA/SecondTableInA.jl b/tests/namespace_test/NamespaceA/SecondTableInA.jl index 12bc220fa04..49897bf3760 100644 --- a/tests/namespace_test/NamespaceA/SecondTableInA.jl +++ b/tests/namespace_test/NamespaceA/SecondTableInA.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceA, quote +NamespaceA.eval(quote import ..NamespaceC diff --git a/tests/namespace_test/NamespaceA/TableInFirstNS.jl b/tests/namespace_test/NamespaceA/TableInFirstNS.jl index 2a88bd366ff..24edc2cf6f1 100644 --- a/tests/namespace_test/NamespaceA/TableInFirstNS.jl +++ b/tests/namespace_test/NamespaceA/TableInFirstNS.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceA, quote +NamespaceA.eval(quote import .NamespaceB diff --git a/tests/namespace_test/NamespaceC/TableInC.jl b/tests/namespace_test/NamespaceC/TableInC.jl index 567610c128a..23411cd64b2 100644 --- a/tests/namespace_test/NamespaceC/TableInC.jl +++ b/tests/namespace_test/NamespaceC/TableInC.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(NamespaceC, quote +NamespaceC.eval(quote import ..NamespaceA diff --git a/tests/namespace_test/namespace_test1_generated.jl b/tests/namespace_test/namespace_test1_generated.jl index c607cec73fc..71ee74a1f42 100644 --- a/tests/namespace_test/namespace_test1_generated.jl +++ b/tests/namespace_test/namespace_test1_generated.jl @@ -1,9 +1,7 @@ # automatically generated by the FlatBuffers compiler, do not modify if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end -if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end -# module: NamespaceA -# module: NamespaceA/NamespaceB +if !isdefined(NamespaceA, :NamespaceB) NamespaceA.eval(:(module NamespaceB import FlatBuffers end)) end include("NamespaceA/NamespaceB/EnumInNestedNS.jl") include("NamespaceA/NamespaceB/TableInNestedNS.jl") include("NamespaceA/NamespaceB/StructInNestedNS.jl") diff --git a/tests/namespace_test/namespace_test2_generated.jl b/tests/namespace_test/namespace_test2_generated.jl index df2299a8d64..ffae8fd5a97 100644 --- a/tests/namespace_test/namespace_test2_generated.jl +++ b/tests/namespace_test/namespace_test2_generated.jl @@ -1,11 +1,9 @@ # automatically generated by the FlatBuffers compiler, do not modify +include("namespace_test1_generated.jl") if !isdefined(@__MODULE__(), :NamespaceA) @__MODULE__().eval(:(module NamespaceA import FlatBuffers end)) end -if !isdefined(NamespaceA, :NamespaceB) Core.eval(NamespaceA, :(module NamespaceB import FlatBuffers end)) end +if !isdefined(NamespaceA, :NamespaceB) NamespaceA.eval(:(module NamespaceB import FlatBuffers end)) end if !isdefined(@__MODULE__(), :NamespaceC) @__MODULE__().eval(:(module NamespaceC import FlatBuffers end)) end -# module: NamespaceA include("NamespaceA/TableInFirstNS.jl") include("NamespaceA/SecondTableInA.jl") -# module: NamespaceA/NamespaceB -# module: NamespaceC include("NamespaceC/TableInC.jl") diff --git a/tests/union_vector/UnionVector/Attacker.jl b/tests/union_vector/UnionVector/Attacker.jl index cc025dd14f2..036738eb343 100644 --- a/tests/union_vector/UnionVector/Attacker.jl +++ b/tests/union_vector/UnionVector/Attacker.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(UnionVector, quote +UnionVector.eval(quote FlatBuffers.@with_kw mutable struct Attacker diff --git a/tests/union_vector/UnionVector/BookReader.jl b/tests/union_vector/UnionVector/BookReader.jl index 8d21d80c61e..cf6a66b6b0a 100644 --- a/tests/union_vector/UnionVector/BookReader.jl +++ b/tests/union_vector/UnionVector/BookReader.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(UnionVector, quote +UnionVector.eval(quote FlatBuffers.@STRUCT struct BookReader diff --git a/tests/union_vector/UnionVector/Character.jl b/tests/union_vector/UnionVector/Character.jl index 34076927d22..086a1473dc1 100644 --- a/tests/union_vector/UnionVector/Character.jl +++ b/tests/union_vector/UnionVector/Character.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(UnionVector, quote +UnionVector.eval(quote FlatBuffers.@UNION(Character, ( diff --git a/tests/union_vector/UnionVector/Movie.jl b/tests/union_vector/UnionVector/Movie.jl index d1ea33b3d88..16420f18eae 100644 --- a/tests/union_vector/UnionVector/Movie.jl +++ b/tests/union_vector/UnionVector/Movie.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(UnionVector, quote +UnionVector.eval(quote FlatBuffers.@with_kw mutable struct Movie diff --git a/tests/union_vector/UnionVector/Rapunzel.jl b/tests/union_vector/UnionVector/Rapunzel.jl index 6ce331a07f4..9738d47d9c8 100644 --- a/tests/union_vector/UnionVector/Rapunzel.jl +++ b/tests/union_vector/UnionVector/Rapunzel.jl @@ -1,6 +1,6 @@ # automatically generated by the FlatBuffers compiler, do not modify -Core.eval(UnionVector, quote +UnionVector.eval(quote FlatBuffers.@STRUCT struct Rapunzel From 40e0b78ba2d65a08da167d6c6d4819b6ddd9e3c8 Mon Sep 17 00:00:00 2001 From: rowan Date: Thu, 10 Jan 2019 09:52:52 +1100 Subject: [PATCH 15/19] clang format --- src/idl_gen_julia.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index fa211a2baf6..2f44de1f5d0 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -207,7 +207,8 @@ class JuliaGenerator : public BaseGenerator { std::string DefineModule(std::string scope, std::string mod) { return "if !isdefined(" + scope + ", :" + mod + ") " + scope + - ".eval(:(module " + mod + " import " + JuliaPackageName + " end)) end\n"; + ".eval(:(module " + mod + " import " + JuliaPackageName + + " end)) end\n"; } bool GenTopLevel(void) { @@ -221,8 +222,10 @@ class JuliaGenerator : public BaseGenerator { it != parser_.included_files_.end(); ++it) { if (it->second.empty()) continue; auto dir = flatbuffers::StripFileName(it->second); - auto basename = flatbuffers::StripPath(flatbuffers::StripExtension(it->second)); - auto toinclude = ConCatPathFileName(dir, basename + "_generated" + JuliaFileExtension); + auto basename = + flatbuffers::StripPath(flatbuffers::StripExtension(it->second)); + auto toinclude = + ConCatPathFileName(dir, basename + "_generated" + JuliaFileExtension); auto fullpath = ConCatPathFileName(path_, toinclude); if (!FileExists(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; @@ -628,7 +631,7 @@ class JuliaGenerator : public BaseGenerator { static std::string GenTypeBasic(const Type &type) { static const char *ctypename[] = { -// clang-format off + // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ #JLTYPE, From 14849748149c78569f2860773187aec7bf98b7e2 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Tue, 22 Jan 2019 14:09:39 +1100 Subject: [PATCH 16/19] disable precompilation for generated modules --- src/idl_gen_julia.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index 2f44de1f5d0..cc6a049887e 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -207,7 +207,7 @@ class JuliaGenerator : public BaseGenerator { std::string DefineModule(std::string scope, std::string mod) { return "if !isdefined(" + scope + ", :" + mod + ") " + scope + - ".eval(:(module " + mod + " import " + JuliaPackageName + + ".eval(:(module " + mod + " __precompile__(false); import " + JuliaPackageName + " end)) end\n"; } @@ -720,6 +720,10 @@ class JuliaGenerator : public BaseGenerator { ++it) { std::string child = *it; + if (included.find(child) != included.end()) continue; + + included.insert(child); + // if this module depends on another module, go and generate that module // first if (module_table_.IsModule(child)) { @@ -733,7 +737,6 @@ class JuliaGenerator : public BaseGenerator { child.substr(mod.length() + 1).find(kPathSeparator) != std::string::npos) continue; - if (included.find(child) != included.end()) continue; // If the file doesn't exist, don't include it // TODO: this doesn't allow types which reference each other, // but Julia doesn't support this yet anyway @@ -741,7 +744,6 @@ class JuliaGenerator : public BaseGenerator { std::string fullpath = ConCatPathFileName(path_, toinclude); if (!module_table_.IsFile(fullpath.c_str())) continue; code += "include(\"" + toinclude + "\")\n"; - included.insert(child); } return true; } From 6a69c14d8a5b981bc5690c039280242261b3ed12 Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Mon, 18 Nov 2019 10:43:06 +1100 Subject: [PATCH 17/19] get flatc building again, bring in latest FlatBuffers.jl --- julia/.gitignore | 6 + julia/LICENSE.md | 0 julia/Manifest.toml | 54 -- julia/Project.toml | 6 - julia/README.md | 15 +- julia/REQUIRE | 3 +- julia/appveyor.yml | 30 +- julia/docs/.documenter.enc | Bin 0 -> 1680 bytes julia/docs/build/assets/Documenter.css | 18 + julia/docs/build/assets/mathjaxhelper.js | 25 + julia/docs/build/index.md | 0 julia/docs/make.jl | 17 + julia/docs/mkdocs.yml | 29 + julia/docs/src/index.md | 71 ++ julia/src/FlatBuffers.jl | 630 ++++++------------ julia/src/internals.jl | 185 +++--- julia/src/macros.jl | 327 ++++----- julia/test/internals.jl | 811 +++++++++++++++++++++++ julia/test/monster.jl | 77 +++ julia/test/runtests.jl | 274 ++++++++ src/flatc_main.cpp | 2 +- src/idl_gen_csharp.cpp | 2 +- src/idl_gen_java.cpp | 2 +- src/idl_gen_julia.cpp | 12 +- src/idl_gen_kotlin.cpp | 2 +- src/idl_gen_text.cpp | 48 +- src/idl_parser.cpp | 6 +- 27 files changed, 1820 insertions(+), 832 deletions(-) create mode 100755 julia/.gitignore mode change 100644 => 100755 julia/LICENSE.md delete mode 100644 julia/Manifest.toml delete mode 100644 julia/Project.toml mode change 100644 => 100755 julia/README.md mode change 100644 => 100755 julia/REQUIRE mode change 100644 => 100755 julia/appveyor.yml create mode 100755 julia/docs/.documenter.enc create mode 100755 julia/docs/build/assets/Documenter.css create mode 100755 julia/docs/build/assets/mathjaxhelper.js create mode 100755 julia/docs/build/index.md create mode 100755 julia/docs/make.jl create mode 100755 julia/docs/mkdocs.yml create mode 100755 julia/docs/src/index.md mode change 100644 => 100755 julia/src/FlatBuffers.jl mode change 100644 => 100755 julia/src/internals.jl mode change 100644 => 100755 julia/src/macros.jl create mode 100755 julia/test/internals.jl create mode 100755 julia/test/monster.jl create mode 100755 julia/test/runtests.jl diff --git a/julia/.gitignore b/julia/.gitignore new file mode 100755 index 00000000000..3a82f27df64 --- /dev/null +++ b/julia/.gitignore @@ -0,0 +1,6 @@ +*.jl.cov +*.jl.*.cov +*.jl.mem +/deps/builds +/deps/usr +CMakeLists.txt.user diff --git a/julia/LICENSE.md b/julia/LICENSE.md old mode 100644 new mode 100755 diff --git a/julia/Manifest.toml b/julia/Manifest.toml deleted file mode 100644 index 57aea68c746..00000000000 --- a/julia/Manifest.toml +++ /dev/null @@ -1,54 +0,0 @@ -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[Distributed]] -deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[InteractiveUtils]] -deps = ["LinearAlgebra", "Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[OrderedCollections]] -deps = ["Random", "Serialization", "Test"] -git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.0.2" - -[[Parameters]] -deps = ["Markdown", "OrderedCollections", "REPL", "Test"] -git-tree-sha1 = "40f540ec96e50c0b2b9efdb11b5e4d0c63f90923" -uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" -version = "0.10.1" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/julia/Project.toml b/julia/Project.toml deleted file mode 100644 index 3df258589e9..00000000000 --- a/julia/Project.toml +++ /dev/null @@ -1,6 +0,0 @@ -name = "FlatBuffers" -uuid = "6a5b833c-db2f-11e8-18d6-cf74c1981d29" - -[deps] -Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/julia/README.md b/julia/README.md old mode 100644 new mode 100755 index d3677956a48..186ea792775 --- a/julia/README.md +++ b/julia/README.md @@ -1,12 +1,11 @@ # FlatBuffers -*A Julia implementation of [google flatbuffers](https://google.github.io/flatbuffers/)* - +*A Julia implementation of google flatbuffers* | **Documentation** | **PackageEvaluator** | **Build Status** | |:-------------------------------------------------------------------------------:|:---------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| -| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.6-img]][pkg-0.6-url] [![][pkg-0.7-img]][pkg-0.7-url] | [![][travis-img]][travis-url] [![][appveyor-img]][appveyor-url] [![][codecov-img]][codecov-url] | +| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.4-img]][pkg-0.4-url] [![][pkg-0.5-img]][pkg-0.5-url] | [![][travis-img]][travis-url] [![][appveyor-img]][appveyor-url] [![][codecov-img]][codecov-url] | ## Installation @@ -24,7 +23,7 @@ julia> Pkg.add("FlatBuffers") ## Project Status -The package is tested against Julia `1.0` and nightly on Linux, OS X, and Windows. +The package is tested against Julia `0.4` and *current* `0.5-dev` on Linux, OS X, and Windows. ## Contributing and Questions @@ -50,7 +49,7 @@ Contributions are very welcome, as are feature requests and suggestions. Please [issues-url]: https://github.com/JuliaData/FlatBuffers.jl/issues -[pkg-0.6-img]: https://pkg.julialang.org/badges/FlatBuffers_0.6.svg -[pkg-0.6-url]: https://pkg.julialang.org/?pkg=FlatBuffers -[pkg-0.7-img]: https://pkg.julialang.org/badges/FlatBuffers_0.7.svg -[pkg-0.7-url]: https://pkg.julialang.org/?pkg=FlatBuffers +[pkg-0.4-img]: http://pkg.julialang.org/badges/FlatBuffers_0.4.svg +[pkg-0.4-url]: http://pkg.julialang.org/?pkg=FlatBuffers +[pkg-0.5-img]: http://pkg.julialang.org/badges/FlatBuffers_0.5.svg +[pkg-0.5-url]: http://pkg.julialang.org/?pkg=FlatBuffers diff --git a/julia/REQUIRE b/julia/REQUIRE old mode 100644 new mode 100755 index 3b034579702..4aa321c1e26 --- a/julia/REQUIRE +++ b/julia/REQUIRE @@ -1,2 +1 @@ -julia 0.7 1.1- -Parameters 0.10.2 0.11- +julia 0.7- diff --git a/julia/appveyor.yml b/julia/appveyor.yml old mode 100644 new mode 100755 index 9aa56ab8dee..cef728064a3 --- a/julia/appveyor.yml +++ b/julia/appveyor.yml @@ -1,17 +1,17 @@ environment: matrix: - - julia_version: 1.0 - - julia_version: nightly + - julia_version: 0.7 + - julia_version: latest platform: - - x86 # 32-bit - - x64 # 64-bit + - x86 + - x64 -# # Uncomment the following lines to allow failures on nightly julia -# # (tests will run but not make your overall status red) -# matrix: -# allow_failures: -# - julia_version: nightly +## uncomment the following lines to allow failures on nightly julia +## (tests will run but not make your overall status red) +#matrix: +# allow_failures: +# - julia_version: latest branches: only: @@ -25,18 +25,12 @@ notifications: on_build_status_changed: false install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) + - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/master/bin/install.ps1')) build_script: - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" + - julia -e "%JL_BUILD_SCRIPT%" test_script: - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" - -# # Uncomment to support code coverage upload. Should only be enabled for packages -# # which would have coverage gaps without running on Windows -# on_success: -# - echo "%JL_CODECOV_SCRIPT%" -# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" + - julia -e "%JL_TEST_SCRIPT%" diff --git a/julia/docs/.documenter.enc b/julia/docs/.documenter.enc new file mode 100755 index 0000000000000000000000000000000000000000..cb724fb7921d7186658c763e465528f08582d291 GIT binary patch literal 1680 zcmV;B25MQYn&IRW5A`JcFxS$JgT zLINDpI=D;>iyo8T&g)$X0=L6uySZf~y7G^9Vfk*n+Sqra39+U5sfXgH{p@sIwm-|wa@xWCZM=+j143)CPCMH;*?f=YUMoP!#u)bS)mTN+e{9xGt!89LKetcbnsP}et+sYZ z?l3LlJ~Jp<4>ACng%K>|^D!}E3m9<#Z!?t!aYJ>ZWU5RAfHthmwl}mgJ#Ve|x)JV& zAyI0FdM2q=y(#7oFBIwB37`Ip6|rWF8h#eh*vgEDcb&X^Dst6(HaR^cXltK^eY@T% z4XrO}r)xI7?GIN}s3iV9g3kYR>ibgl>RBVc$~@mar9NU4zy}tEwPLA6{i3KoSHm*X zr4K<8b;<)!(a^|a*p9)Y3yGr0o#}1pn$Ot+ugZnhq+q+=`$th@UCLQO#T~`<_+kzF zOvNT8ogoH^#0O>JLB4Ds4*`rBJPdlqK;MZdUg1+PR)K{xjU zB!-+Q^PGvNmNIw@nFNrY8YkpgFOXoSu`Y3AW6>YNCGo|rgXJ|P*a3d>)0VlBE%&tS zLP;F0h<9iP802dqJsyY8XL5G|Gf@H&>`&NwEBA%2-Etjf5c-XOwQS0=-2E{$*O%E{ zCx67i3v%>u3T)Irwp*A`hd(Dvc@99heNtSE6Bv|%uNbK*w6a9=WCY+NVKm3>9YP>E zemM$M(E|(5^#ReEdcI3|qxO%zf61h=SjOZ;I*pGGfgGbhu?JD1NRxc*PB+9sj`#%( zvOkWka7&n~+DB3*eivKo63i~9eq~3b))c;eT18dQ(0#b+MRXS0-0?UMN#ktr zKdNAj+d@LMSOA2{cOyAg_C=w z!9GnL@(YTou1i|#__sy1XTJ-?`9BBqqZyXv>ReKy$fs;52S=eLZ?4}oz^+N>kLB{D ze?m`gi+5M=y8Jvy{MVquHCNsE{81)Z#a6I3!827CQp#tLqn?OBvkwVLw&sFhD-L(s z#pRPcW4vPohmQmnt^(1*=GmDA+q{UVx^S&kh8qE31C`6oI{}nVUx;EFkUfRN#7|+f zUV8p2HS9Z$X$SA*5#M7=Pu5E(I^8QC>W;oqK2ls-;gA}Pe=>KNe*>ekn7X@xs^E5Y z0K3{>N!3eil&aC+u>U!9@4-E0}oS!Be;FHMy172GjE|m}uF}eJnP*Xqq z#y`PzAcfh67j0SplDd)$u@2mfuOa_=u-!`)**s8A&VMP;+sWXTj8)HLmby@55K-ZY zZ>cf3Lmzi@@z*le2Zup~(_5*A7bxq^6}JZ5^1 z*wP3UtLUvoL1-n{{C(ti9{Jb?cb7ys08AGV#UJ^jYQsEIIT1j?PR8Wlw4Pc>Zbk4K z_G|re)e$~o^ZlGV;n;;c^ZaOmDxxWX`nsX6LIG~fu5q6P%~9}*s|)7-hJK&T`QoxI zijB54b|%;|e>@Q|iKf{!@rHYG1^|JOZ80Ll?_^LF>dxJF(c&J;o$nm_(abDC1S9_3 z(KN8kx|uRg-8GTCHRS6s8feeP0U2I~qppf{Z3N3Z7|;eYVE6#buQO4soP~t}=3ewd z65;~nUh{y;lkQs;Tvm_CEd(y)11Af2k<3dPkP})Wl1in`p?@^3{@_@ErtG+rQa=uP aQ1JFJreDvSjEOA+8vvT9 literal 0 HcmV?d00001 diff --git a/julia/docs/build/assets/Documenter.css b/julia/docs/build/assets/Documenter.css new file mode 100755 index 00000000000..f7ae84ab159 --- /dev/null +++ b/julia/docs/build/assets/Documenter.css @@ -0,0 +1,18 @@ +div.wy-menu-vertical ul.current li.toctree-l3 a { + font-weight: bold; +} + +a.documenter-source { + float: right; +} + +.documenter-methodtable pre { + margin-left: 0px; + margin-right: 0px; + margin-top: 0px; + padding: 0px; +} + +.documenter-methodtable pre.documenter-inline { + display: inline; +} diff --git a/julia/docs/build/assets/mathjaxhelper.js b/julia/docs/build/assets/mathjaxhelper.js new file mode 100755 index 00000000000..3561b109f9f --- /dev/null +++ b/julia/docs/build/assets/mathjaxhelper.js @@ -0,0 +1,25 @@ +MathJax.Hub.Config({ + "tex2jax": { + inlineMath: [['$','$'], ['\\(','\\)']], + processEscapes: true + } +}); +MathJax.Hub.Config({ + config: ["MMLorHTML.js"], + jax: [ + "input/TeX", + "output/HTML-CSS", + "output/NativeMML" + ], + extensions: [ + "MathMenu.js", + "MathZoom.js", + "TeX/AMSmath.js", + "TeX/AMSsymbols.js", + "TeX/autobold.js", + "TeX/autoload-all.js" + ] +}); +MathJax.Hub.Config({ + TeX: { equationNumbers: { autoNumber: "AMS" } } +}); diff --git a/julia/docs/build/index.md b/julia/docs/build/index.md new file mode 100755 index 00000000000..e69de29bb2d diff --git a/julia/docs/make.jl b/julia/docs/make.jl new file mode 100755 index 00000000000..6dd30421836 --- /dev/null +++ b/julia/docs/make.jl @@ -0,0 +1,17 @@ +using Documenter, FlatBuffers + +makedocs( + modules = [FlatBuffers], + format = :html, + sitename = "FlatBuffers.jl", + pages = ["Home" => "index.md"] +) + +deploydocs( + repo = "github.com/JuliaData/FlatBuffers.jl.git", + target = "build", + deps = nothing, + make = nothing, + julia = "0.5", + osname = "linux" +) diff --git a/julia/docs/mkdocs.yml b/julia/docs/mkdocs.yml new file mode 100755 index 00000000000..da901f9e2d0 --- /dev/null +++ b/julia/docs/mkdocs.yml @@ -0,0 +1,29 @@ +site_name: FlatBuffers.jl +repo_url: https://github.com/dmbates/FlatBuffers.jl +site_description: Julia implementation of google flatbuffers +site_author: Jacob Quinn + +theme: material + +extra: + palette: + primary: 'indigo' + accent: 'blue' + +extra_css: + - assets/Documenter.css + +extra_javascript: + - https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML + - assets/mathjaxhelper.js + +markdown_extensions: + - extra + - tables + - fenced_code + - mdx_math + +docs_dir: 'build' + +pages: + - Home: index.md diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md new file mode 100755 index 00000000000..4f374d292d0 --- /dev/null +++ b/julia/docs/src/index.md @@ -0,0 +1,71 @@ +# FlatBuffers.jl Documentation + +#### Usage + +FlatBuffers.jl provides native Julia support for reading and writing binary structures following the google flatbuffer schema (see [here](https://google.github.io/flatbuffers/flatbuffers_internals.html) for a more in-depth review of the binary format). + +The typical language support for flatbuffers involves utilizing the `flatc` compiler to translate a flatbuffer schema file (.fbs) into a langugage-specific set of types/classes and methods. See [here](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) for the official guide on writing schemas. + +Currently in Julia, the `flatc` compiler isn't supported, but FlatBuffers.jl provides a native implementation of reading/writing the binary format directly with native Julia types. What does this mean exactly? Basically you can take a schema like: + +``` +namespace example; + +table SimpleType { + x: int = 1; +} + +root_type SimpleType; +``` + +and do a straightforward Julia translation like: + +```julia +module Example + +using FlatBuffers + +mutable struct SimpleType + x::Int32 +end + +@DEFAULT SimpleType x=1 + +end +``` + +A couple of things to point out: +* `using FlatBuffers` was included near the top to bring in the FlatBuffers module; this defines the necessary FlatBuffers.jl machinery for making the schema definitions easier +* `int` translates to a Julia `Int32`, see more info on flatbuffer types [here](https://google.github.io/flatbuffers/md__schemas.html) +* A default value for the `x` field in `SimpleType` was declared after the type with the `@DEFAULT` macro +* No `root_type` definition is necessary in Julia; basically any type defined with `type` (i.e. not abstract or immutable) can be a valid root table type in Julia. + +So let's see how we can actually use a flatbuffer in Julia: + +```julia +using FlatBuffers, Example # the schema module we defined above + +val = Example.SimpleType(2) # create an instance of our type + +flatbuffer = FlatBuffers.build!(val) # start and build a flatbuffer for our SimpleType +val2 = FlatBuffers.read(flatbuffer) # now we can deserialize the value from our flatbuffer, `val2` == `val` +flatbytes = FlatBuffers.bytes(flatbuffer) # get the serialized bytes of the flatbuffer +val3 = Flatbuffers.read(Example.SimpleType, flatbytes) # now we can deserialize directly from flatbytes +``` + +For more involved examples, see the test suite [here](https://github.com/dmbates/FlatBuffers.jl/tree/master/test). + +#### Reference + +Documentation is included inline for each type/method, these can be accessed at the REPL by type `?foo` where `foo` is the name of the type or method you'd like more information on. + +List of types/methods: + +* `FlatBuffers.Table{T}`: type for deserializing a Julia type `T` from a flatbuffer +* `FlatBuffers.Builder{T}`: type for serializing a Julia type `T` to a flatbuffer +* `FlatBuffers.read`: performs the actual deserializing on a `FlatBuffer.Table` +* `FlatBuffers.build!`: performs the actual serializing on a `FlatBuffer.Builder` +* `@ALIGN T size_in_bytes`: convenience macro for forcing a flatbuffer alignment on the Julia type `T` to `size_in_bytes` +* `@DEFAULT T field1=val1 field2=val2 ...`: convenience macro for defining default field values for Julia type `T` +* `@UNION T Union{T1,T2,...}`: convenience macro for defining a flatbuffer union type `T` +* `@STRUCT immutable T fields... end`: convenience macro for defining flatbuffer struct types, ensuring any necessary padding gets added to the type definition diff --git a/julia/src/FlatBuffers.jl b/julia/src/FlatBuffers.jl old mode 100644 new mode 100755 index 1c9804be5f6..fdc003be367 --- a/julia/src/FlatBuffers.jl +++ b/julia/src/FlatBuffers.jl @@ -1,71 +1,38 @@ +__precompile__(true) module FlatBuffers # utils -""" -serialize(stream::IO, value::T) where {T} -Serialize `value` to `stream` using the `FlatBuffer` format. -""" -function serialize(stream::IO, value::T) where {T} - write(stream, bytes(build!(value))) -end - -""" -deserialize(stream::IO, ::Type{T}) where {T} -Read a `T` from the flatbuffer-formatted `stream`. -""" -function deserialize(stream::IO, ::Type{T}) where {T} - read(T, read(stream)) -end - struct UndefinedType end const Undefined = UndefinedType() + getfieldvalue(obj::T, i) where {T} = isdefined(obj, i) ? getfield(obj, i) : Undefined -getprevfieldvalue(obj::T, i) where {T} = i == 1 ? missing : getfieldvalue(obj, i - 1) """ -Scalar + Scalar A Union of the Julia types `T <: Number` that are allowed in FlatBuffers schema """ -const Scalar = Union{Bool, -Int8, Int16, Int32, Int64, -UInt8, UInt16, UInt32, UInt64, -Float32, Float64} +const Scalar = Union{UndefinedType, Bool, + Int8, Int16, Int32, Int64, + UInt8, UInt16, UInt32, UInt64, + Float32, Float64} -isstruct(T) = isconcretetype(T) && !T.mutable -isbitstype(T) = fieldcount(T) == 0 -isunionwithnothing(T) = T isa Union && T.a == Nothing && !(isa(T.b, Union)) +if VERSION < v"0.7-DEV" + isconcrete = isleaftype +else + isconcrete = isconcretetype +end -file_identifier(T) = "" -file_extension(T) = "" -slot_offsets(T) = [4 + ((i - 1) * 2) for i = 1:length(T.types)] +isstruct(T) = !T.mutable && isconcrete(T) +isbitstype(T) = fieldcount(T) == 0 default(T, TT, sym) = default(TT) +default(::Type{UndefinedType}) = Undefined default(::Type{T}) where {T <: Scalar} = zero(T) -default(::Type{T}) where {T <: AbstractString} = nothing +default(::Type{T}) where {T <: AbstractString} = "" default(::Type{T}) where {T <: Enum} = enumtype(T)(T(0)) -default(::Type{Vector{T}}) where {T} = nothing - -# attempt to call default constructors for the type, -# use above methods as fallback -function default(::Type{T}, i::Integer) where {T} - TT = T.types[i] - try - return FlatBuffers.default(T, TT, fieldnames(T)[i]) - # catch because Parameters throws an error if there is no - # default value defined... - catch - end - return default(TT) -end - +default(::Type{Vector{T}}) where {T} = T[] # fallback that recursively builds a default; for structs/tables -function default(::Type{T}) where {T} - if isa(T, Union) || isa(T, UnionAll) - return nothing - else - return T([default(T, i) for i = 1:length(T.types)]...) - end -end +default(::Type{T}) where {T} = isa(T, Union) ? nothing : T(map(TT->TT == T ? TT() : default(TT),T.types)...) function typeorder end @@ -73,18 +40,18 @@ enumtype(::Type{<:Enum}) = UInt8 # Types """ -Table + Table The object containing the flatbuffer and positional information specific to the table. The `vtable` containing the offsets for specific members precedes `pos`. The actual values in the table follow `pos` offset and size of the vtable. -- `bytes::AbstractVector{UInt8}`: the flatbuffer itself -- `pos::Integer`: the base position in `bytes` of the table +- `bytes::Vector{UInt8}`: the flatbuffer itself +- `pos::Int`: the base position in `bytes` of the table """ mutable struct Table{T} - bytes::AbstractVector{UInt8} - pos::Integer + bytes::Vector{UInt8} + pos::Int end """ @@ -95,9 +62,9 @@ A Builder constructs byte buffers in a last-first manner for simplicity and performance. """ mutable struct Builder{T} - bytes::AbstractVector{UInt8} - minalign::Int - vtable::Vector{Int} + bytes::Vector{UInt8} + minalign::Int + vtable::Vector{Int} objectend::Int vtables::Vector{Int} head::Int @@ -105,187 +72,123 @@ mutable struct Builder{T} finished::Bool end -function hexloc(x) - "0x" * lpad("$(string(x-1, base=16)) ", 6, '0') -end - -function hexbyte(io, z) - printstyled(io, lpad("$(string(z, base=16)) ", 3, '0'), color=Int(z)) -end - -function hexoffset(x) - "0x$(lpad(string(x, base=16), 4, '0'))" -end - -function stringify(io, buf, offset, x, y, msg="", msgcolor=:blue) - y = min(y, length(buf)) - printstyled(io, hexloc(x + offset), color=:blue) - for i = x:y - hexbyte(io, buf[i]) - end - if length(msg) > 0 - printstyled(io, " " * msg, color=msgcolor) - end - println(io) -end - -function showvtable(io::IO, T, buffer, vtabstart, vtabsize) - syms = T.name.names - printstyled(io, "vtable start pos: $(hexoffset(vtabstart))\n", color=:green) - printstyled(io, "vtable size: $vtabsize\n", color=:green) - i = vtabstart + 4 - soff = slot_offsets(T) - numslots = div(soff[end] - 4, 2) + 1 - field = 1 - slot = 1 - numfields = length(T.types) - while slot <= numslots - # leave holes for deprecated fields - j = 2 - start = field == 1 ? soff[1] : soff[field - 1] - while (start + j) < soff[field] - # empty slot - stringify(io, buffer, 1, i, i+1, "[deprecated field]", :red) - slot += 1 - j += 2 - i += 2 - if (i - vtabstart) > vtabsize - break - end - end - if (i - vtabstart) > vtabsize - break - end - stringify(io, buffer, 1, i, i+1, "[$(fieldnames(T)[field])]") - slot += 1 - field += 1 - i += 2 - if (i - vtabstart) > vtabsize - break - end - end - # now we're pointing at data - printstyled(io, "payload:\n", color=:green) - while i < length(buffer) - stringify(io, buffer, 1, i, i+7, "") - i += 8 - end -end - -function Base.show(io::IO, x::Union{Builder{T}, Table{T}}) where {T} - printstyled(io, "FlatBuffers.$(typeof(x)):\n", color=:green) - buffer = x isa Builder ? x.bytes[x.head+1:end] : x.bytes - if isempty(buffer) - printstyled(io, " (empty flatbuffer)", color=:red) - else - pos = Int(typeof(x) <: Table ? x.pos : readbuffer(buffer, 0, Int32)) - printstyled(io, "root offset: $(hexoffset(pos))\n", color=:green) - vtaboff = readbuffer(buffer, pos, Int32) - vtabstart = pos - vtaboff - vtabsize = readbuffer(buffer, vtabstart, Int16) - showvtable(io, T, buffer, vtabstart, vtabsize) - end +function Base.show(io::IO, x::Union{Builder{T},Table{T}}) where {T} + println(io, "FlatBuffers.$(typeof(x)): ") + buffer = typeof(x) <: Table ? x.bytes : x.bytes[x.head+1:end] + if isempty(buffer) + print(io, " (empty flatbuffer)") + else + pos = Int(typeof(x) <: Table ? x.pos : readbuffer(buffer, 0, Int32)) + # print vtable offset + syms = T.name.names + maxpad = max(length(" vtable rel. start pos: "), maximum(map(x->length(string(x)), syms))) + stringify(buf, x, y, msg) = replace(string(rpad(string(lpad("$(x): ", 6, ' '),lpad(msg, maxpad, ' ')),maxpad+6,' '),string(map(z->lpad(string(Int(z)), 4, ' '),buf[x:y]))[9:end-1]),'"',"") + println(io, stringify(buffer, 1, 4, " root position: ")) + vtaboff = readbuffer(buffer, pos, Int32) + vtabstart = pos - vtaboff + 5 + + println(io, stringify(buffer, 5, 6, " vtable size: ")) + println(io, stringify(buffer, 7, 8, " data size: ")) + i = vtabstart + x = 1 + for y = 1:length(syms) + println(io, stringify(buffer, i, i+1, "$(syms[x]): ")) + i += 2 + x += 1 + end + # print rel pos. of vtable + println(io, stringify(buffer, i, i+3, " vtable rel. start pos: ")) + i += 4 + # now we're pointing at data + while i < length(buffer) + println(io, stringify(buffer, i, i+3, " ")) + i += 4 + end + end end include("internals.jl") -function Table(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T} - return Table{T}(buffer, pos) +function Table(::Type{T}, buffer::Vector{UInt8}, pos::Integer) where {T} + return Table{T}(buffer, pos) end Table(b::Builder{T}) where {T} = Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32)) +function untilindex(func, itr) + for (i,x) in enumerate(itr) + func(x) && return i + end + return 0 +end + getvalue(t, o, ::Type{Nothing}) = nothing getvalue(t, o, ::Type{T}) where {T <: Scalar} = get(t, t.pos + o, T) getvalue(t, o, ::Type{T}) where {T <: Enum} = T(get(t, t.pos + o, enumtype(T))) function getvalue(t, o, ::Type{T}) where {T <: AbstractString} - o += get(t, t.pos + o, Int32) - strlen = get(t, t.pos + o, Int32) - o += t.pos + sizeof(Int32) - return String(t.bytes[o + 1:o + strlen]) + o += get(t, t.pos + o, Int32) + strlen = get(t, t.pos + o, Int32) + o += t.pos + sizeof(Int32) + return String(t.bytes[o + 1:o + strlen]) end function getvalue(t, o, ::Type{Vector{UInt8}}) - o += get(t, t.pos + o, Int32) - len = get(t, t.pos + o, Int32) - o += t.pos + sizeof(Int32) - return t.bytes[o + 1:o + len] #TODO: maybe not make copy here? + o += get(t, t.pos + o, Int32) + len = get(t, t.pos + o, Int32) + o += t.pos + sizeof(Int32) + return t.bytes[o + 1:o + len] #TODO: maybe not make copy here? end getarray(t, vp, len, ::Type{T}) where {T <: Scalar} = (ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len]) getarray(t, vp, len, ::Type{T}) where {T <: Enum} = (ptr = convert(Ptr{enumtype(T)}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len]) function getarray(t, vp, len, ::Type{T}) where {T <: Union{AbstractString, Vector{UInt8}}} - A = Vector{T}(undef, len) - for i = 1:len - A[i] = getvalue(t, vp - t.pos, T) - vp += sizeof(Int32) - end - return A + A = Vector{T}(undef, len) + for i = 1:len + A[i] = getvalue(t, vp - t.pos, T) + vp += sizeof(Int32) + end + return A end function getarray(t, vp, len, ::Type{T}) where {T} - if isstruct(T) - ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)) - return [unsafe_load(ptr, i) for i = 1:len] - else - A = Vector{T}(undef, len) - for i = 1:len - A[i] = getvalue(t, vp - t.pos, T) - vp += sizeof(Int32) - end - return A - end + if isstruct(T) + ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)) + return [unsafe_load(ptr, i) for i = 1:len] + else + A = Vector{T}(undef, len) + for i = 1:len + A[i] = getvalue(t, vp - t.pos, T) + vp += sizeof(Int32) + end + return A + end end function getvalue(t, o, ::Type{Vector{T}}) where {T} - vl = vectorlen(t, o) - vp = vector(t, o) - return getarray(t, vp, vl, T) + vl = vectorlen(t, o) + vp = vector(t, o) + return getarray(t, vp, vl, T) end Base.convert(::Type{T}, e::Integer) where {T <: Enum} = T(e) # fallback which recursively calls read function getvalue(t, o, ::Type{T}) where {T} - if isstruct(T) - if any(x-> x <: Enum, T.types) - args = [] - o = t.pos + o + 1 - for typ in T.types - val = unsafe_load(convert(Ptr{typ <: Enum ? enumtype(typ) : typ}, pointer(view(t.bytes, o:length(t.bytes))))) - push!(args, val) - o += sizeof(typ <: Enum ? enumtype(typ) : typ) - end - return T(args...) - else - return unsafe_load(convert(Ptr{T}, pointer(view(t.bytes, (t.pos + o + 1):length(t.bytes))))) - end - else - o += t.pos - newt = Table{T}(t.bytes, indirect(t, o)) - return FlatBuffers.read(newt, T) - end -end - -function typetoread(prevfield, ::Type{T}, ::Type{V}) where {T <:Any, V <: AbstractVector} - # hacks! if it's a union all, assume it's because we're working around circular dependencies - if isa(eltype(V), UnionAll) - return Vector{T}, false - # if it's a vector of Unions, use the previous field to figure out the types of all the elements - elseif isa(eltype(V), Union) - types = typeorder.(eltype(V), prevfield) - R = definestruct(types) - return R, true - end - return V, false -end - -function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT} - # if it's a Union type, use the previous arg to figure out the true type that was serialized - if !isunionwithnothing(TT) && TT isa Union - R = typeorder(TT, prevfield) + if isstruct(T) + if any(x-> x <: Enum, T.types) + args = [] + o = t.pos + o + 1 + for typ in T.types + val = unsafe_load(convert(Ptr{typ <: Enum ? enumtype(typ) : typ}, pointer(view(t.bytes, o:length(t.bytes))))) + push!(args, val) + o += sizeof(typ <: Enum ? enumtype(typ) : typ) + end + return T(args...) + else + return unsafe_load(convert(Ptr{T}, pointer(view(t.bytes, (t.pos + o + 1):length(t.bytes))))) + end else - R = TT + newt = Table{T}(t.bytes, t.pos + o + get(t, t.pos + o, Int32)) + return FlatBuffers.read(newt, T) end - R, false end """ @@ -293,71 +196,58 @@ end Will recurse as necessary for nested types (Arrays, Tables, etc.) """ function FlatBuffers.read(t::Table{T1}, ::Type{T}=T1) where {T1, T} - args = [] - numfields = length(T.types) - soff = slot_offsets(T) - for i = 1:numfields - TT = T.types[i] - o = offset(t, soff[i]) - R, isunionvector = typetoread(i == 1 ? nothing : args[end], T, TT) - nullable = false - if isunionwithnothing(R) - nullable = true - R = R.b + args = [] + numfields = length(T.types) + for i = 1:numfields + TT = T.types[i] + # if it's a Union type, use the previous arg to figure out the true type that was serialized + if isa(TT, Union) + TT = typeorder(TT, args[end]) end + o = offset(t, 4 + ((i - 1) * 2)) if o == 0 - push!(args, nullable ? nothing : default(T, TT, T.name.names[i])) + push!(args, default(T, TT, T.name.names[i])) else - if isunionvector - eval(:(newr = getvalue($t, $o, $R))) - eval(:(n = length($R.types))) - push!(args, [getfieldvalue(newr, j) for j = 1:n]) - else - push!(args, getvalue(t, o, R)) - end + push!(args, getvalue(t, o, TT)) end end - - return T(args...) + return T(args...) end -FlatBuffers.read(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T} = FlatBuffers.read(Table(T, buffer, pos)) +FlatBuffers.read(::Type{T}, buffer::Vector{UInt8}, pos::Integer) where {T} = FlatBuffers.read(Table(T, buffer, pos)) FlatBuffers.read(b::Builder{T}) where {T} = FlatBuffers.read(Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32))) # assume `bytes` is a pure flatbuffer buffer where we can read the root position at the beginning FlatBuffers.read(::Type{T}, bytes) where {T} = FlatBuffers.read(T, bytes, read(IOBuffer(bytes), Int32)) -has_identifier(::Type{T}, bytes) where {T} = length(bytes) >= 8 && String(bytes[5:8]) == FlatBuffers.file_identifier(T) -root_type(::Type{T}) where {T} = false - """ -flat_bytes = bytes(b) + flat_bytes = bytes(b) `flat_bytes` are the serialized bytes for the FlatBuffer. This discards the Julia specific `head`. """ bytes(b::Builder) = unsafe_wrap(Array{UInt8,1}, pointer(b.bytes, b.head+1), (length(b.bytes)-b.head)) function Builder(::Type{T}=Any, size=0) where {T} - objectend = 0 - vtables = zeros(Int, 0) - head = size - nested = false - bytes = zeros(UInt8, size) - minalign = 1 - vtable = zeros(Int, 0) - finished = false - b = Builder{T}(bytes, minalign, vtable, objectend, - vtables, head, nested, finished) - return b + objectend = 0 + vtables = zeros(Int, 0) + head = size + nested = false + bytes = zeros(UInt8, size) + minalign = 1 + vtable = zeros(Int, 0) + finished = false + b = Builder{T}(bytes, minalign, vtable, objectend, + vtables, head, nested, finished) + return b end # build! "`alignment` looks for the largest scalar member of `T` that represents a flatbuffer Struct" function alignment(::Type{T}) where {T} - largest = 0 - for typ in T.types - largest = isbitstype(typ) ? max(largest,sizeof(typ)) : alignment(typ) - end - return largest + largest = 0 + for typ in T.types + largest = isbitstype(typ) ? max(largest,sizeof(typ)) : alignment(typ) + end + return largest end """ @@ -367,79 +257,53 @@ even building its elements recursively if needed (Array of Arrays, Array of tabl function buildvector! end # empty vector -function buildvector!(b, A::Vector{Nothing}, len, prev) - startvector(b, 1, 0, 1) - return endvector(b, 0) +function buildvector!(b, A::Vector{Nothing}, len) + startvector(b, 1, 0, 1) + return endvector(b, 0) end # scalar type vector -function buildvector!(b, A::Vector{T}, len, prev) where {T <: Scalar} - startvector(b, sizeof(T), len, sizeof(T)) - foreach(x->prepend!(b, A[x]), len:-1:1) - return endvector(b, len) +function buildvector!(b, A::Vector{T}, len) where {T <: Scalar} + startvector(b, sizeof(T), len, sizeof(T)) + foreach(x->prepend!(b, A[x]), len:-1:1) + return endvector(b, len) end -function buildvector!(b, A::Vector{T}, len, prev) where {T <: Enum} - startvector(b, sizeof(enumtype(T)), len, sizeof(enumtype(T))) - foreach(x->prepend!(b, enumtype(T)(A[x])), len:-1:1) - return endvector(b, len) +function buildvector!(b, A::Vector{T}, len) where {T <: Enum} + startvector(b, sizeof(enumtype(T)), len, sizeof(enumtype(T))) + foreach(x->prepend!(b, enumtype(T)(A[x])), len:-1:1) + return endvector(b, len) end function putoffsetvector!(b, offsets, len) - startvector(b, 4, len, 4) #TODO: elsize/alignment correct here? - foreach(x->prependoffset!(b, offsets[x]), len:-1:1) - return endvector(b, len) + startvector(b, 4, len, 4) #TODO: elsize/alignment correct here? + foreach(x->prependoffset!(b, offsets[x]), len:-1:1) + return endvector(b, len) end # byte vector vector -function buildvector!(b, A::Vector{Vector{UInt8}}, len, prev) - offsets = map(x->createbytevector(b, A[x]), 1:len) - return putoffsetvector!(b, offsets, len) +function buildvector!(b, A::Vector{Vector{UInt8}}, len) + offsets = map(x->createbytevector(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) end # string vector -function buildvector!(b, A::Vector{T}, len, prev) where {T <: AbstractString} - offsets = map(x->createstring(b, A[x]), 1:len) - return putoffsetvector!(b, offsets, len) +function buildvector!(b, A::Vector{T}, len) where {T <: AbstractString} + offsets = map(x->createstring(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) end # array vector -function buildvector!(b, A::Vector{Vector{T}}, len, prev) where {T} - offsets = map(x->buildbuffer!(b, A[x]), 1:len) - return putoffsetvector!(b, offsets, len) +function buildvector!(b, A::Vector{Vector{T}}, len) where {T} + offsets = map(x->buildbuffer!(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) end - -# make a new struct which has fields of the given type -function definestruct(types::Vector{DataType}) - fields = [:($(gensym())::$(TT)) for TT in types] - T1 = gensym() - eval(:(mutable struct $T1 - $(fields...) - end)) - return T1 -end - -# make a new struct which has fields of the given type -# and populate them with values from the vector -function createstruct(types::Vector{DataType}, A::Vector{T}) where {T} - T1 = definestruct(types) - eval(:(newt = $T1($(A...)))) - return newt -end - # struct or table/object vector -function buildvector!(b, A::Vector{T}, len, prev) where {T} - if isstruct(T) - # struct - startvector(b, sizeof(T), len, alignment(T)) #TODO: forced elsize/alignment correct here? - foreach(x->buildbuffer!(b, A[x]), len:-1:1) - return endvector(b, len) - elseif isa(T, Union) - types = typeorder.(T, prev) - - # define a new type, construct one, and pack it into the buffer - newt = createstruct(types, A) - buildbuffer!(b, newt) - else - # table/object - offsets = map(x->buildbuffer!(b, A[x]), 1:len) - return putoffsetvector!(b, offsets, len) - end +function buildvector!(b, A::Vector{T}, len) where {T} + if isstruct(T) + # struct + startvector(b, sizeof(T), len, alignment(T)) #TODO: forced elsize/alignment correct here? + foreach(x->buildbuffer!(b, A[x]), len:-1:1) + return endvector(b, len) + else # table/object + offsets = map(x->buildbuffer!(b, A[x]), 1:len) + return putoffsetvector!(b, offsets, len) + end end """ @@ -450,15 +314,14 @@ down to their last leaf scalar types before returning the highest-level offset. """ function getoffset end -getoffset(b, arg::Nothing, prev=nothing) = 0 -getoffset(b, arg::T, prev=nothing) where {T <: Scalar} = 0 -getoffset(b, arg::T, prev=nothing) where {T <: Enum} = 0 -getoffset(b, arg::AbstractString, prev=nothing) = createstring(b, arg) -getoffset(b, arg::Vector{UInt8}, prev) = createbytevector(b, arg) -getoffset(b, arg::Vector{T}, prev) where {T} = buildbuffer!(b, arg, prev) - +getoffset(b, arg::Nothing) = 0 +getoffset(b, arg::T) where {T <: Scalar} = 0 +getoffset(b, arg::T) where {T <: Enum} = 0 +getoffset(b, arg::Vector{UInt8}) = createbytevector(b, arg) +getoffset(b, arg::AbstractString) = createstring(b, arg) +getoffset(b, arg::Vector{T}) where {T} = buildbuffer!(b, arg) # structs or table/object -getoffset(b, arg::T, prev=nothing) where {T} = isstruct(T) ? 0 : buildbuffer!(b, arg, prev) +getoffset(b, arg::T) where {T} = isstruct(T) ? 0 : buildbuffer!(b, arg) """ `putslot!` is one of the final steps in building a flatbuffer. @@ -468,66 +331,20 @@ to the actual data (Arrays, Strings, other tables) """ function putslot! end -putslot!(b, i, arg::T, off, default, prev) where {T <: Scalar} = prependslot!(b, i, arg, default) -putslot!(b, i, arg::T, off, default, prev) where {T <: Enum} = prependslot!(b, i, enumtype(T)(arg), default) -putslot!(b, i, arg::AbstractString, off, default, prev) = prependoffsetslot!(b, i, off, 0) -putslot!(b, i, arg::Vector{T}, off, default, prev) where {T} = prependoffsetslot!(b, i, off, 0) +putslot!(b, i, arg::T, off) where {T <: Scalar} = prependslot!(b, i, arg, default(T)) +putslot!(b, i, arg::T, off) where {T <: Enum} = prependslot!(b, i, enumtype(T)(arg), default(T)) +putslot!(b, i, arg::AbstractString, off) = prependoffsetslot!(b, i, off, 0) +putslot!(b, i, arg::Vector{T}, off) where {T} = prependoffsetslot!(b, i, off, 0) # structs or table/object -function putslot!(b, i, arg::T, off, default, prev) where {T} - if isstruct(T) - prependstructslot!(b, i, buildbuffer!(b, arg, prev), 0) - else - prependoffsetslot!(b, i, off, 0) - end -end - -function needreconstruct(T) - for TT in T.types - if TT <: Vector && eltype(TT) isa Union && !(eltype(TT) isa UnionAll) - return true - elseif TT isa Union && !isunionwithnothing(TT) - return true - end - end - return false -end - -function reconstructkwargs(arg::T) where {T} - kwargs = Dict{Symbol, Any}() - numfields = length(T.types) - fnames = fieldnames(T) - for i = 2:numfields - field = getfield(arg, i) - prevname = fnames[i - 1] - # hack to make the example work - TT = field isa Vector ? eltype(field) : typeof(field) - if :parameters in propertynames(TT) && length(TT.parameters) > 0 - TT = TT.name.wrapper - end - if field isa Vector && eltype(field) isa Union && !(eltype(field) isa UnionAll) - kwargs[prevname] = [FlatBuffers.typeorder(TT, typeof(x)) for x in field] - elseif (T.types[i] isa Union && !isunionwithnothing(T.types[i])) - kwargs[prevname] = FlatBuffers.typeorder(T.types[i], TT) - end - end - return kwargs -end - -function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Array} - # array of things - buildvector!(b, arg, length(arg), prev) -end - -function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Any} - # populate the _type field before unions/vectors of unions - if needreconstruct(T) - # reconstruct it so the types before the fields - # are populated correctly - kwargs = reconstructkwargs(arg) - arg = Parameters.reconstruct(arg; kwargs...) - end - if isstruct(T) - # build a struct type with provided `arg` +putslot!(b, i, arg::T, off) where {T} = + isstruct(T) ? prependstructslot!(b, i, buildbuffer!(b, arg), 0) : prependoffsetslot!(b, i, off, 0) + +function buildbuffer!(b::Builder{T1}, arg::T) where {T1, T} + if T <: Array + # array of things + n = buildvector!(b, arg, length(arg)) + elseif isstruct(T) + # build a struct type with provided `arg` all(isstruct, T.types) || throw(ArgumentError("can't seralize flatbuffer, $T is not a pure struct")) align = alignment(T) prep!(b, align, 2align) @@ -538,7 +355,7 @@ function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:A elseif isbitstype(typ) prepend!(b, getfield(arg,i)) else - buildbuffer!(b, getfield(arg, i), getprevfieldvalue(arg, i)) + buildbuffer!(b, getfield(arg,i)) end end n = offset(b) @@ -546,61 +363,20 @@ function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:A # build a table type # check for string/array/table types numfields = length(T.types) - # early exit for empty objects - if numfields == 0 - startobject(b, 0) - return endobject(b) - end - os = Int[] - isdefault = falses(numfields) - for i = 1:numfields - push!(os, getoffset(b, getfieldvalue(arg, i), getprevfieldvalue(arg, i))) - end - # all nested have been written, with offsets in `os[]` - # don't use slots for the last N members if they are all default - # also leave slots for deprecated fields - i = numfields - isdefault = getfieldvalue(arg, i) == default(T, i) - while isdefault && i > 0 - i -= 1 - isdefault = getfieldvalue(arg, i) == default(T, i) - end - soff = slot_offsets(T) - numslots = div(soff[i] - 4, 2) + 1 - startobject(b, numslots) - i = 1 - field = 1 - while i <= numslots - # leave holes for deprecated fields - j = 2 - start = field == 1 ? soff[1] : soff[field - 1] - while (start + j) < soff[field] - # empty slot - i += 1 - j += 2 - end - val = getfieldvalue(arg, field) - d = default(T, field) - if !(isunionwithnothing(T.types[field]) && val == nothing) - putslot!(b, i, - val, - os[field], - d, - getprevfieldvalue(arg, field) - ) - end - field += 1 - i += 1 - end + offsets = [getoffset(b, getfieldvalue(arg,i)) for i = 1:numfields] + + # all nested have been written, with offsets in `offsets[]` + startobject(b, numfields) + foreach(i->putslot!(b, i, getfieldvalue(arg,i), offsets[i]), 1:numfields) n = endobject(b) end - return n + return n end function build!(b, arg) - n = buildbuffer!(b, arg) - finish!(b, n) - return b + n = buildbuffer!(b, arg) + finish!(b, n) + return b end build!(arg::T) where {T} = build!(Builder(T), arg) diff --git a/julia/src/internals.jl b/julia/src/internals.jl old mode 100644 new mode 100755 index 054b573d29e..0bfae214010 --- a/julia/src/internals.jl +++ b/julia/src/internals.jl @@ -6,7 +6,7 @@ const TableOrBuilder = Union{Table,Builder} const Bytes2Type = Dict{Int, DataType}(1=>UInt8, 2=>UInt16, 4=>UInt32, 8=>UInt64) Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T} = read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), T) -readbuffer(t::AbstractVector{UInt8}, pos::Int, ::Type{T}) where {T} = read(IOBuffer(view(t, (pos+1):length(t))), T) +readbuffer(t::Vector{UInt8}, pos::Int, ::Type{T}) where {T} = read(IOBuffer(view(t, (pos+1):length(t))), T) Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), Bytes2Type[sizeof(T)])) """ @@ -15,8 +15,8 @@ Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(read(IOBuffer( Deprecated fields are ignored by checking against the vtable's length. """ function offset(t::Table, vtableoffset) - vtable = t.pos - get(t, t.pos, Int32) - return vtableoffset < get(t, vtable, Int16) ? get(t, vtable + vtableoffset, Int16) : 0 + vtable = t.pos - get(t, t.pos, Int32) + return vtableoffset < get(t, vtable, Int16) ? get(t, vtable + vtableoffset, Int16) : 0 end "`indirect` retrieves the relative offset stored at `offset`." @@ -40,7 +40,7 @@ function vector(t::Table, off) off += t.pos off += get(t, off, Int32) # data starts after metadata containing the vector length - return off + sizeof(Int32) + return off + sizeof(Int32) end """ @@ -75,8 +75,8 @@ value(x::T) where {T <: Enum} = length(T.types) == 0 ? Int(x) : getfield(x,1) Base.write(sink::Builder, o, x::Union{Bool,UInt8}) = sink.bytes[o+1] = UInt8(x) function Base.write(sink::Builder, off, x::T) where {T} - off += 1 - for (i,ind) = enumerate(off:(off + sizeof(T) - 1)) + off += 1 + for (i,ind) = enumerate(off:(off + sizeof(T) - 1)) sink.bytes[ind] = (x >> ((i-1) * 8)) % UInt8 end end @@ -95,27 +95,27 @@ Panics if the builder is not in a finished state (which is caused by calling `finish!()`). """ function finishedbytes(b::Builder) - assertfinished(b) - return b.bytes[b.head+1:end] + assertfinished(b) + return b.bytes[b.head+1:end] end -function startobject(b::Builder, numslots) - assertnotnested(b) - b.nested = true - b.vtable = zeros(Int, numslots) - b.objectend = offset(b) - b.minalign = 1 - return b +function startobject(b::Builder, numfields) + assertnotnested(b) + b.nested = true + b.vtable = zeros(Int, numfields) + b.objectend = offset(b) + b.minalign = 1 + return b end """ `endobject` writes data necessary to finish object construction. """ -function endobject(b::Builder{T}) where {T} - assertnested(b) - n = writevtable!(b) - b.nested = false - return n +function endobject(b::Builder) + assertnested(b) + n = writevtable!(b) + b.nested = false + return n end """ @@ -132,17 +132,17 @@ function prep!(b::Builder, size, additionalbytes) end # Find the amount of alignment needed such that `size` is properly # aligned after `additionalBytes`: - alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1 + alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1 alignsize &= (size - 1) # Reallocate the buffer if needed: - totalsize = alignsize + size + additionalbytes - if b.head <= totalsize - len = length(b.bytes) - prepend!(b.bytes, zeros(UInt8, totalsize)) - b.head += length(b.bytes) - len - end - pad!(b, alignsize) + totalsize = alignsize + size + additionalbytes + if b.head <= totalsize + len = length(b.bytes) + prepend!(b.bytes, zeros(UInt8, totalsize)) + b.head += length(b.bytes) - len + end + pad!(b, alignsize) return end @@ -153,7 +153,7 @@ Aligns and checks for space. function Base.prepend!(b::Builder, x::T) where {T} prep!(b, sizeof(T), 0) place!(b, x) - return + return end """ @@ -161,33 +161,40 @@ end """ function place!(b::Builder, x::T) where {T} b.head -= sizeof(T) - write(b, b.head, x) - return + write(b, b.head, x) + return end """ `startvector` initializes bookkeeping for writing a new vector. A vector has the following format: - -+, where T is the type of elements of this vector. + + +, where T is the type of elements of this vector. """ function startvector(b::Builder, elemSize, numElems, alignment) - assertnotnested(b) - b.nested = true - prep!(b, sizeof(UInt32), elemSize * numElems) - prep!(b, alignment, elemSize * numElems) - return offset(b) + assertnotnested(b) + b.nested = true + prep!(b, sizeof(UInt32), elemSize * numElems) + prep!(b, alignment, elemSize * numElems) + return offset(b) end """ `endvector` writes data necessary to finish vector construction. """ function endvector(b::Builder, vectorNumElems) - assertnested(b) - place!(b, UInt32(vectorNumElems)) - b.nested = false - return offset(b) + assertnested(b) + place!(b, UInt32(vectorNumElems)) + b.nested = false + return offset(b) +end + +if !isdefined(Base, :codeunits) + codeunits = Vector{UInt8} +end +if !isdefined(Base, :copyto!) + copyto! = copy! end """ @@ -196,14 +203,16 @@ end function createstring(b::Builder, s::AbstractString) assertnotnested(b) b.nested = true - s = codeunits(s) + s = codeunits(s) + prep!(b, sizeof(UInt32), length(s) + 1) - place!(b, UInt8(0)) + place!(b, UInt8(0)) l = length(s) b.head -= l - copyto!(b.bytes, b.head+1, s, 1, l) + copyto!(b.bytes, b.head+1, s, 1, l) + return endvector(b, length(s)) end @@ -211,7 +220,7 @@ end `createbytevector` writes a byte vector """ function createbytevector(b::Builder, v::AbstractVector{UInt8}) - assertnotnested(b) + assertnotnested(b) b.nested = true prep!(b, sizeof(UInt32), length(v)) @@ -219,7 +228,7 @@ function createbytevector(b::Builder, v::AbstractVector{UInt8}) l = length(v) b.head -= l - copyto!(b.bytes, b.head+1, v, 1, l) + copyto!(b.bytes, b.head+1, v, 1, l) return endvector(b, length(v)) end @@ -234,15 +243,15 @@ function prependoffset!(b::Builder, off) end off2 = offset(b) - off + sizeof(Int32) place!(b, Int32(off2)) - return + return end -function prependoffsetslot!(b::Builder, o::Int, x::T, d) where {T} - if x != T(d) +function prependoffsetslot!(b::Builder, o::Int, x::T, d::T) where {T} + if x != d prependoffset!(b, x) slot!(b, o) end - return + return end """ @@ -250,12 +259,12 @@ end If value `x` equals default `d`, then the slot will be set to zero and no other data will be written. """ -function prependslot!(b::Builder, o::Int, x::T, d) where {T} - if x != T(d) +function prependslot!(b::Builder, o::Int, x::T, d::T) where {T} + if x != d prepend!(b, x) slot!(b, o) end - return + return end """ @@ -271,7 +280,7 @@ function prependstructslot!(b::Builder, voffset, x, d) end slot!(b, voffset) end - return + return end """ @@ -284,17 +293,12 @@ end """ `finish!` finalizes a buffer, pointing to the given `rootTable`. """ -function finish!(b::Builder{T}, rootTable) where {T} +function finish!(b::Builder, rootTable) assertnotnested(b) - identifier = file_identifier(T) - n = length(identifier) prep!(b, b.minalign, sizeof(UInt32)) - for i = 0:(n-1) - prepend!(b, UInt8(identifier[n - i])) - end prependoffset!(b, Int32(rootTable)) b.finished = true - return + return end function assertnested(b::Builder) @@ -305,7 +309,7 @@ function assertnested(b::Builder) if !b.nested throw(ArgumentError("Incorrect creation order: must be inside object.")) end - return + return end function assertnotnested(b::Builder) @@ -320,7 +324,7 @@ function assertnotnested(b::Builder) if b.nested throw(ArgumentError("Incorrect creation order: object must not be nested.")) end - return + return end function assertfinished(b::Builder) @@ -335,30 +339,30 @@ function assertfinished(b::Builder) end """ -WriteVtable serializes the vtable for the current object, if applicable. + WriteVtable serializes the vtable for the current object, if applicable. -Before writing out the vtable, this checks pre-existing vtables for equality -to this one. If an equal vtable is found, point the object to the existing -vtable and return. + Before writing out the vtable, this checks pre-existing vtables for equality + to this one. If an equal vtable is found, point the object to the existing + vtable and return. -Because vtable values are sensitive to alignment of object data, not all -logically-equal vtables will be deduplicated. + Because vtable values are sensitive to alignment of object data, not all + logically-equal vtables will be deduplicated. -A vtable has the following format: - - - * N, where N is the number of fields in -the schema for this type. Includes deprecated fields. -Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. + A vtable has the following format: + + + * N, where N is the number of fields in + the schema for this type. Includes deprecated fields. + Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide. -An object has the following format: - -+ + An object has the following format: + + + """ -function writevtable!(b::Builder{T}) where {T} +function writevtable!(b::Builder) # Prepend a zero scalar to the object. Later in this function we'll # write an offset here that points to the object's vtable: - prepend!(b, Int32(0)) + prepend!(b, Int32(0)) objectOffset = offset(b) existingVtable = 0 @@ -368,11 +372,11 @@ function writevtable!(b::Builder{T}) where {T} # BenchmarkVtableDeduplication for a case in which this heuristic # saves about 30% of the time used in writing objects with duplicate # tables. - for i = length(b.vtables):-1:1 + for i = length(b.vtables):-1:1 # Find the other vtable, which is associated with `i`: vt2Offset = b.vtables[i] vt2Start = length(b.bytes) - vt2Offset - vt2Len = readbuffer(b.bytes, vt2Start, Int16) + vt2Len = readbuffer(b.bytes, vt2Start, Int16) metadata = VtableMetadataFields * sizeof(Int16) vt2End = vt2Start + vt2Len @@ -391,14 +395,15 @@ function writevtable!(b::Builder{T}) where {T} # Write out the current vtable in reverse , because # serialization occurs in last-first order: - for i = length(b.vtable):-1:1 - off::Int16 = 0 + for i = length(b.vtable):-1:1 + off::Int16 = 0 if b.vtable[i] != 0 # Forward reference to field; # use 32bit number to assert no overflow: off = objectOffset - b.vtable[i] end - prepend!(b, Int16(off)) + + prepend!(b, off) end # The two metadata fields are written last. @@ -414,7 +419,7 @@ function writevtable!(b::Builder{T}) where {T} # Next, write the offset to the new vtable in the # already-allocated SOffsetT at the beginning of this object: objectStart::Int32 = length(b.bytes) - objectOffset - write(b, objectStart, Int32(offset(b) - objectOffset)) + write(b, objectStart, Int32(offset(b) - objectOffset)) # Finally, store this vtable in memory for future # deduplication: @@ -427,7 +432,7 @@ function writevtable!(b::Builder{T}) where {T} # Write the offset to the found vtable in the # already-allocated SOffsetT at the beginning of this object: - write(b, b.head, Int32(existingVtable - objectOffset)) + write(b, b.head, Int32(existingVtable - objectOffset)) end empty!(b.vtable) @@ -440,8 +445,8 @@ function vtableEqual(a::Vector{Int}, objectStart, b::AbstractVector{UInt8}) return false end - for i = 0:(length(a)-1) - x = read(IOBuffer(view(b, (i * sizeof(Int16) + 1):length(b))), Int16) + for i = 0:(length(a)-1) + x = read(IOBuffer(view(b, (i * sizeof(Int16) + 1):length(b))), Int16) # Skip vtable entries that indicate a default value. x == 0 && a[i+1] == 0 && continue diff --git a/julia/src/macros.jl b/julia/src/macros.jl old mode 100644 new mode 100755 index 281fefc2f83..69b20512776 --- a/julia/src/macros.jl +++ b/julia/src/macros.jl @@ -1,35 +1,40 @@ -export @UNION, @DEFAULT, @ALIGN, @STRUCT, @with_kw - -const __module__ = 0 +export @UNION, @DEFAULT, @ALIGN, @STRUCT + +if VERSION < v"0.7-DEV" + const __module__ = 0 + __mod__(x) = current_module() + fieldcount(x) = nfields(x) +else + __mod__(x) = x +end function indexof(needle, haystack) - for (i, v) in enumerate(haystack) - v == needle && return i-1 - end - return -1 + for (i, v) in enumerate(haystack) + v == needle && return i-1 + end + return -1 end macro UNION(T, TT) - typeof(T) == Symbol || throw(ArgumentError("1st argument must be a symbol to represent a Union type")) - TT.head == :tuple || throw(ArgumentError("2nd argument must be a tuple of types like `(T1,T2,...)`")) - return esc(quote - const $T = $(Expr(:curly, :Union, TT.args...)) - FlatBuffers.typeorder(::Type{$T}, ::Type{TT}) where {TT} = FlatBuffers.indexof(TT, $TT) - FlatBuffers.typeorder(::Type{$T}, i::Integer) = ($TT)[i+1] - FlatBuffers.isunionwithnothing(::Type{$T}) = false - end) + typeof(T) == Symbol || throw(ArgumentError("1st argument must be a symbol to represent a Union type")) + TT.head == :tuple || throw(ArgumentError("2nd argument must be a tuple of types like `(T1,T2,...)`")) + return esc(quote + const $T = $(Expr(:curly, :Union, TT.args...)) + FlatBuffers.typeorder(::Type{$T}, ::Type{TT}) where {TT} = FlatBuffers.indexof(TT, $TT) + FlatBuffers.typeorder(::Type{$T}, i::Integer) = ($TT)[i+1] + end) end macro ALIGN(T, sz) - return esc(quote - FlatBuffers.alignment(::Type{$T}) = $sz - end) + return esc(quote + FlatBuffers.alignment(::Type{$T}) = $sz + end) end macro enumtype(T, typ) - return esc(quote - FlatBuffers.enumtype(::Type{$T}) = $typ - end) + return esc(quote + FlatBuffers.enumtype(::Type{$T}) = $typ + end) end # recursively finds largest field of a STRUCT @@ -42,184 +47,126 @@ maxsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : maximum(map(x->maxs nextsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : nextsizeof(T.types[1]) function fieldlayout(mod, typ, exprs...) - fields = Expr[] - values = [] - largest_field = maximum(map(x->maxsizeof(Core.eval(mod, x.args[2])), exprs)) - sz = cur_sz = 0 - x = 0 - for (i,expr) in enumerate(exprs) - T = Core.eval(mod, expr.args[2]) - if !isbitstype(T) - exprs2 = [Expr(:(::), nm, typ) for (nm,typ) in zip(fieldnames(T),T.types)] - fields2, values2 = fieldlayout(mod, T, exprs2...) - append!(fields, map(x->Expr(:(::), Symbol(string(expr.args[1],'_',x.args[1])), x.args[2]), fields2)) - append!(values, map(x->x == 0 ? 0 : Expr(:call, :getfield, expr.args[1], QuoteNode(x)), values2)) - else - push!(fields, expr) - push!(values, expr.args[1]) - end - sz += cur_sz = fbsizeof(T) - if sz % largest_field == 0 - sz = cur_sz = 0 - continue - end - nextsz = i == length(exprs) ? 0 : nextsizeof(Core.eval(mod, exprs[i+1].args[2])) - if i == length(exprs) || cur_sz < nextsz || (sz + nextsz) > largest_field - # this is the last field and we're not `sz % largest_field` - # potential diffs = 7, 6, 5, 4, 3, 2, 1 - sym = expr.args[1] - diff = cur_sz < nextsz ? nextsz - cur_sz : largest_field - sz - if diff == 7 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 - push!(values, 0); push!(values, 0); push!(values, 0) - elseif diff == 6 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 - push!(values, 0); push!(values, 0) - elseif diff == 5 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 - push!(values, 0); push!(values, 0) - elseif diff == 4 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 - push!(values, 0) - elseif diff == 3 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 - push!(values, 0); push!(values, 0) - elseif diff == 2 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 - push!(values, 0) - elseif diff == 1 - push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 - push!(values, 0) - end - sz = (sz + diff) % largest_field == 0 ? 0 : (cur_sz < nextsz ? sz + diff : 0) - cur_sz = 0 - end - end - return fields, values + fields = Expr[] + values = [] + largest_field = maximum(map(x->maxsizeof(Core.eval(mod, x.args[2])), exprs)) + sz = cur_sz = 0 + x = 0 + for (i,expr) in enumerate(exprs) + T = Core.eval(mod, expr.args[2]) + if !isbitstype(T) + exprs2 = [Expr(:(::), nm, typ) for (nm,typ) in zip(fieldnames(T),T.types)] + fields2, values2 = fieldlayout(mod, T, exprs2...) + append!(fields, map(x->Expr(:(::), Symbol(string(expr.args[1],'_',T,'_',x.args[1])), x.args[2]), fields2)) + append!(values, map(x->x == 0 ? 0 : Expr(:call, :getfield, expr.args[1], QuoteNode(x)), values2)) + else + push!(fields, expr) + push!(values, expr.args[1]) + end + sz += cur_sz = fbsizeof(T) + if sz % largest_field == 0 + sz = cur_sz = 0 + continue + end + nextsz = i == length(exprs) ? 0 : nextsizeof(Core.eval(mod, exprs[i+1].args[2])) + if i == length(exprs) || cur_sz < nextsz || (sz + nextsz) > largest_field + # this is the last field and we're not `sz % largest_field` + # potential diffs = 7, 6, 5, 4, 3, 2, 1 + sym = expr.args[1] + diff = cur_sz < nextsz ? nextsz - cur_sz : largest_field - sz + if diff == 7 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0); push!(values, 0) + elseif diff == 6 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 5 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 4 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1 + push!(values, 0) + elseif diff == 3 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(values, 0); push!(values, 0) + elseif diff == 2 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1 + push!(values, 0) + elseif diff == 1 + push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1 + push!(values, 0) + end + sz = (sz + diff) % largest_field == 0 ? 0 : (cur_sz < nextsz ? sz + diff : 0) + cur_sz = 0 + end + end + return fields, values end -linefilter = x->typeof(x) != LineNumberNode +if VERSION < v"0.7" + linefilter = x->isa(x, Expr) && x.head !== :line +else + linefilter = x->typeof(x) != LineNumberNode +end macro STRUCT(expr) - !expr.args[1] || throw(ArgumentError("@STRUCT is only applicable for immutable types")) - exprs = filter(linefilter, expr.args[3].args) - fields, values = FlatBuffers.fieldlayout(__module__, expr.args[2], exprs...) - expr.args[3].args = fields - # generate convenience outer constructors if necessary - # if there are nested structs or padding: - # build an outer constructor that takes all direct, original fields - # recursively flatten/splat all nested structs into one big args tuple - # adding zeros for padded arguments - # pass big, flat, args tuple to inner constructor - T = expr.args[2] - if any(x->!FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])), exprs) || - length(fields) > length(exprs) - exprs2 = map(x->FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])) ? x.args[1] : x, exprs) - sig = Expr(:call, T, exprs2...) - body = Expr(:call, T, values...) - outer = Expr(:function, sig, body) - else - outer = :(nothing) - end - return esc(quote - $expr - $outer - end) + !expr.args[1] || throw(ArgumentError("@STRUCT is only applicable for immutable types")) + exprs = filter(linefilter, expr.args[3].args) + fields, values = FlatBuffers.fieldlayout(__mod__(__module__), expr.args[2], exprs...) + expr.args[3].args = fields + # generate convenience outer constructors if necessary + # if there are nested structs or padding: + # build an outer constructor that takes all direct, original fields + # recursively flatten/splat all nested structs into one big args tuple + # adding zeros for padded arguments + # pass big, flat, args tuple to inner constructor + T = expr.args[2] + if any(x->!FlatBuffers.isbitstype(Core.eval(__mod__(__module__), x.args[2])), exprs) || + length(fields) > length(exprs) + exprs2 = map(x->FlatBuffers.isbitstype(Core.eval(__mod__(__module__), x.args[2])) ? x.args[1] : x, exprs) + sig = Expr(:call, T, exprs2...) + body = Expr(:call, T, values...) + outer = Expr(:function, sig, body) + else + outer = :(nothing) + end + return esc(quote + $expr + $outer + end) end macro DEFAULT(T, kwargs...) - ifblock = quote end - if length(kwargs) > 0 && isa(kwargs[1], Expr) && length(kwargs[1].args) > 1 - for kw in kwargs - push!(ifblock.args, :(if sym == $(QuoteNode(kw.args[1])) - return $(kw.args[2]) - end)) - end - end - esc(quote - if $T <: Enum - FlatBuffers.default(::Type{$T}) = FlatBuffers.enumtype($T)($(kwargs[1])) - else - function FlatBuffers.default(::Type{$T}, TT, sym) - $ifblock - return FlatBuffers.default(TT) - end - end - end) -end - -import Parameters - -# TODO: this is so evil and will fall over in the slightest breeze. -function getdef(typedef::Expr) - typedef.args[end-16].args[end].args[end] -end - -function getfielddefs(typedef::Expr) - [a for a in getdef(typedef).args[1:end-2] if a isa Expr] -end - -function getconstructor(typedef::Expr) - cons = getdef(typedef).args[end-1].args[1] - typevars = [] - if :head in propertynames(cons) && cons.head == :where - typevars = cons.args[2:end] - cons = cons.args[1] - end - cons, typevars -end - -function getkwtype(defs, name) - for d in defs - if :args in propertynames(d) && d.args[1] == name - return d.args[end] - end - end - return nothing -end - -function createdefaultfns(typedef::Expr) - cons, typevars = getconstructor(typedef) - T = cons.args[1] - params = cons.args[2] - - @assert params.head == :parameters - - kwargs = [] - defs = getfielddefs(typedef) - kwdict = Dict{Any, Any}() - for p in params.args - name = p.args[1] - t = getkwtype(defs, name) - if t == nothing - continue - end - value = p.args[end] - ifblock = get(kwdict, t, quote end) - push!(ifblock.args, :(if sym == $(QuoteNode(name)) - return convert($t, $value) - end)) - kwdict[t] = ifblock - end - - [:(function FlatBuffers.default(::Type{$T}, ::Type{$TT}, sym) where {$(typevars...)} - $(kwdict[TT]) - return FlatBuffers.default($TT) - end) for TT in keys(kwdict)] -end - -macro with_kw(typedef) - body = Parameters.with_kw(typedef, __module__, true) - defaults = createdefaultfns(body) - defaultsblock = Expr(:block, body, defaults...) - esc(defaultsblock) + ifblock = quote end + if length(kwargs) > 0 && isa(kwargs[1], Expr) && length(kwargs[1].args) > 1 + for kw in kwargs + push!(ifblock.args, :(if sym == $(QuoteNode(kw.args[1])) + return $(kw.args[2]) + end)) + end + end + esc(quote + if $T <: Enum + FlatBuffers.default(::Type{$T}) = FlatBuffers.enumtype($T)($(kwargs[1])) + else + function FlatBuffers.default(::Type{$T}, TT, sym) + $ifblock + return FlatBuffers.default(TT) + end + end + end) end #TODO: +# handle default values # handle id? +# handle deprecated # nested_flatbuffer +# macro table(expr) +# +# end diff --git a/julia/test/internals.jl b/julia/test/internals.jl new file mode 100755 index 00000000000..c0ba29ed9b9 --- /dev/null +++ b/julia/test/internals.jl @@ -0,0 +1,811 @@ +# Store specific byte patterns in these variables for the fuzzer. These +# values are taken verbatim from the C++ function FuzzTest1. +const overflowingInt32Val = read(IOBuffer(UInt8[0x83, 0x33, 0x33, 0x33]), Int32) +const overflowingInt64Val = read(IOBuffer(UInt8[0x84, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44]), Int64) + +# CheckByteLayout verifies the bytes of a Builder in various scenarios. +function CheckByteLayout() + check = want-> begin + got = b.bytes[b.head+1:end] + @test want == got + return + end + + # test 1: numbers + + b = FlatBuffers.Builder() + check(UInt8[]) + FlatBuffers.prepend!(b, true) + check(UInt8[1]) + FlatBuffers.prepend!(b, Int8(-127)) + check(UInt8[129, 1]) + FlatBuffers.prepend!(b, UInt8(255)) + check(UInt8[255, 129, 1]) + FlatBuffers.prepend!(b, Int16(-32222)) + check(UInt8[0x22, 0x82, 0, 255, 129, 1]) # first pad + FlatBuffers.prepend!(b, UInt16(0xFEEE)) + check(UInt8[0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) # no pad this time + FlatBuffers.prepend!(b, Int32(-53687092)) + check(UInt8[204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) + FlatBuffers.prepend!(b, UInt32(0x98765432)) + check(UInt8[0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) + + # test 1b: numbers 2 + + b = FlatBuffers.Builder() + prepend!(b, 0x1122334455667788) + check(UInt8[0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11]) + + # test 2: 1xbyte vector + + b = FlatBuffers.Builder() + check(UInt8[]) + FlatBuffers.startvector(b, sizeof(Bool), 1, 1) + check(UInt8[0, 0, 0]) # align to 4bytes + FlatBuffers.prepend!(b, UInt8(1)) + check(UInt8[1, 0, 0, 0]) + FlatBuffers.endvector(b, 1) + check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding + + # test 3: 2xbyte vector + + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(UInt8), 2, 1) + check(UInt8[0, 0]) # align to 4bytes + FlatBuffers.prepend!(b, UInt8(1)) + check(UInt8[1, 0, 0]) + FlatBuffers.prepend!(b, UInt8(2)) + check(UInt8[2, 1, 0, 0]) + FlatBuffers.endvector(b, 2) + check(UInt8[2, 0, 0, 0, 2, 1, 0, 0]) # padding + + # test 3b: 11xbyte vector matches builder size + + b = FlatBuffers.Builder(Any, 12) + FlatBuffers.startvector(b, sizeof(UInt8), 8, 1) + start = UInt8[] + check(start) + for i = 1:11 + FlatBuffers.prepend!(b, UInt8(i)) + start = append!(UInt8[i], start) + check(start) + end + FlatBuffers.endvector(b, 8) + check(append!(UInt8[8, 0, 0, 0], start)) + + # test 4: 1xuint16 vector + + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(UInt16), 1, 1) + check(UInt8[0, 0]) # align to 4bytes + FlatBuffers.prepend!(b, UInt16(1)) + check(UInt8[1, 0, 0, 0]) + FlatBuffers.endvector(b, 1) + check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding + + # test 5: 2xuint16 vector + + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(UInt16), 2, 1) + check(UInt8[]) # align to 4bytes + FlatBuffers.prepend!(b, UInt16(0xABCD)) + check(UInt8[0xCD, 0xAB]) + FlatBuffers.prepend!(b, UInt16(0xDCBA)) + check(UInt8[0xBA, 0xDC, 0xCD, 0xAB]) + FlatBuffers.endvector(b, 2) + check(UInt8[2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB]) + + # test 6: CreateString + + b = FlatBuffers.Builder() + FlatBuffers.createstring(b, "foo") + check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad + FlatBuffers.createstring(b, "moop") + check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0]) + + # test 6b: CreateString unicode + + b = FlatBuffers.Builder() + # These characters are chinese from blog.golang.org/strings + # We use escape codes here so that editors without unicode support + # aren't bothered: + uni_str = "\u65e5\u672c\u8a9e" + FlatBuffers.createstring(b, uni_str) + check(UInt8[9, 0, 0, 0, 230, 151, 165, 230, 156, 172, 232, 170, 158, 0, # null-terminated, 2-byte pad + 0, 0]) + + # test 6c: CreateUInt8String + + b = FlatBuffers.Builder() + FlatBuffers.createstring(b, "foo") + check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad + FlatBuffers.createstring(b, "moop") + check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad + 3, 0, 0, 0, 'f', 'o', 'o', 0]) + + # test 7: empty vtable + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + check(UInt8[]) + FlatBuffers.endobject(b) + check(UInt8[4, 0, 4, 0, 4, 0, 0, 0]) + + # test 8: vtable with one true bool + b = FlatBuffers.Builder() + check(UInt8[]) + FlatBuffers.startobject(b, 1) + check(UInt8[]) + FlatBuffers.prependslot!(b, 1, true, false) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # length of object including vtable offset + 7, 0, # start of bool value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, 0, # padded to 4 bytes + 1, # bool value + ]) + + # test 9: vtable with one default bool + b = FlatBuffers.Builder() + check(UInt8[]) + FlatBuffers.startobject(b, 1) + check(UInt8[]) + FlatBuffers.prependslot!(b, 1, false, false) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 4, 0, # end of object from here + 0, 0, # entry 1 is zero + 6, 0, 0, 0, # offset for start of vtable (int32) + ]) + + # test 10: vtable with one int16 + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 1) + FlatBuffers.prependslot!(b, 1, Int16(0x789A), Int16(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value + 6, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding to 4 bytes + 0x9A, 0x78, + ]) + + # test 11: vtable with two int16 + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 2) + FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) + FlatBuffers.prependslot!(b, 2, Int16(0x789A), Int16(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 4, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0x9A, 0x78, # value 1 + 0x56, 0x34, # value 0 + ]) + + # test 12: vtable with int16 and bool + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 2) + FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0)) + FlatBuffers.prependslot!(b, 2, true, false) + FlatBuffers.endobject(b) + check(UInt8[ + 8, 0, # vtable bytes + 8, 0, # end of object from here + 6, 0, # offset to value 0 + 5, 0, # offset to value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 1, # value 1 + 0x56, 0x34, # value 0 + ]) + + # test 12: vtable with empty vector + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) + vecend = FlatBuffers.endvector(b, 0) + FlatBuffers.startobject(b, 1) + FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset to vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, + 0, 0, 0, 0, # length of vector (not in struct) + ]) + + # test 12b: vtable with empty vector of byte and some scalars + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(UInt8), 0, 1) + vecend = FlatBuffers.endvector(b, 0) + FlatBuffers.startobject(b, 2) + FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) + FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 8, 0, # vtable bytes + 12, 0, + 10, 0, # offset to value 0 + 4, 0, # offset to vector offset + 8, 0, 0, 0, # vtable loc + 8, 0, 0, 0, # value 1 + 0, 0, 55, 0, # value 0 + + 0, 0, 0, 0, # length of vector (not in struct) + ]) + + # test 13: vtable with 1 int16 and 2-vector of int16 + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(Int16), 2, 1) + FlatBuffers.prepend!(b, Int16(0x1234)) + FlatBuffers.prepend!(b, Int16(0x5678)) + vecend = FlatBuffers.endvector(b, 2) + FlatBuffers.startobject(b, 2) + FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0)) + FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 8, 0, # vtable bytes + 12, 0, # length of object + 6, 0, # start of value 0 from end of vtable + 8, 0, # start of value 1 from end of buffer + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 55, 0, # value 0 + 4, 0, 0, 0, # vector position from here + 2, 0, 0, 0, # length of vector (uint32) + 0x78, 0x56, # vector value 1 + 0x34, 0x12, # vector value 0 + ]) + + # test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32 + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 1) + FlatBuffers.prep!(b, 4+4+4, 0) + FlatBuffers.prepend!(b, Int8(55)) + FlatBuffers.pad!(b, 3) + FlatBuffers.prepend!(b, Int16(0x1234)) + FlatBuffers.pad!(b, 2) + FlatBuffers.prepend!(b, Int32(0x12345678)) + structStart = FlatBuffers.offset(b) + FlatBuffers.prependstructslot!(b, 1, structStart, 0) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 16, 0, # end of object from here + 4, 0, # start of struct from here + 6, 0, 0, 0, # offset for start of vtable (int32) + 0x78, 0x56, 0x34, 0x12, # value 2 + 0, 0, # padding + 0x34, 0x12, # value 1 + 0, 0, 0, # padding + 55, # value 0 + ]) + + # test 15: vtable with 1 vector of 2 struct of 2 int8 + b = FlatBuffers.Builder() + FlatBuffers.startvector(b, sizeof(Int8)*2, 2, 1) + FlatBuffers.prepend!(b, Int8(33)) + FlatBuffers.prepend!(b, Int8(44)) + FlatBuffers.prepend!(b, Int8(55)) + FlatBuffers.prepend!(b, Int8(66)) + vecend = FlatBuffers.endvector(b, 2) + FlatBuffers.startobject(b, 1) + FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0)) + FlatBuffers.endobject(b) + check(UInt8[ + 6, 0, # vtable bytes + 8, 0, + 4, 0, # offset of vector offset + 6, 0, 0, 0, # offset for start of vtable (int32) + 4, 0, 0, 0, # vector start offset + + 2, 0, 0, 0, # vector length + 66, # vector value 1,1 + 55, # vector value 1,0 + 44, # vector value 0,1 + 33, # vector value 0,0 + ]) + + # test 16: table with some elements + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 2) + FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) + FlatBuffers.prependslot!(b, 2, Int16(66), Int16(0)) + off = FlatBuffers.endobject(b) + FlatBuffers.finish!(b, off) #TODO + + check(UInt8[ + 12, 0, 0, 0, # root of table: points to vtable offset + + 8, 0, # vtable bytes + 8, 0, # end of object from here + 7, 0, # start of value 0 + 4, 0, # start of value 1 + + 8, 0, 0, 0, # offset for start of vtable (int32) + + 66, 0, # value 1 + 0, # padding + 33, # value 0 + ]) + + # test 17: one unfinished table and one finished table + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 2) + FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0)) + FlatBuffers.prependslot!(b, 2, Int8(44), Int8(0)) + off = FlatBuffers.endobject(b) + FlatBuffers.finish!(b, off) + + FlatBuffers.startobject(b, 3) + FlatBuffers.prependslot!(b, 1, Int8(55), Int8(0)) + FlatBuffers.prependslot!(b, 2, Int8(66), Int8(0)) + FlatBuffers.prependslot!(b, 3, Int8(77), Int8(0)) + off = FlatBuffers.endobject(b) + FlatBuffers.finish!(b, off) + + check(UInt8[ + 16, 0, 0, 0, # root of table: points to object + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # offset for start of vtable (int32) + 0, # padding + 77, # value 2 + 66, # value 1 + 55, # value 0 + + 12, 0, 0, 0, # root of table: points to object + + 8, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 8, 0, 0, 0, # offset for start of vtable (int32) + 0, 0, # padding + 44, # value 1 + 33, # value 0 + ]) + + # test 18: a bunch of bools + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 8) + FlatBuffers.prependslot!(b, 1, true, false) + FlatBuffers.prependslot!(b, 2, true, false) + FlatBuffers.prependslot!(b, 3, true, false) + FlatBuffers.prependslot!(b, 4, true, false) + FlatBuffers.prependslot!(b, 5, true, false) + FlatBuffers.prependslot!(b, 6, true, false) + FlatBuffers.prependslot!(b, 7, true, false) + FlatBuffers.prependslot!(b, 8, true, false) + off = FlatBuffers.endobject(b) + FlatBuffers.finish!(b, off) + + check(UInt8[ + 24, 0, 0, 0, # root of table: points to vtable offset + + 20, 0, # vtable bytes + 12, 0, # size of object + 11, 0, # start of value 0 + 10, 0, # start of value 1 + 9, 0, # start of value 2 + 8, 0, # start of value 3 + 7, 0, # start of value 4 + 6, 0, # start of value 5 + 5, 0, # start of value 6 + 4, 0, # start of value 7 + 20, 0, 0, 0, # vtable offset + + 1, # value 7 + 1, # value 6 + 1, # value 5 + 1, # value 4 + 1, # value 3 + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + + # test 19: three bools + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 3) + FlatBuffers.prependslot!(b, 1, true, false) + FlatBuffers.prependslot!(b, 2, true, false) + FlatBuffers.prependslot!(b, 3, true, false) + off = FlatBuffers.endobject(b) + FlatBuffers.finish!(b, off) + + check(UInt8[ + 16, 0, 0, 0, # root of table: points to vtable offset + + 0, 0, # padding + + 10, 0, # vtable bytes + 8, 0, # size of object + 7, 0, # start of value 0 + 6, 0, # start of value 1 + 5, 0, # start of value 2 + 10, 0, 0, 0, # vtable offset from here + + 0, # padding + 1, # value 2 + 1, # value 1 + 1, # value 0 + ]) + + # test 20: some floats + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 1) + FlatBuffers.prependslot!(b, 1, Float32(1.0), Float32(0.0)) + off = FlatBuffers.endobject(b) + + check(UInt8[ + 6, 0, # vtable bytes + 8, 0, # size of object + 4, 0, # start of value 0 + 6, 0, 0, 0, # vtable offset + + 0, 0, 128, 63, # value 0 + ]) +end + +# CheckManualBuild builds a Monster manually. +function CheckManualBuild() + b = FlatBuffers.Builder() + str = FlatBuffers.createstring(b, "MyMonster") + + FlatBuffers.startvector(b, 1, 5, 1) + FlatBuffers.prepend!(b, UInt8(4)) + FlatBuffers.prepend!(b, UInt8(3)) + FlatBuffers.prepend!(b, UInt8(2)) + FlatBuffers.prepend!(b, UInt8(1)) + FlatBuffers.prepend!(b, UInt8(0)) + inv = FlatBuffers.endvector(b, 5) + + FlatBuffers.startobject(b, 13) + FlatBuffers.prependslot!(b, 2, Int16(20), Int16(100)) + mon2 = FlatBuffers.endobject(b) + + # Test4Vector + FlatBuffers.startvector(b, 4, 2, 1) + + # Test 0 + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, Int8(20)) + FlatBuffers.place!(b, Int16(10)) + + # Test 1 + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, Int8(40)) + FlatBuffers.place!(b, Int16(30)) + + # end testvector + test4 = FlatBuffers.endvector(b, 2) + + FlatBuffers.startobject(b, 13) + + # a vec3 + FlatBuffers.prep!(b, 16, 32) + FlatBuffers.pad!(b, 2) + FlatBuffers.prep!(b, 2, 4) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, UInt8(6)) + FlatBuffers.place!(b, Int16(5)) + FlatBuffers.pad!(b, 1) + FlatBuffers.place!(b, UInt8(4)) + FlatBuffers.place!(b, Float64(3.0)) + FlatBuffers.pad!(b, 4) + FlatBuffers.place!(b, Float32(3.0)) + FlatBuffers.place!(b, Float32(2.0)) + FlatBuffers.place!(b, Float32(1.0)) + vec3Loc = FlatBuffers.offset(b) + # end vec3 + + FlatBuffers.prependstructslot!(b, 1, vec3Loc, 0) # vec3. noop + FlatBuffers.prependslot!(b, 3, Int16(80), Int16(100)) # hp + FlatBuffers.prependoffsetslot!(b, 4, Int32(str), Int32(0)) + FlatBuffers.prependoffsetslot!(b, 6, Int32(inv), Int32(0)) # inventory + FlatBuffers.prependslot!(b, 8, UInt8(1), UInt8(0)) + FlatBuffers.prependoffsetslot!(b, 9, Int32(mon2), Int32(0)) + FlatBuffers.prependoffsetslot!(b, 10, Int32(test4), Int32(0)) + mon = FlatBuffers.endobject(b) + + FlatBuffers.finish!(b, mon) + + return b.bytes, b.head +end + +# CheckVtableDeduplication verifies that vtables are deduplicated. +function CheckVtableDeduplication() + b = FlatBuffers.Builder() + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(11), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(22), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(33), Int16(0)) + obj0 = FlatBuffers.endobject(b) + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(44), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(55), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(66), Int16(0)) + obj1 = FlatBuffers.endobject(b) + + FlatBuffers.startobject(b, 4) + FlatBuffers.prependslot!(b, 1, UInt8(0), UInt8(0)) + FlatBuffers.prependslot!(b, 2, UInt8(77), UInt8(0)) + FlatBuffers.prependslot!(b, 3, UInt8(88), UInt8(0)) + FlatBuffers.prependslot!(b, 4, Int16(99), Int16(0)) + obj2 = FlatBuffers.endobject(b) + + got = b.bytes[b.head+1:end] + + want = UInt8[ + 240, 255, 255, 255, # == -12. offset to dedupped vtable. + 99, 0, + 88, + 77, + 248, 255, 255, 255, # == -8. offset to dedupped vtable. + 66, 0, + 55, + 44, + 12, 0, # start of vtable + 8, 0, + 0, 0, + 7, 0, + 6, 0, + 4, 0, + 12, 0, 0, 0, # table0 + 33, 0, + 22, + 11 + ] + + @test got == want + + table0 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj0) + table1 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj1) + table2 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj2) + + function testTable(tab, a, b, c, d) + # vtable size + got = FlatBuffers.getoffsetslot(tab, 0, Int16(0)) + @test 12 == got + # object size + got = FlatBuffers.getoffsetslot(tab, 2, Int16(0)) + @test 8 == got + # default value + got = FlatBuffers.getoffsetslot(tab, 4, Int16(0)) + @test a == got + got = FlatBuffers.getslot(tab, 6, UInt8(0)) + @test b == got + val = FlatBuffers.getslot(tab, 8, UInt8(0)) + c != val && throw(ArgumentError("failed 8, 0: $got")) + got = FlatBuffers.getslot(tab, 10, UInt8(0)) + @test d == got + return + end + + testTable(table0, UInt16(0), UInt8(11), UInt8(22), UInt8(33)) + testTable(table1, UInt16(0), UInt8(44), UInt8(55), UInt8(66)) + testTable(table2, UInt16(0), UInt8(77), UInt8(88), UInt8(99)) +end + +# CheckNotInObjectError verifies that `endobject` fails if not inside an +# object. +function CheckNotInObjectError() + b = FlatBuffers.Builder() + + @test_throws ArgumentError FlatBuffers.endobject(b) +end + +# CheckStringIsNestedError verifies that a string can not be created inside +# another object. +function CheckStringIsNestedError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.createstring(b, "foo") +end + +# CheckByteStringIsNestedError verifies that a bytestring can not be created +# inside another object. +function CheckByteStringIsNestedError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.createstring(b, "foo") +end + +# CheckStructIsNotInlineError verifies that writing a struct in a location +# away from where it is used will cause a panic. +function CheckStructIsNotInlineError() + b = FlatBuffers.Builder() + FlatBuffers.startobject(b, 0) + @test_throws ArgumentError FlatBuffers.prependstructslot!(b, 0, 1, 0) +end + +# CheckFinishedBytesError verifies that `FinishedBytes` panics if the table +# is not finished. +function CheckFinishedBytesError() + b = FlatBuffers.Builder() + + @test_throws ArgumentError FlatBuffers.finishedbytes(b) +end + +function CheckCreateByteVector() + raw = UInt8(0):UInt8(29) + + for size = 1:30 + b1 = FlatBuffers.Builder() + b2 = FlatBuffers.Builder() + FlatBuffers.startvector(b1, 1, size, 1) + for i = size:-1:1 + FlatBuffers.prepend!(b1, raw[i]) + end + FlatBuffers.endvector(b1, size) + FlatBuffers.createbytevector(b2, raw[1:size]) + @test b1.bytes == b2.bytes + end +end + +const InitialLCGSeed = 48271 +mutable struct LCG + val::UInt32 + LCG() = new(UInt32(InitialLCGSeed)) +end +reset!(lcg::LCG) = lcg.val = UInt32(InitialLCGSeed) +function Base.next(lcg::LCG) + n = UInt32((UInt64(lcg.val) * UInt64(279470273)) % UInt64(4294967291)) + lcg.val = n + return n +end + +# Low level stress/fuzz test: serialize/deserialize a variety of +# different kinds of data in different combinations +function checkFuzz(fuzzFields, fuzzObjects, verbose=true) + + # Values we're testing against: chosen to ensure no bits get chopped + # off anywhere, and also be different from eachother. + boolVal = true + int8Val = Int8(-127) # 0x81 + uint8Val = UInt8(0xFF) + int16Val = Int16(-32222) # 0x8222 + uint16Val = UInt16(0xFEEE) + int32Val = Int32(overflowingInt32Val) + uint32Val = UInt32(0xFDDDDDDD) + int64Val = Int64(overflowingInt64Val) + uint64Val = UInt64(0xFCCCCCCCCCCCCCCC) + float32Val = Float32(3.14159) + float64Val = Float64(3.14159265359) + + testValuesMax = 11 # hardcoded to the number of scalar types + + b = FlatBuffers.Builder() + l = LCG() + + objects = fill(0, fuzzObjects) + + # Generate fuzzObjects random objects each consisting of + # fuzzFields fields, each of a random type. + for i = 1:fuzzObjects + FlatBuffers.startobject(b, fuzzFields) + + for f = 1:fuzzFields + choice = next(l) % UInt32(testValuesMax) + if choice == 0 + FlatBuffers.prependslot!(b, f, boolVal, false) + elseif choice == 1 + FlatBuffers.prependslot!(b, f, int8Val, Int8(0)) + elseif choice == 2 + FlatBuffers.prependslot!(b, f, uint8Val, UInt8(0)) + elseif choice == 3 + FlatBuffers.prependslot!(b, f, int16Val, Int16(0)) + elseif choice == 4 + FlatBuffers.prependslot!(b, f, uint16Val, UInt16(0)) + elseif choice == 5 + FlatBuffers.prependslot!(b, f, int32Val, Int32(0)) + elseif choice == 6 + FlatBuffers.prependslot!(b, f, uint32Val, UInt32(0)) + elseif choice == 7 + FlatBuffers.prependslot!(b, f, int64Val, Int64(0)) + elseif choice == 8 + FlatBuffers.prependslot!(b, f, uint64Val, UInt64(0)) + elseif choice == 9 + FlatBuffers.prependslot!(b, f, float32Val, Float32(0)) + elseif choice == 10 + FlatBuffers.prependslot!(b, f, float64Val, Float64(0)) + end + end + + off = FlatBuffers.endobject(b) + + # store the offset from the end of the builder buffer, + # since it will keep growing: + objects[i] = off + end + + # Do some bookkeeping to generate stats on fuzzes: + stats = Dict{String,Int}() + function check(desc, want, got) + v = get!(stats, desc, 0) + stats[desc] = v + 1 + @test want == got + end + + l = LCG() # Reset. + + # Test that all objects we generated are readable and return the + # expected values. We generate random objects in the same order + # so this is deterministic. + for i = 1:fuzzObjects + + table = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - objects[i]) + + for j = 0:(fuzzFields - 1) + f = (FlatBuffers.VtableMetadataFields + j) * sizeof(Int16) + choice = next(l) % UInt32(testValuesMax) + + if choice == 0 + check("bool", boolVal, FlatBuffers.getslot(table, f, false)) + elseif choice == 1 + check("int8", int8Val, FlatBuffers.getslot(table, f, Int8(0))) + elseif choice == 2 + check("uint8", uint8Val, FlatBuffers.getslot(table, f, UInt8(0))) + elseif choice == 3 + check("int16", int16Val, FlatBuffers.getslot(table, f, Int16(0))) + elseif choice == 4 + check("uint16", uint16Val, FlatBuffers.getslot(table, f, UInt16(0))) + elseif choice == 5 + check("int32", int32Val, FlatBuffers.getslot(table, f, Int32(0))) + elseif choice == 6 + check("uint32", uint32Val, FlatBuffers.getslot(table, f, UInt32(0))) + elseif choice == 7 + check("int64", int64Val, FlatBuffers.getslot(table, f, Int64(0))) + elseif choice == 8 + check("uint64", uint64Val, FlatBuffers.getslot(table, f, UInt64(0))) + elseif choice == 9 + check("float32", float32Val, FlatBuffers.getslot(table, f, Float32(0))) + elseif choice == 10 + check("float64", float64Val, FlatBuffers.getslot(table, f, Float64(0))) + end + end + end + + # If enough checks were made, verify that all scalar types were used: + if fuzzFields*fuzzObjects >= testValuesMax + if length(stats) != testValuesMax + throw(ArgumentError("fuzzing failed to test all scalar types")) + end + end + + # Print some counts, if needed: + if verbose + if fuzzFields == 0 || fuzzObjects == 0 + println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t[none]\t 0") + else + ks = sort!(collect(keys(stats))) + for k in ks + println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t$(k): $(stats[k])") + end + end + end + return +end diff --git a/julia/test/monster.jl b/julia/test/monster.jl new file mode 100755 index 00000000000..cfbdd707cc8 --- /dev/null +++ b/julia/test/monster.jl @@ -0,0 +1,77 @@ +module Example + +using FlatBuffers + +@enum(Color, Red = 1, Green = 2, Blue = 8) +@DEFAULT Color Red + +@STRUCT struct Test + a::Int16 + b::UInt8 + # _1::UInt8 # padding +end + +mutable struct TestSimpleTableWithEnum + color::Color +end + +@DEFAULT TestSimpleTableWithEnum color=Green + +@STRUCT struct Vec3 + x::Float32 + y::Float32 + z::Float32 + # _::UInt32 # padding + test1::Float64 + test2::Color + # __::UInt8 # padding + test3::Test + # ___::UInt16 # padding +end + +@ALIGN Vec3 16 + +mutable struct Stat + id::String + val::Int64 + count::UInt16 +end + +# Julia doesn't support forward referencing of types +# @union Any_ Union{Monster, TestSimpleTableWithEnum} + +mutable struct Monster + pos::Vec3 + mana::Int16 + hp::Int16 + name::String + friendly::Bool # deprecated + inventory::Vector{UInt8} + color::Color + # test_type::Any_ + # test::Vector{UInt8} + test4::Vector{Test} + testarrayofstring::Vector{String} + testarrayoftables::Vector{Monster} + # don't support nested circulr reference objects yet + # enemy::Monster + testnestedflatbuffer::Vector{UInt8} + testempty::Stat + testbool::Bool + testhashs32_fnv1::Int32 + testhashu32_fnv1::UInt32 + testhashs64_fnv1::Int64 + testhashu64_fnv1::UInt64 + testhashs32_fnv1a::Int32 + testhashu32_fnv1a::UInt32 + testhashs64_fnv1a::Int64 + testhashu64_fnv1a::UInt64 + testarrayofbools::Vector{Bool} + testf::Float32 + testf2::Float32 + testf3::Float32 +end + +@DEFAULT Monster hp=100 mana=150 color=Blue friendly=false testf=Float32(3.14159) testf2=Float32(3) + +end # module diff --git a/julia/test/runtests.jl b/julia/test/runtests.jl new file mode 100755 index 00000000000..128aebecbb3 --- /dev/null +++ b/julia/test/runtests.jl @@ -0,0 +1,274 @@ +using FlatBuffers +@static if VERSION < v"0.7.0-DEV.2005" + using Base.Test +else + using Test +end +if !isdefined(Base, :Nothing) + const Nothing = Void +end + +include("internals.jl") +CheckByteLayout() +CheckManualBuild() +CheckVtableDeduplication() +CheckNotInObjectError() +CheckStringIsNestedError() +CheckByteStringIsNestedError() +CheckStructIsNotInlineError() +CheckFinishedBytesError() +CheckCreateByteVector() +checkFuzz(100, 100, true) + +include("monster.jl") + +vec3 = Example.Vec3(1.0, 2.0, 3.0, 3.0, Example.Color(1), Example.Test(5, 6)) +test4 = Example.Test[Example.Test(10, 20), Example.Test(30, 40)] +testArrayOfString = ["test1","test2"] + +mon = Example.Monster(vec3, 150, 80, "MyMonster", false, collect(0x00:0x04), + Example.Blue, test4, testArrayOfString, Example.Monster[], + UInt8[], Example.Stat("",0,0), false, 0, 0, 0, 0, 0, 0, 0, 0, + Bool[], 0, 0, 0) +b = FlatBuffers.build!(mon) +monst = FlatBuffers.read(b) +monst2 = FlatBuffers.read(Example.Monster, FlatBuffers.bytes(b)) + +@test mon.pos == monst.pos +@test mon.pos == monst2.pos + +# create test types +# types (Scalar, Enum, struct, T, String, Vector{UInt8}) +mutable struct TestInt8T + x::Int8 +end + +inst1 = TestInt8T(1) + +b = FlatBuffers.Builder(TestInt8T) +FlatBuffers.build!(b, inst1) +t = FlatBuffers.Table(b) +inst1_2 = FlatBuffers.read(t) + +@test inst1.x === inst1_2.x + +struct TestInt8I + x::Int8 +end + +inst2 = TestInt8I(2) + +mutable struct TestInt8A + x::Vector{Int8} +end + +inst3 = TestInt8A([1,2,3]) + +b = FlatBuffers.Builder(TestInt8A) +FlatBuffers.build!(b, inst3) +t = FlatBuffers.Table(b) +inst3_2 = FlatBuffers.read(t) + +@test inst3.x == inst3_2.x + +mutable struct TestMixT + x::Int8 + y::String + z::Vector{Int8} +end + +inst4 = TestMixT(10,"hey there sailor",[1,2,3]) + +b = FlatBuffers.Builder(TestMixT) +FlatBuffers.build!(b, inst4) +t = FlatBuffers.Table(b) +inst4_2 = FlatBuffers.read(t) + +@test inst4.x == inst4_2.x && inst4.y == inst4_2.y && inst4.z == inst4_2.z + +# simple sub-table/type (Stat) +mutable struct TestSubT + x::TestInt8T + y::TestInt8I + z::TestInt8A +end + +inst5 = TestSubT(inst1, inst2, inst3) + +b = FlatBuffers.Builder(TestSubT) +FlatBuffers.build!(b, inst5) +t = FlatBuffers.Table(b) +inst5_2 = FlatBuffers.read(t) + +@test inst5.x.x == inst5_2.x.x && inst5.y.x == inst5_2.y.x && inst5.z.x == inst5_2.z.x + +# vtable duplicates +mutable struct TestDupT + x::TestInt8T + y::TestInt8I + z::TestInt8T +end + +inst6 = TestDupT(inst1, inst2, TestInt8T(2)) + +b = FlatBuffers.Builder(TestDupT) +FlatBuffers.build!(b, inst6) +t = FlatBuffers.Table(b) +inst6_2 = FlatBuffers.read(t) + +@test inst6.x.x == inst6_2.x.x && inst6.y.x == inst6_2.y.x && inst6.z.x == inst6_2.z.x + +mutable struct TestDup2T + x::Vector{TestInt8T} +end + +inst7 = TestDup2T([inst1, TestInt8T(2), TestInt8T(3), TestInt8T(4)]) + +b = FlatBuffers.Builder(TestDup2T) +FlatBuffers.build!(b, inst7) +t = FlatBuffers.Table(b) +inst7_2 = FlatBuffers.read(t) + +@test all(map(x->x.x, inst7.x) .== map(x->x.x, inst7_2.x)) + +# self-referential type test (type has subtype of itself) +# type TestCircT +# x::Int8 +# y::TestCircT +# end +# +# struct TestCircI +# x::Int8 +# y::TestCircI +# end + +# simple Union (Any_) + +# fbs +# table TestUnionT { +# x::TestUnionI +# } + +@UNION TestUnionU (Nothing,TestInt8T,TestInt8A) + +mutable struct TestUnionT + x_type::Int8 + x::TestUnionU +end + +TestUnionT(x::TestUnionU) = TestUnionT(FlatBuffers.typeorder(TestUnionU, typeof(x)), x) + +inst8 = TestUnionT(inst1) + +b = FlatBuffers.Builder(TestUnionT) +FlatBuffers.build!(b, inst8) +t = FlatBuffers.Table(b) +inst8_2 = FlatBuffers.read(t) + +@test inst8.x_type == inst8_2.x_type && inst8.x.x == inst8_2.x.x + +inst9 = TestUnionT(inst3) + +b = FlatBuffers.Builder(TestUnionT) +FlatBuffers.build!(b, inst9) +t = FlatBuffers.Table(b) +inst9_2 = FlatBuffers.read(t) + +@test inst9.x_type == inst9_2.x_type && inst9.x.x == inst9_2.x.x + +# test @STRUCT macro +@STRUCT struct A + a::Int32 +end +@test sizeof(A) == 4 +@test all(fieldnames(A) .== [:a]) +@test A(1) == A(1) + +@STRUCT struct B + a::Int8 + b::Int32 +end +@test sizeof(B) == 8 +@test all(fieldnames(B) .== [:a, :_pad_a_B_0, :_pad_a_B_1, :b]) +@test B(1,2) == B(1,2) + +@STRUCT struct C + a::Int16 + b::Int32 + c::Int16 +end +@test sizeof(C) == 12 +@test all(fieldnames(C) .== [:a, :_pad_a_C_0, :b, :c, :_pad_c_C_1]) +@test C(1,2,3) == C(1,2,3) + +@STRUCT struct D + a::Int8 + b::Int64 +end +@test sizeof(D) == 16 +@test all(fieldnames(D) .== [:a, :_pad_a_D_0, :_pad_a_D_1, :_pad_a_D_2, :b]) +@test D(1,2) == D(1,2) + +@STRUCT struct E + a::Int64 + b::Int32 +end +@test sizeof(E) == 16 +@test all(fieldnames(E) .== [:a, :b, :_pad_b_E_0]) +@test E(1,2) == E(1,2) + +@STRUCT struct F + a::Int32 + b::Int16 + c::Int32 + d::Int32 + e::Int64 +end +@test sizeof(F) == 24 +@test all(fieldnames(F) .== [:a, :b, :_pad_b_F_0, :c, :d, :e]) +@test F(1,2,3,4,5) == F(1,2,3,4,5) + +@STRUCT struct G + a::Float64 + b::Int8 + c::Int16 + d::Int32 +end +@test sizeof(G) == 24 +@test all(fieldnames(G) .== [:a, :b, :_pad_b_G_0, :c, :_pad_c_G_1, :d]) +@test G(1,2,3,4) == G(1,2,3,4) + +@STRUCT struct H + a::Float32 + b::Int8 + c::Int16 +end +@test sizeof(H) == 8 +@test all(fieldnames(H) .== [:a, :b, :_pad_b_H_0, :c]) +@test H(1,2,3) == H(1,2,3) + +@STRUCT struct I + a::Float64 + b::Int8 + c::Int32 +end +@test sizeof(I) == 16 +@test all(fieldnames(I) .== [:a, :b, :_pad_b_I_0, :_pad_b_I_1, :c]) +@test I(1,2,3) == I(1,2,3) + +@STRUCT struct J + a::Int8 + b::A +end +@test sizeof(J) == 8 +@test all(fieldnames(J) .== [:a, :_pad_a_J_0, :_pad_a_J_1, :b_A_a]) +@test J(1,A(2)) == J(1,A(2)) + +@STRUCT struct K + a::J + b::I + c::J +end +@test sizeof(K) == 48 +@test all(fieldnames(K) .== [:a_J_a, :a_J__pad_a_J_0, :a_J__pad__pad_a_J_0_J_0, :a_J__pad_a_J_1, :a_J__pad__pad_a_J_1_J_1, :a_J_b_A_a, :b_I_a, :b_I_b, :b_I__pad_b_I_0, :b_I__pad__pad_b_I_0_I_0, :b_I__pad_b_I_1, :b_I__pad__pad_b_I_1_I_1, :b_I_c, :c_J_a, :c_J__pad_a_J_0, :c_J__pad__pad_a_J_0_J_0, :c_J__pad_a_J_1, :c_J__pad__pad_a_J_1_J_1, :c_J_b_A_a]) +@test K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) == K(J(1,A(2)), I(3.0, 4, 5), J(6, A(7))) diff --git a/src/flatc_main.cpp b/src/flatc_main.cpp index 5b7a8c7d6de..152b4504412 100644 --- a/src/flatc_main.cpp +++ b/src/flatc_main.cpp @@ -66,7 +66,7 @@ int main(int argc, const char *argv[]) { { flatbuffers::GenerateJulia, nullptr, "--julia", "Julia", true, nullptr, flatbuffers::IDLOptions::kJulia, "Generate Julia modules for tables/structs", - flatbuffers::GeneralMakeRule }, + nullptr }, { flatbuffers::GenerateDart, "-d", "--dart", "Dart", true, nullptr, flatbuffers::IDLOptions::kDart, "Generate Dart classes for tables/structs", flatbuffers::DartMakeRule }, diff --git a/src/idl_gen_csharp.cpp b/src/idl_gen_csharp.cpp index 5045bd95d92..095876a9ae4 100644 --- a/src/idl_gen_csharp.cpp +++ b/src/idl_gen_csharp.cpp @@ -120,7 +120,7 @@ class CSharpGenerator : public BaseGenerator { // clang-format off static const char * const csharp_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ #NTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_java.cpp b/src/idl_gen_java.cpp index b257111f824..3e189733bb0 100644 --- a/src/idl_gen_java.cpp +++ b/src/idl_gen_java.cpp @@ -144,7 +144,7 @@ class JavaGenerator : public BaseGenerator { // clang-format off static const char * const java_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ #JTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp index cc6a049887e..991bf73dee0 100644 --- a/src/idl_gen_julia.cpp +++ b/src/idl_gen_julia.cpp @@ -347,10 +347,10 @@ class JuliaGenerator : public BaseGenerator { *code_ptr += "@enum " + enum_name + "::" + enum_type + " begin\n"; } - void EnumMember(const std::string enum_name, const EnumVal ev, + void EnumMember(const EnumDef &enum_def, const std::string &enum_name, const EnumVal ev, std::string *code_ptr) { *code_ptr += Indent + enum_name + NormalizedName(ev); - *code_ptr += " = " + NumToString(ev.value) + "\n"; + *code_ptr += " = " + enum_def.ToString(ev) + "\n"; } static void EndEnum(std::string *code_ptr) { *code_ptr += "end\n\n"; } @@ -600,7 +600,7 @@ class JuliaGenerator : public BaseGenerator { std::set imports; auto union_name = NormalizedName(enum_def); BeginUnion(union_name, code_ptr); - for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; std::string type_name = GenTypeGet(ev.union_type, &enum_def); @@ -620,11 +620,11 @@ class JuliaGenerator : public BaseGenerator { GenComment(enum_def.doc_comment, code_ptr, &JuliaCommentConfig); auto enum_name = NormalizedName(enum_def); BeginEnum(enum_name, GenTypeBasic(enum_def.underlying_type), code_ptr); - for (auto it = enum_def.vals.vec.begin(); it != enum_def.vals.vec.end(); + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig); - EnumMember(enum_name, ev, code_ptr); + EnumMember(enum_def, enum_name, ev, code_ptr); } EndEnum(code_ptr); } @@ -633,7 +633,7 @@ class JuliaGenerator : public BaseGenerator { static const char *ctypename[] = { // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, JLTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ #JLTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_kotlin.cpp b/src/idl_gen_kotlin.cpp index b12e554d313..c72bbe2f715 100644 --- a/src/idl_gen_kotlin.cpp +++ b/src/idl_gen_kotlin.cpp @@ -141,7 +141,7 @@ class KotlinGenerator : public BaseGenerator { // clang-format off static const char * const kotlin_typename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ #KTYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD diff --git a/src/idl_gen_text.cpp b/src/idl_gen_text.cpp index 46e494545cb..626aa5b0ee9 100644 --- a/src/idl_gen_text.cpp +++ b/src/idl_gen_text.cpp @@ -188,7 +188,7 @@ bool Print(const void *val, Type type, int indent, // clang-format off switch (vec_type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (!PrintArray( \ *reinterpret_cast *>(val), \ @@ -285,31 +285,31 @@ static bool GenStruct(const StructDef &struct_def, const Table *table, text += ":"; text += " "; switch (fd.value.type.base_type) { - // clang-format off - #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - - case BASE_TYPE_ ## ENUM: \ - if (!GenField(fd, table, struct_def.fixed, \ - opts, indent + Indent(opts), _text)) { \ - return false; \ - } \ - break; - FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD - // Generate drop-thru case statements for all pointer types: + // clang-format off #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - - case BASE_TYPE_ ## ENUM: - FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) - FLATBUFFERS_GEN_TYPE_ARRAY(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD - if (!GenFieldOffset(fd, table, struct_def.fixed, indent + Indent(opts), - prev_val, opts, _text)) { - return false; - } + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ + case BASE_TYPE_ ## ENUM: \ + if (!GenField(fd, table, struct_def.fixed, \ + opts, indent + Indent(opts), _text)) { \ + return false; \ + } \ break; - // clang-format on - } + FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + // Generate drop-thru case statements for all pointer types: + #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ + case BASE_TYPE_ ## ENUM: + FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) + FLATBUFFERS_GEN_TYPE_ARRAY(FLATBUFFERS_TD) + #undef FLATBUFFERS_TD + if (!GenFieldOffset(fd, table, struct_def.fixed, indent + Indent(opts), + prev_val, opts, _text)) { + return false; + } + break; + // clang-format on + } // Track prev val for use with union types. if (struct_def.fixed) { prev_val = reinterpret_cast(table) + fd.value.offset; diff --git a/src/idl_parser.cpp b/src/idl_parser.cpp index 5d6e9949549..cc10c784356 100644 --- a/src/idl_parser.cpp +++ b/src/idl_parser.cpp @@ -1259,7 +1259,7 @@ CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) { static bool CompareType(const uint8_t *a, const uint8_t *b, BaseType ftype) { switch (ftype) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, \ - RTYPE, KTYPE) \ + RTYPE, KTYPE, JLTYPE) \ case BASE_TYPE_##ENUM: return ReadScalar(a) < ReadScalar(b); FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) #undef FLATBUFFERS_TD @@ -1430,7 +1430,7 @@ CheckedError Parser::ParseArray(Value &array) { // clang-format off switch (val.type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, JLTYPE) \ case BASE_TYPE_ ## ENUM: \ if (IsStruct(val.type)) { \ SerializeStruct(builder, *val.type.struct_def, val); \ @@ -2019,7 +2019,7 @@ struct EnumValBuilder { // clang-format off switch (enum_def.underlying_type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, \ - PTYPE, RTYPE, KTYPE) \ + PTYPE, RTYPE, KTYPE, JLTYPE) \ case BASE_TYPE_##ENUM: { \ if (!IsInteger(BASE_TYPE_##ENUM)) break; \ return ValidateImpl(ev, next ? 1 : 0); \ From 6ae0bf49fc1157d4f49b659c0ccf4d67b82e572d Mon Sep 17 00:00:00 2001 From: Rowan Katekar Date: Mon, 18 Nov 2019 11:18:28 +1100 Subject: [PATCH 18/19] bring in julia tests --- julia/.gitignore | 2 + julia/LICENSE.md | 0 julia/Manifest.toml | 45 + julia/Project.toml | 11 + julia/README.md | 17 +- julia/REQUIRE | 1 - julia/appveyor.yml | 33 +- julia/docs/.documenter.enc | Bin julia/docs/Manifest.toml | 79 ++ julia/docs/Project.toml | 2 + julia/docs/build/assets/Documenter.css | 605 ++++++++++- julia/docs/build/assets/mathjaxhelper.js | 25 - julia/docs/build/index.md | 0 julia/docs/make.jl | 6 +- julia/docs/mkdocs.yml | 0 julia/docs/src/index.md | 104 +- julia/src/FlatBuffers.jl | 627 ++++++++---- julia/src/internals.jl | 185 ++-- julia/src/macros.jl | 334 ++++--- julia/test/defaults.jl | 40 + julia/test/flatc.jl | 91 ++ julia/test/internals.jl | 946 +++++++++--------- julia/test/monster.jl | 94 +- julia/test/monsterdata_python_wire.mon | Bin 0 -> 352 bytes julia/test/monsterdata_test.mon | Bin 0 -> 448 bytes julia/test/runtests.jl | 165 +-- src/idl_parser.cpp | 2 +- tests/JuliaTest.sh | 3 + tests/MyGame/Example/Color.jl | 12 +- tests/MyGame/Example/Monster.jl | 3 +- tests/MyGame/Example/Race.jl | 14 + tests/MyGame/Example/Vec3.jl | 2 +- tests/TestAll.sh | 4 + tests/juliatest.jl | 3 + tests/monster_test_generated.jl | 12 +- .../namespace_test1_generated.jl | 4 +- .../namespace_test2_generated.jl | 7 +- 37 files changed, 2345 insertions(+), 1133 deletions(-) mode change 100755 => 100644 julia/.gitignore mode change 100755 => 100644 julia/LICENSE.md create mode 100644 julia/Manifest.toml create mode 100644 julia/Project.toml mode change 100755 => 100644 julia/README.md delete mode 100755 julia/REQUIRE mode change 100755 => 100644 julia/appveyor.yml mode change 100755 => 100644 julia/docs/.documenter.enc create mode 100644 julia/docs/Manifest.toml create mode 100644 julia/docs/Project.toml mode change 100755 => 100644 julia/docs/build/assets/Documenter.css delete mode 100755 julia/docs/build/assets/mathjaxhelper.js delete mode 100755 julia/docs/build/index.md mode change 100755 => 100644 julia/docs/make.jl mode change 100755 => 100644 julia/docs/mkdocs.yml mode change 100755 => 100644 julia/docs/src/index.md mode change 100755 => 100644 julia/src/FlatBuffers.jl mode change 100755 => 100644 julia/src/internals.jl mode change 100755 => 100644 julia/src/macros.jl create mode 100644 julia/test/defaults.jl create mode 100644 julia/test/flatc.jl mode change 100755 => 100644 julia/test/internals.jl mode change 100755 => 100644 julia/test/monster.jl create mode 100644 julia/test/monsterdata_python_wire.mon create mode 100644 julia/test/monsterdata_test.mon mode change 100755 => 100644 julia/test/runtests.jl create mode 100644 tests/JuliaTest.sh create mode 100644 tests/MyGame/Example/Race.jl create mode 100644 tests/juliatest.jl diff --git a/julia/.gitignore b/julia/.gitignore old mode 100755 new mode 100644 index 3a82f27df64..e450ffdc9ed --- a/julia/.gitignore +++ b/julia/.gitignore @@ -4,3 +4,5 @@ /deps/builds /deps/usr CMakeLists.txt.user +docs/build/** +*~ diff --git a/julia/LICENSE.md b/julia/LICENSE.md old mode 100755 new mode 100644 diff --git a/julia/Manifest.toml b/julia/Manifest.toml new file mode 100644 index 00000000000..d1c6f0f9a7d --- /dev/null +++ b/julia/Manifest.toml @@ -0,0 +1,45 @@ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[OrderedCollections]] +deps = ["Random", "Serialization", "Test"] +git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.1.0" + +[[Parameters]] +deps = ["OrderedCollections"] +git-tree-sha1 = "b62b2558efb1eef1fa44e4be5ff58a515c287e38" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.0" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/julia/Project.toml b/julia/Project.toml new file mode 100644 index 00000000000..03f3cbca131 --- /dev/null +++ b/julia/Project.toml @@ -0,0 +1,11 @@ +name = "FlatBuffers" +uuid = "53afe959-3a16-52fa-a8da-cf864710bae9" +version = "0.5.4" + +[deps] +Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +julia = "1" +Parameters = "0.12.0" diff --git a/julia/README.md b/julia/README.md old mode 100755 new mode 100644 index 186ea792775..55e7786f197 --- a/julia/README.md +++ b/julia/README.md @@ -3,9 +3,10 @@ *A Julia implementation of google flatbuffers* -| **Documentation** | **PackageEvaluator** | **Build Status** | -|:-------------------------------------------------------------------------------:|:---------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| -| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][pkg-0.4-img]][pkg-0.4-url] [![][pkg-0.5-img]][pkg-0.5-url] | [![][travis-img]][travis-url] [![][appveyor-img]][appveyor-url] [![][codecov-img]][codecov-url] | + +| **Documentation** | **Build Status** +|:-------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| +| [![][docs-stable-img]][docs-stable-url] [![][docs-latest-img]][docs-latest-url] | [![][travis-img]][travis-url] [![][appveyor-img]][appveyor-url] [![][codecov-img]][codecov-url] | ## Installation @@ -23,7 +24,7 @@ julia> Pkg.add("FlatBuffers") ## Project Status -The package is tested against Julia `0.4` and *current* `0.5-dev` on Linux, OS X, and Windows. +The package is tested against Julia `1.0`, `1.1`, `1.2`, `1.3`, and nightly on Linux, OS X, and Windows. ## Contributing and Questions @@ -49,7 +50,7 @@ Contributions are very welcome, as are feature requests and suggestions. Please [issues-url]: https://github.com/JuliaData/FlatBuffers.jl/issues -[pkg-0.4-img]: http://pkg.julialang.org/badges/FlatBuffers_0.4.svg -[pkg-0.4-url]: http://pkg.julialang.org/?pkg=FlatBuffers -[pkg-0.5-img]: http://pkg.julialang.org/badges/FlatBuffers_0.5.svg -[pkg-0.5-url]: http://pkg.julialang.org/?pkg=FlatBuffers +[pkg-0.6-img]: https://pkg.julialang.org/badges/FlatBuffers_0.6.svg +[pkg-0.6-url]: https://pkg.julialang.org/?pkg=FlatBuffers +[pkg-0.7-img]: https://pkg.julialang.org/badges/FlatBuffers_0.7.svg +[pkg-0.7-url]: https://pkg.julialang.org/?pkg=FlatBuffers diff --git a/julia/REQUIRE b/julia/REQUIRE deleted file mode 100755 index 4aa321c1e26..00000000000 --- a/julia/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -julia 0.7- diff --git a/julia/appveyor.yml b/julia/appveyor.yml old mode 100755 new mode 100644 index cef728064a3..b1e6618db31 --- a/julia/appveyor.yml +++ b/julia/appveyor.yml @@ -1,17 +1,20 @@ environment: matrix: - - julia_version: 0.7 - - julia_version: latest + - julia_version: 1.0 + - julia_version: 1.1 + - julia_version: 1.2 + - julia_version: 1.3 + - julia_version: nightly platform: - - x86 - - x64 + - x86 # 32-bit + - x64 # 64-bit -## uncomment the following lines to allow failures on nightly julia -## (tests will run but not make your overall status red) -#matrix: -# allow_failures: -# - julia_version: latest +# # Uncomment the following lines to allow failures on nightly julia +# # (tests will run but not make your overall status red) +# matrix: +# allow_failures: +# - julia_version: nightly branches: only: @@ -25,12 +28,18 @@ notifications: on_build_status_changed: false install: - - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/master/bin/install.ps1')) + - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) build_script: - echo "%JL_BUILD_SCRIPT%" - - julia -e "%JL_BUILD_SCRIPT%" + - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" test_script: - echo "%JL_TEST_SCRIPT%" - - julia -e "%JL_TEST_SCRIPT%" + - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" + +# # Uncomment to support code coverage upload. Should only be enabled for packages +# # which would have coverage gaps without running on Windows +# on_success: +# - echo "%JL_CODECOV_SCRIPT%" +# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" diff --git a/julia/docs/.documenter.enc b/julia/docs/.documenter.enc old mode 100755 new mode 100644 diff --git a/julia/docs/Manifest.toml b/julia/docs/Manifest.toml new file mode 100644 index 00000000000..b341c92ea83 --- /dev/null +++ b/julia/docs/Manifest.toml @@ -0,0 +1,79 @@ +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[DocStringExtensions]] +deps = ["LibGit2", "Markdown", "Pkg", "Test"] +git-tree-sha1 = "1df01539a1c952cef21f2d2d1c092c2bcf0177d7" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.6.0" + +[[Documenter]] +deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] +git-tree-sha1 = "9f2135e0e7ecb63f9c3ef73ea15a31d8cdb79bb7" +uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +version = "0.20.0" + +[[InteractiveUtils]] +deps = ["LinearAlgebra", "Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[LibGit2]] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Pkg]] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[Test]] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[UUIDs]] +deps = ["Random"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" diff --git a/julia/docs/Project.toml b/julia/docs/Project.toml new file mode 100644 index 00000000000..dfa65cd107d --- /dev/null +++ b/julia/docs/Project.toml @@ -0,0 +1,2 @@ +[deps] +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/julia/docs/build/assets/Documenter.css b/julia/docs/build/assets/Documenter.css old mode 100755 new mode 100644 index f7ae84ab159..7cd26628de8 --- a/julia/docs/build/assets/Documenter.css +++ b/julia/docs/build/assets/Documenter.css @@ -1,18 +1,601 @@ -div.wy-menu-vertical ul.current li.toctree-l3 a { - font-weight: bold; +/* + * The default CSS style for Documenter.jl generated sites + * + * Heavily inspired by the Julia Sphinx theme + * https://github.com/JuliaLang/JuliaDoc + * which extends the sphinx_rtd_theme + * https://github.com/snide/sphinx_rtd_theme + * + * Part of Documenter.jl + * https://github.com/JuliaDocs/Documenter.jl + * + * License: MIT + */ + +/* fonts */ +body, input { + font-family: 'Lato', 'Helvetica Neue', Arial, sans-serif; + font-size: 16px; + color: #222; + text-rendering: optimizeLegibility; +} + +pre, code, kbd { + font-family: 'Roboto Mono', Monaco, courier, monospace; + font-size: 0.90em; +} + +pre code { + font-size: 1em; +} + +a { + color: #2980b9; + text-decoration: none; +} + +a:hover { + color: #3091d1; +} + +a:visited { + color: #9b59b6; +} + +body { + line-height: 1.5; +} + +h1 { + font-size: 1.75em; +} + +/* Unless the

the is very first thing on the page (i.e. the second element + * in the
, * after the
, we add some additional styling to it + * to make it stand out a bit more. This way we get a reasonable fallback if CSS3 + * selectors are not supported in the browser. + */ +article > h1:not(:nth-child(2)) { + margin: 2.5em 0 0; + padding-bottom: 0.30em; + border-bottom: 1px solid #e5e5e5; +} +h2 { + font-size: 1.50em; + margin: 2.3em 0 0; + padding-bottom: 0.25em; + border-bottom: 1px solid #e5e5e5; +} +h3 { + font-size: 1.25em; + margin: 2.0em 0 0; +} +h4 { font-size: 1.15em; } +h5 { font-size: 1.10em; } +h6 { font-size: 1em; } + +h4, h5, h6 { + margin-top: 1.5em; + margin-bottom: 1em; +} + +img { + max-width: 100%; +} + +table { + border-collapse: collapse; + margin: 1em 0; +} + +th, td { + border: 1px solid #e1e4e5; + padding: 0.5em 1em; +} + +th { + border-bottom-width: 2px; +} + +tr:nth-child(even) { + background-color: #f3f6f6; +} + +hr { + border: 0; + border-top: 1px solid #e5e5e5; +} + +/* Inline code and code blocks */ + +code { + padding: 0.1em; + background-color: rgba(0,0,0,.04); + border-radius: 3px; +} + +pre { + background-color: #f5f5f5; + border: 1px solid #dddddd; + border-radius: 3px; + padding: 0.5em; + overflow: auto; +} + +pre code { + padding: 0; + background-color: initial; +} + +kbd { + font-size: 0.70em; + display: inline-block; + padding: 0.1em 0.5em 0.4em 0.5em; + line-height: 1.0em; + color: #444d56; + vertical-align: middle; + background-color: #fafbfc; + border: solid 1px #c6cbd1; + border-bottom-color: #959da5; + border-radius: 3px; + box-shadow: inset 0 -1px 0 #959da5; +} + +/* Headers in admonitions and docstrings */ +.admonition h1, +article section.docstring h1 { + font-size: 1.25em; +} + +.admonition h2, +article section.docstring h2 { + font-size: 1.10em; +} + +.admonition h3, +.admonition h4, +.admonition h5, +.admonition h6, +article section.docstring h3, +article section.docstring h4, +article section.docstring h5, +article section.docstring h6 { + font-size: 1em; +} + +/* Navigation */ +nav.toc { + position: fixed; + top: 0; + left: 0; + bottom: 0; + width: 20em; + display: flex; + flex-flow: column nowrap; + overflow-y: auto; + padding: 1em 0 0 0; + background-color: #fcfcfc; + box-shadow: inset -14px 0px 5px -12px rgb(210,210,210); +} + +nav.toc .logo { + margin: 0 auto; + display: block; + max-height: 6em; + max-width: 18em; +} + +nav.toc h1 { + text-align: center; + margin-top: .57em; + margin-bottom: 0; +} + +nav.toc select { + display: block; + height: 2em; + flex-shrink: 0; + padding: 0 1.6em 0 1em; + min-width: 7em; + max-width: 90%; + max-width: calc(100% - 5em); + margin: 0 auto; + font-size: .83em; + border: 1px solid #c9c9c9; + border-radius: 1em; + + /* TODO: doesn't seem to be centered on Safari */ + text-align: center; + text-align-last: center; + + appearance: none; + -moz-appearance: none; + -webkit-appearance: none; + + background: white url("arrow.svg"); + background-size: 1.155em; + background-repeat: no-repeat; + background-position: right; +} + +nav.toc select:hover { + border: 1px solid #a0a0a0; +} + +nav.toc select option { + text-align: center; +} + +nav.toc input { + display: block; + height: 2em; + width: 90%; + width: calc(100% - 5em); + margin: 1.2em auto; + padding: 0 1em; + border: 1px solid #c9c9c9; + border-radius: 1em; + font-size: .83em; +} + +nav.toc > ul * { + margin: 0; +} + +nav.toc > ul { + min-height: 2em; + overflow-y: auto; + margin: 0; +} + +nav.toc > ul > li:last-child { + padding-bottom: 1em; +} + +nav.toc ul { + color: #404040; + padding: 0; + list-style: none; +} + +nav.toc ul .toctext { + color: inherit; + display: block; +} + +nav.toc ul a:hover { + color: #fcfcfc; + background-color: #4e4a4a; +} + +nav.toc ul.internal a { + color: inherit; + display: block; +} + +nav.toc ul.internal a:hover { + background-color: #d6d6d6; +} + +nav.toc ul.internal { + background-color: #e3e3e3; + box-shadow: inset -14px 0px 5px -12px rgb(210,210,210); + list-style: none; +} + +nav.toc ul.internal li.toplevel { + border-top: 1px solid #909090; + font-weight: bold; +} + +nav.toc ul.internal li.toplevel:first-child { + border-top: none; +} + +nav.toc .toctext { + padding-top: 0.3em; + padding-bottom: 0.3em; + padding-right: 1em; } -a.documenter-source { - float: right; +nav.toc ul .toctext { + padding-left: 1em; } -.documenter-methodtable pre { - margin-left: 0px; - margin-right: 0px; - margin-top: 0px; - padding: 0px; +nav.toc ul ul .toctext { + padding-left: 2em; +} + +nav.toc ul ul ul .toctext { + padding-left: 3em; +} + +nav.toc li.current > .toctext { + border-top: 1px solid #c9c9c9; + border-bottom: 1px solid #c9c9c9; + color: #404040; + font-weight: bold; + background-color: white; +} + +nav.toc ul::-webkit-scrollbar { + width: .4em; + background: none; +} + +nav.toc ul::-webkit-scrollbar-thumb { + border-radius: 5px; + background: #c9c9c9; +} + +nav.toc ul::-webkit-scrollbar-thumb:hover { + border-radius: 5px; + background: #aaaaaa; +} + +article { + margin-left: 20em; + min-width: 20em; + max-width: 48em; + padding: 2em; +} + +article > header {} + +article > header div#topbar { + display: none; +} + +article > header nav ul { + display: inline-block; + list-style: none; + margin: 0; + padding: 0; +} + +article > header nav li { + display: inline-block; + padding-right: 0.2em; +} + +article > header nav li:before { + content: "»"; + padding-right: 0.2em; +} + +article > header .edit-page { + float: right; +} + +article > footer {} + +article > footer a.prev { + float: left; +} +article > footer a.next { + float: right; +} + +article > footer a .direction:after { + content: ": "; +} + +article hr { + margin: 1em 0; +} + +article section.docstring { + border: 1px solid #ddd; + margin: 0.5em 0; + padding: 0.5em; + border-radius: 3px; +} + +article section.docstring .docstring-header { + margin-bottom: 1em; +} + +article section.docstring .docstring-binding { + color: #333; + font-weight: bold; +} + +article section.docstring .docstring-category { + font-style: italic; +} + +article section.docstring a.source-link { + display: block; + font-weight: bold; +} + +.nav-anchor, +.nav-anchor:hover, +.nav-anchor:visited { + color: #333; +} + +/* + * Admonitions + * + * Colors (title, body) + * warning: #f0b37e #ffedcc (orange) + * note: #6ab0de #e7f2fa (blue) + * tip: #1abc9c #dbfaf4 (green) +*/ +.admonition { + border-radius: 3px; + background-color: #eeeeee; + margin: 1em 0; +} + +.admonition-title { + border-radius: 3px 3px 0 0; + background-color: #9b9b9b; + padding: 0.15em 0.5em; +} + +.admonition-text { + padding: 0.5em; +} + +.admonition-text > :first-child { + margin-top: 0; +} + +.admonition-text > :last-child { + margin-bottom: 0; +} + +.admonition > .admonition-title:before { + font-family: "FontAwesome"; + margin-right: 5px; + content: "\f06a"; +} + +.admonition.warning > .admonition-title { + background-color: #f0b37e; +} + +.admonition.warning { + background-color: #ffedcc; +} + +.admonition.note > .admonition-title { + background-color: #6ab0de; +} + +.admonition.note { + background-color: #e7f2fa; +} + +.admonition.tip > .admonition-title { + background-color: #1abc9c; +} + +.admonition.tip { + background-color: #dbfaf4; +} + + +/* footnotes */ +.footnote { + padding-left: 0.8em; + border-left: 2px solid #ccc; +} + +/* Search page */ +#search-results .category { + font-size: smaller; +} + +/* Overriding the block style of highligh.js. + * We have to override the padding and the background-color, since we style this + * part ourselves. Specifically, we style the 
 surrounding the , while
+ * highlight.js applies the .hljs style directly to the  tag.
+ */
+.hljs {
+    background-color: transparent;
+    padding: 0;
+}
+
+@media only screen and (max-width: 768px) {
+    nav.toc {
+        position: fixed;
+        width: 16em;
+        left: -16em;
+        -webkit-overflow-scrolling: touch;
+        -webkit-transition-property: left; /* Safari */
+        -webkit-transition-duration: 0.3s; /* Safari */
+        transition-property: left;
+        transition-duration: 0.3s;
+        -webkit-transition-timing-function: ease-out; /* Safari */
+        transition-timing-function: ease-out;
+        z-index: 2;
+        box-shadow: 5px 0px 5px 0px rgb(210,210,210);
+    }
+
+    nav.toc.show {
+        left: 0;
+    }
+
+    article {
+        margin-left: 0;
+        padding: 3em 0.9em 0 0.9em; /* top right bottom left */
+        overflow-wrap: break-word;
+    }
+
+    article > header {
+        position: fixed;
+        left: 0;
+        z-index: 1;
+    }
+
+    article > header nav, hr {
+        display: none;
+    }
+
+    article > header div#topbar {
+        display: block; /* is mobile */
+        position: fixed;
+        width: 100%;
+        height: 1.5em;
+        padding-top: 1em;
+        padding-bottom: 1em;
+        background-color: #fcfcfc;
+        box-shadow: 0 1px 3px rgba(0,0,0,.26);
+        top: 0;
+        -webkit-transition-property: top; /* Safari */
+        -webkit-transition-duration: 0.3s; /* Safari */
+        transition-property: top;
+        transition-duration: 0.3s;
+    }
+
+    article > header div#topbar.headroom--unpinned.headroom--not-top.headroom--not-bottom {
+        top: -4em;
+        -webkit-transition-property: top; /* Safari */
+        -webkit-transition-duration: 0.7s; /* Safari */
+        transition-property: top;
+        transition-duration: 0.7s;
+    }
+
+    article > header div#topbar span {
+        width: 80%;
+        height: 1.5em;
+        margin-top: -0.1em;
+        margin-left: 0.9em;
+        font-size: 1.2em;
+        overflow: hidden;
+    }
+
+    article > header div#topbar a.fa-bars {
+        float: right;
+        padding: 0.6em;
+        margin-top: -0.6em;
+        margin-right: 0.3em;
+        font-size: 1.5em;
+    }
+
+    article > header div#topbar a.fa-bars:visited {
+        color: #3091d1;
+    }
+
+    article table {
+        overflow-x: auto;
+        display: block;
+    }
+
+    article div.MathJax_Display {
+        overflow: scroll;
+    }
+
+    article span.MathJax {
+        overflow: hidden;
+    }
 }
 
-.documenter-methodtable pre.documenter-inline {
-    display: inline;
+@media only screen and (max-width: 320px) {
+    body {
+        font-size: 15px;
+    }
 }
diff --git a/julia/docs/build/assets/mathjaxhelper.js b/julia/docs/build/assets/mathjaxhelper.js
deleted file mode 100755
index 3561b109f9f..00000000000
--- a/julia/docs/build/assets/mathjaxhelper.js
+++ /dev/null
@@ -1,25 +0,0 @@
-MathJax.Hub.Config({
-  "tex2jax": {
-    inlineMath: [['$','$'], ['\\(','\\)']],
-    processEscapes: true
-  }
-});
-MathJax.Hub.Config({
-  config: ["MMLorHTML.js"],
-  jax: [
-    "input/TeX",
-    "output/HTML-CSS",
-    "output/NativeMML"
-  ],
-  extensions: [
-    "MathMenu.js",
-    "MathZoom.js",
-    "TeX/AMSmath.js",
-    "TeX/AMSsymbols.js",
-    "TeX/autobold.js",
-    "TeX/autoload-all.js"
-  ]
-});
-MathJax.Hub.Config({
-  TeX: { equationNumbers: { autoNumber: "AMS" } }
-});
diff --git a/julia/docs/build/index.md b/julia/docs/build/index.md
deleted file mode 100755
index e69de29bb2d..00000000000
diff --git a/julia/docs/make.jl b/julia/docs/make.jl
old mode 100755
new mode 100644
index 6dd30421836..2384c6a66d3
--- a/julia/docs/make.jl
+++ b/julia/docs/make.jl
@@ -1,3 +1,5 @@
+import Pkg
+Pkg.instantiate()
 using Documenter, FlatBuffers
 
 makedocs(
@@ -11,7 +13,5 @@ deploydocs(
     repo = "github.com/JuliaData/FlatBuffers.jl.git",
     target = "build",
     deps = nothing,
-    make = nothing,
-    julia = "0.5",
-    osname = "linux"
+    make = nothing
 )
diff --git a/julia/docs/mkdocs.yml b/julia/docs/mkdocs.yml
old mode 100755
new mode 100644
diff --git a/julia/docs/src/index.md b/julia/docs/src/index.md
old mode 100755
new mode 100644
index 4f374d292d0..6e06b214606
--- a/julia/docs/src/index.md
+++ b/julia/docs/src/index.md
@@ -1,13 +1,13 @@
 # FlatBuffers.jl Documentation
 
-#### Usage
-
+#### Overview
 FlatBuffers.jl provides native Julia support for reading and writing binary structures following the google flatbuffer schema (see [here](https://google.github.io/flatbuffers/flatbuffers_internals.html) for a more in-depth review of the binary format).
 
 The typical language support for flatbuffers involves utilizing the `flatc` compiler to translate a flatbuffer schema file (.fbs) into a langugage-specific set of types/classes and methods. See [here](https://google.github.io/flatbuffers/flatbuffers_guide_writing_schema.html) for the official guide on writing schemas.
 
-Currently in Julia, the `flatc` compiler isn't supported, but FlatBuffers.jl provides a native implementation of reading/writing the binary format directly with native Julia types. What does this mean exactly? Basically you can take a schema like:
+This Julia package provides the serialization primitives used by code that has been generated by `flatc`. Since it was originally built without `flatc` support, it can also be used as a minimal set of macros to provide flatbuffer-compatible serialization of existing Julia types. This has led to the Julia code generated by `flatc` appearing somewhat more readable than for other languages.
 
+For example, for this schema:
 ```
 namespace example;
 
@@ -17,55 +17,81 @@ table SimpleType {
 
 root_type SimpleType;
 ```
-
-and do a straightforward Julia translation like:
-
+the code generated by `flatc` looks like this:
 ```julia
 module Example
 
 using FlatBuffers
-
-mutable struct SimpleType
-    x::Int32
+@with_kw mutable struct SimpleType
+    x::Int32 = 1
 end
 
-@DEFAULT SimpleType x=1
-
+# ... other generated stuff
 end
 ```
+If you don't want to write a schema, you can pepper your existing Julia types
+with these macros and then call the functions below to produce flatbuffer-compatible
+binaries.
 
-A couple of things to point out:
-* `using FlatBuffers` was included near the top to bring in the FlatBuffers module; this defines the necessary FlatBuffers.jl machinery for making the schema definitions easier
-* `int` translates to a Julia `Int32`, see more info on flatbuffer types [here](https://google.github.io/flatbuffers/md__schemas.html)
-* A default value for the `x` field in `SimpleType` was declared after the type with the `@DEFAULT` macro
-* No `root_type` definition is necessary in Julia; basically any type defined with `type` (i.e. not abstract or immutable) can be a valid root table type in Julia.
-
-So let's see how we can actually use a flatbuffer in Julia:
+#### Usage
+`FlatBuffers` provides the following functions for reading and writing flatbuffers:
+```
+FlatBuffers.serialize(stream::IO, value::T) 
+FlatBuffers.deserialize(stream::IO, ::Type{T})
+```
+These methods are not exported to avoid naming clashes with the `Serialization` module.
+For convenience, there are also two additional constructors defined for each generated type:
+* `T(buf::AbstractVector{UInt8}, pos::Integer=0)`
+* `T(io::IO)`
 
+Here is an example showing how to use them to serialize the example type above.
 ```julia
-using FlatBuffers, Example # the schema module we defined above
-
-val = Example.SimpleType(2) # create an instance of our type
-
-flatbuffer = FlatBuffers.build!(val) # start and build a flatbuffer for our SimpleType
-val2 = FlatBuffers.read(flatbuffer) # now we can deserialize the value from our flatbuffer, `val2` == `val`
-flatbytes = FlatBuffers.bytes(flatbuffer) # get the serialized bytes of the flatbuffer
-val3 = Flatbuffers.read(Example.SimpleType, flatbytes) # now we can deserialize directly from flatbytes
-```
+import FlatBuffers, Example
 
-For more involved examples, see the test suite [here](https://github.com/dmbates/FlatBuffers.jl/tree/master/test).
+# create an instance of our type
+val = Example.SimpleType(2)
 
-#### Reference
+# serialize it to example.bin
+open("example.bin", "w") do f FlatBuffers.serialize(f, val) end
 
-Documentation is included inline for each type/method, these can be accessed at the REPL by type `?foo` where `foo` is the name of the type or method you'd like more information on.
+# read the value back again from file
+val2 = open("example.bin", "r") do f Example.SimpleType(f) end
+```
+In addition, this package provides the following types and methods, which are useful
+when inspecting and constructing flatbuffers:
+* `FlatBuffers.Table{T}` - type for deserializing a Julia type `T` from a flatbuffer
+* `FlatBuffers.Builder{T}` - type for serializing a Julia type `T` to a flatbuffer
+* `FlatBuffers.read` - performs the actual deserializing on a `FlatBuffer.Table`
+* `FlatBuffers.build!` - performs the actual serializing on a `FlatBuffer.Builder`
+
+#### Methods for Generated Types
+For a generated type `T`, in addition to the constructors mentioned above:
+* if `T` has default values, constructors will be defined as per the `@with_kw` macro in [Parameters.jl](https://github.com/mauro3/Parameters.jl)
+* `FlatBuffers.file_extension(T)` - returns the `file_extension` specified in the schema (if any)
+* `FlatBuffers.file_identifier(T)` - returns the `file_identifier` specified in the schema (if any)
+* `FlatBuffers.has_identifier(T, bytes)` - returns whether the given bytes contain the identifier for `T` at the offset designated by the flatbuffers specification
+* `FlatBuffers.slot_offsets(T)` - an array containing the positions of the slots in the vtable for type `T`, accounting for gaps caused by deprecated fields
+* `FlatBuffers.root_type(T)` - returns whether the type is designated as the root type by the schema. Also note however that no `root_type` definition is necessary in Julia; any of the generated `mutable struct`s can be a valid root table type.
+
+#### Circular References
+It's a bit unfortunate that the flatbuffers example uses mutually referential types, something which Julia doesn't have support for yet.
+However, there is a [workaround](https://github.com/JuliaLang/julia/issues/269#issuecomment-68421745) - by modifying the
+code generated by `flatc` slightly to add a type parameter, we can refer to a type that hasn't yet been defined.
+```julia
+FlatBuffers.@with_kw mutable struct Monster{T}
+    # ...
+    test::T = nothing
+    # ...
+end
+```
+In general though, try to avoid schemas which introduce these kinds of circular references.
+For the full `Monster` example see the test suite [here](https://github.com/JuliaData/FlatBuffers.jl/blob/master/test/MyGame/Example/Monster.jl).
 
-List of types/methods:
+#### Internal Utilities
+These functions are used by the code generated by `flatc`. Documentation is also included for many
+internal methods and may be queried using `?` at the REPL.
+* `@ALIGN T size_in_bytes` - convenience macro for forcing a flatbuffer alignment on the Julia type `T` to `size_in_bytes`
+* `@with_kw mutable struct T fields...` - convenience macro for defining default field values for Julia type `T`
+* `@UNION T Union{T1,T2,...}` - convenience macro for defining a flatbuffer union type `T`
+* `@STRUCT struct T fields... end` - convenience macro for defining flatbuffer struct types, ensuring any necessary padding gets added to the type definition
 
-* `FlatBuffers.Table{T}`: type for deserializing a Julia type `T` from a flatbuffer
-* `FlatBuffers.Builder{T}`: type for serializing a Julia type `T` to a flatbuffer
-* `FlatBuffers.read`: performs the actual deserializing on a `FlatBuffer.Table`
-* `FlatBuffers.build!`: performs the actual serializing on a `FlatBuffer.Builder`
-* `@ALIGN T size_in_bytes`: convenience macro for forcing a flatbuffer alignment on the Julia type `T` to `size_in_bytes`
-* `@DEFAULT T field1=val1 field2=val2 ...`: convenience macro for defining default field values for Julia type `T`
-* `@UNION T Union{T1,T2,...}`: convenience macro for defining a flatbuffer union type `T`
-* `@STRUCT immutable T fields... end`: convenience macro for defining flatbuffer struct types, ensuring any necessary padding gets added to the type definition
diff --git a/julia/src/FlatBuffers.jl b/julia/src/FlatBuffers.jl
old mode 100755
new mode 100644
index fdc003be367..fa1bad5eb8f
--- a/julia/src/FlatBuffers.jl
+++ b/julia/src/FlatBuffers.jl
@@ -1,38 +1,71 @@
-__precompile__(true)
 module FlatBuffers
 
 # utils
+"""
+serialize(stream::IO, value::T) where {T}
+Serialize `value` to `stream` using the `FlatBuffer` format.
+"""
+function serialize(stream::IO, value::T) where {T}
+	write(stream, bytes(build!(value)))
+end
+
+"""
+deserialize(stream::IO, ::Type{T}) where {T}
+Read a `T` from the flatbuffer-formatted `stream`.
+"""
+function deserialize(stream::IO, ::Type{T}) where {T}
+	read(T, read(stream))
+end
+
 struct UndefinedType end
 const Undefined = UndefinedType()
-
 getfieldvalue(obj::T, i) where {T} = isdefined(obj, i) ? getfield(obj, i) : Undefined
+getprevfieldvalue(obj::T, i) where {T} = i == 1 ? missing : getfieldvalue(obj, i - 1)
 
 """
-     Scalar
+Scalar
 A Union of the Julia types `T <: Number` that are allowed in FlatBuffers schema
 """
-const Scalar = Union{UndefinedType, Bool,
-                        Int8, Int16, Int32, Int64,
-                        UInt8, UInt16, UInt32, UInt64,
-                        Float32, Float64}
-
-if VERSION < v"0.7-DEV"
-    isconcrete = isleaftype
-else
-    isconcrete = isconcretetype
-end
+const Scalar = Union{Bool,
+Int8, Int16, Int32, Int64,
+UInt8, UInt16, UInt32, UInt64,
+Float32, Float64}
 
-isstruct(T) = !T.mutable && isconcrete(T)
+isstruct(T) = isconcretetype(T) && !T.mutable
 isbitstype(T) = fieldcount(T) == 0
+isunionwithnothing(T) = T isa Union && T.a == Nothing && !(isa(T.b, Union))
+
+file_identifier(T) = ""
+file_extension(T) = ""
+slot_offsets(T) = [4 + ((i - 1) * 2) for i = 1:length(T.types)]
 
 default(T, TT, sym) = default(TT)
-default(::Type{UndefinedType}) = Undefined
 default(::Type{T}) where {T <: Scalar} = zero(T)
 default(::Type{T}) where {T <: AbstractString} = ""
 default(::Type{T}) where {T <: Enum} = enumtype(T)(T(0))
 default(::Type{Vector{T}}) where {T} = T[]
+
+# attempt to call default constructors for the type,
+# use above methods as fallback
+function default(::Type{T}, i::Integer) where {T}
+	TT = T.types[i]
+	try
+		return FlatBuffers.default(T, TT, fieldnames(T)[i])
+		# catch because Parameters throws an error if there is no
+		# default value defined...
+	catch
+	end
+	return default(TT)
+end
+
 # fallback that recursively builds a default; for structs/tables
-default(::Type{T}) where {T} = isa(T, Union) ? nothing : T(map(TT->TT == T ? TT() : default(TT),T.types)...)
+function default(::Type{T}) where {T}
+	if isa(T, Union) || isa(T, UnionAll)
+		return nothing
+	else
+		return T([default(T, i) for i = 1:length(T.types)]...)
+	end
+end
 
 function typeorder end
 
@@ -40,18 +73,18 @@ enumtype(::Type{<:Enum}) = UInt8
 
 # Types
 """
-    Table
+Table
 
 The object containing the flatbuffer and positional information specific to the table.
 The `vtable` containing the offsets for specific members precedes `pos`.
 The actual values in the table follow `pos` offset and size of the vtable.
 
-- `bytes::Vector{UInt8}`: the flatbuffer itself
-- `pos::Int`:  the base position in `bytes` of the table
+- `bytes::AbstractVector{UInt8}`: the flatbuffer itself
+- `pos::Integer`:  the base position in `bytes` of the table
 """
 mutable struct Table{T}
-    bytes::Vector{UInt8}
-    pos::Int
+	bytes::AbstractVector{UInt8}
+	pos::Integer
 end
 
 """
@@ -62,9 +95,9 @@ A Builder constructs byte buffers in a last-first manner for simplicity and
 performance.
 """
 mutable struct Builder{T}
-    bytes::Vector{UInt8}
-    minalign::Int
-    vtable::Vector{Int}
+	bytes::AbstractVector{UInt8}
+	minalign::Int
+	vtable::Vector{Int}
 	objectend::Int
 	vtables::Vector{Int}
 	head::Int
@@ -72,123 +105,191 @@ mutable struct Builder{T}
 	finished::Bool
 end
 
-function Base.show(io::IO, x::Union{Builder{T},Table{T}}) where {T}
-    println(io, "FlatBuffers.$(typeof(x)): ")
-    buffer = typeof(x) <: Table ? x.bytes : x.bytes[x.head+1:end]
-    if isempty(buffer)
-        print(io, " (empty flatbuffer)")
-    else
-        pos = Int(typeof(x) <: Table ? x.pos : readbuffer(buffer, 0, Int32))
-        # print vtable offset
-        syms = T.name.names
-        maxpad = max(length(" vtable rel. start pos: "), maximum(map(x->length(string(x)), syms)))
-        stringify(buf, x, y, msg) = replace(string(rpad(string(lpad("$(x): ", 6, ' '),lpad(msg, maxpad, ' ')),maxpad+6,' '),string(map(z->lpad(string(Int(z)), 4, ' '),buf[x:y]))[9:end-1]),'"',"")
-        println(io, stringify(buffer, 1, 4, " root position: "))
-        vtaboff = readbuffer(buffer, pos, Int32)
-        vtabstart = pos - vtaboff + 5
-
-        println(io, stringify(buffer, 5, 6, " vtable size: "))
-        println(io, stringify(buffer, 7, 8, " data size: "))
-        i = vtabstart
-        x = 1
-        for y = 1:length(syms)
-            println(io, stringify(buffer, i, i+1, "$(syms[x]): "))
-            i += 2
-            x += 1
-        end
-        # print rel pos. of vtable
-        println(io, stringify(buffer, i, i+3, " vtable rel. start pos: "))
-        i += 4
-        # now we're pointing at data
-        while i < length(buffer)
-            println(io, stringify(buffer, i, i+3, " "))
-            i += 4
-        end
-    end
+function hexloc(x)
+	"0x" * lpad("$(string(x-1, base=16)) ", 6, '0')
 end
 
-include("internals.jl")
+function hexbyte(io, z)
+	printstyled(io, lpad("$(string(z, base=16)) ", 3, '0'), color=Int(z))
+end
 
-function Table(::Type{T}, buffer::Vector{UInt8}, pos::Integer) where {T}
-    return Table{T}(buffer, pos)
+function hexoffset(x)
+	"0x$(lpad(string(x, base=16), 4, '0'))"
 end
 
-Table(b::Builder{T}) where {T} = Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32))
+function stringify(io, buf, offset, x, y, msg="", msgcolor=:blue)
+	y = min(y, length(buf))
+	printstyled(io, hexloc(x + offset), color=:blue)
+	for i = x:y
+		hexbyte(io, buf[i])
+	end
+	if length(msg) > 0
+		printstyled(io, " " * msg, color=msgcolor)
+	end
+	println(io)
+end
 
-function untilindex(func, itr)
-    for (i,x) in enumerate(itr)
-        func(x) && return i
-    end
-    return 0
+function showvtable(io::IO, T, buffer, vtabstart, vtabsize)
+	syms = T.name.names
+	printstyled(io, "vtable start pos: $(hexoffset(vtabstart))\n", color=:green)
+	printstyled(io, "vtable size: $vtabsize\n", color=:green)
+	i = vtabstart + 4
+	soff = slot_offsets(T)
+    numslots = div(soff[end] - 4, 2) + 1
+	field = 1
+	slot = 1
+	numfields = length(T.types)
+	while slot <= numslots
+		# leave holes for deprecated fields
+		j = 2
+		start = field == 1 ? soff[1] : soff[field - 1]
+		while (start + j) < soff[field]
+			# empty slot
+			stringify(io, buffer, 1, i, i+1, "[deprecated field]", :red)
+			slot += 1
+			j += 2
+			i += 2
+			if (i - vtabstart) > vtabsize
+				break
+			end
+		end
+		if (i - vtabstart) > vtabsize
+			break
+		end
+		stringify(io, buffer, 1, i, i+1, "[$(fieldnames(T)[field])]")
+		slot += 1
+		field += 1
+		i += 2
+		if (i - vtabstart) > vtabsize
+			break
+		end
+	end
+	# now we're pointing at data
+	printstyled(io, "payload:\n", color=:green)
+	while i < length(buffer)
+		stringify(io, buffer, 1, i, i+7, "")
+		i += 8
+	end
 end
 
+function Base.show(io::IO, x::Union{Builder{T}, Table{T}}) where {T}
+	printstyled(io, "FlatBuffers.$(typeof(x)):\n", color=:green)
+	buffer = x isa Builder ? x.bytes[x.head+1:end] : x.bytes
+	if isempty(buffer)
+		printstyled(io, " (empty flatbuffer)", color=:red)
+	else
+		pos = Int(typeof(x) <: Table ? x.pos : readbuffer(buffer, 0, Int32))
+		printstyled(io, "root offset: $(hexoffset(pos))\n", color=:green)
+		vtaboff = readbuffer(buffer, pos, Int32)
+		vtabstart = pos - vtaboff
+		vtabsize = readbuffer(buffer, vtabstart, Int16)
+		showvtable(io, T, buffer, vtabstart, vtabsize)
+	end
+end
+
+include("internals.jl")
+
+function Table(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T}
+	return Table{T}(buffer, pos)
+end
+
+Table(b::Builder{T}) where {T} = Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32))
+
 getvalue(t, o, ::Type{Nothing}) = nothing
 getvalue(t, o, ::Type{T}) where {T <: Scalar} = get(t, t.pos + o, T)
 getvalue(t, o, ::Type{T}) where {T <: Enum} = T(get(t, t.pos + o, enumtype(T)))
 function getvalue(t, o, ::Type{T}) where {T <: AbstractString}
-    o += get(t, t.pos + o, Int32)
-    strlen = get(t, t.pos + o, Int32)
-    o += t.pos + sizeof(Int32)
-    return String(t.bytes[o + 1:o + strlen])
+	o += get(t, t.pos + o, Int32)
+	strlen = get(t, t.pos + o, Int32)
+	o += t.pos + sizeof(Int32)
+	return String(t.bytes[o + 1:o + strlen])
 end
 function getvalue(t, o, ::Type{Vector{UInt8}})
-    o += get(t, t.pos + o, Int32)
-    len = get(t, t.pos + o, Int32)
-    o += t.pos + sizeof(Int32)
-    return t.bytes[o + 1:o + len] #TODO: maybe not make copy here?
+	o += get(t, t.pos + o, Int32)
+	len = get(t, t.pos + o, Int32)
+	o += t.pos + sizeof(Int32)
+	return t.bytes[o + 1:o + len] #TODO: maybe not make copy here?
 end
 
 getarray(t, vp, len, ::Type{T}) where {T <: Scalar} = (ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len])
 getarray(t, vp, len, ::Type{T}) where {T <: Enum} = (ptr = convert(Ptr{enumtype(T)}, pointer(t.bytes, vp + 1)); return [unsafe_load(ptr, i) for i = 1:len])
 function getarray(t, vp, len, ::Type{T}) where {T <: Union{AbstractString, Vector{UInt8}}}
-    A = Vector{T}(undef, len)
-    for i = 1:len
-        A[i] = getvalue(t, vp - t.pos, T)
-        vp += sizeof(Int32)
-    end
-    return A
+	A = Vector{T}(undef, len)
+	for i = 1:len
+		A[i] = getvalue(t, vp - t.pos, T)
+		vp += sizeof(Int32)
+	end
+	return A
 end
 function getarray(t, vp, len, ::Type{T}) where {T}
-    if isstruct(T)
-        ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1))
-        return [unsafe_load(ptr, i) for i = 1:len]
-    else
-        A = Vector{T}(undef, len)
-        for i = 1:len
-            A[i] = getvalue(t, vp - t.pos, T)
-            vp += sizeof(Int32)
-        end
-        return A
-    end
+	if isstruct(T)
+		ptr = convert(Ptr{T}, pointer(t.bytes, vp + 1))
+		return [unsafe_load(ptr, i) for i = 1:len]
+	else
+		A = Vector{T}(undef, len)
+		for i = 1:len
+			A[i] = getvalue(t, vp - t.pos, T)
+			vp += sizeof(Int32)
+		end
+		return A
+	end
 end
 
 function getvalue(t, o, ::Type{Vector{T}}) where {T}
-    vl = vectorlen(t, o)
-    vp = vector(t, o)
-    return getarray(t, vp, vl, T)
+	vl = vectorlen(t, o)
+	vp = vector(t, o)
+	return getarray(t, vp, vl, T)
 end
 
 Base.convert(::Type{T}, e::Integer) where {T <: Enum} = T(e)
 
 # fallback which recursively calls read
 function getvalue(t, o, ::Type{T}) where {T}
-    if isstruct(T)
-        if any(x-> x <: Enum, T.types)
-            args = []
-            o = t.pos + o + 1
-            for typ in T.types
-                val = unsafe_load(convert(Ptr{typ <: Enum ? enumtype(typ) : typ}, pointer(view(t.bytes, o:length(t.bytes)))))
-                push!(args, val)
-                o += sizeof(typ <: Enum ? enumtype(typ) : typ)
-            end
-            return T(args...)
-        else
-            return unsafe_load(convert(Ptr{T}, pointer(view(t.bytes, (t.pos + o + 1):length(t.bytes)))))
+	if isstruct(T)
+		if any(x-> x <: Enum, T.types)
+			args = []
+			o = t.pos + o + 1
+			for typ in T.types
+				val = unsafe_load(convert(Ptr{typ <: Enum ? enumtype(typ) : typ}, pointer(view(t.bytes, o:length(t.bytes)))))
+				push!(args, val)
+				o += sizeof(typ <: Enum ? enumtype(typ) : typ)
+			end
+			return T(args...)
+		else
+			return unsafe_load(convert(Ptr{T}, pointer(view(t.bytes, (t.pos + o + 1):length(t.bytes)))))
+		end
+	else
+		o += t.pos
+		newt = Table{T}(t.bytes, indirect(t, o))
+		return FlatBuffers.read(newt, T)
+	end
+end
+
+function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT}
+    R = TT
+    nullable = false
+    if isunionwithnothing(R)
+        nullable = true
+        R = TT.b
+    end
+
+    # if it's a Union type, use the previous arg to figure out the true type that was serialized
+    if !isunionwithnothing(R) && R isa Union
+        R = typeorder(R, prevfield)
+    end
+
+    if R <: AbstractVector
+        # hacks! if it's a union all, assume it's because we're working around circular dependencies
+        if isa(eltype(R), UnionAll)
+            return Vector{T}, false, nullable
+        # if it's a vector of Unions, use the previous field to figure out the types of all the elements
+        elseif isa(eltype(R), Union)
+            types = typeorder.(eltype(R), prevfield)
+            R = definestruct(types)
+            return R, true, nullable
         end
-    else
-        newt = Table{T}(t.bytes, t.pos + o + get(t, t.pos + o, Int32))
-        return FlatBuffers.read(newt, T)
     end
+    return R, false, nullable
 end
 
 """
@@ -196,58 +297,66 @@ end
 Will recurse as necessary for nested types (Arrays, Tables, etc.)
 """
 function FlatBuffers.read(t::Table{T1}, ::Type{T}=T1) where {T1, T}
-    args = []
-    numfields = length(T.types)
-    for i = 1:numfields
-        TT = T.types[i]
-        # if it's a Union type, use the previous arg to figure out the true type that was serialized
-        if isa(TT, Union)
-            TT = typeorder(TT, args[end])
-        end
-        o = offset(t, 4 + ((i - 1) * 2))
+	args = []
+	numfields = length(T.types)
+	soff = slot_offsets(T)
+	for i = 1:numfields
+		TT = T.types[i]
+		o = offset(t, soff[i])
+        R, isunionvector, nullable = typetoread(i == 1 ? nothing : args[end], T, TT)
         if o == 0
-            push!(args, default(T, TT, T.name.names[i]))
+            push!(args, nullable ? nothing : default(T, TT, T.name.names[i]))
         else
-            push!(args, getvalue(t, o, TT))
+            if isunionvector
+                eval(:(newr = getvalue($t, $o, $R)))
+                eval(:(n = length($R.types)))
+                push!(args, [getfieldvalue(newr, j) for j = 1:n]) 
+            else
+                push!(args, getvalue(t, o, R))
+            end
         end
     end
-    return T(args...)
+
+	return T(args...)
 end
 
-FlatBuffers.read(::Type{T}, buffer::Vector{UInt8}, pos::Integer) where {T} = FlatBuffers.read(Table(T, buffer, pos))
+FlatBuffers.read(::Type{T}, buffer::AbstractVector{UInt8}, pos::Integer) where {T} = FlatBuffers.read(Table(T, buffer, pos))
 FlatBuffers.read(b::Builder{T}) where {T} = FlatBuffers.read(Table(T, b.bytes[b.head+1:end], get(b, b.head, Int32)))
 # assume `bytes` is a pure flatbuffer buffer where we can read the root position at the beginning
 FlatBuffers.read(::Type{T}, bytes) where {T} = FlatBuffers.read(T, bytes, read(IOBuffer(bytes), Int32))
 
+has_identifier(::Type{T}, bytes) where {T} = length(bytes) >= 8 && String(bytes[5:8]) == FlatBuffers.file_identifier(T)
+root_type(::Type{T}) where {T} = false
+
 """
-    flat_bytes = bytes(b)
+flat_bytes = bytes(b)
 
 `flat_bytes` are the serialized bytes for the FlatBuffer.  This discards the Julia specific `head`.
 """
 bytes(b::Builder) = unsafe_wrap(Array{UInt8,1}, pointer(b.bytes, b.head+1), (length(b.bytes)-b.head))
 
 function Builder(::Type{T}=Any, size=0) where {T}
-    objectend = 0
-    vtables = zeros(Int, 0)
-    head = size
-    nested = false
-    bytes = zeros(UInt8, size)
-    minalign = 1
-    vtable = zeros(Int, 0)
-    finished = false
-    b = Builder{T}(bytes, minalign, vtable, objectend,
-                vtables, head, nested, finished)
-    return b
+	objectend = 0
+	vtables = zeros(Int, 0)
+	head = size
+	nested = false
+	bytes = zeros(UInt8, size)
+	minalign = 1
+	vtable = zeros(Int, 0)
+	finished = false
+	b = Builder{T}(bytes, minalign, vtable, objectend,
+		vtables, head, nested, finished)
+	return b
 end
 
 # build!
 "`alignment` looks for the largest scalar member of `T` that represents a flatbuffer Struct"
 function alignment(::Type{T}) where {T}
-    largest = 0
-    for typ in T.types
-        largest = isbitstype(typ) ? max(largest,sizeof(typ)) : alignment(typ)
-    end
-    return largest
+	largest = 0
+	for typ in T.types
+		largest = isbitstype(typ) ? max(largest,sizeof(typ)) : alignment(typ)
+	end
+	return largest
 end
 
 """
@@ -257,53 +366,79 @@ even building its elements recursively if needed (Array of Arrays, Array of tabl
 function buildvector! end
 
 # empty vector
-function buildvector!(b, A::Vector{Nothing}, len)
-    startvector(b, 1, 0, 1)
-    return endvector(b, 0)
+function buildvector!(b, A::Vector{Nothing}, len, prev)
+	startvector(b, 1, 0, 1)
+	return endvector(b, 0)
 end
 # scalar type vector
-function buildvector!(b, A::Vector{T}, len) where {T <: Scalar}
-    startvector(b, sizeof(T), len, sizeof(T))
-    foreach(x->prepend!(b, A[x]), len:-1:1)
-    return endvector(b, len)
+function buildvector!(b, A::Vector{T}, len, prev) where {T <: Scalar}
+	startvector(b, sizeof(T), len, sizeof(T))
+	foreach(x->prepend!(b, A[x]), len:-1:1)
+	return endvector(b, len)
 end
-function buildvector!(b, A::Vector{T}, len) where {T <: Enum}
-    startvector(b, sizeof(enumtype(T)), len, sizeof(enumtype(T)))
-    foreach(x->prepend!(b, enumtype(T)(A[x])), len:-1:1)
-    return endvector(b, len)
+function buildvector!(b, A::Vector{T}, len, prev) where {T <: Enum}
+	startvector(b, sizeof(enumtype(T)), len, sizeof(enumtype(T)))
+	foreach(x->prepend!(b, enumtype(T)(A[x])), len:-1:1)
+	return endvector(b, len)
 end
 
 function putoffsetvector!(b, offsets, len)
-    startvector(b, 4, len, 4) #TODO: elsize/alignment correct here?
-    foreach(x->prependoffset!(b, offsets[x]), len:-1:1)
-    return endvector(b, len)
+	startvector(b, 4, len, 4) #TODO: elsize/alignment correct here?
+	foreach(x->prependoffset!(b, offsets[x]), len:-1:1)
+	return endvector(b, len)
 end
 # byte vector vector
-function buildvector!(b, A::Vector{Vector{UInt8}}, len)
-    offsets = map(x->createbytevector(b, A[x]), 1:len)
-    return putoffsetvector!(b, offsets, len)
+function buildvector!(b, A::Vector{Vector{UInt8}}, len, prev)
+	offsets = map(x->createbytevector(b, A[x]), 1:len)
+	return putoffsetvector!(b, offsets, len)
 end
 # string vector
-function buildvector!(b, A::Vector{T}, len) where {T <: AbstractString}
-    offsets = map(x->createstring(b, A[x]), 1:len)
-    return putoffsetvector!(b, offsets, len)
+function buildvector!(b, A::Vector{T}, len, prev) where {T <: AbstractString}
+	offsets = map(x->createstring(b, A[x]), 1:len)
+	return putoffsetvector!(b, offsets, len)
 end
 # array vector
-function buildvector!(b, A::Vector{Vector{T}}, len) where {T}
-    offsets = map(x->buildbuffer!(b, A[x]), 1:len)
-    return putoffsetvector!(b, offsets, len)
+function buildvector!(b, A::Vector{Vector{T}}, len, prev) where {T}
+	offsets = map(x->buildbuffer!(b, A[x]), 1:len)
+	return putoffsetvector!(b, offsets, len)
+end
+
+# make a new struct which has fields of the given type
+function definestruct(types::Vector{DataType})
+	fields = [:($(gensym())::$(TT)) for TT in types]
+	T1 = gensym()
+	eval(:(mutable struct $T1
+		$(fields...)
+	end))
+	return T1
+end
+
+# make a new struct which has fields of the given type
+# and populate them with values from the vector
+function createstruct(types::Vector{DataType}, A::Vector{T}) where {T}
+	T1 = definestruct(types)
+	eval(:(newt = $T1($(A...))))
+	return newt
 end
+
 # struct or table/object vector
-function buildvector!(b, A::Vector{T}, len) where {T}
-    if isstruct(T)
-        # struct
-        startvector(b, sizeof(T), len, alignment(T)) #TODO: forced elsize/alignment correct here?
-        foreach(x->buildbuffer!(b, A[x]), len:-1:1)
-        return endvector(b, len)
-    else # table/object
-        offsets = map(x->buildbuffer!(b, A[x]), 1:len)
-        return putoffsetvector!(b, offsets, len)
-    end
+function buildvector!(b, A::Vector{T}, len, prev) where {T}
+	if isstruct(T)
+		# struct
+		startvector(b, sizeof(T), len, alignment(T)) #TODO: forced elsize/alignment correct here?
+		foreach(x->buildbuffer!(b, A[x]), len:-1:1)
+		return endvector(b, len)
+	elseif isa(T, Union)
+		types = typeorder.(T, prev)
+
+		# define a new type, construct one, and pack it into the buffer
+		newt = createstruct(types, A)
+		buildbuffer!(b, newt)
+	else
+		# table/object
+		offsets = map(x->buildbuffer!(b, A[x]), 1:len)
+		return putoffsetvector!(b, offsets, len)
+	end
 end
 
 """
@@ -314,14 +449,15 @@ down to their last leaf scalar types before returning the highest-level offset.
 """
 function getoffset end
 
-getoffset(b, arg::Nothing) = 0
-getoffset(b, arg::T) where {T <: Scalar} = 0
-getoffset(b, arg::T) where {T <: Enum} = 0
-getoffset(b, arg::Vector{UInt8}) = createbytevector(b, arg)
-getoffset(b, arg::AbstractString) = createstring(b, arg)
-getoffset(b, arg::Vector{T}) where {T} = buildbuffer!(b, arg)
+getoffset(b, arg::Nothing, prev=nothing) = 0
+getoffset(b, arg::T, prev=nothing) where {T <: Scalar} = 0
+getoffset(b, arg::T, prev=nothing) where {T <: Enum} = 0
+getoffset(b, arg::AbstractString, prev=nothing) = createstring(b, arg)
+getoffset(b, arg::Vector{UInt8}, prev) = createbytevector(b, arg)
+getoffset(b, arg::Vector{T}, prev) where {T} = buildbuffer!(b, arg, prev)
+
 # structs or table/object
-getoffset(b, arg::T) where {T} = isstruct(T) ? 0 : buildbuffer!(b, arg)
+getoffset(b, arg::T, prev=nothing) where {T} = isstruct(T) ? 0 : buildbuffer!(b, arg, prev)
 
 """
 `putslot!` is one of the final steps in building a flatbuffer.
@@ -331,20 +467,66 @@ to the actual data (Arrays, Strings, other tables)
 """
 function putslot! end
 
-putslot!(b, i, arg::T, off) where {T <: Scalar} = prependslot!(b, i, arg, default(T))
-putslot!(b, i, arg::T, off) where {T <: Enum} = prependslot!(b, i, enumtype(T)(arg), default(T))
-putslot!(b, i, arg::AbstractString, off) = prependoffsetslot!(b, i, off, 0)
-putslot!(b, i, arg::Vector{T}, off) where {T} = prependoffsetslot!(b, i, off, 0)
+putslot!(b, i, arg::T, off, default, prev) where {T <: Scalar} = prependslot!(b, i, arg, default)
+putslot!(b, i, arg::T, off, default, prev) where {T <: Enum} = prependslot!(b, i, enumtype(T)(arg), default)
+putslot!(b, i, arg::AbstractString, off, default, prev) = prependoffsetslot!(b, i, off, 0)
+putslot!(b, i, arg::Vector{T}, off, default, prev) where {T} = prependoffsetslot!(b, i, off, 0)
 # structs or table/object
-putslot!(b, i, arg::T, off) where {T} =
-    isstruct(T) ? prependstructslot!(b, i, buildbuffer!(b, arg), 0) : prependoffsetslot!(b, i, off, 0)
-
-function buildbuffer!(b::Builder{T1}, arg::T) where {T1, T}
-    if T <: Array
-        # array of things
-        n = buildvector!(b, arg, length(arg))
-    elseif isstruct(T)
-        # build a struct type with provided `arg`
+function putslot!(b, i, arg::T, off, default, prev) where {T}
+	if isstruct(T)
+		prependstructslot!(b, i, buildbuffer!(b, arg, prev), 0)
+	else
+		prependoffsetslot!(b, i, off, 0)
+	end
+end
+
+function needreconstruct(T)
+	for TT in T.types 
+        if TT <: Vector && eltype(TT) isa Union && !(eltype(TT) isa UnionAll)
+            return true
+        elseif TT isa Union && !isunionwithnothing(TT)
+            return true
+        end
+	end
+    return false	
+end
+
+function reconstructkwargs(arg::T) where {T}
+    kwargs = Dict{Symbol, Any}()
+    numfields = length(T.types)
+    fnames = fieldnames(T)
+    for i = 2:numfields
+        field = getfield(arg, i)
+        prevname = fnames[i - 1]
+        # hack to make the example work
+        TT = field isa Vector ? eltype(field) : typeof(field)
+        if :parameters in propertynames(TT) && length(TT.parameters) > 0
+            TT = TT.name.wrapper
+        end
+        if field isa Vector && eltype(field) isa Union && !(eltype(field) isa UnionAll)
+            kwargs[prevname] = [FlatBuffers.typeorder(TT, typeof(x)) for x in field]
+        elseif (T.types[i] isa Union && !isunionwithnothing(T.types[i]))
+            kwargs[prevname] = FlatBuffers.typeorder(T.types[i], TT)
+        end
+    end
+    return kwargs
+end
+
+function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Array}
+    # array of things
+    buildvector!(b, arg, length(arg), prev)
+end
+
+function buildbuffer!(b::Builder{T1}, arg::T, prev=nothing) where {T1<:Any, T<:Any}
+    # populate the _type field before unions/vectors of unions
+    if needreconstruct(T)
+        # reconstruct it so the types before the fields
+        # are populated correctly
+        kwargs = reconstructkwargs(arg)
+        arg = Parameters.reconstruct(arg; kwargs...)
+    end
+    if isstruct(T)
+    # build a struct type with provided `arg`
         all(isstruct, T.types) || throw(ArgumentError("can't seralize flatbuffer, $T is not a pure struct"))
         align = alignment(T)
         prep!(b, align, 2align)
@@ -355,7 +537,7 @@ function buildbuffer!(b::Builder{T1}, arg::T) where {T1, T}
             elseif isbitstype(typ)
                 prepend!(b, getfield(arg,i))
             else
-                buildbuffer!(b, getfield(arg,i))
+                buildbuffer!(b, getfield(arg, i), getprevfieldvalue(arg, i))
             end
         end
         n = offset(b)
@@ -363,20 +545,61 @@ function buildbuffer!(b::Builder{T1}, arg::T) where {T1, T}
         # build a table type
         # check for string/array/table types
         numfields = length(T.types)
-        offsets = [getoffset(b, getfieldvalue(arg,i)) for i = 1:numfields]
-
-        # all nested have been written, with offsets in `offsets[]`
-        startobject(b, numfields)
-        foreach(i->putslot!(b, i, getfieldvalue(arg,i), offsets[i]), 1:numfields)
+        # early exit for empty objects
+        if numfields == 0
+            startobject(b, 0)
+            return endobject(b)
+        end
+        os = Int[]
+        isdefault = falses(numfields)
+        for i = 1:numfields
+            push!(os, getoffset(b, getfieldvalue(arg, i), getprevfieldvalue(arg, i)))
+        end
+        # all nested have been written, with offsets in `os[]`
+        # don't use slots for the last N members if they are all default
+        # also leave slots for deprecated fields
+        i = numfields
+        isdefault = getfieldvalue(arg, i) == default(T, i)
+        while isdefault && i > 0
+            i -= 1
+            isdefault = getfieldvalue(arg, i) == default(T, i)
+        end
+        soff = slot_offsets(T)
+        numslots = div(soff[i] - 4, 2) + 1
+        startobject(b, numslots)
+        i = 1
+        field = 1
+        while i <= numslots
+            # leave holes for deprecated fields
+            j = 2
+            start = field == 1 ? soff[1] : soff[field - 1]
+            while (start + j) < soff[field]
+                # empty slot
+                i += 1
+                j += 2
+            end
+            val = getfieldvalue(arg, field)
+            d = default(T, field)
+            if !(isunionwithnothing(T.types[field]) && val == nothing)
+                putslot!(b, i,
+                    val,
+                    os[field],
+                    d,
+                    getprevfieldvalue(arg, field)
+                    )
+            end
+            field += 1
+            i += 1
+        end
         n = endobject(b)
     end
-    return n
+	return n
 end
 
 function build!(b, arg)
-    n = buildbuffer!(b, arg)
-    finish!(b, n)
-    return b
+	n = buildbuffer!(b, arg)
+	finish!(b, n)
+	return b
 end
 
 build!(arg::T) where {T} = build!(Builder(T), arg)
diff --git a/julia/src/internals.jl b/julia/src/internals.jl
old mode 100755
new mode 100644
index 0bfae214010..054b573d29e
--- a/julia/src/internals.jl
+++ b/julia/src/internals.jl
@@ -6,7 +6,7 @@ const TableOrBuilder = Union{Table,Builder}
 const Bytes2Type = Dict{Int, DataType}(1=>UInt8, 2=>UInt16, 4=>UInt32, 8=>UInt64)
 
 Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T} = read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), T)
-readbuffer(t::Vector{UInt8}, pos::Int, ::Type{T}) where {T} = read(IOBuffer(view(t, (pos+1):length(t))), T)
+readbuffer(t::AbstractVector{UInt8}, pos::Int, ::Type{T}) where {T} = read(IOBuffer(view(t, (pos+1):length(t))), T)
 Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(read(IOBuffer(view(t.bytes, (pos+1):length(t.bytes))), Bytes2Type[sizeof(T)]))
 
 """
@@ -15,8 +15,8 @@ Base.get(t::TableOrBuilder, pos, ::Type{T}) where {T <: Enum} = T(read(IOBuffer(
 Deprecated fields are ignored by checking against the vtable's length.
 """
 function offset(t::Table, vtableoffset)
-    vtable = t.pos - get(t, t.pos, Int32)
-    return vtableoffset < get(t, vtable, Int16) ? get(t, vtable + vtableoffset, Int16) : 0
+	vtable = t.pos - get(t, t.pos, Int32)
+	return vtableoffset < get(t, vtable, Int16) ? get(t, vtable + vtableoffset, Int16) : 0
 end
 
 "`indirect` retrieves the relative offset stored at `offset`."
@@ -40,7 +40,7 @@ function vector(t::Table, off)
 	off += t.pos
 	off += get(t, off, Int32)
 	# data starts after metadata containing the vector length
-    return off + sizeof(Int32)
+	return off + sizeof(Int32)
 end
 
 """
@@ -75,8 +75,8 @@ value(x::T) where {T <: Enum} = length(T.types) == 0 ? Int(x) : getfield(x,1)
 
 Base.write(sink::Builder, o, x::Union{Bool,UInt8}) = sink.bytes[o+1] = UInt8(x)
 function Base.write(sink::Builder, off, x::T) where {T}
-    off += 1
-    for (i,ind) = enumerate(off:(off + sizeof(T) - 1))
+	off += 1
+	for (i,ind) = enumerate(off:(off + sizeof(T) - 1))
 		sink.bytes[ind] = (x >> ((i-1) * 8)) % UInt8
 	end
 end
@@ -95,27 +95,27 @@ Panics if the builder is not in a finished state (which is caused by calling
 `finish!()`).
 """
 function finishedbytes(b::Builder)
-    assertfinished(b)
-    return b.bytes[b.head+1:end]
+	assertfinished(b)
+	return b.bytes[b.head+1:end]
 end
 
-function startobject(b::Builder, numfields)
-    assertnotnested(b)
-    b.nested = true
-    b.vtable = zeros(Int, numfields)
-    b.objectend = offset(b)
-    b.minalign = 1
-    return b
+function startobject(b::Builder, numslots)
+	assertnotnested(b)
+	b.nested = true
+	b.vtable = zeros(Int, numslots)
+	b.objectend = offset(b)
+	b.minalign = 1
+	return b
 end
 
 """
 `endobject` writes data necessary to finish object construction.
 """
-function endobject(b::Builder)
-    assertnested(b)
-    n = writevtable!(b)
-    b.nested = false
-    return n
+function endobject(b::Builder{T}) where {T}
+	assertnested(b)
+	n = writevtable!(b)
+	b.nested = false
+	return n
 end
 
 """
@@ -132,17 +132,17 @@ function prep!(b::Builder, size, additionalbytes)
 	end
 	# Find the amount of alignment needed such that `size` is properly
 	# aligned after `additionalBytes`:
-    alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1
+	alignsize = xor(Int(-1), (length(b.bytes) - b.head) + additionalbytes) + 1
 	alignsize &= (size - 1)
 
 	# Reallocate the buffer if needed:
-    totalsize = alignsize + size + additionalbytes
-    if b.head <= totalsize
-        len = length(b.bytes)
-        prepend!(b.bytes, zeros(UInt8, totalsize))
-        b.head += length(b.bytes) - len
-    end
-    pad!(b, alignsize)
+	totalsize = alignsize + size + additionalbytes
+	if b.head <= totalsize
+		len = length(b.bytes)
+		prepend!(b.bytes, zeros(UInt8, totalsize))
+		b.head += length(b.bytes) - len
+	end
+	pad!(b, alignsize)
 	return
 end
 
@@ -153,7 +153,7 @@ Aligns and checks for space.
 function Base.prepend!(b::Builder, x::T) where {T}
 	prep!(b, sizeof(T), 0)
 	place!(b, x)
-    return
+	return
 end
 
 """
@@ -161,40 +161,33 @@ end
 """
 function place!(b::Builder, x::T) where {T}
 	b.head -= sizeof(T)
-    write(b, b.head, x)
-    return
+	write(b, b.head, x)
+	return
 end
 
 """
 `startvector` initializes bookkeeping for writing a new vector.
 
 A vector has the following format:
-  
-  +, where T is the type of elements of this vector.
+
++, where T is the type of elements of this vector.
 """
 function startvector(b::Builder, elemSize, numElems, alignment)
-    assertnotnested(b)
-    b.nested = true
-    prep!(b, sizeof(UInt32), elemSize * numElems)
-    prep!(b, alignment, elemSize * numElems)
-    return offset(b)
+	assertnotnested(b)
+	b.nested = true
+	prep!(b, sizeof(UInt32), elemSize * numElems)
+	prep!(b, alignment, elemSize * numElems)
+	return offset(b)
 end
 
 """
 `endvector` writes data necessary to finish vector construction.
 """
 function endvector(b::Builder, vectorNumElems)
-    assertnested(b)
-    place!(b, UInt32(vectorNumElems))
-    b.nested = false
-    return offset(b)
-end
-
-if !isdefined(Base, :codeunits)
-	codeunits = Vector{UInt8}
-end
-if !isdefined(Base, :copyto!)
-	copyto! = copy!
+	assertnested(b)
+	place!(b, UInt32(vectorNumElems))
+	b.nested = false
+	return offset(b)
 end
 
 """
@@ -203,16 +196,14 @@ end
 function createstring(b::Builder, s::AbstractString)
 	assertnotnested(b)
 	b.nested = true
-    s = codeunits(s)
-
+	s = codeunits(s)
 	prep!(b, sizeof(UInt32), length(s) + 1)
-    place!(b, UInt8(0))
+	place!(b, UInt8(0))
 
 	l = length(s)
 
 	b.head -= l
-    copyto!(b.bytes, b.head+1, s, 1, l)
-
+	copyto!(b.bytes, b.head+1, s, 1, l)
 	return endvector(b, length(s))
 end
 
@@ -220,7 +211,7 @@ end
 `createbytevector` writes a byte vector
 """
 function createbytevector(b::Builder, v::AbstractVector{UInt8})
-    assertnotnested(b)
+	assertnotnested(b)
 	b.nested = true
 
 	prep!(b, sizeof(UInt32), length(v))
@@ -228,7 +219,7 @@ function createbytevector(b::Builder, v::AbstractVector{UInt8})
 	l = length(v)
 
 	b.head -= l
-    copyto!(b.bytes, b.head+1, v, 1, l)
+	copyto!(b.bytes, b.head+1, v, 1, l)
 
 	return endvector(b, length(v))
 end
@@ -243,15 +234,15 @@ function prependoffset!(b::Builder, off)
 	end
 	off2 = offset(b) - off + sizeof(Int32)
 	place!(b, Int32(off2))
-    return
+	return
 end
 
-function prependoffsetslot!(b::Builder, o::Int, x::T, d::T) where {T}
-	if x != d
+function prependoffsetslot!(b::Builder, o::Int, x::T, d) where {T}
+	if x != T(d)
 		prependoffset!(b, x)
 		slot!(b, o)
 	end
-    return
+	return
 end
 
 """
@@ -259,12 +250,12 @@ end
 If value `x` equals default `d`, then the slot will be set to zero and no
 other data will be written.
 """
-function prependslot!(b::Builder, o::Int, x::T, d::T) where {T}
-	if x != d
+function prependslot!(b::Builder, o::Int, x::T, d) where {T}
+	if x != T(d)
 		prepend!(b, x)
 		slot!(b, o)
 	end
-    return
+	return
 end
 
 """
@@ -280,7 +271,7 @@ function prependstructslot!(b::Builder, voffset, x, d)
 		end
 		slot!(b, voffset)
 	end
-    return
+	return
 end
 
 """
@@ -293,12 +284,17 @@ end
 """
 `finish!` finalizes a buffer, pointing to the given `rootTable`.
 """
-function finish!(b::Builder, rootTable)
+function finish!(b::Builder{T}, rootTable) where {T}
 	assertnotnested(b)
+	identifier = file_identifier(T)
+	n = length(identifier)
 	prep!(b, b.minalign, sizeof(UInt32))
+	for i = 0:(n-1)
+		prepend!(b, UInt8(identifier[n - i]))
+	end
 	prependoffset!(b, Int32(rootTable))
 	b.finished = true
-    return
+	return
 end
 
 function assertnested(b::Builder)
@@ -309,7 +305,7 @@ function assertnested(b::Builder)
 	if !b.nested
 		throw(ArgumentError("Incorrect creation order: must be inside object."))
 	end
-    return
+	return
 end
 
 function assertnotnested(b::Builder)
@@ -324,7 +320,7 @@ function assertnotnested(b::Builder)
 	if b.nested
 		throw(ArgumentError("Incorrect creation order: object must not be nested."))
 	end
-    return
+	return
 end
 
 function assertfinished(b::Builder)
@@ -339,30 +335,30 @@ function assertfinished(b::Builder)
 end
 
 """
- WriteVtable serializes the vtable for the current object, if applicable.
+WriteVtable serializes the vtable for the current object, if applicable.
 
- Before writing out the vtable, this checks pre-existing vtables for equality
- to this one. If an equal vtable is found, point the object to the existing
- vtable and return.
+Before writing out the vtable, this checks pre-existing vtables for equality
+to this one. If an equal vtable is found, point the object to the existing
+vtable and return.
 
- Because vtable values are sensitive to alignment of object data, not all
- logically-equal vtables will be deduplicated.
+Because vtable values are sensitive to alignment of object data, not all
+logically-equal vtables will be deduplicated.
 
- A vtable has the following format:
-   
-   
-    * N, where N is the number of fields in
-	        the schema for this type. Includes deprecated fields.
- Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide.
+A vtable has the following format:
+
+
+ * N, where N is the number of fields in
+the schema for this type. Includes deprecated fields.
+Thus, a vtable is made of 2 + N elements, each SizeVOffsetT bytes wide.
 
- An object has the following format:
-   
-   +
+An object has the following format:
+
++
 """
-function writevtable!(b::Builder)
+function writevtable!(b::Builder{T}) where {T}
 	# Prepend a zero scalar to the object. Later in this function we'll
 	# write an offset here that points to the object's vtable:
-    prepend!(b, Int32(0))
+	prepend!(b, Int32(0))
 
 	objectOffset = offset(b)
 	existingVtable = 0
@@ -372,11 +368,11 @@ function writevtable!(b::Builder)
 	# BenchmarkVtableDeduplication for a case in which this heuristic
 	# saves about 30% of the time used in writing objects with duplicate
 	# tables.
-    for i = length(b.vtables):-1:1
+	for i = length(b.vtables):-1:1
 		# Find the other vtable, which is associated with `i`:
 		vt2Offset = b.vtables[i]
 		vt2Start = length(b.bytes) - vt2Offset
-        vt2Len = readbuffer(b.bytes, vt2Start, Int16)
+		vt2Len = readbuffer(b.bytes, vt2Start, Int16)
 
 		metadata = VtableMetadataFields * sizeof(Int16)
 		vt2End = vt2Start + vt2Len
@@ -395,15 +391,14 @@ function writevtable!(b::Builder)
 
 		# Write out the current vtable in reverse , because
 		# serialization occurs in last-first order:
-        for i = length(b.vtable):-1:1
-            off::Int16 = 0
+		for i = length(b.vtable):-1:1
+			off::Int16 = 0
 			if b.vtable[i] != 0
 				# Forward reference to field;
 				# use 32bit number to assert no overflow:
 				off = objectOffset - b.vtable[i]
 			end
-
-			prepend!(b, off)
+			prepend!(b, Int16(off))
 		end
 
 		# The two metadata fields are written last.
@@ -419,7 +414,7 @@ function writevtable!(b::Builder)
 		# Next, write the offset to the new vtable in the
 		# already-allocated SOffsetT at the beginning of this object:
 		objectStart::Int32 = length(b.bytes) - objectOffset
-        write(b, objectStart, Int32(offset(b) - objectOffset))
+		write(b, objectStart, Int32(offset(b) - objectOffset))
 
 		# Finally, store this vtable in memory for future
 		# deduplication:
@@ -432,7 +427,7 @@ function writevtable!(b::Builder)
 
 		# Write the offset to the found vtable in the
 		# already-allocated SOffsetT at the beginning of this object:
-        write(b, b.head, Int32(existingVtable - objectOffset))
+		write(b, b.head, Int32(existingVtable - objectOffset))
 	end
 
 	empty!(b.vtable)
@@ -445,8 +440,8 @@ function vtableEqual(a::Vector{Int}, objectStart, b::AbstractVector{UInt8})
 		return false
 	end
 
-    for i = 0:(length(a)-1)
-        x = read(IOBuffer(view(b, (i * sizeof(Int16) + 1):length(b))), Int16)
+	for i = 0:(length(a)-1)
+		x = read(IOBuffer(view(b, (i * sizeof(Int16) + 1):length(b))), Int16)
 
 		# Skip vtable entries that indicate a default value.
 		x == 0 && a[i+1] == 0 && continue
diff --git a/julia/src/macros.jl b/julia/src/macros.jl
old mode 100755
new mode 100644
index 69b20512776..f1c0cf12f26
--- a/julia/src/macros.jl
+++ b/julia/src/macros.jl
@@ -1,40 +1,35 @@
-export @UNION, @DEFAULT, @ALIGN, @STRUCT
-
-if VERSION < v"0.7-DEV"
-    const __module__ = 0
-    __mod__(x) = current_module()
-    fieldcount(x) = nfields(x)
-else
-    __mod__(x) = x
-end
+export @UNION, @DEFAULT, @ALIGN, @STRUCT, @with_kw
+
+const __module__ = 0
 
 function indexof(needle, haystack)
-    for (i, v) in enumerate(haystack)
-        v == needle && return i-1
-    end
-    return -1
+	for (i, v) in enumerate(haystack)
+		v == needle && return i-1
+	end
+	return -1
 end
 
 macro UNION(T, TT)
-    typeof(T) == Symbol || throw(ArgumentError("1st argument must be a symbol to represent a Union type"))
-    TT.head == :tuple || throw(ArgumentError("2nd argument must be a tuple of types like `(T1,T2,...)`"))
-    return esc(quote
-        const $T = $(Expr(:curly, :Union, TT.args...))
-        FlatBuffers.typeorder(::Type{$T}, ::Type{TT}) where {TT} = FlatBuffers.indexof(TT, $TT)
-        FlatBuffers.typeorder(::Type{$T}, i::Integer) = ($TT)[i+1]
-    end)
+	typeof(T) == Symbol || throw(ArgumentError("1st argument must be a symbol to represent a Union type"))
+	TT.head == :tuple || throw(ArgumentError("2nd argument must be a tuple of types like `(T1,T2,...)`"))
+	return esc(quote
+		const $T = $(Expr(:curly, :Union, TT.args...))
+		FlatBuffers.typeorder(::Type{$T}, ::Type{TT}) where {TT} = FlatBuffers.indexof(TT, $TT)
+		FlatBuffers.typeorder(::Type{$T}, i::Integer) = ($TT)[i+1]
+		FlatBuffers.isunionwithnothing(::Type{$T}) = false
+	end)
 end
 
 macro ALIGN(T, sz)
-    return esc(quote
-        FlatBuffers.alignment(::Type{$T}) = $sz
-    end)
+	return esc(quote
+		FlatBuffers.alignment(::Type{$T}) = $sz
+	end)
 end
 
 macro enumtype(T, typ)
-    return esc(quote
-        FlatBuffers.enumtype(::Type{$T}) = $typ
-    end)
+	return esc(quote
+		FlatBuffers.enumtype(::Type{$T}) = $typ
+	end)
 end
 
 # recursively finds largest field of a STRUCT
@@ -47,126 +42,191 @@ maxsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : maximum(map(x->maxs
 nextsizeof(::Type{T}) where {T} = isbitstype(T) ? sizeof(T) : nextsizeof(T.types[1])
 
 function fieldlayout(mod, typ, exprs...)
-    fields = Expr[]
-    values = []
-    largest_field = maximum(map(x->maxsizeof(Core.eval(mod, x.args[2])), exprs))
-    sz = cur_sz = 0
-    x = 0
-    for (i,expr) in enumerate(exprs)
-        T = Core.eval(mod, expr.args[2])
-        if !isbitstype(T)
-            exprs2 = [Expr(:(::), nm, typ) for (nm,typ) in zip(fieldnames(T),T.types)]
-            fields2, values2 = fieldlayout(mod, T, exprs2...)
-            append!(fields, map(x->Expr(:(::), Symbol(string(expr.args[1],'_',T,'_',x.args[1])), x.args[2]), fields2))
-            append!(values, map(x->x == 0 ? 0 : Expr(:call, :getfield, expr.args[1], QuoteNode(x)), values2))
-        else
-            push!(fields, expr)
-            push!(values, expr.args[1])
-        end
-        sz += cur_sz = fbsizeof(T)
-        if sz % largest_field == 0
-            sz = cur_sz = 0
-            continue
-        end
-        nextsz = i == length(exprs) ? 0 : nextsizeof(Core.eval(mod, exprs[i+1].args[2]))
-        if i == length(exprs) || cur_sz < nextsz || (sz + nextsz) > largest_field
-            # this is the last field and we're not `sz % largest_field`
-            # potential diffs = 7, 6, 5, 4, 3, 2, 1
-            sym = expr.args[1]
-            diff = cur_sz < nextsz ? nextsz - cur_sz : largest_field - sz
-            if diff == 7
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
-                push!(values, 0); push!(values, 0); push!(values, 0)
-            elseif diff == 6
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
-                push!(values, 0); push!(values, 0)
-            elseif diff == 5
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
-                push!(values, 0); push!(values, 0)
-            elseif diff == 4
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
-                push!(values, 0)
-            elseif diff == 3
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
-                push!(values, 0); push!(values, 0)
-            elseif diff == 2
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
-                push!(values, 0)
-            elseif diff == 1
-                push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
-                push!(values, 0)
-            end
-            sz = (sz + diff) % largest_field == 0 ? 0 : (cur_sz < nextsz ? sz + diff : 0)
-            cur_sz = 0
-        end
-    end
-    return fields, values
+	fields = Expr[]
+	values = []
+	largest_field = maximum(map(x->maxsizeof(Core.eval(mod, x.args[2])), exprs))
+	sz = cur_sz = 0
+	x = 0
+	for (i,expr) in enumerate(exprs)
+		T = Core.eval(mod, expr.args[2])
+		if !isbitstype(T)
+			exprs2 = [Expr(:(::), nm, typ) for (nm,typ) in zip(fieldnames(T),T.types)]
+			fields2, values2 = fieldlayout(mod, T, exprs2...)
+			append!(fields, map(x->Expr(:(::), Symbol(string(expr.args[1],'_',x.args[1])), x.args[2]), fields2))
+			append!(values, map(x->x == 0 ? 0 : Expr(:call, :getfield, expr.args[1], QuoteNode(x)), values2))
+		else
+			push!(fields, expr)
+			push!(values, expr.args[1])
+		end
+		sz += cur_sz = fbsizeof(T)
+		if sz % largest_field == 0
+			sz = cur_sz = 0
+			continue
+		end
+		nextsz = i == length(exprs) ? 0 : nextsizeof(Core.eval(mod, exprs[i+1].args[2]))
+		if i == length(exprs) || cur_sz < nextsz || (sz + nextsz) > largest_field
+			# this is the last field and we're not `sz % largest_field`
+			# potential diffs = 7, 6, 5, 4, 3, 2, 1
+			sym = expr.args[1]
+			diff = cur_sz < nextsz ? nextsz - cur_sz : largest_field - sz
+			if diff == 7
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
+				push!(values, 0); push!(values, 0); push!(values, 0)
+			elseif diff == 6
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
+				push!(values, 0); push!(values, 0)
+			elseif diff == 5
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
+				push!(values, 0); push!(values, 0)
+			elseif diff == 4
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt32)); x += 1
+				push!(values, 0)
+			elseif diff == 3
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
+				push!(values, 0); push!(values, 0)
+			elseif diff == 2
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt16)); x += 1
+				push!(values, 0)
+			elseif diff == 1
+				push!(fields, Expr(:(::), Symbol("_pad_$(sym)_$(typ)_$x"), :UInt8)); x += 1
+				push!(values, 0)
+			end
+			sz = (sz + diff) % largest_field == 0 ? 0 : (cur_sz < nextsz ? sz + diff : 0)
+			cur_sz = 0
+		end
+	end
+	return fields, values
 end
 
-if VERSION < v"0.7"
-    linefilter = x->isa(x, Expr) && x.head !== :line
-else
-    linefilter = x->typeof(x) != LineNumberNode
-end
+linefilter = x->typeof(x) != LineNumberNode
 
 macro STRUCT(expr)
-    !expr.args[1] || throw(ArgumentError("@STRUCT is only applicable for immutable types"))
-    exprs = filter(linefilter, expr.args[3].args)
-    fields, values = FlatBuffers.fieldlayout(__mod__(__module__), expr.args[2], exprs...)
-    expr.args[3].args = fields
-    # generate convenience outer constructors if necessary
-     # if there are nested structs or padding:
-        # build an outer constructor that takes all direct, original fields
-        # recursively flatten/splat all nested structs into one big args tuple
-        # adding zeros for padded arguments
-        # pass big, flat, args tuple to inner constructor
-    T = expr.args[2]
-    if any(x->!FlatBuffers.isbitstype(Core.eval(__mod__(__module__), x.args[2])), exprs) ||
-       length(fields) > length(exprs)
-       exprs2 = map(x->FlatBuffers.isbitstype(Core.eval(__mod__(__module__), x.args[2])) ? x.args[1] : x, exprs)
-       sig = Expr(:call, T, exprs2...)
-       body = Expr(:call, T, values...)
-       outer = Expr(:function, sig, body)
-    else
-        outer = :(nothing)
-    end
-    return esc(quote
-        $expr
-        $outer
-    end)
+	!expr.args[1] || throw(ArgumentError("@STRUCT is only applicable for immutable types"))
+	exprs = filter(linefilter, expr.args[3].args)
+	fields, values = FlatBuffers.fieldlayout(__module__, expr.args[2], exprs...)
+	expr.args[3].args = fields
+	# generate convenience outer constructors if necessary
+	# if there are nested structs or padding:
+	# build an outer constructor that takes all direct, original fields
+	# recursively flatten/splat all nested structs into one big args tuple
+	# adding zeros for padded arguments
+	# pass big, flat, args tuple to inner constructor
+	T = expr.args[2]
+	if any(x->!FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])), exprs) ||
+		length(fields) > length(exprs)
+		exprs2 = map(x->FlatBuffers.isbitstype(Core.eval(__module__, x.args[2])) ? x.args[1] : x, exprs)
+		sig = Expr(:call, T, exprs2...)
+		body = Expr(:call, T, values...)
+		outer = Expr(:function, sig, body)
+	else
+		outer = :(nothing)
+	end
+	return esc(quote
+		$expr
+		$outer
+	end)
 end
 
 macro DEFAULT(T, kwargs...)
-    ifblock = quote end
-    if length(kwargs) > 0 && isa(kwargs[1], Expr) && length(kwargs[1].args) > 1
-        for kw in kwargs
-            push!(ifblock.args, :(if sym == $(QuoteNode(kw.args[1]))
-                                    return $(kw.args[2])
-                                end))
-        end
-    end
-    esc(quote
-        if $T <: Enum
-            FlatBuffers.default(::Type{$T}) = FlatBuffers.enumtype($T)($(kwargs[1]))
-        else
-            function FlatBuffers.default(::Type{$T}, TT, sym)
-                $ifblock
-                return FlatBuffers.default(TT)
-            end
-        end
-    end)
+	ifblock = quote end
+	if length(kwargs) > 0 && isa(kwargs[1], Expr) && length(kwargs[1].args) > 1
+		for kw in kwargs
+			push!(ifblock.args, :(if sym == $(QuoteNode(kw.args[1]))
+				return $(kw.args[2])
+			end))
+		end
+	end
+	esc(quote
+		if $T <: Enum
+			FlatBuffers.default(::Type{$T}) = FlatBuffers.enumtype($T)($(kwargs[1]))
+		else
+			function FlatBuffers.default(::Type{$T}, TT, sym)
+				$ifblock
+				return FlatBuffers.default(TT)
+			end
+		end
+	end)
+end
+
+import Parameters
+
+function getdef(typedef::Expr)
+	isstructexpr(x) = x isa Expr && x.head == :struct
+	i = findfirst(x -> x isa Expr && any(isstructexpr.(x.args)), typedef.args)
+	if i == nothing
+		throw(ArgumentError("malformed @with_kw expression"))
+	end
+	wrapper = typedef.args[i]
+	i = findfirst(isstructexpr, wrapper.args)
+	def = wrapper.args[i]
+	return def
+end
+
+function getfielddefs(typedef::Expr)
+	[a for a in getdef(typedef).args[end].args if a isa Expr && a.head == :(::)]
+end
+
+function getconstructor(typedef::Expr)
+	cons = getdef(typedef).args[end].args[end-1].args[1]
+	typevars = []
+	if :head in propertynames(cons) && cons.head == :where
+		typevars = cons.args[2:end]
+		cons = cons.args[1]
+	end
+	cons, typevars
+end
+
+function getkwtype(defs, name)
+	for d in defs
+		if :args in propertynames(d) && d.args[1] == name
+			return d.args[end]
+		end
+	end
+	return nothing
+end
+
+function createdefaultfns(typedef::Expr)
+	cons, typevars = getconstructor(typedef)
+	T = cons.args[1]
+	params = cons.args[2]
+
+	@assert params.head == :parameters
+
+	kwargs = []
+	defs = getfielddefs(typedef)
+	kwdict = Dict{Any, Any}()
+	for p in params.args
+		name = p.args[1]
+		t = getkwtype(defs, name)
+		if t == nothing
+			continue
+		end
+		value = p.args[end]
+		ifblock = get(kwdict, t, quote end)
+		push!(ifblock.args, :(if sym == $(QuoteNode(name))
+			return convert($t, $value)
+		end))
+		kwdict[t] = ifblock
+	end
+
+	[:(function FlatBuffers.default(::Type{$T}, ::Type{$TT}, sym) where {$(typevars...)}
+		$(kwdict[TT])
+		return FlatBuffers.default($TT)
+	end) for TT in keys(kwdict)]
+end
+
+macro with_kw(typedef)
+	body = Parameters.with_kw(typedef, __module__, true)
+	defaults = createdefaultfns(body)
+	defaultsblock = Expr(:block, body, defaults...)
+	esc(defaultsblock)
 end
 
 #TODO:
-# handle default values
 # handle id?
-# handle deprecated
 # nested_flatbuffer
-# macro table(expr)
-#
-# end
diff --git a/julia/test/defaults.jl b/julia/test/defaults.jl
new file mode 100644
index 00000000000..ee978f8da3d
--- /dev/null
+++ b/julia/test/defaults.jl
@@ -0,0 +1,40 @@
+using FlatBuffers
+using Test
+import Parameters
+
+# test default fields
+@with_kw mutable struct UltimateAnswer
+    answer::Int32 = 42
+    question::String
+    highwaysbuilt::Int32 = 7
+end
+
+x = UltimateAnswer(;question="How many roads must a man walk down?")
+@test x.answer == 42
+@test FlatBuffers.default(UltimateAnswer, Int32, :answer) == 42
+@test FlatBuffers.default(UltimateAnswer, Int32, :highwaysbuilt) == 7
+b = FlatBuffers.Builder(UltimateAnswer)
+FlatBuffers.build!(b, x)
+xbytes = FlatBuffers.bytes(b)
+y = FlatBuffers.read(UltimateAnswer, xbytes)
+
+@test y.answer == x.answer
+@test y.question == x.question
+@test y.highwaysbuilt == x.highwaysbuilt
+@test x.highwaysbuilt == 7
+
+y = Parameters.reconstruct(x, highwaysbuilt = 0)
+b = FlatBuffers.Builder(UltimateAnswer)
+FlatBuffers.build!(b, y)
+ybytes = FlatBuffers.bytes(b)
+
+# check that we save bytes with default integer values
+@test length(ybytes) > length(xbytes)
+
+@test y.answer == x.answer
+@test y.question == x.question
+@test y.highwaysbuilt == 0
+
+y = FlatBuffers.read(UltimateAnswer, ybytes)
+@test y.highwaysbuilt == 0
+
diff --git a/julia/test/flatc.jl b/julia/test/flatc.jl
new file mode 100644
index 00000000000..088cac6e6c3
--- /dev/null
+++ b/julia/test/flatc.jl
@@ -0,0 +1,91 @@
+using Test
+import FlatBuffers
+
+# generated code
+include(joinpath(@__DIR_, "..", "..", "tests", "MyGame", "MyGame.jl"))
+import .MyGame
+import .MyGame.Example
+import .MyGame.Example2
+import .MyGame.Example.Any_
+import .MyGame.Example.Monster
+import .MyGame.Example.TestSimpleTableWithEnum
+
+# override the typeorder function, since we are working around
+# circular type definitions using type parameters
+FlatBuffers.typeorder(::Type{Any_}, i::Integer) = [Nothing, Monster{Any_}, TestSimpleTableWithEnum, Example2.Monster][i+1]
+
+function loadmonsterfile(filename)
+    open(joinpath(@__DIR__, filename), "r") do f Monster{Any_}(f) end
+end
+
+function checkmonster(monster)
+    @test monster.hp == 80
+    @test monster.mana == 150
+    @test monster.name == "MyMonster"
+
+    vec = monster.pos
+
+    @test vec.x == 1.0
+    @test vec.y == 2.0
+    @test vec.z == 3.0
+    @test vec.test1 == 3.0
+    @test vec.test2 == MyGame.Example.ColorGreen
+    @test vec.test3_a == 5
+    @test vec.test3_b == 6
+
+    monster2 = monster.test
+    @test monster2.name == "Fred"
+
+    @test length(monster.inventory) == 5
+    @test sum(monster.inventory) == 10
+
+    @test monster.vector_of_longs == [10 ^ (2*i) for i = 0:4]
+    @test monster.vector_of_doubles == [-1.7976931348623157e+308, 0, 1.7976931348623157e+308]
+
+    @test length(monster.test4) == 2
+
+    (test0, test1) = monster.test4
+    @test sum([test0.a, test0.b, test1.a, test1.b]) == 100
+
+    @test monster.testarrayofstring == ["test1", "test2"]
+    @test monster.testarrayoftables == []
+    @test monster.testf == 3.14159f0
+end
+
+function checkpassthrough(monster)
+    b = FlatBuffers.Builder(Monster{Any_})
+    FlatBuffers.build!(b, monster)
+    bytes = FlatBuffers.bytes(b)
+    @test FlatBuffers.has_identifier(Monster{Any_}, bytes)
+    newmonster = FlatBuffers.read(Monster{Any_}, bytes)
+    checkmonster(newmonster)
+end
+
+function checkserialize(monster)
+    io = IOBuffer()
+    FlatBuffers.serialize(io, monster)
+    bytes = take!(io)
+    newmonster = FlatBuffers.deserialize(IOBuffer(bytes), Monster{Any_})
+    checkmonster(newmonster)
+end
+
+@test FlatBuffers.root_type(Monster) == true
+@test FlatBuffers.file_identifier(Monster) == "MONS"
+@test FlatBuffers.file_extension(Monster) == "mon"
+
+for testcase in ["test", "python_wire"]
+    mon = loadmonsterfile("monsterdata_$testcase.mon")
+    checkmonster(mon)
+    checkpassthrough(mon)
+    checkserialize(mon)
+end
+
+# test printing
+mon = loadmonsterfile("monsterdata_test.mon")
+b = FlatBuffers.Builder(Monster{Any_})
+FlatBuffers.build!(b, mon)
+io = IOBuffer()
+show(io, b)
+output = String(take!(io))
+@test occursin("deprecated field", split(output, "\n")[9])
+
diff --git a/julia/test/internals.jl b/julia/test/internals.jl
old mode 100755
new mode 100644
index c0ba29ed9b9..392f7ca7e93
--- a/julia/test/internals.jl
+++ b/julia/test/internals.jl
@@ -5,462 +5,462 @@ const overflowingInt64Val = read(IOBuffer(UInt8[0x84, 0x44, 0x44, 0x44, 0x44, 0x
 
 # CheckByteLayout verifies the bytes of a Builder in various scenarios.
 function CheckByteLayout()
-    check = want-> begin
-        got = b.bytes[b.head+1:end]
-        @test want == got
-        return
-    end
-
-	# test 1: numbers
-
-    b = FlatBuffers.Builder()
-	check(UInt8[])
-	FlatBuffers.prepend!(b, true)
-	check(UInt8[1])
-	FlatBuffers.prepend!(b, Int8(-127))
-	check(UInt8[129, 1])
-	FlatBuffers.prepend!(b, UInt8(255))
-	check(UInt8[255, 129, 1])
-	FlatBuffers.prepend!(b, Int16(-32222))
-	check(UInt8[0x22, 0x82, 0, 255, 129, 1]) # first pad
-	FlatBuffers.prepend!(b, UInt16(0xFEEE))
-	check(UInt8[0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) # no pad this time
-	FlatBuffers.prepend!(b, Int32(-53687092))
-	check(UInt8[204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1])
-	FlatBuffers.prepend!(b, UInt32(0x98765432))
-	check(UInt8[0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1])
-
-	# test 1b: numbers 2
-
-	b = FlatBuffers.Builder()
-	prepend!(b, 0x1122334455667788)
-	check(UInt8[0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11])
-
-	# test 2: 1xbyte vector
-
-	b = FlatBuffers.Builder()
-	check(UInt8[])
-	FlatBuffers.startvector(b, sizeof(Bool), 1, 1)
-	check(UInt8[0, 0, 0]) # align to 4bytes
-	FlatBuffers.prepend!(b, UInt8(1))
-	check(UInt8[1, 0, 0, 0])
-	FlatBuffers.endvector(b, 1)
-	check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding
-
-	# test 3: 2xbyte vector
-
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(UInt8), 2, 1)
-	check(UInt8[0, 0]) # align to 4bytes
-	FlatBuffers.prepend!(b, UInt8(1))
-	check(UInt8[1, 0, 0])
-	FlatBuffers.prepend!(b, UInt8(2))
-	check(UInt8[2, 1, 0, 0])
-	FlatBuffers.endvector(b, 2)
-	check(UInt8[2, 0, 0, 0, 2, 1, 0, 0]) # padding
-
-	# test 3b: 11xbyte vector matches builder size
+	check = want-> begin
+	got = b.bytes[b.head+1:end]
+	@test want == got
+	return
+end
 
-	b = FlatBuffers.Builder(Any, 12)
-	FlatBuffers.startvector(b, sizeof(UInt8), 8, 1)
-	start = UInt8[]
+# test 1: numbers
+
+b = FlatBuffers.Builder()
+check(UInt8[])
+FlatBuffers.prepend!(b, true)
+check(UInt8[1])
+FlatBuffers.prepend!(b, Int8(-127))
+check(UInt8[129, 1])
+FlatBuffers.prepend!(b, UInt8(255))
+check(UInt8[255, 129, 1])
+FlatBuffers.prepend!(b, Int16(-32222))
+check(UInt8[0x22, 0x82, 0, 255, 129, 1]) # first pad
+FlatBuffers.prepend!(b, UInt16(0xFEEE))
+check(UInt8[0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1]) # no pad this time
+FlatBuffers.prepend!(b, Int32(-53687092))
+check(UInt8[204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1])
+FlatBuffers.prepend!(b, UInt32(0x98765432))
+check(UInt8[0x32, 0x54, 0x76, 0x98, 204, 204, 204, 252, 0xEE, 0xFE, 0x22, 0x82, 0, 255, 129, 1])
+
+# test 1b: numbers 2
+
+b = FlatBuffers.Builder()
+prepend!(b, 0x1122334455667788)
+check(UInt8[0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11])
+
+# test 2: 1xbyte vector
+
+b = FlatBuffers.Builder()
+check(UInt8[])
+FlatBuffers.startvector(b, sizeof(Bool), 1, 1)
+check(UInt8[0, 0, 0]) # align to 4bytes
+FlatBuffers.prepend!(b, UInt8(1))
+check(UInt8[1, 0, 0, 0])
+FlatBuffers.endvector(b, 1)
+check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding
+
+# test 3: 2xbyte vector
+
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(UInt8), 2, 1)
+check(UInt8[0, 0]) # align to 4bytes
+FlatBuffers.prepend!(b, UInt8(1))
+check(UInt8[1, 0, 0])
+FlatBuffers.prepend!(b, UInt8(2))
+check(UInt8[2, 1, 0, 0])
+FlatBuffers.endvector(b, 2)
+check(UInt8[2, 0, 0, 0, 2, 1, 0, 0]) # padding
+
+# test 3b: 11xbyte vector matches builder size
+
+b = FlatBuffers.Builder(Any, 12)
+FlatBuffers.startvector(b, sizeof(UInt8), 8, 1)
+start = UInt8[]
+check(start)
+for i = 1:11
+	FlatBuffers.prepend!(b, UInt8(i))
+	start = append!(UInt8[i], start)
 	check(start)
-    for i = 1:11
-		FlatBuffers.prepend!(b, UInt8(i))
-		start = append!(UInt8[i], start)
-		check(start)
-	end
-	FlatBuffers.endvector(b, 8)
-	check(append!(UInt8[8, 0, 0, 0], start))
-
-	# test 4: 1xuint16 vector
-
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(UInt16), 1, 1)
-	check(UInt8[0, 0]) # align to 4bytes
-	FlatBuffers.prepend!(b, UInt16(1))
-	check(UInt8[1, 0, 0, 0])
-	FlatBuffers.endvector(b, 1)
-	check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding
-
-	# test 5: 2xuint16 vector
-
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(UInt16), 2, 1)
-	check(UInt8[]) # align to 4bytes
-	FlatBuffers.prepend!(b, UInt16(0xABCD))
-	check(UInt8[0xCD, 0xAB])
-	FlatBuffers.prepend!(b, UInt16(0xDCBA))
-	check(UInt8[0xBA, 0xDC, 0xCD, 0xAB])
-	FlatBuffers.endvector(b, 2)
-	check(UInt8[2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB])
-
-	# test 6: CreateString
-
-	b = FlatBuffers.Builder()
-	FlatBuffers.createstring(b, "foo")
-	check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad
-	FlatBuffers.createstring(b, "moop")
-	check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad
-		3, 0, 0, 0, 'f', 'o', 'o', 0])
-
-	# test 6b: CreateString unicode
-
-	b = FlatBuffers.Builder()
-	# These characters are chinese from blog.golang.org/strings
-	# We use escape codes here so that editors without unicode support
-	# aren't bothered:
-	uni_str = "\u65e5\u672c\u8a9e"
-	FlatBuffers.createstring(b, uni_str)
-	check(UInt8[9, 0, 0, 0, 230, 151, 165, 230, 156, 172, 232, 170, 158, 0, #  null-terminated, 2-byte pad
-		0, 0])
-
-	# test 6c: CreateUInt8String
-
-	b = FlatBuffers.Builder()
-	FlatBuffers.createstring(b, "foo")
-	check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad
-	FlatBuffers.createstring(b, "moop")
-	check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad
-		3, 0, 0, 0, 'f', 'o', 'o', 0])
-
-	# test 7: empty vtable
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 0)
-	check(UInt8[])
-	FlatBuffers.endobject(b)
-	check(UInt8[4, 0, 4, 0, 4, 0, 0, 0])
-
-	# test 8: vtable with one true bool
-	b = FlatBuffers.Builder()
-	check(UInt8[])
-    FlatBuffers.startobject(b, 1)
-    check(UInt8[])
-    FlatBuffers.prependslot!(b, 1, true, false)
-    FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		8, 0, # length of object including vtable offset
-		7, 0, # start of bool value
-		6, 0, 0, 0, # offset for start of vtable (int32)
-		0, 0, 0, # padded to 4 bytes
-		1, # bool value
+end
+FlatBuffers.endvector(b, 8)
+check(append!(UInt8[8, 0, 0, 0], start))
+
+# test 4: 1xuint16 vector
+
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(UInt16), 1, 1)
+check(UInt8[0, 0]) # align to 4bytes
+FlatBuffers.prepend!(b, UInt16(1))
+check(UInt8[1, 0, 0, 0])
+FlatBuffers.endvector(b, 1)
+check(UInt8[1, 0, 0, 0, 1, 0, 0, 0]) # padding
+
+# test 5: 2xuint16 vector
+
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(UInt16), 2, 1)
+check(UInt8[]) # align to 4bytes
+FlatBuffers.prepend!(b, UInt16(0xABCD))
+check(UInt8[0xCD, 0xAB])
+FlatBuffers.prepend!(b, UInt16(0xDCBA))
+check(UInt8[0xBA, 0xDC, 0xCD, 0xAB])
+FlatBuffers.endvector(b, 2)
+check(UInt8[2, 0, 0, 0, 0xBA, 0xDC, 0xCD, 0xAB])
+
+# test 6: CreateString
+
+b = FlatBuffers.Builder()
+FlatBuffers.createstring(b, "foo")
+check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad
+FlatBuffers.createstring(b, "moop")
+check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad
+	3, 0, 0, 0, 'f', 'o', 'o', 0])
+
+# test 6b: CreateString unicode
+
+b = FlatBuffers.Builder()
+# These characters are chinese from blog.golang.org/strings
+# We use escape codes here so that editors without unicode support
+# aren't bothered:
+uni_str = "\u65e5\u672c\u8a9e"
+FlatBuffers.createstring(b, uni_str)
+check(UInt8[9, 0, 0, 0, 230, 151, 165, 230, 156, 172, 232, 170, 158, 0, #  null-terminated, 2-byte pad
+	0, 0])
+
+# test 6c: CreateUInt8String
+
+b = FlatBuffers.Builder()
+FlatBuffers.createstring(b, "foo")
+check(UInt8[3, 0, 0, 0, 'f', 'o', 'o', 0]) # 0-terminated, no pad
+FlatBuffers.createstring(b, "moop")
+check(UInt8[4, 0, 0, 0, 'm', 'o', 'o', 'p', 0, 0, 0, 0, # 0-terminated, 3-byte pad
+	3, 0, 0, 0, 'f', 'o', 'o', 0])
+
+# test 7: empty vtable
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 0)
+check(UInt8[])
+FlatBuffers.endobject(b)
+check(UInt8[4, 0, 4, 0, 4, 0, 0, 0])
+
+# test 8: vtable with one true bool
+b = FlatBuffers.Builder()
+check(UInt8[])
+FlatBuffers.startobject(b, 1)
+check(UInt8[])
+FlatBuffers.prependslot!(b, 1, true, false)
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	8, 0, # length of object including vtable offset
+	7, 0, # start of bool value
+	6, 0, 0, 0, # offset for start of vtable (int32)
+	0, 0, 0, # padded to 4 bytes
+	1, # bool value
 	])
 
-	# test 9: vtable with one default bool
-	b = FlatBuffers.Builder()
-	check(UInt8[])
-    FlatBuffers.startobject(b, 1)
-    check(UInt8[])
-	FlatBuffers.prependslot!(b, 1, false, false)
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		4, 0, # end of object from here
-		0, 0, # entry 1 is zero
-		6, 0, 0, 0, # offset for start of vtable (int32)
+# test 9: vtable with one default bool
+b = FlatBuffers.Builder()
+check(UInt8[])
+FlatBuffers.startobject(b, 1)
+check(UInt8[])
+FlatBuffers.prependslot!(b, 1, false, false)
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	4, 0, # end of object from here
+	0, 0, # entry 1 is zero
+	6, 0, 0, 0, # offset for start of vtable (int32)
 	])
 
-	# test 10: vtable with one int16
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 1)
-	FlatBuffers.prependslot!(b, 1, Int16(0x789A), Int16(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		8, 0, # end of object from here
-		6, 0, # offset to value
-		6, 0, 0, 0, # offset for start of vtable (int32)
-		0, 0, # padding to 4 bytes
-		0x9A, 0x78,
+# test 10: vtable with one int16
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 1)
+FlatBuffers.prependslot!(b, 1, Int16(0x789A), Int16(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	8, 0, # end of object from here
+	6, 0, # offset to value
+	6, 0, 0, 0, # offset for start of vtable (int32)
+	0, 0, # padding to 4 bytes
+	0x9A, 0x78,
 	])
 
-	# test 11: vtable with two int16
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0))
-	FlatBuffers.prependslot!(b, 2, Int16(0x789A), Int16(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		8, 0, # vtable bytes
-		8, 0, # end of object from here
-		6, 0, # offset to value 0
-		4, 0, # offset to value 1
-		8, 0, 0, 0, # offset for start of vtable (int32)
-		0x9A, 0x78, # value 1
-		0x56, 0x34, # value 0
+# test 11: vtable with two int16
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0))
+FlatBuffers.prependslot!(b, 2, Int16(0x789A), Int16(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	8, 0, # vtable bytes
+	8, 0, # end of object from here
+	6, 0, # offset to value 0
+	4, 0, # offset to value 1
+	8, 0, 0, 0, # offset for start of vtable (int32)
+	0x9A, 0x78, # value 1
+	0x56, 0x34, # value 0
 	])
 
-	# test 12: vtable with int16 and bool
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0))
-	FlatBuffers.prependslot!(b, 2, true, false)
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		8, 0, # vtable bytes
-		8, 0, # end of object from here
-		6, 0, # offset to value 0
-		5, 0, # offset to value 1
-		8, 0, 0, 0, # offset for start of vtable (int32)
-		0,          # padding
-		1,          # value 1
-		0x56, 0x34, # value 0
+# test 12: vtable with int16 and bool
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependslot!(b, 1, Int16(0x3456), Int16(0))
+FlatBuffers.prependslot!(b, 2, true, false)
+FlatBuffers.endobject(b)
+check(UInt8[
+	8, 0, # vtable bytes
+	8, 0, # end of object from here
+	6, 0, # offset to value 0
+	5, 0, # offset to value 1
+	8, 0, 0, 0, # offset for start of vtable (int32)
+	0,          # padding
+	1,          # value 1
+	0x56, 0x34, # value 0
 	])
 
-	# test 12: vtable with empty vector
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(UInt8), 0, 1)
-	vecend = FlatBuffers.endvector(b, 0)
-    FlatBuffers.startobject(b, 1)
-	FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		8, 0,
-		4, 0, # offset to vector offset
-		6, 0, 0, 0, # offset for start of vtable (int32)
-		4, 0, 0, 0,
-		0, 0, 0, 0, # length of vector (not in struct)
+# test 12: vtable with empty vector
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(UInt8), 0, 1)
+vecend = FlatBuffers.endvector(b, 0)
+FlatBuffers.startobject(b, 1)
+FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	8, 0,
+	4, 0, # offset to vector offset
+	6, 0, 0, 0, # offset for start of vtable (int32)
+	4, 0, 0, 0,
+	0, 0, 0, 0, # length of vector (not in struct)
 	])
 
-	# test 12b: vtable with empty vector of byte and some scalars
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(UInt8), 0, 1)
-	vecend = FlatBuffers.endvector(b, 0)
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0))
-	FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		8, 0, # vtable bytes
-		12, 0,
-		10, 0, # offset to value 0
-		4, 0, # offset to vector offset
-		8, 0, 0, 0, # vtable loc
-		8, 0, 0, 0, # value 1
-		0, 0, 55, 0, # value 0
-
-		0, 0, 0, 0, # length of vector (not in struct)
+# test 12b: vtable with empty vector of byte and some scalars
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(UInt8), 0, 1)
+vecend = FlatBuffers.endvector(b, 0)
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0))
+FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	8, 0, # vtable bytes
+	12, 0,
+	10, 0, # offset to value 0
+	4, 0, # offset to vector offset
+	8, 0, 0, 0, # vtable loc
+	8, 0, 0, 0, # value 1
+	0, 0, 55, 0, # value 0
+
+	0, 0, 0, 0, # length of vector (not in struct)
 	])
 
-	# test 13: vtable with 1 int16 and 2-vector of int16
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(Int16), 2, 1)
-	FlatBuffers.prepend!(b, Int16(0x1234))
-	FlatBuffers.prepend!(b, Int16(0x5678))
-	vecend = FlatBuffers.endvector(b, 2)
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0))
-	FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		8, 0, # vtable bytes
-		12, 0, # length of object
-		6, 0, # start of value 0 from end of vtable
-		8, 0, # start of value 1 from end of buffer
-		8, 0, 0, 0, # offset for start of vtable (int32)
-		0, 0, # padding
-		55, 0, # value 0
-		4, 0, 0, 0, # vector position from here
-		2, 0, 0, 0, # length of vector (uint32)
-		0x78, 0x56, # vector value 1
-		0x34, 0x12, # vector value 0
+# test 13: vtable with 1 int16 and 2-vector of int16
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(Int16), 2, 1)
+FlatBuffers.prepend!(b, Int16(0x1234))
+FlatBuffers.prepend!(b, Int16(0x5678))
+vecend = FlatBuffers.endvector(b, 2)
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependoffsetslot!(b, 2, UInt32(vecend), UInt32(0))
+FlatBuffers.prependslot!(b, 1, Int16(55), Int16(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	8, 0, # vtable bytes
+	12, 0, # length of object
+	6, 0, # start of value 0 from end of vtable
+	8, 0, # start of value 1 from end of buffer
+	8, 0, 0, 0, # offset for start of vtable (int32)
+	0, 0, # padding
+	55, 0, # value 0
+	4, 0, 0, 0, # vector position from here
+	2, 0, 0, 0, # length of vector (uint32)
+	0x78, 0x56, # vector value 1
+	0x34, 0x12, # vector value 0
 	])
 
-	# test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 1)
-	FlatBuffers.prep!(b, 4+4+4, 0)
-	FlatBuffers.prepend!(b, Int8(55))
-	FlatBuffers.pad!(b, 3)
-	FlatBuffers.prepend!(b, Int16(0x1234))
-	FlatBuffers.pad!(b, 2)
-	FlatBuffers.prepend!(b, Int32(0x12345678))
-	structStart = FlatBuffers.offset(b)
-	FlatBuffers.prependstructslot!(b, 1, structStart, 0)
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		16, 0, # end of object from here
-		4, 0, # start of struct from here
-		6, 0, 0, 0, # offset for start of vtable (int32)
-		0x78, 0x56, 0x34, 0x12, # value 2
-		0, 0, # padding
-		0x34, 0x12, # value 1
-		0, 0, 0, # padding
-		55, # value 0
+# test 14: vtable with 1 struct of 1 int8, 1 int16, 1 int32
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 1)
+FlatBuffers.prep!(b, 4+4+4, 0)
+FlatBuffers.prepend!(b, Int8(55))
+FlatBuffers.pad!(b, 3)
+FlatBuffers.prepend!(b, Int16(0x1234))
+FlatBuffers.pad!(b, 2)
+FlatBuffers.prepend!(b, Int32(0x12345678))
+structStart = FlatBuffers.offset(b)
+FlatBuffers.prependstructslot!(b, 1, structStart, 0)
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	16, 0, # end of object from here
+	4, 0, # start of struct from here
+	6, 0, 0, 0, # offset for start of vtable (int32)
+	0x78, 0x56, 0x34, 0x12, # value 2
+	0, 0, # padding
+	0x34, 0x12, # value 1
+	0, 0, 0, # padding
+	55, # value 0
 	])
 
-	# test 15: vtable with 1 vector of 2 struct of 2 int8
-	b = FlatBuffers.Builder()
-	FlatBuffers.startvector(b, sizeof(Int8)*2, 2, 1)
-	FlatBuffers.prepend!(b, Int8(33))
-	FlatBuffers.prepend!(b, Int8(44))
-	FlatBuffers.prepend!(b, Int8(55))
-	FlatBuffers.prepend!(b, Int8(66))
-	vecend = FlatBuffers.endvector(b, 2)
-    FlatBuffers.startobject(b, 1)
-	FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0))
-	FlatBuffers.endobject(b)
-	check(UInt8[
-		6, 0, # vtable bytes
-		8, 0,
-		4, 0, # offset of vector offset
-		6, 0, 0, 0, # offset for start of vtable (int32)
-		4, 0, 0, 0, # vector start offset
-
-		2, 0, 0, 0, # vector length
-		66, # vector value 1,1
-		55, # vector value 1,0
-		44, # vector value 0,1
-		33, # vector value 0,0
+# test 15: vtable with 1 vector of 2 struct of 2 int8
+b = FlatBuffers.Builder()
+FlatBuffers.startvector(b, sizeof(Int8)*2, 2, 1)
+FlatBuffers.prepend!(b, Int8(33))
+FlatBuffers.prepend!(b, Int8(44))
+FlatBuffers.prepend!(b, Int8(55))
+FlatBuffers.prepend!(b, Int8(66))
+vecend = FlatBuffers.endvector(b, 2)
+FlatBuffers.startobject(b, 1)
+FlatBuffers.prependoffsetslot!(b, 1, UInt32(vecend), UInt32(0))
+FlatBuffers.endobject(b)
+check(UInt8[
+	6, 0, # vtable bytes
+	8, 0,
+	4, 0, # offset of vector offset
+	6, 0, 0, 0, # offset for start of vtable (int32)
+	4, 0, 0, 0, # vector start offset
+
+	2, 0, 0, 0, # vector length
+	66, # vector value 1,1
+	55, # vector value 1,0
+	44, # vector value 0,1
+	33, # vector value 0,0
 	])
 
-	# test 16: table with some elements
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0))
-	FlatBuffers.prependslot!(b, 2, Int16(66), Int16(0))
-	off = FlatBuffers.endobject(b)
-	FlatBuffers.finish!(b, off) #TODO
+# test 16: table with some elements
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0))
+FlatBuffers.prependslot!(b, 2, Int16(66), Int16(0))
+off = FlatBuffers.endobject(b)
+FlatBuffers.finish!(b, off) #TODO
 
-	check(UInt8[
-		12, 0, 0, 0, # root of table: points to vtable offset
+check(UInt8[
+	12, 0, 0, 0, # root of table: points to vtable offset
 
-		8, 0, # vtable bytes
-		8, 0, # end of object from here
-		7, 0, # start of value 0
-		4, 0, # start of value 1
+	8, 0, # vtable bytes
+	8, 0, # end of object from here
+	7, 0, # start of value 0
+	4, 0, # start of value 1
 
-		8, 0, 0, 0, # offset for start of vtable (int32)
+	8, 0, 0, 0, # offset for start of vtable (int32)
 
-		66, 0, # value 1
-		0,  # padding
-		33, # value 0
+	66, 0, # value 1
+	0,  # padding
+	33, # value 0
 	])
 
-	# test 17: one unfinished table and one finished table
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 2)
-	FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0))
-	FlatBuffers.prependslot!(b, 2, Int8(44), Int8(0))
-	off = FlatBuffers.endobject(b)
-	FlatBuffers.finish!(b, off)
-
-	FlatBuffers.startobject(b, 3)
-	FlatBuffers.prependslot!(b, 1, Int8(55), Int8(0))
-	FlatBuffers.prependslot!(b, 2, Int8(66), Int8(0))
-	FlatBuffers.prependslot!(b, 3, Int8(77), Int8(0))
-	off = FlatBuffers.endobject(b)
-	FlatBuffers.finish!(b, off)
-
-	check(UInt8[
-		16, 0, 0, 0, # root of table: points to object
-		0, 0, # padding
-
-		10, 0, # vtable bytes
-		8, 0, # size of object
-		7, 0, # start of value 0
-		6, 0, # start of value 1
-		5, 0, # start of value 2
-		10, 0, 0, 0, # offset for start of vtable (int32)
-		0,  # padding
-		77, # value 2
-		66, # value 1
-		55, # value 0
-
-		12, 0, 0, 0, # root of table: points to object
-
-		8, 0, # vtable bytes
-		8, 0, # size of object
-		7, 0, # start of value 0
-		6, 0, # start of value 1
-		8, 0, 0, 0, # offset for start of vtable (int32)
-		0, 0, # padding
-		44, # value 1
-		33, # value 0
+# test 17: one unfinished table and one finished table
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 2)
+FlatBuffers.prependslot!(b, 1, Int8(33), Int8(0))
+FlatBuffers.prependslot!(b, 2, Int8(44), Int8(0))
+off = FlatBuffers.endobject(b)
+FlatBuffers.finish!(b, off)
+
+FlatBuffers.startobject(b, 3)
+FlatBuffers.prependslot!(b, 1, Int8(55), Int8(0))
+FlatBuffers.prependslot!(b, 2, Int8(66), Int8(0))
+FlatBuffers.prependslot!(b, 3, Int8(77), Int8(0))
+off = FlatBuffers.endobject(b)
+FlatBuffers.finish!(b, off)
+
+check(UInt8[
+	16, 0, 0, 0, # root of table: points to object
+	0, 0, # padding
+
+	10, 0, # vtable bytes
+	8, 0, # size of object
+	7, 0, # start of value 0
+	6, 0, # start of value 1
+	5, 0, # start of value 2
+	10, 0, 0, 0, # offset for start of vtable (int32)
+	0,  # padding
+	77, # value 2
+	66, # value 1
+	55, # value 0
+
+	12, 0, 0, 0, # root of table: points to object
+
+	8, 0, # vtable bytes
+	8, 0, # size of object
+	7, 0, # start of value 0
+	6, 0, # start of value 1
+	8, 0, 0, 0, # offset for start of vtable (int32)
+	0, 0, # padding
+	44, # value 1
+	33, # value 0
 	])
 
-	# test 18: a bunch of bools
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 8)
-	FlatBuffers.prependslot!(b, 1, true, false)
-	FlatBuffers.prependslot!(b, 2, true, false)
-	FlatBuffers.prependslot!(b, 3, true, false)
-	FlatBuffers.prependslot!(b, 4, true, false)
-	FlatBuffers.prependslot!(b, 5, true, false)
-	FlatBuffers.prependslot!(b, 6, true, false)
-	FlatBuffers.prependslot!(b, 7, true, false)
-	FlatBuffers.prependslot!(b, 8, true, false)
-	off = FlatBuffers.endobject(b)
-	FlatBuffers.finish!(b, off)
-
-	check(UInt8[
-		24, 0, 0, 0, # root of table: points to vtable offset
-
-		20, 0, # vtable bytes
-		12, 0, # size of object
-		11, 0, # start of value 0
-		10, 0, # start of value 1
-		9, 0, # start of value 2
-		8, 0, # start of value 3
-		7, 0, # start of value 4
-		6, 0, # start of value 5
-		5, 0, # start of value 6
-		4, 0, # start of value 7
-		20, 0, 0, 0, # vtable offset
-
-		1, # value 7
-		1, # value 6
-		1, # value 5
-		1, # value 4
-		1, # value 3
-		1, # value 2
-		1, # value 1
-		1, # value 0
+# test 18: a bunch of bools
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 8)
+FlatBuffers.prependslot!(b, 1, true, false)
+FlatBuffers.prependslot!(b, 2, true, false)
+FlatBuffers.prependslot!(b, 3, true, false)
+FlatBuffers.prependslot!(b, 4, true, false)
+FlatBuffers.prependslot!(b, 5, true, false)
+FlatBuffers.prependslot!(b, 6, true, false)
+FlatBuffers.prependslot!(b, 7, true, false)
+FlatBuffers.prependslot!(b, 8, true, false)
+off = FlatBuffers.endobject(b)
+FlatBuffers.finish!(b, off)
+
+check(UInt8[
+	24, 0, 0, 0, # root of table: points to vtable offset
+
+	20, 0, # vtable bytes
+	12, 0, # size of object
+	11, 0, # start of value 0
+	10, 0, # start of value 1
+	9, 0, # start of value 2
+	8, 0, # start of value 3
+	7, 0, # start of value 4
+	6, 0, # start of value 5
+	5, 0, # start of value 6
+	4, 0, # start of value 7
+	20, 0, 0, 0, # vtable offset
+
+	1, # value 7
+	1, # value 6
+	1, # value 5
+	1, # value 4
+	1, # value 3
+	1, # value 2
+	1, # value 1
+	1, # value 0
 	])
 
-	# test 19: three bools
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 3)
-	FlatBuffers.prependslot!(b, 1, true, false)
-	FlatBuffers.prependslot!(b, 2, true, false)
-	FlatBuffers.prependslot!(b, 3, true, false)
-	off = FlatBuffers.endobject(b)
-	FlatBuffers.finish!(b, off)
-
-	check(UInt8[
-		16, 0, 0, 0, # root of table: points to vtable offset
-
-		0, 0, # padding
-
-		10, 0, # vtable bytes
-		8, 0, # size of object
-		7, 0, # start of value 0
-		6, 0, # start of value 1
-		5, 0, # start of value 2
-		10, 0, 0, 0, # vtable offset from here
-
-		0, # padding
-		1, # value 2
-		1, # value 1
-		1, # value 0
+# test 19: three bools
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 3)
+FlatBuffers.prependslot!(b, 1, true, false)
+FlatBuffers.prependslot!(b, 2, true, false)
+FlatBuffers.prependslot!(b, 3, true, false)
+off = FlatBuffers.endobject(b)
+FlatBuffers.finish!(b, off)
+
+check(UInt8[
+	16, 0, 0, 0, # root of table: points to vtable offset
+
+	0, 0, # padding
+
+	10, 0, # vtable bytes
+	8, 0, # size of object
+	7, 0, # start of value 0
+	6, 0, # start of value 1
+	5, 0, # start of value 2
+	10, 0, 0, 0, # vtable offset from here
+
+	0, # padding
+	1, # value 2
+	1, # value 1
+	1, # value 0
 	])
 
-	# test 20: some floats
-	b = FlatBuffers.Builder()
-    FlatBuffers.startobject(b, 1)
-	FlatBuffers.prependslot!(b, 1, Float32(1.0), Float32(0.0))
-	off = FlatBuffers.endobject(b)
+# test 20: some floats
+b = FlatBuffers.Builder()
+FlatBuffers.startobject(b, 1)
+FlatBuffers.prependslot!(b, 1, Float32(1.0), Float32(0.0))
+off = FlatBuffers.endobject(b)
 
-	check(UInt8[
-		6, 0, # vtable bytes
-		8, 0, # size of object
-		4, 0, # start of value 0
-		6, 0, 0, 0, # vtable offset
+check(UInt8[
+	6, 0, # vtable bytes
+	8, 0, # size of object
+	4, 0, # start of value 0
+	6, 0, 0, 0, # vtable offset
 
-		0, 0, 128, 63, # value 0
+	0, 0, 128, 63, # value 0
 	])
 end
 
@@ -560,27 +560,27 @@ function CheckVtableDeduplication()
 	got = b.bytes[b.head+1:end]
 
 	want = UInt8[
-		240, 255, 255, 255, # == -12. offset to dedupped vtable.
-		99, 0,
-		88,
-		77,
-		248, 255, 255, 255, # == -8. offset to dedupped vtable.
-		66, 0,
-		55,
-		44,
-		12, 0, # start of vtable
-		8, 0,
-		0, 0,
-		7, 0,
-		6, 0,
-		4, 0,
-		12, 0, 0, 0, # table0
-		33, 0,
-		22,
-		11
+	240, 255, 255, 255, # == -12. offset to dedupped vtable.
+	99, 0,
+	88,
+	77,
+	248, 255, 255, 255, # == -8. offset to dedupped vtable.
+	66, 0,
+	55,
+	44,
+	12, 0, # start of vtable
+	8, 0,
+	0, 0,
+	7, 0,
+	6, 0,
+	4, 0,
+	12, 0, 0, 0, # table0
+	33, 0,
+	22,
+	11
 	]
 
-    @test got == want
+	@test got == want
 
 	table0 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj0)
 	table1 = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - obj1)
@@ -588,21 +588,21 @@ function CheckVtableDeduplication()
 
 	function testTable(tab, a, b, c, d)
 		# vtable size
-        got = FlatBuffers.getoffsetslot(tab, 0, Int16(0))
+		got = FlatBuffers.getoffsetslot(tab, 0, Int16(0))
 		@test 12 == got
 		# object size
-        got = FlatBuffers.getoffsetslot(tab, 2, Int16(0))
+		got = FlatBuffers.getoffsetslot(tab, 2, Int16(0))
 		@test 8 == got
 		# default value
-        got = FlatBuffers.getoffsetslot(tab, 4, Int16(0))
+		got = FlatBuffers.getoffsetslot(tab, 4, Int16(0))
 		@test a == got
-        got = FlatBuffers.getslot(tab, 6, UInt8(0))
+		got = FlatBuffers.getslot(tab, 6, UInt8(0))
 		@test b == got
-        val = FlatBuffers.getslot(tab, 8, UInt8(0))
+		val = FlatBuffers.getslot(tab, 8, UInt8(0))
 		c != val && throw(ArgumentError("failed 8, 0: $got"))
-        got = FlatBuffers.getslot(tab, 10, UInt8(0))
+		got = FlatBuffers.getslot(tab, 10, UInt8(0))
 		@test d == got
-        return
+		return
 	end
 
 	testTable(table0, UInt16(0), UInt8(11), UInt8(22), UInt8(33))
@@ -653,29 +653,29 @@ end
 function CheckCreateByteVector()
 	raw = UInt8(0):UInt8(29)
 
-    for size = 1:30
+	for size = 1:30
 		b1 = FlatBuffers.Builder()
 		b2 = FlatBuffers.Builder()
 		FlatBuffers.startvector(b1, 1, size, 1)
-        for i = size:-1:1
+		for i = size:-1:1
 			FlatBuffers.prepend!(b1, raw[i])
 		end
 		FlatBuffers.endvector(b1, size)
 		FlatBuffers.createbytevector(b2, raw[1:size])
-        @test b1.bytes == b2.bytes
+		@test b1.bytes == b2.bytes
 	end
 end
 
 const InitialLCGSeed = 48271
 mutable struct LCG
-    val::UInt32
-    LCG() = new(UInt32(InitialLCGSeed))
+	val::UInt32
+	LCG() = new(UInt32(InitialLCGSeed))
 end
 reset!(lcg::LCG) = lcg.val = UInt32(InitialLCGSeed)
-function Base.next(lcg::LCG)
-    n = UInt32((UInt64(lcg.val) * UInt64(279470273)) % UInt64(4294967291))
-    lcg.val = n
-    return n
+function next(lcg::LCG)
+	n = UInt32((UInt64(lcg.val) * UInt64(279470273)) % UInt64(4294967291))
+	lcg.val = n
+	return n
 end
 
 # Low level stress/fuzz test: serialize/deserialize a variety of
@@ -701,16 +701,16 @@ function checkFuzz(fuzzFields, fuzzObjects, verbose=true)
 	b = FlatBuffers.Builder()
 	l = LCG()
 
-    objects = fill(0, fuzzObjects)
+	objects = fill(0, fuzzObjects)
 
 	# Generate fuzzObjects random objects each consisting of
 	# fuzzFields fields, each of a random type.
-    for i = 1:fuzzObjects
+	for i = 1:fuzzObjects
 		FlatBuffers.startobject(b, fuzzFields)
 
-        for f = 1:fuzzFields
+		for f = 1:fuzzFields
 			choice = next(l) % UInt32(testValuesMax)
-            if choice ==  0
+			if choice ==  0
 				FlatBuffers.prependslot!(b, f, boolVal, false)
 			elseif choice ==  1
 				FlatBuffers.prependslot!(b, f, int8Val, Int8(0))
@@ -743,27 +743,27 @@ function checkFuzz(fuzzFields, fuzzObjects, verbose=true)
 	end
 
 	# Do some bookkeeping to generate stats on fuzzes:
-    stats = Dict{String,Int}()
-    function check(desc, want, got)
-        v = get!(stats, desc, 0)
-        stats[desc] = v + 1
-        @test want == got
-    end
+	stats = Dict{String,Int}()
+	function check(desc, want, got)
+		v = get!(stats, desc, 0)
+		stats[desc] = v + 1
+		@test want == got
+	end
 
 	l = LCG() # Reset.
 
 	# Test that all objects we generated are readable and return the
 	# expected values. We generate random objects in the same order
 	# so this is deterministic.
-    for i = 1:fuzzObjects
+	for i = 1:fuzzObjects
 
 		table = FlatBuffers.Table{Any}(b.bytes, length(b.bytes) - objects[i])
 
-        for j = 0:(fuzzFields - 1)
-            f = (FlatBuffers.VtableMetadataFields + j) * sizeof(Int16)
+		for j = 0:(fuzzFields - 1)
+			f = (FlatBuffers.VtableMetadataFields + j) * sizeof(Int16)
 			choice = next(l) % UInt32(testValuesMax)
 
-            if choice == 0
+			if choice == 0
 				check("bool", boolVal, FlatBuffers.getslot(table, f, false))
 			elseif choice == 1
 				check("int8", int8Val, FlatBuffers.getslot(table, f, Int8(0)))
@@ -797,13 +797,13 @@ function checkFuzz(fuzzFields, fuzzObjects, verbose=true)
 	end
 
 	# Print some counts, if needed:
-    if verbose
+	if verbose
 		if fuzzFields == 0 || fuzzObjects == 0
-            println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t[none]\t 0")
+			println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t[none]\t 0")
 		else
 			ks = sort!(collect(keys(stats)))
-            for k in ks
-                println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t$(k): $(stats[k])")
+			for k in ks
+				println("fuzz\tfields: $fuzzFields \tobjects: $fuzzObjects \t$(k): $(stats[k])")
 			end
 		end
 	end
diff --git a/julia/test/monster.jl b/julia/test/monster.jl
old mode 100755
new mode 100644
index cfbdd707cc8..b37a5685f9f
--- a/julia/test/monster.jl
+++ b/julia/test/monster.jl
@@ -1,4 +1,4 @@
-module Example
+module SmallExample
 
 using FlatBuffers
 
@@ -6,72 +6,72 @@ using FlatBuffers
 @DEFAULT Color Red
 
 @STRUCT struct Test
-    a::Int16
-    b::UInt8
-    # _1::UInt8 # padding
+	a::Int16
+	b::UInt8
+	# _1::UInt8 # padding
 end
 
 mutable struct TestSimpleTableWithEnum
-    color::Color
+	color::Color
 end
 
 @DEFAULT TestSimpleTableWithEnum color=Green
 
 @STRUCT struct Vec3
-    x::Float32
-    y::Float32
-    z::Float32
-    # _::UInt32 # padding
-    test1::Float64
-    test2::Color
-    # __::UInt8 # padding
-    test3::Test
-    # ___::UInt16 # padding
+	x::Float32
+	y::Float32
+	z::Float32
+	# _::UInt32 # padding
+	test1::Float64
+	test2::Color
+	# __::UInt8 # padding
+	test3::Test
+	# ___::UInt16 # padding
 end
 
 @ALIGN Vec3 16
 
 mutable struct Stat
-    id::String
-    val::Int64
-    count::UInt16
+	id::Union{Nothing, String}
+	val::Int64
+	count::UInt16
 end
 
 # Julia doesn't support forward referencing of types
 # @union Any_ Union{Monster, TestSimpleTableWithEnum}
 
 mutable struct Monster
-    pos::Vec3
-    mana::Int16
-    hp::Int16
-    name::String
-    friendly::Bool # deprecated
-    inventory::Vector{UInt8}
-    color::Color
-    # test_type::Any_
-    # test::Vector{UInt8}
-    test4::Vector{Test}
-    testarrayofstring::Vector{String}
-    testarrayoftables::Vector{Monster}
-    # don't support nested circulr reference objects yet
-    # enemy::Monster
-    testnestedflatbuffer::Vector{UInt8}
-    testempty::Stat
-    testbool::Bool
-    testhashs32_fnv1::Int32
-    testhashu32_fnv1::UInt32
-    testhashs64_fnv1::Int64
-    testhashu64_fnv1::UInt64
-    testhashs32_fnv1a::Int32
-    testhashu32_fnv1a::UInt32
-    testhashs64_fnv1a::Int64
-    testhashu64_fnv1a::UInt64
-    testarrayofbools::Vector{Bool}
-    testf::Float32
-    testf2::Float32
-    testf3::Float32
+	pos::Vec3
+	mana::Int16
+	hp::Int16
+	name::Union{Nothing, String}
+	friendly::Bool # deprecated
+	inventory::Union{Nothing, Vector{UInt8}}
+	color::Color
+	# test_type::Any_
+	# test::Union{Nothing, Vector{UInt8}}
+	test4::Union{Nothing, Vector{Test}}
+	testarrayofstring::Union{Nothing, Vector{String}}
+	testarrayoftables::Union{Nothing, Vector{Monster}}
+	# don't support nested circulr reference objects yet
+	# enemy::Monster
+	testnestedflatbuffer::Union{Nothing, Vector{UInt8}}
+	testempty::Union{Nothing, Stat}
+	testbool::Bool
+	testhashs32_fnv1::Int32
+	testhashu32_fnv1::UInt32
+	testhashs64_fnv1::Int64
+	testhashu64_fnv1::UInt64
+	testhashs32_fnv1a::Int32
+	testhashu32_fnv1a::UInt32
+	testhashs64_fnv1a::Int64
+	testhashu64_fnv1a::UInt64
+	testarrayofbools::Union{Nothing, Vector{Bool}}
+	testf::Float32
+	testf2::Float32
+	testf3::Float32
 end
 
-@DEFAULT Monster hp=100 mana=150 color=Blue friendly=false testf=Float32(3.14159) testf2=Float32(3)
+@DEFAULT Monster hp=Int16(100) mana=Int16(150) color=convert(UInt8, Blue) friendly=false testf=Float32(3.14159) testf2=Float32(3)
 
 end # module
diff --git a/julia/test/monsterdata_python_wire.mon b/julia/test/monsterdata_python_wire.mon
new file mode 100644
index 0000000000000000000000000000000000000000..55e37bf03961f9afdc7bd84220dcde2c8a5e8264
GIT binary patch
literal 352
zcma!GKm{HQJ`5T_AvFdCASnaH(hMRD0t`Gr7O{YXfrY^Xs2GG(fOrBB&j8{zKnw(o
zAo>QF82}_3?16*>5CJg^a5ykAurjbgxy(TNKN!6K4^si<*Ms=1K+Fi00NDou0_qUj
z!HFNjc=#2h3=Einqz@3=05OOy2lO!)(3d<696%ljuz>w50AztRF)}f;fJGS?+=^1c
gR)EAyQj1HBplU#DLm-f!oUG!u`y@>NgW_&VK87YVXy$QY=D@7ftA65!G$4#A%nq#
z!H1!Mp@Ja*EDj?=7$O*AfN~5BjEo?e274gk07O6x0~`)aKy%oDTmwc11_dDA0c6hs
z;tU`L>2f?-c^8BWmo)Z)unADC0EmwO@d6-joLK$oYy*EDjQ)5x{rDY^pUyBEsDyzT
zh#7%!!~g&PS%7R1b}LFv0a6T1Kr8~pATd@TE=eseF+yS+0@)xnTtHQFz_37O^MFGL
zC<)Zf113?;2H6X=5(wV^2dM%9D8C-W2H6MnG8ljz17r!PL--C({1C>&uOMY$0A?^U
VF@xO73B
Date: Mon, 18 Nov 2019 16:40:44 +1100
Subject: [PATCH 19/19] work on circular type dependencies

---
 julia/src/FlatBuffers.jl                    |   8 +-
 julia/src/macros.jl                         |   2 +-
 julia/test/flatc.jl                         |  20 ++--
 src/idl_gen_julia.cpp                       | 106 ++++++++++++++++----
 tests/MyGame/Example/AnyAmbiguousAliases.jl |  15 ++-
 tests/MyGame/Example/AnyUniqueAliases.jl    |  15 ++-
 tests/MyGame/Example/Any_.jl                |  15 ++-
 tests/MyGame/Example/Monster.jl             |  32 +++---
 tests/monster_test_generated.jl             |   3 +
 9 files changed, 157 insertions(+), 59 deletions(-)

diff --git a/julia/src/FlatBuffers.jl b/julia/src/FlatBuffers.jl
index fa1bad5eb8f..0471e20eca3 100644
--- a/julia/src/FlatBuffers.jl
+++ b/julia/src/FlatBuffers.jl
@@ -273,6 +273,11 @@ function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT}
         R = TT.b
     end
 
+    # if it's a self-referential UnionAll, use the more specific type
+    if isa(R, UnionAll) && T <: R
+        R = T
+    end
+
     # if it's a Union type, use the previous arg to figure out the true type that was serialized
     if !isunionwithnothing(R) && R isa Union
         R = typeorder(R, prevfield)
@@ -280,7 +285,7 @@ function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT}
 
     if R <: AbstractVector
         # hacks! if it's a union all, assume it's because we're working around circular dependencies
-        if isa(eltype(R), UnionAll)
+        if isa(eltype(R), UnionAll) && T <: eltype(R)
             return Vector{T}, false, nullable
         # if it's a vector of Unions, use the previous field to figure out the types of all the elements
         elseif isa(eltype(R), Union)
@@ -289,6 +294,7 @@ function typetoread(prevfield, ::Type{T}, ::Type{TT}) where {T, TT}
             return R, true, nullable
         end
     end
+
     return R, false, nullable
 end
 
diff --git a/julia/src/macros.jl b/julia/src/macros.jl
index f1c0cf12f26..2ae6e3a2690 100644
--- a/julia/src/macros.jl
+++ b/julia/src/macros.jl
@@ -183,7 +183,7 @@ end
 
 function getkwtype(defs, name)
 	for d in defs
-		if :args in propertynames(d) && d.args[1] == name
+		if :args in propertynames(d) && d.args[1] == name && d.args[end] isa Symbol
 			return d.args[end]
 		end
 	end
diff --git a/julia/test/flatc.jl b/julia/test/flatc.jl
index 088cac6e6c3..68bf44316dc 100644
--- a/julia/test/flatc.jl
+++ b/julia/test/flatc.jl
@@ -2,7 +2,7 @@ using Test
 import FlatBuffers
 
 # generated code
-include(joinpath(@__DIR_, "..", "..", "tests", "MyGame", "MyGame.jl"))
+include(joinpath(@__DIR__, "..", "..", "tests", "monster_test_generated.jl"))
 import .MyGame
 import .MyGame.Example
 import .MyGame.Example2
@@ -10,13 +10,7 @@ import .MyGame.Example.Any_
 import .MyGame.Example.Monster
 import .MyGame.Example.TestSimpleTableWithEnum
 
-# override the typeorder function, since we are working around
-# circular type definitions using type parameters
-FlatBuffers.typeorder(::Type{Any_}, i::Integer) = [Nothing, Monster{Any_}, TestSimpleTableWithEnum, Example2.Monster][i+1]
-
-function loadmonsterfile(filename)
-    open(joinpath(@__DIR__, filename), "r") do f Monster{Any_}(f) end
-end
+loadmonsterfile(filename) = open(joinpath(@__DIR__, filename), "r") do f Monster(f) end
 
 function checkmonster(monster)
     @test monster.hp == 80
@@ -53,11 +47,11 @@ function checkmonster(monster)
 end
 
 function checkpassthrough(monster)
-    b = FlatBuffers.Builder(Monster{Any_})
+    b = FlatBuffers.Builder(Monster)
     FlatBuffers.build!(b, monster)
     bytes = FlatBuffers.bytes(b)
-    @test FlatBuffers.has_identifier(Monster{Any_}, bytes)
-    newmonster = FlatBuffers.read(Monster{Any_}, bytes)
+    @test FlatBuffers.has_identifier(Monster, bytes)
+    newmonster = FlatBuffers.read(Monster, bytes)
     checkmonster(newmonster)
 end
 
@@ -65,7 +59,7 @@ function checkserialize(monster)
     io = IOBuffer()
     FlatBuffers.serialize(io, monster)
     bytes = take!(io)
-    newmonster = FlatBuffers.deserialize(IOBuffer(bytes), Monster{Any_})
+    newmonster = FlatBuffers.deserialize(IOBuffer(bytes), Monster)
     checkmonster(newmonster)
 end
 
@@ -82,7 +76,7 @@ end
 
 # test printing
 mon = loadmonsterfile("monsterdata_test.mon")
-b = FlatBuffers.Builder(Monster{Any_})
+b = FlatBuffers.Builder(Monster)
 FlatBuffers.build!(b, mon)
 io = IOBuffer()
 show(io, b)
diff --git a/src/idl_gen_julia.cpp b/src/idl_gen_julia.cpp
index 991bf73dee0..6a4888cd8ca 100644
--- a/src/idl_gen_julia.cpp
+++ b/src/idl_gen_julia.cpp
@@ -120,6 +120,53 @@ class DepGraph {
   size_t size() { return adj.size(); }
 };
 
+class TypeVariableTable {
+public:
+  std::string GetOrCreate(std::string type_name) {
+    if (variables_.find(type_name) != variables_.end())
+      return (*variables_.find(type_name)).first;
+    variables_[current_name_] = type_name;
+    std::string ret = current_name_;
+    int i = current_name_.length() - 1;
+    while (i >= 0) {
+      if (current_name_[i] == 'Z') {
+        current_name_[i] = 'A';
+        i--;
+        continue;
+      }
+      current_name_[i]++;
+      break;
+    }
+    if (i < 0)
+      current_name_.append("A");
+    return ret;
+  }
+  
+  std::string TypeSignature(bool concrete = false) {
+    std::string signature = "";
+    if (variables_.size() == 0) {
+      return signature;
+    }
+    signature += "{";
+    bool first = true;
+    for (auto it = variables_.begin(); it != variables_.end(); ++it) {
+      if (!first)
+        signature += ", ";
+      signature += concrete ? (*it).second : (*it).first;
+      first = false;
+    }
+    signature += "}";
+    return signature;
+  }
+
+  TypeVariableTable() : current_name_("A") {}
+  ~TypeVariableTable() {}
+  
+private:
+  std::string current_name_;
+  std::map variables_;
+};
+
 class ModuleTable {
  public:
   bool IsModule(const std::string &m) {
@@ -273,7 +320,7 @@ class JuliaGenerator : public BaseGenerator {
   }
 
   // Begin an object declaration.
-  void BeginObject(const StructDef &struct_def, std::string *code_ptr,
+  void BeginObject(TypeVariableTable &vars, const StructDef &struct_def, std::string *code_ptr,
                    bool has_defaults) {
     auto &code = *code_ptr;
     if (has_defaults) code += JuliaPackageName + ".@with_kw ";
@@ -281,7 +328,8 @@ class JuliaGenerator : public BaseGenerator {
       code += "mutable struct ";
     else
       code += JuliaPackageName + ".@STRUCT struct ";
-    code += NormalizedName(struct_def) + "\n";
+    code += NormalizedName(struct_def);
+    code += vars.TypeSignature() + "\n";
   }
 
   void EndObject(const StructDef &struct_def,
@@ -365,13 +413,13 @@ class JuliaGenerator : public BaseGenerator {
 
   static void EndUnion(std::string *code_ptr) { *code_ptr += "))\n\n"; }
 
-  void NewObjectFromBuffer(const StructDef &struct_def, std::string *code_ptr) {
+  void NewObjectFromBuffer(TypeVariableTable &vars, const StructDef &struct_def, std::string *code_ptr) {
     auto &code = *code_ptr;
     code += NormalizedName(struct_def) + "(buf::AbstractVector{UInt8})";
-    code += " = " + JuliaPackageName + ".read(" + NormalizedName(struct_def) +
+    code += " = " + JuliaPackageName + ".read(" + NormalizedName(struct_def) + vars.TypeSignature(true) +
             ", buf)\n";
     code += NormalizedName(struct_def) + "(io::IO) = " + JuliaPackageName;
-    code += ".deserialize(io, " + NormalizedName(struct_def) + ")\n";
+    code += ".deserialize(io, " + NormalizedName(struct_def) + vars.TypeSignature(true) + ")\n";
   }
 
   void GenScalarField(const StructDef &struct_def, const FieldDef &field,
@@ -495,12 +543,19 @@ class JuliaGenerator : public BaseGenerator {
   }
 
   // generate a field which depends upon generated types
-  void GenDependentField(const StructDef &struct_def, const FieldDef &field,
+  void GenDependentField(TypeVariableTable &vars,
+                         const StructDef &struct_def, const FieldDef &field,
                          std::string *code_ptr, bool *has_defaults,
                          std::set *imports_ptr) {
-    std::string type_name = GenTypeGet(field.value.type, &struct_def);
-
+    
     BaseType bt = field.value.type.base_type;
+    Type vartype = (bt == BASE_TYPE_VECTOR) ? field.value.type.VectorType() : field.value.type;
+    std::string type_name = GenTypeGet(field.value.type, &struct_def);
+    if (!struct_def.fixed && !IsScalar(vartype.base_type) && vartype.base_type != BASE_TYPE_STRING) {
+      std::string type_variable = vars.GetOrCreate(GenTypeGet(vartype, &struct_def));
+      type_name = (bt == BASE_TYPE_VECTOR) ? "Vector{" + type_variable + "}" : type_variable;
+    }
+    
     if (bt == BASE_TYPE_STRUCT && !struct_def.fixed) {
       type_name = "Union{" + type_name + ", Nothing}";
     }
@@ -533,7 +588,7 @@ class JuliaGenerator : public BaseGenerator {
   }
 
   // Generate a field, conditioned on its child type(s).
-  void GenField(const StructDef &struct_def, const FieldDef &field,
+  void GenField(TypeVariableTable &vars, const StructDef &struct_def, const FieldDef &field,
                 std::string *code_ptr, std::set *imports_ptr,
                 bool *has_defaults) {
     GenComment(field.doc_comment, code_ptr, &JuliaCommentConfig);
@@ -547,7 +602,7 @@ class JuliaGenerator : public BaseGenerator {
         case BASE_TYPE_STRUCT:
         case BASE_TYPE_VECTOR:
         case BASE_TYPE_UNION:
-          GenDependentField(struct_def, field, code_ptr, has_defaults,
+          GenDependentField(vars, struct_def, field, code_ptr, has_defaults,
                             imports_ptr);
           break;
         default: FLATBUFFERS_ASSERT(0);
@@ -573,12 +628,13 @@ class JuliaGenerator : public BaseGenerator {
     // generate all the fields
     std::vector offsets;
 
+    TypeVariableTable vars;
     GenComment(struct_def.doc_comment, code_ptr, &JuliaCommentConfig);
     for (auto it = struct_def.fields.vec.begin();
          it != struct_def.fields.vec.end(); ++it) {
       auto &field = **it;
       if (field.deprecated) continue;
-      GenField(struct_def, field, code_ptr, &imports, &has_defaults);
+      GenField(vars, struct_def, field, code_ptr, &imports, &has_defaults);
       offsets.push_back("0x" + IntToStringHex(field.value.offset, 8));
     }
     EndObject(struct_def, offsets, code_ptr);
@@ -586,12 +642,12 @@ class JuliaGenerator : public BaseGenerator {
     // need to call BeginObject after EndObject because we don't know
     // the defaults until we've looked at all the fields.
     std::string struct_begin;
-    BeginObject(struct_def, &struct_begin, has_defaults);
+    BeginObject(vars, struct_def, &struct_begin, has_defaults);
 
     *code_ptr = GenImports(imports) + "\n" + struct_begin + *code_ptr;
 
     // Generate a functions for constructing the object from a buffer
-    NewObjectFromBuffer(struct_def, code_ptr);
+    NewObjectFromBuffer(vars, struct_def, code_ptr);
   }
 
   void GenUnion(const EnumDef &enum_def, std::string *code_ptr) {
@@ -599,12 +655,21 @@ class JuliaGenerator : public BaseGenerator {
 
     std::set imports;
     auto union_name = NormalizedName(enum_def);
-    BeginUnion(union_name, code_ptr);
     for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end();
          ++it) {
       auto &ev = **it;
-      std::string type_name = GenTypeGet(ev.union_type, &enum_def);
+      if (ev.name == "NONE") continue;
       GenComment(ev.doc_comment, code_ptr, &JuliaCommentConfig);
+      std::string val_type = union_name + NormalizedName(ev);
+      *code_ptr += "struct " + val_type + " end\n";
+      std::string type_name = GenTypeGet(ev.union_type, &enum_def);
+      *code_ptr += val_type + "(args...; kwargs...) = " + type_name + "(args...; kwargs...)\n\n";
+    }
+    BeginUnion(union_name, code_ptr);
+    for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end();
+         ++it) {
+      auto &ev = **it;
+      std::string type_name = ev.name == "NONE" ? GenTypeGet(ev.union_type, &enum_def) : union_name + NormalizedName(ev);
       UnionMember(type_name, code_ptr);
       // special case, instead make every Union a union with Nothing
       if (ev.name == "NONE") continue;
@@ -719,15 +784,14 @@ class JuliaGenerator : public BaseGenerator {
     for (auto it = sorted_children.rbegin(); it != sorted_children.rend();
          ++it) {
       std::string child = *it;
-
+      
       if (included.find(child) != included.end()) continue;
 
-      included.insert(child);
-
       // if this module depends on another module, go and generate that module
       // first
       if (module_table_.IsModule(child)) {
         GenIncludes(child, included, code_ptr);
+        included.insert(child);
         continue;
       }
 
@@ -735,8 +799,10 @@ class JuliaGenerator : public BaseGenerator {
       if (child.find(mod) == std::string::npos ||
           child.length() < (mod.length() + 1) ||
           child.substr(mod.length() + 1).find(kPathSeparator) !=
-              std::string::npos)
+          std::string::npos) {
         continue;
+      }
+      
       // If the file doesn't exist, don't include it
       // TODO: this doesn't allow types which reference each other,
       // but Julia doesn't support this yet anyway
@@ -744,6 +810,8 @@ class JuliaGenerator : public BaseGenerator {
       std::string fullpath = ConCatPathFileName(path_, toinclude);
       if (!module_table_.IsFile(fullpath.c_str())) continue;
       code += "include(\"" + toinclude + "\")\n";
+      
+      included.insert(child);
     }
     return true;
   }
diff --git a/tests/MyGame/Example/AnyAmbiguousAliases.jl b/tests/MyGame/Example/AnyAmbiguousAliases.jl
index 768d12cdc61..ee10a22bf12 100644
--- a/tests/MyGame/Example/AnyAmbiguousAliases.jl
+++ b/tests/MyGame/Example/AnyAmbiguousAliases.jl
@@ -3,11 +3,20 @@
 MyGame.Example.eval(quote
 
 
+struct AnyAmbiguousAliasesM1 end
+AnyAmbiguousAliasesM1(args...; kwargs...) = Monster(args...; kwargs...)
+
+struct AnyAmbiguousAliasesM2 end
+AnyAmbiguousAliasesM2(args...; kwargs...) = Monster(args...; kwargs...)
+
+struct AnyAmbiguousAliasesM3 end
+AnyAmbiguousAliasesM3(args...; kwargs...) = Monster(args...; kwargs...)
+
 FlatBuffers.@UNION(AnyAmbiguousAliases, (
     Nothing,
-    Monster,
-    Monster,
-    Monster,
+    AnyAmbiguousAliasesM1,
+    AnyAmbiguousAliasesM2,
+    AnyAmbiguousAliasesM3,
 ))
 
 
diff --git a/tests/MyGame/Example/AnyUniqueAliases.jl b/tests/MyGame/Example/AnyUniqueAliases.jl
index 2744542c82a..730d3f5fc21 100644
--- a/tests/MyGame/Example/AnyUniqueAliases.jl
+++ b/tests/MyGame/Example/AnyUniqueAliases.jl
@@ -4,11 +4,20 @@ MyGame.Example.eval(quote
 
 import ..Example2
 
+struct AnyUniqueAliasesM end
+AnyUniqueAliasesM(args...; kwargs...) = Monster(args...; kwargs...)
+
+struct AnyUniqueAliasesTS end
+AnyUniqueAliasesTS(args...; kwargs...) = TestSimpleTableWithEnum(args...; kwargs...)
+
+struct AnyUniqueAliasesM2 end
+AnyUniqueAliasesM2(args...; kwargs...) = Example2.Monster(args...; kwargs...)
+
 FlatBuffers.@UNION(AnyUniqueAliases, (
     Nothing,
-    Monster,
-    TestSimpleTableWithEnum,
-    Example2.Monster,
+    AnyUniqueAliasesM,
+    AnyUniqueAliasesTS,
+    AnyUniqueAliasesM2,
 ))
 
 
diff --git a/tests/MyGame/Example/Any_.jl b/tests/MyGame/Example/Any_.jl
index 33d7ea894c5..29a3e1b7b20 100644
--- a/tests/MyGame/Example/Any_.jl
+++ b/tests/MyGame/Example/Any_.jl
@@ -4,11 +4,20 @@ MyGame.Example.eval(quote
 
 import ..Example2
 
+struct Any_Monster end
+Any_Monster(args...; kwargs...) = Monster(args...; kwargs...)
+
+struct Any_TestSimpleTableWithEnum end
+Any_TestSimpleTableWithEnum(args...; kwargs...) = TestSimpleTableWithEnum(args...; kwargs...)
+
+struct Any_MyGame_Example2_Monster end
+Any_MyGame_Example2_Monster(args...; kwargs...) = Example2.Monster(args...; kwargs...)
+
 FlatBuffers.@UNION(Any_, (
     Nothing,
-    Monster,
-    TestSimpleTableWithEnum,
-    Example2.Monster,
+    Any_Monster,
+    Any_TestSimpleTableWithEnum,
+    Any_MyGame_Example2_Monster,
 ))
 
 
diff --git a/tests/MyGame/Example/Monster.jl b/tests/MyGame/Example/Monster.jl
index 4567f5b7402..8cda252027f 100644
--- a/tests/MyGame/Example/Monster.jl
+++ b/tests/MyGame/Example/Monster.jl
@@ -4,28 +4,28 @@ MyGame.Example.eval(quote
 
 import ..InParentNamespace
 
-FlatBuffers.@with_kw mutable struct Monster
+FlatBuffers.@with_kw mutable struct Monster{A, B, C, D, E, F, G, H, I, J, K, L, M}
 #=
 #  an example documentation comment: monster object
 =#
-    pos::Union{Vec3, Nothing} = nothing
+    pos::Union{A, Nothing} = nothing
     mana::Int16 = 150
     hp::Int16 = 100
     name::String = ""
     inventory::Vector{UInt8} = []
     color::Color = 8
     test_type::UInt8 = 0
-    test::Any_ = nothing
-    test4::Vector{Test} = []
+    test::B = nothing
+    test4::Vector{C} = []
     testarrayofstring::Vector{String} = []
 #=
 #  an example documentation comment: this will end up in the generated code
 #  multiline too
 =#
-    testarrayoftables::Vector{Monster} = []
-    enemy::Union{Monster, Nothing} = nothing
+    testarrayoftables::Vector{D} = []
+    enemy::Union{E, Nothing} = nothing
     testnestedflatbuffer::Vector{UInt8} = []
-    testempty::Union{Stat, Nothing} = nothing
+    testempty::Union{F, Nothing} = nothing
     testbool::Bool = false
     testhashs32_fnv1::Int32 = 0
     testhashu32_fnv1::UInt32 = 0
@@ -40,24 +40,24 @@ FlatBuffers.@with_kw mutable struct Monster
     testf2::Float32 = 3.0
     testf3::Float32 = 0.0
     testarrayofstring2::Vector{String} = []
-    testarrayofsortedstruct::Vector{Ability} = []
+    testarrayofsortedstruct::Vector{G} = []
     flex::Vector{UInt8} = []
-    test5::Vector{Test} = []
+    test5::Vector{H} = []
     vector_of_longs::Vector{Int64} = []
     vector_of_doubles::Vector{Float64} = []
-    parent_namespace_test::Union{InParentNamespace, Nothing} = nothing
-    vector_of_referrables::Vector{Referrable} = []
+    parent_namespace_test::Union{I, Nothing} = nothing
+    vector_of_referrables::Vector{J} = []
     single_weak_reference::UInt64 = 0
     vector_of_weak_references::Vector{UInt64} = []
-    vector_of_strong_referrables::Vector{Referrable} = []
+    vector_of_strong_referrables::Vector{K} = []
     co_owning_reference::UInt64 = 0
     vector_of_co_owning_references::Vector{UInt64} = []
     non_owning_reference::UInt64 = 0
     vector_of_non_owning_references::Vector{UInt64} = []
     any_unique_type::UInt8 = 0
-    any_unique::AnyUniqueAliases = nothing
+    any_unique::L = nothing
     any_ambiguous_type::UInt8 = 0
-    any_ambiguous::AnyAmbiguousAliases = nothing
+    any_ambiguous::M = nothing
     vector_of_enums::Vector{Color} = []
     signed_enum::Race = -1
 end
@@ -80,8 +80,8 @@ FlatBuffers.root_type(::Type{T}) where {T<:Monster} = true
 FlatBuffers.file_identifier(::Type{T}) where {T<:Monster} = "MONS"
 FlatBuffers.file_extension(::Type{T}) where {T<:Monster} = "mon"
 
-Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster, buf)
-Monster(io::IO) = FlatBuffers.deserialize(io, Monster)
+Monster(buf::AbstractVector{UInt8}) = FlatBuffers.read(Monster{Vec3, Any_, Test, Monster, Monster, Stat, Ability, Test, InParentNamespace, Referrable, Referrable, AnyUniqueAliases, AnyAmbiguousAliases}, buf)
+Monster(io::IO) = FlatBuffers.deserialize(io, Monster{Vec3, Any_, Test, Monster, Monster, Stat, Ability, Test, InParentNamespace, Referrable, Referrable, AnyUniqueAliases, AnyAmbiguousAliases})
 
 end)
 
diff --git a/tests/monster_test_generated.jl b/tests/monster_test_generated.jl
index 6ad57b03577..092756ac0e1 100644
--- a/tests/monster_test_generated.jl
+++ b/tests/monster_test_generated.jl
@@ -14,5 +14,8 @@ include("MyGame/Example/Stat.jl")
 include("MyGame/Example/Ability.jl")
 include("MyGame/Example/Referrable.jl")
 include("MyGame/Example/TestSimpleTableWithEnum.jl")
+include("MyGame/Example/AnyUniqueAliases.jl")
 include("MyGame/Example/AnyAmbiguousAliases.jl")
+include("MyGame/Example/Monster.jl")
+include("MyGame/Example/Any_.jl")
 include("MyGame/Example/TypeAliases.jl")