Introduce scalar fixed size arrays

dvidelabs · May 8, 2019 · 6ef1459 · 6ef1459
1 parent 0c219f5
commit 6ef1459
Show file tree

Hide file tree

Showing 15 changed files with 508 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,7 @@
 - Make `flatbuffers_not_found` and `flatbuffers_end` constant values both
   because it is more correct, and to silence warnings on some systems.
 - Fix flatcc endless loop with our of order field id's (#112).
+- Add support for scalar fixed size arrays in struct fields.
 
 ## [0.5.3]
 - BREAKING: 0.5.3 changes behavour of builder create calls so arguments

diff --git a/README.md b/README.md
@@ -46,6 +46,7 @@ executable also handle optional json parsing or printing in less than 2 us for a
   * [Type Identifiers](#type-identifiers)
 * [JSON Parsing and Printing](#json-parsing-and-printing)
   * [Base64 Encoding](#base64-encoding)
+  * [Fixed Size Arrays](#fixed-size-arrays)
   * [Generic Parsing and Printing.](#generic-parsing-and-printing)
   * [Performance Notes](#performance-notes)
 * [Global Scope and Included Schema](#global-scope-and-included-schema)
@@ -54,6 +55,7 @@ executable also handle optional json parsing or printing in less than 2 us for a
 * [Types](#types)
 * [Unions](#unions)
   * [Union Scope Resolution](#union-scope-resolution)
+* [Fixed Size Arrays](#fixed-size-arrays-1)
 * [Endianness](#endianness)
 * [Pitfalls in Error Handling](#pitfalls-in-error-handling)
 * [Searching and Sorting](#searching-and-sorting)
@@ -290,7 +292,8 @@ together with a key attribute to chose default key for finding and
 sorting. If primary is absent, the key with the lowest id becomes
 primary. Tables and vectors can now be sorted recursively on primary
 keys. BREAKING: previously the first listed, not the lowest id, would be
-the primary key.
+the primary key. Also introduces fixed size scalar arrays in struct fields
+(struct and enum elements are not supported).
 
 Release 0.5.3 inlcudes various bug fixes (see changelog) and one
 breaking but likely low impact change: BREAKING: 0.5.3 changes behavour
@@ -1445,7 +1448,6 @@ vector field named `manyany` which is a vector of `Any` unions in the
         "manyany": [{"name": "Joe"}, null]
     }
 
-
 ### Base64 Encoding
 
 As of v0.5.0 it is possible to encode and decode a vector of type
@@ -1481,6 +1483,13 @@ possible, but it is recognized that a `(force_align: n)` attribute on
 `[ubyte]` vectors could be useful, but it can also be handled via nested
 flatbuffers which also align data.
 
+### Fixed Size Arrays
+
+Fixed size scalar arrays introduced in 0.6.0 are parsed like ordinary
+vectors but the element count must match exactly to avoid an array
+overflow or underflow error. Two runtime flags allow the parser to zero
+pad missing elements and to skip extra elements without raising an arror.
+Base64 encoding is not supported for fixed size arrays.
 
 ### Generic Parsing and Printing.
 
@@ -1778,6 +1787,32 @@ monster type to an Any union field, and `MyGame.Example.Any.Monster2`, or just
 `Monster2` when assigning the second monster type. C uses the usual enum
 namespace prefixed symbols like `MyGame_Example_Any_Monster2`.
 
+## Fixed Size Arrays
+
+Fixed Size Arrays is a late feature to the FlatBuffers format introduced in
+flatc and flatcc mid 2019. Currently only scalars arrays are supported, and only
+as struct fields. To use fixed size arrays as a table field wrap it in a struct
+first. It would make sense to support struct elements and enum elements, but
+that has not been implemented. Char arrays are more controversial due to
+verification and zero termination and are also not supported. Arrays are aligned
+to the size of the first field and are equivalent to repeating elements within
+the struct.
+
+The schema syntax is:
+
+```
+struct MyStruct {
+    my_array : [float:10];
+}
+```
+
+See `test_fixed_array` in [monster_test.c] for an example of how to work with
+these arrays.
+
+Flatcc opts to allow arbitrary length fixed size arrays but limit the entire
+struct to 2^16-1 bytes. Tables cannot hold larger structs, and the C language
+does not guarantee support for larger structs. Other implementations might have
+different limits on maximum array size. Arrays of 0 length are not permitted.
 
 ## Endianness
 

diff --git a/include/flatcc/flatcc_json_parser.h b/include/flatcc/flatcc_json_parser.h
@@ -27,6 +27,8 @@ enum flatcc_json_parser_flags {
     flatcc_json_parser_f_skip_unknown = 1,
     flatcc_json_parser_f_force_add = 2,
     flatcc_json_parser_f_with_size = 4,
+    flatcc_json_parser_f_skip_array_overflow = 8,
+    flatcc_json_parser_f_pad_array_underflow = 16
 };
 
 #define FLATCC_JSON_PARSE_ERROR_MAP(XX)                                     \
@@ -63,6 +65,8 @@ enum flatcc_json_parser_flags {
     XX(union_vector_length,     "union vector length mismatch")             \
     XX(base64,                  "invalid base64 content")                   \
     XX(base64url,               "invalid base64url content")                \
+    XX(array_underflow,         "fixed size array underflow")               \
+    XX(array_overflow,          "fixed size array overflow")                \
     XX(runtime,                 "runtime error")                            \
     XX(not_supported,           "not supported")
 

diff --git a/include/flatcc/flatcc_json_printer.h b/include/flatcc/flatcc_json_printer.h
@@ -500,6 +500,12 @@ void flatcc_json_printer_ ## TN ## _struct_field(flatcc_json_printer_t *ctx,\
         int index, const void *p, size_t offset,                            \
         const char *name, int len);
 
+#define __define_print_scalar_array_struct_field_proto(TN, T)               \
+void flatcc_json_printer_ ## TN ## _array_struct_field(                     \
+        flatcc_json_printer_t *ctx,                                         \
+        int index, const void *p, size_t offset,                            \
+        const char *name, int len, size_t count);
+
 #define __define_print_enum_struct_field_proto(TN, T)                       \
 void flatcc_json_printer_ ## TN ## _enum_struct_field(                      \
         flatcc_json_printer_t *ctx,                                         \
@@ -559,6 +565,18 @@ __define_print_scalar_struct_field_proto(bool, flatbuffers_bool_t)
 __define_print_scalar_struct_field_proto(float, float)
 __define_print_scalar_struct_field_proto(double, double)
 
+__define_print_scalar_array_struct_field_proto(uint8, uint8_t)
+__define_print_scalar_array_struct_field_proto(uint16, uint16_t)
+__define_print_scalar_array_struct_field_proto(uint32, uint32_t)
+__define_print_scalar_array_struct_field_proto(uint64, uint64_t)
+__define_print_scalar_array_struct_field_proto(int8, int8_t)
+__define_print_scalar_array_struct_field_proto(int16, int16_t)
+__define_print_scalar_array_struct_field_proto(int32, int32_t)
+__define_print_scalar_array_struct_field_proto(int64, int64_t)
+__define_print_scalar_array_struct_field_proto(bool, flatbuffers_bool_t)
+__define_print_scalar_array_struct_field_proto(float, float)
+__define_print_scalar_array_struct_field_proto(double, double)
+
 __define_print_enum_struct_field_proto(uint8, uint8_t)
 __define_print_enum_struct_field_proto(uint16, uint16_t)
 __define_print_enum_struct_field_proto(uint32, uint32_t)

diff --git a/src/compiler/codegen_c_builder.c b/src/compiler/codegen_c_builder.c
@@ -411,6 +411,17 @@ int fb_gen_common_c_builder_header(fb_output_t *out)
         "#define __%scopy_to_pe(P, P2, N) (*(P) = N ## _cast_to_pe(*P2), (P))\n"
         "#define __%sto_pe(P, N) (*(P) = N ## _cast_to_pe(*P), (P))\n",
         nsc, nsc, nsc, nsc);
+    fprintf(out->fp,
+        "#define __%sdefine_fixed_array_primitives(NS, N, T)\\\n"
+        "static inline T *N ## _array_copy(T *p, const T *p2, size_t n)\\\n"
+        "{ memcpy(p, p2, n * sizeof(T)); return p; }\\\n"
+        "static inline T *N ## _array_copy_from_pe(T *p, const T *p2, size_t n)\\\n"
+        "{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\\\n"
+        "  for (i = 0; i < n; ++i) N ## _copy_from_pe(&p[i], &p2[i]); return p; }\\\n"
+        "static inline T *N ## _array_copy_to_pe(T *p, const T *p2, size_t n)\\\n"
+        "{ size_t i; if (NS ## is_native_pe()) memcpy(p, p2, n * sizeof(T)); else\\\n"
+        "  for (i = 0; i < n; ++i) N ## _copy_to_pe(&p[i], &p2[i]); return p; }\n",
+        nsc);
     fprintf(out->fp,
         "#define __%sdefine_scalar_primitives(NS, N, T)\\\n"
         "static inline T *N ## _from_pe(T *p) { return __ ## NS ## from_pe(p, N); }\\\n"
@@ -427,6 +438,7 @@ int fb_gen_common_c_builder_header(fb_output_t *out)
         "{ *p = N ## _cast_to_pe(v0); return p; }\n"
         "#define __%sbuild_scalar(NS, N, T)\\\n"
         "__ ## NS ## define_scalar_primitives(NS, N, T)\\\n"
+        "__ ## NS ## define_fixed_array_primitives(NS, N, T)\\\n"
         "__ ## NS ## build_vector(NS, N, T, sizeof(T), sizeof(T))\n",
         nsc, nsc);
 
@@ -886,21 +898,23 @@ static int get_total_struct_field_count(fb_compound_type_t *ct)
     fb_member_t *member;
     fb_symbol_t *sym;
     int count = 0;
+
     for (sym = ct->members; sym; sym = sym->link) {
         member = (fb_member_t *)sym;
         if (member->metadata_flags & fb_f_deprecated) {
             continue;
         }
-        /* Fall through comments needed to silence gcc 7 warnings. */
         switch (member->type.type) {
         case vt_compound_type_ref:
             if (member->type.ct->symbol.kind == fb_is_struct) {
                 count += get_total_struct_field_count(member->type.ct);
                 continue;
             }
-            /* Fall through */
+            ++count;
+            break;
         default:
             ++count;
+            break;
         }
     }
     return count;
@@ -953,6 +967,12 @@ static int gen_builder_struct_args(fb_output_t *out, fb_compound_type_t *ct, int
             fb_compound_name(member->type.ct, &snref);
             fprintf(out->fp, "%s_enum_t v%i", snref.text, index++);
             break;
+        case vt_fixed_array_type:
+            gen_comma(out, index, len, is_macro);
+            tname_ns = scalar_type_ns(member->type.st, nsc);
+            tname = scalar_type_name(member->type.st);
+            fprintf(out->fp, "const %s%s v%i[%i]", tname_ns, tname, index++, (int)member->type.len);
+            break;
         case vt_scalar_type:
             gen_comma(out, index, len, is_macro);
             tname_ns = scalar_type_ns(member->type.st, nsc);
@@ -982,12 +1002,13 @@ static int gen_builder_struct_call_list(fb_output_t *out, fb_compound_type_t *ct
 enum { no_conversion, convert_from_pe, convert_to_pe };
 
 /* Note: returned index is not correct when using from_ptr since it doesn't track arguments, but it shouldn't matter. */
-static int gen_builder_struct_field_assign(fb_output_t *out, fb_compound_type_t *ct, int index, int arg_count, int conversion, int from_ptr)
+static int gen_builder_struct_field_assign(fb_output_t *out, fb_compound_type_t *ct, int index, int arg_count,
+        int conversion, int from_ptr)
 {
     const char *nsc = out->nsc;
     fb_member_t *member;
     fb_symbol_t *sym;
-    int n;
+    int n, len;
     const char *s;
     int deprecated_index = 0;
     const char *kind, *tprefix;
@@ -1068,6 +1089,47 @@ static int gen_builder_struct_field_assign(fb_output_t *out, fb_compound_type_t
             }
             ++index;
             continue;
+        case vt_fixed_array_type:
+            tprefix = scalar_type_prefix(member->type.st);
+            len = member->type.len;
+            if (member->metadata_flags & fb_f_deprecated) {
+                fprintf(out->fp, "__%sstruct_clear_field(p->__deprecated%i)",
+                        nsc, deprecated_index);
+                ++deprecated_index;
+                ++index;
+                continue;
+            }
+            switch (member->size == 1 ? no_conversion : conversion) {
+            case convert_from_pe:
+                if (from_ptr) {
+                    fprintf(out->fp, "%s%s_array_copy_from_pe(p->%.*s, p2->%.*s, %d)",
+                            nsc, tprefix, n, s, n, s, len);
+                } else {
+                    fprintf(out->fp, "%s%s_array_copy_from_pe(p->%.*s, v%i, %d)",
+                            nsc, tprefix, n, s, index, len);
+                }
+                break;
+            case convert_to_pe:
+                if (from_ptr) {
+                    fprintf(out->fp, "%s%s_array_copy_to_pe(p->%.*s, p2->%.*s, %d)",
+                            nsc, tprefix, n, s, n, s, len);
+                } else {
+                    fprintf(out->fp, "%s%s_array_copy_to_pe(p->%.*s, v%i, %d)",
+                            nsc, tprefix, n, s, index, len);
+                }
+                break;
+            default:
+                if (from_ptr) {
+                    fprintf(out->fp, "%s%s_array_copy(p->%.*s, p2->%.*s, %d)",
+                            nsc, tprefix, n, s, n, s, len);
+                } else {
+                    fprintf(out->fp, "%s%s_array_copy(p->%.*s, v%i, %d)",
+                            nsc, tprefix, n, s, index, len);
+                }
+                break;
+            }
+            ++index;
+            break;
         case vt_scalar_type:
             tprefix = scalar_type_prefix(member->type.st);
             if (member->metadata_flags & fb_f_deprecated) {
@@ -1323,7 +1385,7 @@ static int gen_builder_table_args(fb_output_t *out, fb_compound_type_t *ct, int
             fprintf(out->fp, "%sstring_vec_ref_t v%llu", nsc, llu(member->id));
             break;
         default:
-            gen_panic(out, "internal error: unexpected struct member type");
+            gen_panic(out, "internal error: unexpected table member type");
             continue;
         }
     }

diff --git a/src/compiler/codegen_c_json_parser.c b/src/compiler/codegen_c_json_parser.c
@@ -416,6 +416,8 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
     int is_base64 = 0;
     int is_base64url = 0;
     int is_nested = 0;
+    int is_array = 0;
+    size_t array_len = 0;
     int st = 0;
     const char *tname_prefix = "n/a", *tname = "n/a"; /* suppress compiler warnigns */
 
@@ -454,7 +456,15 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
         is_nested = member->nest != 0;
         is_base64 = member->metadata_flags & fb_f_base64;
         is_base64url = member->metadata_flags & fb_f_base64url;
-        /* Fall through. */
+        is_scalar = 1;
+        st = member->type.st;
+        break;
+    case vt_fixed_array_type:
+        is_array = 1;
+        is_scalar = 1;
+        array_len = member->type.len;
+        st = member->type.st;
+        break;
     case vt_scalar_type:
         is_scalar = 1;
         st = member->type.st;
@@ -515,6 +525,13 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
                 (uint64_t)member->size, (short)member->align,
                 (uint64_t)FLATBUFFERS_COUNT_MAX(member->size));
         }
+    }
+    if (is_array) {
+        println(out, "size_t count = %d;", array_len);
+        println(out, "%s *base = (%s *)((size_t)struct_base + %"PRIu64");",
+                tname, tname, (uint64_t)member->offset);
+    }
+    if (is_array || is_vector) {
         println(out, "buf = flatcc_json_parser_array_start(ctx, buf, end, &more);");
         /* Note that we reuse `more` which is safe because it is updated at the end of the main loop. */
         println(out, "while (more) {"); indent();
@@ -542,7 +559,7 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
     if (is_vector && !is_offset) {
         println(out, "if (!(pval = flatcc_builder_extend_vector(ctx->ctx, 1))) goto failed;");
     }
-    if (is_struct_container) {
+    if (is_struct_container && !is_array) {
         /* `struct_base` is given as argument to struct parsers. */
         println(out, "pval = (void *)((size_t)struct_base + %"PRIu64");", (uint64_t)member->offset);
     } else if (is_struct && !is_vector) {
@@ -560,7 +577,7 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
         if (!is_struct_container && !is_vector && !is_base64 && !is_base64url) {
 #if !FLATCC_JSON_PARSE_FORCE_DEFAULTS
             /* We need to create a check for the default value and create a table field if not the default. */
-            switch(member->value.type) {
+            switch (member->value.type) {
             case vt_bool:
             case vt_uint:
                 println(out, "if (val != %"PRIu64" || (ctx->flags & flatcc_json_parser_f_force_add)) {", member->value.u); indent();
@@ -597,7 +614,16 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
 #endif
         }
         /* For scalars in table field, and in struct container. */
-        println(out, "%s%s_write_to_pe(pval, val);", out->nsc, tname_prefix);
+        if (is_array) {
+            println(out, "if (count) {"); indent();
+            println(out, "%s%s_write_to_pe(base, val);", out->nsc, tname_prefix);
+            println(out, "--count, ++base;");
+            unindent(); println(out, "} else if (!(ctx->flags & flatcc_json_parser_f_skip_array_overflow)) {"); indent();
+            println(out, "return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_array_overflow);");
+            unindent(); println(out, "}");
+        } else {
+            println(out, "%s%s_write_to_pe(pval, val);", out->nsc, tname_prefix);
+        }
         if (!is_struct_container && !is_vector) {
             unindent(); println(out, "}");
         }
@@ -651,6 +677,16 @@ static int gen_field_match_handler(fb_output_t *out, fb_compound_type_t *ct, voi
             println(out, "ref = flatcc_builder_end_vector(ctx->ctx);");
         }
     }
+    if (is_array) {
+        println(out, "buf = flatcc_json_parser_array_end(ctx, buf, end, &more);");
+        unindent(); println(out, "}");
+        println(out, "if (count) {"); indent();
+        println(out, "if (!(ctx->flags & flatcc_json_parser_f_pad_array_underflow)) {"); indent();
+        println(out, "return flatcc_json_parser_set_error(ctx, buf, end, flatcc_json_parser_error_array_underflow);");
+        unindent(); println(out, "}");
+        println(out, "memset(base, 0, count * sizeof(*base));");
+        unindent(); println(out, "}");
+    }
     if (is_nested == 1) {
         is_nested = 2;
         goto repeat_nested;

diff --git a/src/compiler/codegen_c_json_printer.c b/src/compiler/codegen_c_json_printer.c
@@ -296,6 +296,13 @@ static int gen_json_printer_struct(fb_output_t *out, fb_compound_type_t *ct)
                     "    flatcc_json_printer_%s_struct_field(ctx, %d, p, %"PRIu64", \"%.*s\", %ld);\n",
                     tp, index, (uint64_t)member->offset, (int)sym->ident->len, sym->ident->text, sym->ident->len);
             break;
+        case vt_fixed_array_type:
+            tp = scalar_type_prefix(member->type.st);
+            fprintf(
+                    out->fp,
+                    "    flatcc_json_printer_%s_array_struct_field(ctx, %d, p, %"PRIu64", \"%.*s\", %ld, %d);\n",
+                    tp, index, (uint64_t)member->offset, (int)sym->ident->len, sym->ident->text, sym->ident->len, member->type.len);
+            break;
         case vt_compound_type_ref:
             fb_compound_name(member->type.ct, &snref);
             switch (member->type.ct->symbol.kind) {