Merge pull request #663 from finos/arrow-schema

Allow arrow tables to be created from a schema
finos · Jul 16, 2019 · e821170 · e821170
2 parents 12d10d5 + 356a1a9
commit e821170
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 10 deletions.
diff --git a/cpp/perspective/src/cpp/emscripten.cpp b/cpp/perspective/src/cpp/emscripten.cpp
@@ -1562,18 +1562,27 @@ namespace binding {
             PSP_COMPLAIN_AND_ABORT("Specified index '" + index + "' does not exist in data.")
         }
 
-        // Create the table
-        // TODO assert size > 0
-        t_table tbl(t_schema(colnames, dtypes));
-        tbl.init();
-        tbl.extend(size);
-
         bool is_new_gnode = gnode.isUndefined();
         std::shared_ptr<t_gnode> new_gnode;
         if (!is_new_gnode) {
             new_gnode = gnode.as<std::shared_ptr<t_gnode>>();
+            if (is_arrow && is_update && new_gnode->get_table()->size() == 0) {
+                auto schema = new_gnode->get_table()->get_schema();
+                for (auto idx = 0; idx < schema.m_types.size(); ++idx) {
+                    if (dtypes[idx] == DTYPE_INT64) {
+                        std::cout << "Promoting int64 `" << colnames[idx] << "`" << std::endl;
+                        new_gnode->promote_column(colnames[idx], DTYPE_INT64);
+                    }
+                }
+            }
         }
 
+        // Create the table
+        // TODO assert size > 0
+        t_table tbl(t_schema(colnames, dtypes));
+        tbl.init();
+        tbl.extend(size);
+
         _fill_data(tbl, colnames, accessor, dtypes, offset, is_arrow,
             (is_update || new_gnode->mapping_size() > 0));
 

diff --git a/cpp/perspective/src/cpp/gnode.cpp b/cpp/perspective/src/cpp/gnode.cpp
@@ -737,6 +737,23 @@ t_gnode::get_table() const {
     return m_state->get_table().get();
 }
 
+/**
+ * Convenience method for promoting a column.  This is a hack used to
+ * interop with javascript more efficiently, and does not handle all
+ * possible type conversions.  Non-public.
+ */
+void
+t_gnode::promote_column(const std::string& name, t_dtype new_type) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    get_table()->promote_column(name, new_type, 0, false);
+    _get_otable(0)->promote_column(name, new_type, 0, false);
+    _get_itable(0)->promote_column(name, new_type, 0, false);
+    m_tblschema.retype_column(name, new_type);
+    m_ischemas[0].retype_column(name, new_type);
+    m_oschemas[0].retype_column(name, new_type);
+}
+
 void
 t_gnode::pprint() const {
     PSP_TRACE_SENTINEL();

diff --git a/cpp/perspective/src/cpp/table.cpp b/cpp/perspective/src/cpp/table.cpp
@@ -605,6 +605,11 @@ t_table::promote_column(
     if (fill) {
         for (auto i = 0; i < iter_limit; ++i) {
             switch (new_dtype) {
+                case DTYPE_INT64: {
+                    std::int32_t* val = current_col->get_nth<std::int32_t>(i);
+                    std::int64_t fval = static_cast<std::int64_t>(*val);
+                    promoted_col->set_nth(i, fval);
+                } break;
                 case DTYPE_FLOAT64: {
                     std::int32_t* val = current_col->get_nth<std::int32_t>(i);
                     double fval = static_cast<double>(*val);

diff --git a/cpp/perspective/src/include/perspective/gnode.h b/cpp/perspective/src/include/perspective/gnode.h
@@ -135,6 +135,8 @@ class PERSPECTIVE_EXPORT t_gnode {
     // helper function for tests
     std::shared_ptr<t_table> tstep(std::shared_ptr<const t_table> input_table);
 
+    void promote_column(const std::string& name, t_dtype new_type);
+
     // Gnode will steal a reference to the context
     void register_context(const std::string& name, std::shared_ptr<t_ctx0> ctx);
     void register_context(const std::string& name, std::shared_ptr<t_ctx1> ctx);

diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js
@@ -1165,9 +1165,6 @@ export default function(Module) {
         pdata = accessor;
 
         if (data instanceof ArrayBuffer) {
-            if (this.size() === 0) {
-                throw new Error("Overriding Arrow Schema is not supported.");
-            }
             pdata = load_arrow_buffer(data, cols, types);
             if (meter) {
                 meter(pdata.map(x => x.row_count).reduce((x, y) => x + y));
@@ -1192,7 +1189,7 @@ export default function(Module) {
             }
         }
 
-        if (accessor.row_count === 0) {
+        if (pdata.row_count === 0) {
             console.warn("table.update called with no data - ignoring");
             return;
         }