port get_data_type, add boost::optional

finos · Dec 6, 2018 · 942fce2 · 942fce2
1 parent b7534cb
commit 942fce2
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 29 deletions.
diff --git a/packages/perspective/src/js/parse_data.js b/packages/perspective/src/js/parse_data.js
@@ -39,33 +39,6 @@ export class DataParser {
         }
     }
 
-    column_names(data, format) {
-        let column_names = [];
-        if (format === this.data_formats.row) {
-            let max_check = 50;
-            column_names = Object.keys(data[0]);
-            for (let ix = 0; ix < Math.min(max_check, data.length); ix++) {
-                let next = Object.keys(data[ix]);
-                if (column_names.length !== next.length) {
-                    if (next.length > column_names.length) {
-                        if (max_check === 50) console.warn("Array data has inconsistent rows");
-                        console.warn("Extending from " + column_names.length + " to " + next.length);
-                        column_names = next;
-                        max_check *= 2;
-                    }
-                }
-            }
-        } else if (format === this.data_formats.column) {
-            column_names = Object.keys(data);
-        } else if (format === this.data_formats.schema) {
-            for (let name in data) {
-                column_names.push(name);
-            }
-        }
-
-        return column_names;
-    }
-
     data_types(__MODULE__, data, format, column_names) {
         let types = [];
 
@@ -106,7 +79,7 @@ export class DataParser {
         }
 
         for (let name of column_names) {
-            let type = this.get_data_type(__MODULE__, data, format, name);
+            let type = __MODULE__.get_data_type(data, format, name, moment, DATE_PARSE_CANDIDATES); //this.get_data_type(__MODULE__, data, format, name);
             types.push(type);
         }
 
@@ -239,7 +212,7 @@ export class DataParser {
      */
     parse(__MODULE__, data) {
         const format = this.is_format(data);
-        let names = __MODULE__.column_names(data, format); //this.column_names(data, format);
+        let names = __MODULE__.column_names(data, format);
         let types = this.data_types(__MODULE__, data, format, names);
         let [cdata, row_count] = this.make_columnar_data(__MODULE__, data, format, names, types);
         return {cdata, names, types, row_count, is_arrow: false};

diff --git a/src/cpp/main.cpp b/src/cpp/main.cpp
@@ -23,6 +23,7 @@
 #include <perspective/sym_table.h>
 #include <codecvt>
 #include <boost/format.hpp>
+#include <boost/optional.hpp>
 
 using namespace perspective;
 using namespace emscripten;
@@ -946,6 +947,43 @@ column_names(val data, t_int32 format) {
     return column_names;
 }
 
+t_dtype
+get_data_type(val data, t_int32 format, t_str name, val moment, val candidates) {
+    t_int32 i = 0;
+    boost::optional<t_dtype> inferredType;
+
+    if (format == 1) {
+        // loop parameters differ slightly so rewrite the loop
+        while (!inferredType.is_initialized() && i < 100 && i < data["length"].as<t_int32>()) {
+            if (data[i].call<val>("hasOwnProperty", name).as<t_bool>() == true) {
+                if (!data[i][name].isNull()) {
+                    inferredType = infer_type(data[i][name], moment, candidates);
+                } else {
+                    inferredType = t_dtype::DTYPE_STR;
+                }
+            }
+
+            i++;
+        }
+    } else if (format == 2) {
+        while (!inferredType.is_initialized() && i < 100 && i < data[name]["length"].as<t_int32>()) {
+            if (!data[name][i].isNull()) {
+                inferredType = infer_type(data[name][i], moment, candidates);
+            } else {
+                inferredType = t_dtype::DTYPE_STR;
+            }
+
+            i++;
+        }
+    }
+
+    if (!inferredType.is_initialized()) {
+        return t_dtype::DTYPE_STR;
+    } else {
+        return inferredType.get();
+    }
+}
+
 /**
  * Create a default gnode.
  *
@@ -1538,6 +1576,7 @@ EMSCRIPTEN_BINDINGS(perspective) {
     function("sort", &sort);
     function("infer_type", &infer_type);
     function("column_names", &column_names);
+    function("get_data_type", &get_data_type);
     function("make_table", &make_table, allow_raw_pointers());
     function("make_gnode", &make_gnode);
     function("clone_gnode_table", &clone_gnode_table, allow_raw_pointers());