Refactored remote library into regular perspective.js

package.json

@@ -11,6 +11,7 @@
 		"test:build": "[[ -z \"${PSP_DOCKER}\" ]] && npm run _build_test || npm run _emsdk -- npm run _build_test",
 		"test:run": "npm run _test_perspective && npm run _test_viewer && npm run _test_hypergrid && npm run _test_highcharts",
 		"test": "npm-run-all test:build test:run",
+		"clean": "lerna run clean --stream",
 		"quiet_test": "npm run _puppeteer -- npm run _quiet_test",
 		"write_tests": "WRITE_TESTS=1 npm run test:run",
 		"postinstall": "lerna bootstrap --hoist",

packages/perspective-examples/package.json

@@ -18,6 +18,7 @@
     "bench:build": "echo \"No Benchmarks\"",
     "bench:run": "echo \"No Benchmarks\"",
     "build": "npm run copy && npm-run-all -p build:*",
+    "build:perspective": "webpack --color --config ../perspective/test/config/perspective.config.js --context ../perspective/ --output-path ../perspective-examples/build",
     "build:view": "webpack --color --config ../perspective-viewer/test/config/view.config.js --context ../perspective-viewer/ --output-path ../perspective-examples/build",
     "build:hypergrid": "webpack --color --config ../perspective-viewer-hypergrid/test/config/hypergrid.config.js --context ../perspective-viewer-hypergrid/ --output-path ../perspective-examples/build",
     "build:highcharts": "webpack --color --config ../perspective-viewer-highcharts/test/config/highcharts.config.js --context ../perspective-viewer-highcharts/ --output-path ../perspective-examples/build",

packages/perspective-examples/src/html/remote.html

@@ -17,7 +17,8 @@
         <script src="perspective.view.js"></script>
         <script src="hypergrid.plugin.js"></script>
         <script src="highcharts.plugin.js"></script>
-        <script src="perspective.remote.js"></script>
+
+        <script src="perspective.js"></script>
 
         <link rel='stylesheet' href="demo.css">
 
@@ -30,8 +31,8 @@
 
             window.addEventListener('WebComponentsReady', function () {
                 var elem = document.getElementById('view1');
-                var worker = perspective_remote.default.worker('ws://localhost:3000');
-                elem.load(worker.open('superstore.csv'));
+                var worker = perspective.worker('ws://localhost:3000');
+                elem.load(worker.open('superstore.arrow'));
             });
 
         </script>

packages/perspective-examples/src/js/node_remote.js

@@ -10,7 +10,7 @@
 const {WebSocketHost} = require("@jpmorganchase/perspective/build/perspective.node.js");
 const fs = require("fs");
 
-let host = new WebSocketHost(3000);
-let csv = fs.readFileSync('packages/perspective-examples/build/superstore.csv') + "";
+const host = new WebSocketHost(3000);
+const arr = fs.readFileSync(__dirname +'/../../build/superstore.arrow');
 
-host.open("superstore.csv", csv);
+host.open("superstore.arrow", arr);

packages/perspective-viewer/src/js/view.js

@@ -196,7 +196,7 @@ function drop(ev) {
     // Deselect the dropped column
     if (this._plugin.deselectMode === "pivots" && this._visible_column_count() > 1 && name !== "sort") {
         for (let x of this.querySelectorAll("#active_columns perspective-row")) {
-            if (x.getAttribute('name') === data) {
+            if (x.getAttribute('name') === data[0]) {
                  this._active_columns.removeChild(x);
                  break;
             }

packages/perspective/package.json

@@ -22,11 +22,10 @@
     "build": "npm-run-all build:compile:* build:webpack:* ",
     "build:compile:copy": "mkdir -p obj build build/wasm_async build/wasm_sync build/asmjs",
     "build:compile:emmake": "cd obj/ && emcmake cmake ../ && emmake make -j8",
-    "build:webpack:asmj": "webpack --color --config src/config/perspective.asmjs.config.js",
+    "build:webpack:asmjs": "webpack --color --config src/config/perspective.asmjs.config.js",
     "build:webpack:wasm": "webpack --color --config src/config/perspective.wasm.config.js",
     "build:webpack:parallel": "webpack --color --config src/config/perspective.parallel.config.js",
     "build:webpack:node": "webpack --color --config src/config/perspective.node.config.js",
-    "build:webpack:remote": "webpack --color --config src/config/perspective.remote.config.js",
     "test:build": "npm-run-all test:build:copy test:build:webpack",
     "test:build:copy": "npm-run-all -p test:build:copy:*",
     "test:build:copy:html": "cp test/html/* build",

packages/perspective/src/js/perspective.js

@@ -27,6 +27,7 @@ if (typeof self !== "undefined" && self.performance === undefined) {
 
 let __MODULE__;
 
+const CHUNKED_THRESHOLD = 100000
 
 /******************************************************************************
  *
@@ -1361,7 +1362,21 @@ class Host {
         throw new Error("post() not implemented!");
     }
 
-    process(msg) {
+    clear_views(client_id) {
+        for (let key of Object.keys(this._views)) {
+            if (this._views[key].client_id === client_id) {
+                try {
+                    this._views[key].delete();
+                } catch (e) {
+                    console.error(e);
+                }
+                delete this._views[key];
+            }
+        }
+        console.debug(`GC ${Object.keys(this._views).length} views in memory`);
+    }
+
+    process(msg, client_id) {
         switch (msg.cmd) {
             case 'init':
                 this.init(msg);
@@ -1397,6 +1412,7 @@ class Host {
                 break;
             case 'view':
                 this._views[msg.view_name] = this._tables[msg.table_name].view(msg.config);
+                this._views[msg.view_name].client_id = client_id;
                 break;
             case 'table_method': {
                 let obj = this._tables[msg.name];
@@ -1470,6 +1486,9 @@ class Host {
                     }
                 } else {
                     obj[msg.method].apply(obj, msg.args).then(result => {
+                        if (msg.method === "delete") {
+                            delete this._views[msg.name];
+                        }
                         this.post({
                             id: msg.id,
                             data: result
@@ -1588,40 +1607,60 @@ const perspective = {
     table: function(data, options) {
         options = options || {};
         options.index = options.index || "";
-        let pdata;
+        let pdata, chunked = false;
 
-        if (data instanceof ArrayBuffer) {
+        if (data instanceof ArrayBuffer  || (Buffer && data instanceof Buffer)) {
             // Arrow data
             pdata = load_arrow_buffer(data);
         } else {
             if (typeof data === "string") {
                 if (data[0] === ",") {
                     data = "_" + data;
                 }
-                data = papaparse.parse(data, {dynamicTyping: true, header: true}).data;
+                data = papaparse.parse(data.trim(), {dynamicTyping: true, header: true}).data;
             }
             pdata = parse_data(data);
+            chunked = pdata.row_count > CHUNKED_THRESHOLD;
         }
 
         if (options.index && pdata.names.indexOf(options.index) === -1) {
             throw `Specified index '${options.index}' does not exist in data.`;
         }
 
-        let tbl, gnode, pool;
+        let tbl, gnode, pool, pages;
 
         try {
             // Create perspective pool
             pool = new __MODULE__.t_pool({_update_callback: function() {} } );
 
+            if (chunked) {
+                pages = pdata.cdata.map(x => x.splice(0, CHUNKED_THRESHOLD));
+            } else {
+                pages = pdata.cdata;
+            }
+
             // Fill t_table with data
-            tbl = __MODULE__.make_table(pdata.row_count || 0,
-                pdata.names, pdata.types, pdata.cdata,
+            tbl = __MODULE__.make_table(pages[0].length || 0,
+                pdata.names, pdata.types, pages,
                 0, options.index, pdata.is_arrow, false);
 
             gnode = __MODULE__.make_gnode(tbl);
             pool.register_gnode(gnode);
             __MODULE__.fill(pool, gnode, tbl);
 
+            if (chunked) {
+                while (pdata.cdata[0].length > 0) {
+                    tbl.delete();
+                    pages = pdata.cdata.map(x => x.splice(0, CHUNKED_THRESHOLD));
+
+                    tbl = __MODULE__.make_table(pages[0].length || 0,
+                        pdata.names, pdata.types, pages,
+                        gnode.get_table().size(), options.index, pdata.is_arrow, false);
+
+                    __MODULE__.fill(pool, gnode, tbl);    
+                }
+            }
+
             return new table(gnode, pool, options.index);
         } catch (e) {
             if (pool) {

packages/perspective/src/js/perspective.node.js

@@ -15,26 +15,18 @@ const WebSocket = require('ws');
 
 let Module;
 
-if (typeof WebAssembly === "undefined") {
-    const load_perspective = require("../../build/asmjs/psp.js").load_perspective;
-    Module = load_perspective({
-        wasmJSMethod: "asmjs",
-        memoryInitializerPrefixURL: 'build/asmjs/',
-        asmjsCodeFile: "asmjs/psp.js",
-        ENVIRONMENT: "NODE"
-    });
-} else {
-    const load_perspective = require("../../build/wasm_sync/psp.js").load_perspective;
-    const wasm = fs.readFileSync('./build/wasm_sync/psp.wasm');
-    Module = load_perspective({
-        wasmBinary: wasm,
-        wasmJSMethod: 'native-wasm',
-        ENVIRONMENT: "NODE"
-    });    
-}
+const load_perspective = require("../../build/wasm_sync/psp.js").load_perspective;
+const wasm = fs.readFileSync('./build/wasm_sync/psp.wasm');
+Module = load_perspective({
+    wasmBinary: wasm,
+    wasmJSMethod: 'native-wasm',
+    ENVIRONMENT: "NODE"
+});    
 
 module.exports = perspective(Module);
 
+let CLIENT_ID_GEN = 0;
+
 /**
  * A Server instance for a remote perspective.
  */
@@ -43,17 +35,21 @@ class WebSocketHost extends module.exports.Host {
     constructor(port = 8080) {
         super();
         this.REQS = {};        
-        this._wss = new WebSocket.Server({port: port});
+        this._wss = new WebSocket.Server({port: port, perMessageDeflate: true});
         this._wss.on('connection', ws => {
+            ws.id = CLIENT_ID_GEN++;
             ws.on('message', msg => {
                 msg = JSON.parse(msg);
                 this.REQS[msg.id] = ws;
                 try {
-                    this.process(msg);
+                    this.process(msg, ws.id);
                 } catch (e) {
                     console.error(e);
                 }
             });
+            ws.on('close', () => {
+                this.clear_views(ws.id);
+            });
             ws.on('error', console.error);
         });
         console.log(`Listening on port ${port}`);