From 09c7c890e1a4fb0a2c682180b8c795a8b88ec170 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 12 Aug 2018 15:07:51 -0400 Subject: [PATCH] Refactored remote library into regular `perspective.js` --- package.json | 1 + packages/perspective-examples/package.json | 1 + .../perspective-examples/src/html/remote.html | 7 +- .../src/js/node_remote.js | 6 +- packages/perspective/package.json | 3 +- .../src/config/perspective.remote.config.js | 12 -- packages/perspective/src/js/perspective.js | 22 +- .../perspective/src/js/perspective.node.js | 34 ++-- .../src/js/perspective.parallel.js | 190 ++++++++++-------- .../perspective/src/js/perspective.remote.js | 56 ------ .../test/config/perspective.config.js | 13 ++ 11 files changed, 166 insertions(+), 179 deletions(-) delete mode 100644 packages/perspective/src/config/perspective.remote.config.js delete mode 100644 packages/perspective/src/js/perspective.remote.js create mode 100644 packages/perspective/test/config/perspective.config.js diff --git a/package.json b/package.json index a04a8b8f2a..b8e4908272 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "test:build": "[[ -z \"${PSP_DOCKER}\" ]] && npm run _build_test || npm run _emsdk -- npm run _build_test", "test:run": "npm run _test_perspective && npm run _test_viewer && npm run _test_hypergrid && npm run _test_highcharts", "test": "npm-run-all test:build test:run", + "clean": "lerna run clean --stream", "quiet_test": "npm run _puppeteer -- npm run _quiet_test", "write_tests": "WRITE_TESTS=1 npm run test:run", "postinstall": "lerna bootstrap --hoist", diff --git a/packages/perspective-examples/package.json b/packages/perspective-examples/package.json index de1be2a2db..4cc60a4302 100644 --- a/packages/perspective-examples/package.json +++ b/packages/perspective-examples/package.json @@ -18,6 +18,7 @@ "bench:build": "echo \"No Benchmarks\"", "bench:run": "echo \"No Benchmarks\"", "build": "npm run copy && npm-run-all -p build:*", + "build:perspective": "webpack --color --config ../perspective/test/config/perspective.config.js --context ../perspective/ --output-path ../perspective-examples/build", "build:view": "webpack --color --config ../perspective-viewer/test/config/view.config.js --context ../perspective-viewer/ --output-path ../perspective-examples/build", "build:hypergrid": "webpack --color --config ../perspective-viewer-hypergrid/test/config/hypergrid.config.js --context ../perspective-viewer-hypergrid/ --output-path ../perspective-examples/build", "build:highcharts": "webpack --color --config ../perspective-viewer-highcharts/test/config/highcharts.config.js --context ../perspective-viewer-highcharts/ --output-path ../perspective-examples/build", diff --git a/packages/perspective-examples/src/html/remote.html b/packages/perspective-examples/src/html/remote.html index dd5f5613ed..01c6a3dabc 100644 --- a/packages/perspective-examples/src/html/remote.html +++ b/packages/perspective-examples/src/html/remote.html @@ -17,7 +17,8 @@ - + + @@ -30,8 +31,8 @@ window.addEventListener('WebComponentsReady', function () { var elem = document.getElementById('view1'); - var worker = perspective_remote.default.worker('ws://localhost:3000'); - elem.load(worker.open('superstore.csv')); + var worker = perspective.worker('ws://localhost:3000'); + elem.load(worker.open('superstore.arrow')); }); diff --git a/packages/perspective-examples/src/js/node_remote.js b/packages/perspective-examples/src/js/node_remote.js index 8aeeb7081e..28df30e759 100644 --- a/packages/perspective-examples/src/js/node_remote.js +++ b/packages/perspective-examples/src/js/node_remote.js @@ -10,7 +10,7 @@ const {WebSocketHost} = require("@jpmorganchase/perspective/build/perspective.node.js"); const fs = require("fs"); -let host = new WebSocketHost(3000); -let csv = fs.readFileSync('packages/perspective-examples/build/superstore.csv') + ""; +const host = new WebSocketHost(3000); +const arr = fs.readFileSync(__dirname +'/../../build/superstore.arrow'); -host.open("superstore.csv", csv); \ No newline at end of file +host.open("superstore.arrow", arr); \ No newline at end of file diff --git a/packages/perspective/package.json b/packages/perspective/package.json index c81bd280fc..fdea365d28 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -22,11 +22,10 @@ "build": "npm-run-all build:compile:* build:webpack:* ", "build:compile:copy": "mkdir -p obj build build/wasm_async build/wasm_sync build/asmjs", "build:compile:emmake": "cd obj/ && emcmake cmake ../ && emmake make -j8", - "build:webpack:asmj": "webpack --color --config src/config/perspective.asmjs.config.js", + "build:webpack:asmjs": "webpack --color --config src/config/perspective.asmjs.config.js", "build:webpack:wasm": "webpack --color --config src/config/perspective.wasm.config.js", "build:webpack:parallel": "webpack --color --config src/config/perspective.parallel.config.js", "build:webpack:node": "webpack --color --config src/config/perspective.node.config.js", - "build:webpack:remote": "webpack --color --config src/config/perspective.remote.config.js", "test:build": "npm-run-all test:build:copy test:build:webpack", "test:build:copy": "npm-run-all -p test:build:copy:*", "test:build:copy:html": "cp test/html/* build", diff --git a/packages/perspective/src/config/perspective.remote.config.js b/packages/perspective/src/config/perspective.remote.config.js deleted file mode 100644 index f3101303d3..0000000000 --- a/packages/perspective/src/config/perspective.remote.config.js +++ /dev/null @@ -1,12 +0,0 @@ -const path = require('path'); -const common = require('./common.config.js'); - -module.exports = Object.assign({}, common(), { - entry: './src/js/perspective.remote.js', - output: { - filename: 'perspective.remote.js', - library: "perspective_remote", - libraryTarget: "umd", - path: path.resolve(__dirname, '../../build') - } -}); \ No newline at end of file diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 678e385aec..0efa0fc9cb 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -1361,7 +1361,21 @@ class Host { throw new Error("post() not implemented!"); } - process(msg) { + clear_views(client_id) { + for (let key of Object.keys(this._views)) { + if (this._views[key].client_id === client_id) { + try { + this._views[key].delete(); + } catch (e) { + console.error(e); + } + delete this._views[key]; + } + } + console.debug(`GC ${Object.keys(this._views).length} views in memory`); + } + + process(msg, client_id) { switch (msg.cmd) { case 'init': this.init(msg); @@ -1397,6 +1411,7 @@ class Host { break; case 'view': this._views[msg.view_name] = this._tables[msg.table_name].view(msg.config); + this._views[msg.view_name].client_id = client_id; break; case 'table_method': { let obj = this._tables[msg.name]; @@ -1470,6 +1485,9 @@ class Host { } } else { obj[msg.method].apply(obj, msg.args).then(result => { + if (msg.method === "delete") { + delete this._views[msg.name]; + } this.post({ id: msg.id, data: result @@ -1590,7 +1608,7 @@ const perspective = { options.index = options.index || ""; let pdata; - if (data instanceof ArrayBuffer) { + if (data instanceof ArrayBuffer || (Buffer && data instanceof Buffer)) { // Arrow data pdata = load_arrow_buffer(data); } else { diff --git a/packages/perspective/src/js/perspective.node.js b/packages/perspective/src/js/perspective.node.js index 45a41d7527..37ef544dba 100644 --- a/packages/perspective/src/js/perspective.node.js +++ b/packages/perspective/src/js/perspective.node.js @@ -15,26 +15,18 @@ const WebSocket = require('ws'); let Module; -if (typeof WebAssembly === "undefined") { - const load_perspective = require("../../build/asmjs/psp.js").load_perspective; - Module = load_perspective({ - wasmJSMethod: "asmjs", - memoryInitializerPrefixURL: 'build/asmjs/', - asmjsCodeFile: "asmjs/psp.js", - ENVIRONMENT: "NODE" - }); -} else { - const load_perspective = require("../../build/wasm_sync/psp.js").load_perspective; - const wasm = fs.readFileSync('./build/wasm_sync/psp.wasm'); - Module = load_perspective({ - wasmBinary: wasm, - wasmJSMethod: 'native-wasm', - ENVIRONMENT: "NODE" - }); -} +const load_perspective = require("../../build/wasm_sync/psp.js").load_perspective; +const wasm = fs.readFileSync('./build/wasm_sync/psp.wasm'); +Module = load_perspective({ + wasmBinary: wasm, + wasmJSMethod: 'native-wasm', + ENVIRONMENT: "NODE" +}); module.exports = perspective(Module); +let CLIENT_ID_GEN = 0; + /** * A Server instance for a remote perspective. */ @@ -43,17 +35,21 @@ class WebSocketHost extends module.exports.Host { constructor(port = 8080) { super(); this.REQS = {}; - this._wss = new WebSocket.Server({port: port}); + this._wss = new WebSocket.Server({port: port, perMessageDeflate: true}); this._wss.on('connection', ws => { + ws.id = CLIENT_ID_GEN++; ws.on('message', msg => { msg = JSON.parse(msg); this.REQS[msg.id] = ws; try { - this.process(msg); + this.process(msg, ws.id); } catch (e) { console.error(e); } }); + ws.on('close', () => { + this.clear_views(ws.id); + }); ws.on('error', console.error); }); console.log(`Listening on port ${port}`); diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index 445289b026..e86d54bf9a 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -33,6 +33,11 @@ if (detectIE() && window.location.href.indexOf(__SCRIPT_PATH__.host()) === -1) { }(document)); } +// https://github.com/kripken/emscripten/issues/6042 +function detect_iphone () { + return /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream; +} + function XHRWorker(url, ready, scope) { var oReq = new XMLHttpRequest(); oReq.addEventListener('load', function() { @@ -60,112 +65,133 @@ class WebWorker extends worker { } } -} -WebWorker.prototype.send = function (msg) { - if (this._worker.transferable && msg.args && msg.args[0] instanceof ArrayBuffer) { - this._worker.postMessage(msg, msg.args); - } else { - this._worker.postMessage(msg); + send(msg) { + if (this._worker.transferable && msg.args && msg.args[0] instanceof ArrayBuffer) { + this._worker.postMessage(msg, msg.args); + } else { + this._worker.postMessage(msg); + } } -} -WebWorker.prototype._detect_transferable = function () { - var ab = new ArrayBuffer(1); - this._worker.postMessage(ab, [ab]); - this._worker.transferable = (ab.byteLength === 0); - if (!this._worker.transferable) { - console.warn('Transferable support not detected'); - } else { - console.log('Transferable support detected'); - } -} + terminate() { + this._worker.terminate(); + this._worker = undefined; + }; -WebWorker.prototype._start_embedded = function () { - console.log("Running PSP in embedded mode"); - var w = new window.__PSP_WORKER__(); - for (var key in this._worker) { - w[key] = this._worker[key]; + _detect_transferable() { + var ab = new ArrayBuffer(1); + this._worker.postMessage(ab, [ab]); + this._worker.transferable = (ab.byteLength === 0); + if (!this._worker.transferable) { + console.warn('Transferable support not detected'); + } else { + console.log('Transferable support detected'); + } } - this._worker = w; - this._worker.addEventListener('message', this._handle.bind(this)); - this._worker.postMessage({cmd: 'init', data: window.__PSP_WASM__, path: __SCRIPT_PATH__.path()}); - this._detect_transferable(); -}; -WebWorker.prototype._start_cross_origin = function () { - var dir = (typeof WebAssembly === "undefined" ? 'asmjs' : 'wasm_async'); - XHRWorker(__SCRIPT_PATH__.path() + dir + '/perspective.js', function(worker) { + _start_embedded() { + console.log("Running PSP in embedded mode"); + var w = new window.__PSP_WORKER__(); for (var key in this._worker) { - worker[key] = this._worker[key]; + w[key] = this._worker[key]; } - this._worker.postMessage = worker.postMessage.bind(worker); - this._worker.terminate = worker.terminate.bind(worker); - this._worker = worker; - this._detect_transferable(); + this._worker = w; this._worker.addEventListener('message', this._handle.bind(this)); - if (typeof WebAssembly === 'undefined') { - this._start_cross_origin_asmjs(); - } else { - this._start_cross_origin_wasm(); - } - }, this); -}; - -WebWorker.prototype._start_cross_origin_asmjs = function () { - this._worker.postMessage({ - cmd: 'init', - path: __SCRIPT_PATH__.path() - }); -} + this._worker.postMessage({cmd: 'init', data: window.__PSP_WASM__, path: __SCRIPT_PATH__.path()}); + this._detect_transferable(); + } + + _start_cross_origin() { + var dir = (typeof WebAssembly === "undefined" ? 'asmjs' : 'wasm_async'); + XHRWorker(__SCRIPT_PATH__.path() + dir + '/perspective.js', function(worker) { + for (var key in this._worker) { + worker[key] = this._worker[key]; + } + this._worker.postMessage = worker.postMessage.bind(worker); + this._worker.terminate = worker.terminate.bind(worker); + this._worker = worker; + this._detect_transferable(); + this._worker.addEventListener('message', this._handle.bind(this)); + if (typeof WebAssembly === 'undefined') { + this._start_cross_origin_asmjs(); + } else { + this._start_cross_origin_wasm(); + } + }, this); + } -WebWorker.prototype._start_cross_origin_wasm = function () { - var wasmXHR = new XMLHttpRequest(); - wasmXHR.open('GET', __SCRIPT_PATH__.path() + 'wasm_async/psp.wasm', true); - wasmXHR.responseType = 'arraybuffer'; - wasmXHR.onload = () => { - let msg = { + _start_cross_origin_asmjs() { + this._worker.postMessage({ cmd: 'init', - data: wasmXHR.response, path: __SCRIPT_PATH__.path() + }); + } + + _start_cross_origin_wasm() { + var wasmXHR = new XMLHttpRequest(); + wasmXHR.open('GET', __SCRIPT_PATH__.path() + 'wasm_async/psp.wasm', true); + wasmXHR.responseType = 'arraybuffer'; + wasmXHR.onload = () => { + let msg = { + cmd: 'init', + data: wasmXHR.response, + path: __SCRIPT_PATH__.path() + }; + if (this._worker.transferable) { + this._worker.postMessage(msg, [wasmXHR.response]); + } else { + this._worker.postMessage(msg); + } }; - if (this._worker.transferable) { - this._worker.postMessage(msg, [wasmXHR.response]); - } else { - this._worker.postMessage(msg); + wasmXHR.send(null); + } + + _start_same_origin() { + var dir = (typeof WebAssembly === "undefined" || detect_iphone() ? 'asmjs' : 'wasm_async'); + var w = new Worker(__SCRIPT_PATH__.path() + dir + '/perspective.js'); + for (var key in this._worker) { + w[key] = this._worker[key]; } - }; - wasmXHR.send(null); + this._worker = w; + this._worker.addEventListener('message', this._handle.bind(this)); + this._worker.postMessage({cmd: 'init', path: __SCRIPT_PATH__.path()}); + this._detect_transferable(); + } } -// https://github.com/kripken/emscripten/issues/6042 -function detect_iphone () { - return /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream; -} +class WebSocketWorker extends worker { -WebWorker.prototype._start_same_origin = function () { - var dir = (typeof WebAssembly === "undefined" || detect_iphone() ? 'asmjs' : 'wasm_async'); - var w = new Worker(__SCRIPT_PATH__.path() + dir + '/perspective.js'); - for (var key in this._worker) { - w[key] = this._worker[key]; + constructor(url) { + super(); + this._ws = new WebSocket(url); + this._ws.onopen = () => { + this.send({id: -1, cmd: 'init'}); + }; + this._ws.onmessage = (msg) => { + this._handle({data: JSON.parse(msg.data)}); + } } - this._worker = w; - this._worker.addEventListener('message', this._handle.bind(this)); - this._worker.postMessage({cmd: 'init', path: __SCRIPT_PATH__.path()}); - this._detect_transferable(); -}; -WebWorker.prototype.terminate = function () { - this._worker.terminate(); - this._worker = undefined; -}; + send(msg) { + this._ws.send(JSON.stringify(msg)); + } + + terminate() { + this._ws.close(); + } +} export default { - worker: function () { + worker: function (url) { if (window.location.href.indexOf(__SCRIPT_PATH__.host()) === -1 && detectIE()) { return perspective; } - return new WebWorker(); + if (url) { + return new WebSocketWorker(url); + } else { + return new WebWorker(); + } }, TYPE_AGGREGATES: TYPE_AGGREGATES, diff --git a/packages/perspective/src/js/perspective.remote.js b/packages/perspective/src/js/perspective.remote.js deleted file mode 100644 index 8588f6726f..0000000000 --- a/packages/perspective/src/js/perspective.remote.js +++ /dev/null @@ -1,56 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2017, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS, SORT_ORDERS} from "./defaults.js"; - -import * as api from "./api.js"; - -/****************************************************************************** - * - * Utilities - * - */ - -class WebSocketWorker extends api.worker { - - constructor(url) { - super(); - this._ws = new WebSocket(url); - this._ws.onopen = () => { - this.send({id: -1, cmd: 'init'}); - }; - this._ws.onmessage = (msg) => { - this._handle({data: JSON.parse(msg.data)}); - } - } - - send(msg) { - this._ws.send(JSON.stringify(msg)); - } - - terminate() { - this._ws.close(); - } -} - -export default { - worker: function (url) { - return new WebSocketWorker(url); - }, - - TYPE_AGGREGATES: TYPE_AGGREGATES, - - TYPE_FILTERS: TYPE_FILTERS, - - AGGREGATE_DEFAULTS: AGGREGATE_DEFAULTS, - - FILTER_DEFAULTS: FILTER_DEFAULTS, - - SORT_ORDERS: SORT_ORDERS -}; diff --git a/packages/perspective/test/config/perspective.config.js b/packages/perspective/test/config/perspective.config.js new file mode 100644 index 0000000000..431ea8ec0c --- /dev/null +++ b/packages/perspective/test/config/perspective.config.js @@ -0,0 +1,13 @@ +const path = require('path'); +const common = require('../../src/config/common.config.js'); + +module.exports = Object.assign({}, common(), { + entry: './src/js/perspective.parallel.js', + output: { + filename: 'perspective.js', + library: "perspective", + libraryTarget: "umd", + libraryExport: 'default', + path: path.resolve(__dirname, '../../build') + } +}); \ No newline at end of file