Skip to content

Commit

Permalink
Merge pull request #239 from jpmorganchase/limit
Browse files Browse the repository at this point in the history 
Limit tables
  • Loading branch information
@texodus
texodus authored Sep 15, 2018
2 parents 64aa85b + f37545f commit 5364016
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 44 deletions.
15 changes: 6 additions & 9 deletions packages/perspective-examples/src/html/streaming.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, minimum-scale=1, user-scalable=no">

<script src="mobile_fix.js"></script>
<script src="perspective.js"></script>
<script src="perspective.view.js"></script>
<script src="hypergrid.plugin.js"></script>
<script src="highcharts.plugin.js"></script>
Expand All @@ -23,7 +24,7 @@
</head>
<body>

<perspective-viewer index='id' row-pivots='["name"]' column-pivots='["client"]' columns='["chg","vol"]'></perspective-viewer>
<perspective-viewer row-pivots='["name"]' column-pivots='["client"]' columns='["chg","vol"]'></perspective-viewer>

<script>

Expand Down Expand Up @@ -55,8 +56,6 @@
"Krusty",
]

var id = 0;

function newRows() {
rows = [];
for (var x = 0; x < 5; x ++) {
Expand All @@ -68,22 +67,20 @@
bid: Math.random() * 10 + 90,
ask: Math.random() * 10 + 100,
vol: Math.random() * 10 + 100,
id: id
});
id = (id + 1) % 500;
}
return rows;
}

window.addEventListener('WebComponentsReady', function () {
var elem = document.getElementsByTagName('perspective-viewer')[0];
var table = perspective.worker().table(newRows(), {limit: 500});
elem.load(table);

function postRow() {
(function postRow() {
elem.update(newRows());
setTimeout(postRow, 50);
}

postRow();
})();
});

</script>
Expand Down
20 changes: 5 additions & 15 deletions packages/perspective-viewer/src/js/view.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -800,7 +800,9 @@ class ViewPrivate extends HTMLElement {
if (this._table) {
let table = this._table;
this._table = undefined;
all.push(table.delete());
if (table._owner_viewer && table._owner_viewer === this) {
all.push(table.delete());
}
}
return Promise.all(all);
}
Expand Down Expand Up @@ -1182,19 +1184,6 @@ class View extends ViewPrivate {
this._debounce_update();
}

/**
* The column name to index by. Due to the mutable nature of
* `perspective.table`, `index` cannot be modified once `load` or `update`
* has been called.
*
* @type {string}
*/
set index(index) {
if (this._table) {
console.error(`Setting 'index' attribute after initialization has no effect`);
}
}

/**
* Sets this `perspective.table.view`'s `row_pivots` property.
*
Expand Down Expand Up @@ -1254,7 +1243,7 @@ class View extends ViewPrivate {
*/
get worker() {
if (this._table) {
return this._table.worker;
return this._table._worker;
}
return get_worker();
}
Expand Down Expand Up @@ -1305,6 +1294,7 @@ class View extends ViewPrivate {
table = data;
} else {
table = get_worker().table(data, options);
table._owner_viewer = this;
}
let _promises = [loadTable.call(this, table)];
for (let slave of this._slaves) {
Expand Down
5 changes: 3 additions & 2 deletions packages/perspective/src/cpp/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,6 +538,7 @@ make_table(
val j_dtypes,
val j_data,
t_uint32 offset,
t_uint32 limit,
t_str index,
t_bool is_arrow,
t_bool is_delete
Expand Down Expand Up @@ -572,8 +573,8 @@ make_table(

for (auto ridx = 0; ridx < tbl->size(); ++ridx)
{
key_col->set_nth<t_int32>(ridx, ridx + offset);
okey_col->set_nth<t_int32>(ridx, ridx + offset);
key_col->set_nth<t_int32>(ridx, (ridx + offset) % limit);
okey_col->set_nth<t_int32>(ridx, (ridx + offset) % limit);
}
} else {
tbl->clone_column(index, "psp_pkey");
Expand Down
98 changes: 80 additions & 18 deletions packages/perspective/src/js/perspective.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -896,7 +896,7 @@ view.prototype.on_delete = function (callback) {
* @class
* @hideconstructor
*/
function table(gnode, pool, index, computed) {
function table(gnode, pool, index, computed, limit, limit_index) {
this.gnode = gnode;
this.pool = pool;
this.name = Math.random() + "";
Expand All @@ -906,6 +906,8 @@ function table(gnode, pool, index, computed) {
this.computed = computed || [];
this.callbacks = [];
this.views = [];
this.limit = limit;
this.limit_index = limit_index;
}

table.prototype._update_callback = function () {
Expand Down Expand Up @@ -1325,9 +1327,21 @@ table.prototype.update = function (data) {

let tbl;
try {
tbl = __MODULE__.make_table(pdata.row_count || 0,
pdata.names, pdata.types, pdata.cdata,
this.gnode.get_table().size(), this.index || "", pdata.is_arrow, false);
tbl = __MODULE__.make_table(
pdata.row_count || 0,
pdata.names,
pdata.types,
pdata.cdata,
this.limit_index,
this.limit || 4294967295,
this.index || "",
pdata.is_arrow, false
);

this.limit_index += pdata.row_count;
if (this.limit) {
this.limit_index = this.limit_index % this.limit;
}

// Add any computed columns
this._calculate_computed(tbl, this.computed);
Expand Down Expand Up @@ -1370,9 +1384,22 @@ table.prototype.remove = function (data) {

let tbl;
try {
tbl = __MODULE__.make_table(pdata.row_count || 0,
pdata.names, pdata.types, pdata.cdata,
this.gnode.get_table().size(), this.index || "", pdata.is_arrow, true);
tbl = __MODULE__.make_table(
pdata.row_count || 0,
pdata.names,
pdata.types,
pdata.cdata,
this.limit_index,
this.limit || 4294967295,
this.index || "",
pdata.is_arrow,
true
);

this.limit_index += pdata.cdata.length;
if (this.limit) {
this.limit_index = this.limit_index % this.limit;
}

__MODULE__.fill(this.pool, this.gnode, tbl);
this.initialized = true;
Expand Down Expand Up @@ -1775,8 +1802,13 @@ const perspective = {
* a string is supplied, the parameter as parsed as a CSV.
* @param {Object} [options] An optional options dictionary.
* @param {string} options.index The name of the column in the resulting
* table to treat as an index. When updating this table, rows sharing anb
* index of a new row will be overwritten.
* table to treat as an index. When updating this table, rows sharing an
* index of a new row will be overwritten. `index` is mutually exclusive
* to `limit`
* @param {integer} options.limit The maximum number of rows that can be
* added to this table. When exceeded, old rows will be overwritten in
* the order they were inserted. `limit` is mutually exclusive to
* `index`.
*
* @returns {table} A new {@link table} object.
*/
Expand All @@ -1799,15 +1831,20 @@ const perspective = {
chunked = pdata.row_count > CHUNKED_THRESHOLD;
}


if (options.index && options.limit) {
throw `Cannot specify both index '${options.index}' and limit '${options.limit}'.`;
}

if (options.index && pdata.names.indexOf(options.index) === -1) {
throw `Specified index '${options.index}' does not exist in data.`;
}

let tbl, gnode, pool, pages;
let tbl, gnode, pool, pages, limit_index = 0;

try {
// Create perspective pool
pool = new __MODULE__.t_pool({_update_callback: function() {} } );
pool = new __MODULE__.t_pool({_update_callback: function() {}} );

if (chunked) {
pages = pdata.cdata.map(x => x.splice(0, CHUNKED_THRESHOLD));
Expand All @@ -1816,9 +1853,21 @@ const perspective = {
}

// Fill t_table with data
tbl = __MODULE__.make_table(pages[0].length || 0,
pdata.names, pdata.types, pages,
0, options.index, pdata.is_arrow, false);
tbl = __MODULE__.make_table(
pages[0].length || 0,
pdata.names,
pdata.types,
pages,
0,
options.limit || 4294967295,
options.index,
pdata.is_arrow,
false
);
limit_index = tbl.size();
if (options.limit) {
limit_index = limit_index % options.limit;
}

gnode = __MODULE__.make_gnode(tbl);
pool.register_gnode(gnode);
Expand All @@ -1829,15 +1878,28 @@ const perspective = {
tbl.delete();
pages = pdata.cdata.map(x => x.splice(0, CHUNKED_THRESHOLD));

tbl = __MODULE__.make_table(pages[0].length || 0,
pdata.names, pdata.types, pages,
gnode.get_table().size(), options.index, pdata.is_arrow, false);
tbl = __MODULE__.make_table(
pages[0].length || 0,
pdata.names,
pdata.types,
pages,
limit_index,
options.limit || 4294967295,
options.index,
pdata.is_arrow,
false
);

limit_index += pages[0].length;
if (options.limit) {
limit_index = limit_index % options.limit;
}

__MODULE__.fill(pool, gnode, tbl);
}
}

return new table(gnode, pool, options.index);
return new table(gnode, pool, options.index, undefined, options.limit, limit_index);
} catch (e) {
if (pool) {
pool.delete();
Expand Down
19 changes: 19 additions & 0 deletions packages/perspective/test/js/updates.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,25 @@ module.exports = (perspective) => {

});

describe("Limit", function () {

it("{limit: 2} with table of size 4", async function () {
var table = perspective.table(data, {limit: 2});
var view = table.view();
let result = await view.to_json();
expect(data.slice(2)).toEqual(result);
});

it("{limit: 5} with 2 updates of size 4", async function () {
var table = perspective.table(data, {limit: 5});
table.update(data);
var view = table.view();
let result = await view.to_json();
expect(data.slice(1).concat(data.slice(3, 4)).concat(data.slice(0, 1))).toEqual(result);
});

});

describe("Indexed", function() {

it("{index: 'x'} (int)", async function () {
Expand Down

0 comments on commit 5364016

Please sign in to comment.