Return raw feature data from workers

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Not released
 
+- Return raw feature data from workers [#225](https://github.com/CartoDB/carto-react/pull/225)
+
 ## 1.1.2 (2021-12-01)
 
 - Fix CategoryWidget values during animation [#230](https://github.com/CartoDB/carto-react/pull/230)

packages/react-core/src/filters/Filter.js

@@ -57,15 +57,23 @@ function passesFilter(columns, filters, feature) {
 }
 
 export function buildFeatureFilter({ filters = {}, type = 'boolean' }) {
-  if (!Object.keys(filters).length) {
+  const columns = Object.keys(filters);
+
+  if (!columns.length) {
     return () => (type === 'number' ? 1 : true);
   }
 
   return (feature) => {
-    const columns = Object.keys(filters);
     const f = feature.properties || feature;
     const featurePassesFilter = passesFilter(columns, filters, f);
 
     return type === 'number' ? Number(featurePassesFilter) : featurePassesFilter;
   };
 }
+
+// Apply certain filters to a collection of features
+export function applyFilters(features, filters) {
+  return Object.keys(filters).length
+    ? features.filter(buildFeatureFilter({ filters }))
+    : features;
+}

packages/react-core/src/index.js

@@ -24,7 +24,10 @@ export {
   filtersToSQL as _filtersToSQL,
   getApplicableFilters as _getApplicableFilters
 } from './filters/FilterQueryBuilder';
-export { buildFeatureFilter as _buildFeatureFilter } from './filters/Filter';
+export {
+  buildFeatureFilter as _buildFeatureFilter,
+  applyFilters as _applyFilters
+} from './filters/Filter';
 export { viewportFeatures } from './filters/viewportFeatures';
 export { viewportFeaturesBinary } from './filters/viewportFeaturesBinary';
 export { viewportFeaturesGeoJSON } from './filters/viewportFeaturesGeoJSON';

packages/react-workers/__tests__/sorting.test.js

@@ -0,0 +1,83 @@
+import { applySorting } from '../src/utils/sorting';
+
+const features = [
+  { column1: 'C', column2: 3 },
+  { column1: 'A', column2: 1 },
+  { column1: 'B', column2: 2 },
+  { column1: 'D', column2: 4 },
+  { column1: 'D', column2: 5 }
+];
+
+const commonSortedFeatures = [
+  { column1: 'A', column2: 1 },
+  { column1: 'B', column2: 2 },
+  { column1: 'C', column2: 3 },
+  { column1: 'D', column2: 4 },
+  { column1: 'D', column2: 5 }
+];
+
+const commonSortedFeatures2 = [
+  { column1: 'D', column2: 4 },
+  { column1: 'D', column2: 5 },
+  { column1: 'C', column2: 3 },
+  { column1: 'B', column2: 2 },
+  { column1: 'A', column2: 1 }
+];
+
+describe('Sorting', () => {
+  test('should correctly throw error when sortOptions are invalid', () => {
+    expect(() => applySorting(features, { sortBy: 12345 })).toThrowError(Error);
+  });
+
+  describe('should correctly understand sortOptions', () => {
+    test('if undefined', () => {
+      expect(applySorting(features)).toEqual(features);
+    });
+
+    test('if sortBy is string', () => {
+      expect(applySorting(features, { sortBy: 'column1' })).toEqual(commonSortedFeatures);
+    });
+
+    test('if sortBy uses 2 columns', () => {
+      expect(applySorting(features, { sortBy: ['column1', 'column2'] })).toEqual(
+        commonSortedFeatures
+      );
+    });
+
+    test('if sortBy is array of arrays', () => {
+      expect(applySorting(features, { sortBy: [['column1'], ['column2']] })).toEqual(
+        commonSortedFeatures
+      );
+    });
+  });
+  describe('should correctly sort', () => {
+    test('if sortByDirection is used', () => {
+      expect(
+        applySorting(features, { sortBy: 'column1', sortByDirection: 'desc' })
+      ).toEqual(commonSortedFeatures2);
+    });
+
+    test('if sort direction is applied inside sortBy', () => {
+      expect(applySorting(features, { sortBy: [['column1', 'desc']] })).toEqual(
+        commonSortedFeatures2
+      );
+
+      expect(
+        applySorting(features, { sortBy: [['column1']], sortByDirection: 'desc' })
+      ).toEqual(commonSortedFeatures2);
+    });
+
+    test('if sort direction is applied inside sortBy and sortByDirection is also used, sortBy has priority', () => {
+      expect(
+        applySorting(features, { sortBy: [['column1', 'desc']], sortByDirection: 'asc' })
+      ).toEqual(commonSortedFeatures2);
+
+      expect(
+        applySorting(features, {
+          sortBy: [['column1', { direction: 'desc' }]],
+          sortByDirection: 'asc'
+        })
+      ).toEqual(commonSortedFeatures2);
+    });
+  });
+});

packages/react-workers/package.json

@@ -27,9 +27,9 @@
     "build:watch": "webpack --config webpack.config.js --watch",
     "lint": "eslint 'src/**/*.{js,jsx}'",
     "lint:fix": "eslint 'src/**/*.{js,jsx}' --fix",
-    "test": "jest --passWithNoTests",
+    "test": "jest",
     "test:watch": "jest --watch",
-    "test:coverage": "jest --collectCoverage --passWithNoTests",
+    "test:coverage": "jest --collectCoverage",
     "precommit": "lint-staged"
   },
   "devDependencies": {
@@ -64,6 +64,7 @@
     "@babel/runtime": "^7.13.9",
     "@carto/react-core": "^1.1.2",
     "@turf/bbox-polygon": "^6.3.0",
-    "@turf/boolean-intersects": "^6.3.0"
+    "@turf/boolean-intersects": "^6.3.0",
+    "thenby": "^1.3.4"
   }
 }

packages/react-workers/src/utils/sorting.js

@@ -0,0 +1,72 @@
+import { firstBy } from 'thenby';
+
+/**
+ * Apply sort structure to a collection of features
+ * @param {array} features
+ * @param {object} [sortOptions]
+ * @param {string | string[] | object[]} [sortOptions.sortBy] - One or more columns to sort by
+ * @param {string} [sortOptions.sortByDirection] - Direction by the columns will be sorted
+ */
+export function applySorting(features, { sortBy, sortByDirection = 'asc' } = {}) {
+  // If sortBy is undefined, pass all features
+  if (sortBy === undefined) {
+    return features;
+  }
+
+  // sortOptions exists, but are bad formatted
+  const isValidSortBy =
+    (Array.isArray(sortBy) && sortBy.length) || // sortBy can be an array of columns
+    typeof sortBy === 'string'; // or just one column
+
+  if (!isValidSortBy) {
+    throw new Error('Sorting options are bad formatted');
+  }
+
+  const sortFn = createSortFn({
+    sortBy,
+    sortByDirection
+  });
+
+  return features.sort(sortFn);
+}
+
+// Aux
+function createSortFn({ sortBy, sortByDirection }) {
+  const [firstSortOption, ...othersSortOptions] = normalizeSortByOptions({
+    sortBy,
+    sortByDirection
+  });
+
+  let sortFn = firstBy(...firstSortOption);
+  for (let sortOptions of othersSortOptions) {
+    sortFn = sortFn.thenBy(...sortOptions);
+  }
+
+  return sortFn;
+}
+
+function normalizeSortByOptions({ sortBy, sortByDirection }) {
+  if (!Array.isArray(sortBy)) {
+    sortBy = [sortBy];
+  }
+
+  return sortBy.map((sortByEl) => {
+    // sortByEl is 'column'
+    if (typeof sortByEl === 'string') {
+      return [sortByEl, sortByDirection];
+    }
+
+    if (Array.isArray(sortByEl)) {
+      // sortBy is ['column']
+      if (sortByEl[1] === undefined) {
+        return [sortByEl, sortByDirection];
+      }
+
+      // sortBy is ['column', { ... }]
+      if (typeof sortByEl[1] === 'object') {
+        return [sortByEl[0], { direction: sortByDirection, ...sortByEl[1] }];
+      }
+    }
+    return sortByEl;
+  });
+}

packages/react-workers/src/workerMethods.d.ts

@@ -5,4 +5,5 @@ export enum Methods {
   VIEWPORT_FEATURES_SCATTERPLOT = 'viewportFeaturesScatterPlot',
   VIEWPORT_FEATURES_TIME_SERIES = 'viewportFeaturesTimeSeries',
   VIEWPORT_FEATURES_CATEGORY = 'viewportFeaturesCategory',
+  VIEWPORT_FEATURES_RAW_FEATURES = 'viewportFeaturesRawFeatures'
 }

packages/react-workers/src/workerMethods.js

@@ -4,6 +4,7 @@ export const Methods = Object.freeze({
   VIEWPORT_FEATURES_HISTOGRAM: 'viewportFeaturesHistogram',
   VIEWPORT_FEATURES_CATEGORY: 'viewportFeaturesCategory',
   VIEWPORT_FEATURES_SCATTERPLOT: 'viewportFeaturesScatterPlot',
+  VIEWPORT_FEATURES_RAW_FEATURES: 'viewportFeaturesRawFeatures',
   LOAD_GEOJSON_FEATURES: 'loadGeoJSONFeatures',
   VIEWPORT_FEATURES_GEOJSON: 'viewportFeaturesGeoJSON'
 });

packages/react-workers/src/workerPool.d.ts

@@ -1,6 +1,6 @@
 import { Methods } from './workerMethods';
 import { ViewportFeaturesBinary } from '@carto/react-core';
 
-export function executeTask(source: string, method: Methods, params: ViewportFeaturesBinary): Promise<any>;
+export function executeTask(source: string, method: Methods, params?: ViewportFeaturesBinary): Promise<any>;
 
 export function removeWorker(source: string): void;

packages/react-workers/src/workers/viewportFeatures.worker.js

@@ -2,12 +2,13 @@ import {
   viewportFeaturesBinary,
   viewportFeaturesGeoJSON,
   aggregationFunctions,
-  _buildFeatureFilter,
+  _applyFilters,
   histogram,
   scatterPlot,
   groupValuesByColumn,
   groupValuesByDateColumn
 } from '@carto/react-core';
+import { applySorting } from '../utils/sorting';
 import { Methods } from '../workerMethods';
 
 let currentViewportFeatures;
@@ -33,6 +34,9 @@ onmessage = ({ data: { method, ...params } }) => {
     case Methods.VIEWPORT_FEATURES_TIME_SERIES:
       getTimeSeries(params);
       break;
+    case Methods.VIEWPORT_FEATURES_RAW_FEATURES:
+      getRawFeatures(params);
+      break;
     case Methods.LOAD_GEOJSON_FEATURES:
       loadGeoJSONFeatures(params);
       break;
@@ -144,8 +148,36 @@ function getTimeSeries({ filters, column, stepSize, operation, operationColumn }
   postMessage({ result });
 }
 
-function getFilteredFeatures(filters) {
-  return !Object.keys(currentViewportFeatures).length
-    ? currentViewportFeatures
-    : currentViewportFeatures.filter(_buildFeatureFilter({ filters }));
+// See sorting details in utils/sorting.js
+function getRawFeatures({
+  filters,
+  limit = 10,
+  page = 1,
+  sortBy = [],
+  sortByDirection = 'asc'
+}) {
+  let data = [];
+  let numberPages = 0;
+
+  if (currentViewportFeatures) {
+    data = applySorting(getFilteredFeatures(filters), {
+      sortBy,
+      sortByDirection
+    });
+
+    if (limit) {
+      numberPages = Math.ceil(data.length / limit);
+      data = applyPagination(data, { limit, page });
+    }
+  }
+
+  postMessage({ result: { data, currentPage: page, pages: numberPages } });
+}
+
+function applyPagination (features, { limit, page }) {
+  return features.slice(limit * Math.max(0, page - 1), limit * Math.max(1, page));
+}
+
+function getFilteredFeatures(filters = {}) {
+  return _applyFilters(currentViewportFeatures, filters);
 }

yarn.lock

@@ -16377,6 +16377,11 @@ text-table@0.2.0, text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=
 
+thenby@^1.3.4:
+  version "1.3.4"
+  resolved "https://registry.yarnpkg.com/thenby/-/thenby-1.3.4.tgz#81581f6e1bb324c6dedeae9bfc28e59b1a2201cc"
+  integrity sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==
+
 throat@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b"