diff --git a/CHANGELOG.md b/CHANGELOG.md index e5162a6e2..a7d0a45f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # CHANGELOG ## Not released + - Histogram takes into account null values in filters for selected bars [#234](https://github.com/CartoDB/carto-react/pull/234) +- Return raw feature data from workers [#225](https://github.com/CartoDB/carto-react/pull/225) ## 1.1.2 (2021-12-01) diff --git a/packages/react-core/src/filters/Filter.js b/packages/react-core/src/filters/Filter.js index 47e1d94eb..4fad71ee2 100644 --- a/packages/react-core/src/filters/Filter.js +++ b/packages/react-core/src/filters/Filter.js @@ -57,15 +57,23 @@ function passesFilter(columns, filters, feature) { } export function buildFeatureFilter({ filters = {}, type = 'boolean' }) { - if (!Object.keys(filters).length) { + const columns = Object.keys(filters); + + if (!columns.length) { return () => (type === 'number' ? 1 : true); } return (feature) => { - const columns = Object.keys(filters); const f = feature.properties || feature; const featurePassesFilter = passesFilter(columns, filters, f); return type === 'number' ? Number(featurePassesFilter) : featurePassesFilter; }; } + +// Apply certain filters to a collection of features +export function applyFilters(features, filters) { + return Object.keys(filters).length + ? features.filter(buildFeatureFilter({ filters })) + : features; +} diff --git a/packages/react-core/src/index.js b/packages/react-core/src/index.js index 80a4dac34..c801b12e7 100644 --- a/packages/react-core/src/index.js +++ b/packages/react-core/src/index.js @@ -24,7 +24,10 @@ export { filtersToSQL as _filtersToSQL, getApplicableFilters as _getApplicableFilters } from './filters/FilterQueryBuilder'; -export { buildFeatureFilter as _buildFeatureFilter } from './filters/Filter'; +export { + buildFeatureFilter as _buildFeatureFilter, + applyFilters as _applyFilters +} from './filters/Filter'; export { viewportFeatures } from './filters/viewportFeatures'; export { viewportFeaturesBinary } from './filters/viewportFeaturesBinary'; export { viewportFeaturesGeoJSON } from './filters/viewportFeaturesGeoJSON'; diff --git a/packages/react-workers/__tests__/sorting.test.js b/packages/react-workers/__tests__/sorting.test.js new file mode 100644 index 000000000..b3bfdd9db --- /dev/null +++ b/packages/react-workers/__tests__/sorting.test.js @@ -0,0 +1,83 @@ +import { applySorting } from '../src/utils/sorting'; + +const features = [ + { column1: 'C', column2: 3 }, + { column1: 'A', column2: 1 }, + { column1: 'B', column2: 2 }, + { column1: 'D', column2: 4 }, + { column1: 'D', column2: 5 } +]; + +const commonSortedFeatures = [ + { column1: 'A', column2: 1 }, + { column1: 'B', column2: 2 }, + { column1: 'C', column2: 3 }, + { column1: 'D', column2: 4 }, + { column1: 'D', column2: 5 } +]; + +const commonSortedFeatures2 = [ + { column1: 'D', column2: 4 }, + { column1: 'D', column2: 5 }, + { column1: 'C', column2: 3 }, + { column1: 'B', column2: 2 }, + { column1: 'A', column2: 1 } +]; + +describe('Sorting', () => { + test('should correctly throw error when sortOptions are invalid', () => { + expect(() => applySorting(features, { sortBy: 12345 })).toThrowError(Error); + }); + + describe('should correctly understand sortOptions', () => { + test('if undefined', () => { + expect(applySorting(features)).toEqual(features); + }); + + test('if sortBy is string', () => { + expect(applySorting(features, { sortBy: 'column1' })).toEqual(commonSortedFeatures); + }); + + test('if sortBy uses 2 columns', () => { + expect(applySorting(features, { sortBy: ['column1', 'column2'] })).toEqual( + commonSortedFeatures + ); + }); + + test('if sortBy is array of arrays', () => { + expect(applySorting(features, { sortBy: [['column1'], ['column2']] })).toEqual( + commonSortedFeatures + ); + }); + }); + describe('should correctly sort', () => { + test('if sortByDirection is used', () => { + expect( + applySorting(features, { sortBy: 'column1', sortByDirection: 'desc' }) + ).toEqual(commonSortedFeatures2); + }); + + test('if sort direction is applied inside sortBy', () => { + expect(applySorting(features, { sortBy: [['column1', 'desc']] })).toEqual( + commonSortedFeatures2 + ); + + expect( + applySorting(features, { sortBy: [['column1']], sortByDirection: 'desc' }) + ).toEqual(commonSortedFeatures2); + }); + + test('if sort direction is applied inside sortBy and sortByDirection is also used, sortBy has priority', () => { + expect( + applySorting(features, { sortBy: [['column1', 'desc']], sortByDirection: 'asc' }) + ).toEqual(commonSortedFeatures2); + + expect( + applySorting(features, { + sortBy: [['column1', { direction: 'desc' }]], + sortByDirection: 'asc' + }) + ).toEqual(commonSortedFeatures2); + }); + }); +}); diff --git a/packages/react-workers/package.json b/packages/react-workers/package.json index 21f741bf3..7ddf5df5c 100644 --- a/packages/react-workers/package.json +++ b/packages/react-workers/package.json @@ -27,9 +27,9 @@ "build:watch": "webpack --config webpack.config.js --watch", "lint": "eslint 'src/**/*.{js,jsx}'", "lint:fix": "eslint 'src/**/*.{js,jsx}' --fix", - "test": "jest --passWithNoTests", + "test": "jest", "test:watch": "jest --watch", - "test:coverage": "jest --collectCoverage --passWithNoTests", + "test:coverage": "jest --collectCoverage", "precommit": "lint-staged" }, "devDependencies": { @@ -64,6 +64,7 @@ "@babel/runtime": "^7.13.9", "@carto/react-core": "^1.1.2", "@turf/bbox-polygon": "^6.3.0", - "@turf/boolean-intersects": "^6.3.0" + "@turf/boolean-intersects": "^6.3.0", + "thenby": "^1.3.4" } } diff --git a/packages/react-workers/src/utils/sorting.js b/packages/react-workers/src/utils/sorting.js new file mode 100644 index 000000000..9cb454db0 --- /dev/null +++ b/packages/react-workers/src/utils/sorting.js @@ -0,0 +1,72 @@ +import { firstBy } from 'thenby'; + +/** + * Apply sort structure to a collection of features + * @param {array} features + * @param {object} [sortOptions] + * @param {string | string[] | object[]} [sortOptions.sortBy] - One or more columns to sort by + * @param {string} [sortOptions.sortByDirection] - Direction by the columns will be sorted + */ +export function applySorting(features, { sortBy, sortByDirection = 'asc' } = {}) { + // If sortBy is undefined, pass all features + if (sortBy === undefined) { + return features; + } + + // sortOptions exists, but are bad formatted + const isValidSortBy = + (Array.isArray(sortBy) && sortBy.length) || // sortBy can be an array of columns + typeof sortBy === 'string'; // or just one column + + if (!isValidSortBy) { + throw new Error('Sorting options are bad formatted'); + } + + const sortFn = createSortFn({ + sortBy, + sortByDirection + }); + + return features.sort(sortFn); +} + +// Aux +function createSortFn({ sortBy, sortByDirection }) { + const [firstSortOption, ...othersSortOptions] = normalizeSortByOptions({ + sortBy, + sortByDirection + }); + + let sortFn = firstBy(...firstSortOption); + for (let sortOptions of othersSortOptions) { + sortFn = sortFn.thenBy(...sortOptions); + } + + return sortFn; +} + +function normalizeSortByOptions({ sortBy, sortByDirection }) { + if (!Array.isArray(sortBy)) { + sortBy = [sortBy]; + } + + return sortBy.map((sortByEl) => { + // sortByEl is 'column' + if (typeof sortByEl === 'string') { + return [sortByEl, sortByDirection]; + } + + if (Array.isArray(sortByEl)) { + // sortBy is ['column'] + if (sortByEl[1] === undefined) { + return [sortByEl, sortByDirection]; + } + + // sortBy is ['column', { ... }] + if (typeof sortByEl[1] === 'object') { + return [sortByEl[0], { direction: sortByDirection, ...sortByEl[1] }]; + } + } + return sortByEl; + }); +} diff --git a/packages/react-workers/src/workerMethods.d.ts b/packages/react-workers/src/workerMethods.d.ts index 9f25b80a8..8b36c731c 100644 --- a/packages/react-workers/src/workerMethods.d.ts +++ b/packages/react-workers/src/workerMethods.d.ts @@ -5,4 +5,5 @@ export enum Methods { VIEWPORT_FEATURES_SCATTERPLOT = 'viewportFeaturesScatterPlot', VIEWPORT_FEATURES_TIME_SERIES = 'viewportFeaturesTimeSeries', VIEWPORT_FEATURES_CATEGORY = 'viewportFeaturesCategory', + VIEWPORT_FEATURES_RAW_FEATURES = 'viewportFeaturesRawFeatures' } diff --git a/packages/react-workers/src/workerMethods.js b/packages/react-workers/src/workerMethods.js index 8740b97d5..8fed43957 100644 --- a/packages/react-workers/src/workerMethods.js +++ b/packages/react-workers/src/workerMethods.js @@ -4,6 +4,7 @@ export const Methods = Object.freeze({ VIEWPORT_FEATURES_HISTOGRAM: 'viewportFeaturesHistogram', VIEWPORT_FEATURES_CATEGORY: 'viewportFeaturesCategory', VIEWPORT_FEATURES_SCATTERPLOT: 'viewportFeaturesScatterPlot', + VIEWPORT_FEATURES_RAW_FEATURES: 'viewportFeaturesRawFeatures', LOAD_GEOJSON_FEATURES: 'loadGeoJSONFeatures', VIEWPORT_FEATURES_GEOJSON: 'viewportFeaturesGeoJSON' }); diff --git a/packages/react-workers/src/workerPool.d.ts b/packages/react-workers/src/workerPool.d.ts index 475bd01b2..e23fc3f06 100644 --- a/packages/react-workers/src/workerPool.d.ts +++ b/packages/react-workers/src/workerPool.d.ts @@ -1,6 +1,6 @@ import { Methods } from './workerMethods'; import { ViewportFeaturesBinary } from '@carto/react-core'; -export function executeTask(source: string, method: Methods, params: ViewportFeaturesBinary): Promise; +export function executeTask(source: string, method: Methods, params?: ViewportFeaturesBinary): Promise; export function removeWorker(source: string): void; diff --git a/packages/react-workers/src/workers/viewportFeatures.worker.js b/packages/react-workers/src/workers/viewportFeatures.worker.js index 5eb6f0027..9e1aec08f 100644 --- a/packages/react-workers/src/workers/viewportFeatures.worker.js +++ b/packages/react-workers/src/workers/viewportFeatures.worker.js @@ -2,12 +2,13 @@ import { viewportFeaturesBinary, viewportFeaturesGeoJSON, aggregationFunctions, - _buildFeatureFilter, + _applyFilters, histogram, scatterPlot, groupValuesByColumn, groupValuesByDateColumn } from '@carto/react-core'; +import { applySorting } from '../utils/sorting'; import { Methods } from '../workerMethods'; let currentViewportFeatures; @@ -33,6 +34,9 @@ onmessage = ({ data: { method, ...params } }) => { case Methods.VIEWPORT_FEATURES_TIME_SERIES: getTimeSeries(params); break; + case Methods.VIEWPORT_FEATURES_RAW_FEATURES: + getRawFeatures(params); + break; case Methods.LOAD_GEOJSON_FEATURES: loadGeoJSONFeatures(params); break; @@ -144,8 +148,36 @@ function getTimeSeries({ filters, column, stepSize, operation, operationColumn } postMessage({ result }); } -function getFilteredFeatures(filters) { - return !Object.keys(currentViewportFeatures).length - ? currentViewportFeatures - : currentViewportFeatures.filter(_buildFeatureFilter({ filters })); +// See sorting details in utils/sorting.js +function getRawFeatures({ + filters, + limit = 10, + page = 1, + sortBy = [], + sortByDirection = 'asc' +}) { + let data = []; + let numberPages = 0; + + if (currentViewportFeatures) { + data = applySorting(getFilteredFeatures(filters), { + sortBy, + sortByDirection + }); + + if (limit) { + numberPages = Math.ceil(data.length / limit); + data = applyPagination(data, { limit, page }); + } + } + + postMessage({ result: { data, currentPage: page, pages: numberPages } }); +} + +function applyPagination (features, { limit, page }) { + return features.slice(limit * Math.max(0, page - 1), limit * Math.max(1, page)); +} + +function getFilteredFeatures(filters = {}) { + return _applyFilters(currentViewportFeatures, filters); } diff --git a/yarn.lock b/yarn.lock index 7aa2139fe..4636c87b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16377,6 +16377,11 @@ text-table@0.2.0, text-table@^0.2.0: resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= +thenby@^1.3.4: + version "1.3.4" + resolved "https://registry.yarnpkg.com/thenby/-/thenby-1.3.4.tgz#81581f6e1bb324c6dedeae9bfc28e59b1a2201cc" + integrity sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ== + throat@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/throat/-/throat-5.0.0.tgz#c5199235803aad18754a667d659b5e72ce16764b"