diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e671d056..ff5d2d843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Not released +- Add Global mode to TimeSeriesWidget [#377](https://github.com/CartoDB/carto-react/pull/377) - Add Global mode to HistogramWidget [#371](https://github.com/CartoDB/carto-react/pull/371) - Add Global mode to CategoryWidget & PieWidget [#370](https://github.com/CartoDB/carto-react/pull/370) - Add Global mode to FormulaWidget [#368](https://github.com/CartoDB/carto-react/pull/368) diff --git a/packages/react-core/src/operations/constants/GroupDateTypes.d.ts b/packages/react-core/src/operations/constants/GroupDateTypes.d.ts index d28a4604e..df79f39f0 100644 --- a/packages/react-core/src/operations/constants/GroupDateTypes.d.ts +++ b/packages/react-core/src/operations/constants/GroupDateTypes.d.ts @@ -1,8 +1,8 @@ export enum GroupDateTypes { - YEARS = 'years', - MONTHS = 'months', - WEEKS = 'weeks', - DAYS = 'days', - HOURS = 'hours', - MINUTES = 'minutes' + YEARS = 'year', + MONTHS = 'month', + WEEKS = 'week', + DAYS = 'day', + HOURS = 'hour', + MINUTES = 'minute' } diff --git a/packages/react-core/src/operations/constants/GroupDateTypes.js b/packages/react-core/src/operations/constants/GroupDateTypes.js index 5868c8e0a..71d87ad6c 100644 --- a/packages/react-core/src/operations/constants/GroupDateTypes.js +++ b/packages/react-core/src/operations/constants/GroupDateTypes.js @@ -5,20 +5,20 @@ */ export const GroupDateTypes = Object.freeze({ /** Years */ - YEARS: 'years', + YEARS: 'year', /** Months */ - MONTHS: 'months', + MONTHS: 'month', /** Weeks */ - WEEKS: 'weeks', + WEEKS: 'week', /** Days */ - DAYS: 'days', + DAYS: 'day', /** Hours */ - HOURS: 'hours', + HOURS: 'hour', /** Minutes */ - MINUTES: 'minutes' + MINUTES: 'minute' }); diff --git a/packages/react-widgets/__tests__/models/TimeSeriesModel.test.js b/packages/react-widgets/__tests__/models/TimeSeriesModel.test.js new file mode 100644 index 000000000..ccc72e760 --- /dev/null +++ b/packages/react-widgets/__tests__/models/TimeSeriesModel.test.js @@ -0,0 +1,189 @@ +import { getTimeSeries } from '../../src/models/TimeSeriesModel'; +import { AggregationTypes, GroupDateTypes } from '@carto/react-core'; +import { Methods, executeTask } from '@carto/react-workers'; + +const MOCK_WORKER_RESULT = [ + Date.UTC(1970, 0, 1, 0, 0), + Date.UTC(1970, 0, 1, 0, 30), + Date.UTC(1970, 1, 1, 0, 0), + Date.UTC(1970, 1, 1, 0, 30), + Date.UTC(1971, 0, 1, 1, 0) +].map((name) => ({ name, value: 1 })); + +jest.mock('@carto/react-workers', () => ({ + executeTask: jest + .fn() + .mockImplementation(() => new Promise((resolve) => resolve(MOCK_WORKER_RESULT))), + Methods: { + FEATURES_TIME_SERIES: 'featuresTimeSeries' + } +})); + +const mockedExecuteSQL = jest.fn(); + +jest.mock('@carto/react-api', () => ({ + executeSQL: (...args) => mockedExecuteSQL(...args) +})); + +describe('getTimeSeries', () => { + describe('local mode', () => { + test('should work correctly', async () => { + const props = { + source: { + id: '__test__', + type: 'query', + data: 'SELECT * FROM test', + credentials: { + apiVersion: 'v2' + } + }, + operation: AggregationTypes.SUM, + column: 'date_column', + operationColumn: 'opt_column', + stepSize: GroupDateTypes.DAYS + }; + + const data = await getTimeSeries(props); + + expect(data).toBe(MOCK_WORKER_RESULT); + + expect(executeTask).toHaveBeenCalledWith( + props.source.id, + Methods.FEATURES_TIME_SERIES, + { + filters: props.source.filters, + filtersLogicalOperator: props.source.filtersLogicalOperator, + operation: props.operation, + joinOperation: props.joinOperation, + column: props.column, + operationColumn: props.operationColumn, + stepSize: props.stepSize + } + ); + }); + }); + + describe('global mode', () => { + const DEFAULT_PROPS = { + source: { + id: '__test__', + type: 'table', + data: '__test__', + credentials: { + apiVersion: 'v3', + accessToken: '__test_token__' + }, + connection: '__test_connection__' + }, + operation: AggregationTypes.COUNT, + column: 'date_column', + operationColumn: 'opt_column', + global: true + }; + + describe('no-week stepSize', () => { + const MOCK_API_RESULT = [ + { + ['_agg_' + GroupDateTypes.YEARS]: 1970, + ['_agg_' + GroupDateTypes.MONTHS]: 1, + value: 2 + }, + { + ['_agg_' + GroupDateTypes.YEARS]: 1970, + ['_agg_' + GroupDateTypes.MONTHS]: 2, + value: 3 + } + ]; + + const RESULT = [ + { name: Date.UTC(1970, 0, 1, 0, 0), value: 2 }, + { name: Date.UTC(1970, 1, 1, 0, 0), value: 3 } + ]; + + test('should work correctly', async () => { + mockedExecuteSQL.mockImplementation( + () => new Promise((resolve) => resolve(MOCK_API_RESULT)) + ); + + const props = { + ...DEFAULT_PROPS, + stepSize: GroupDateTypes.MONTHS + }; + + const data = await getTimeSeries(props); + + expect(data).toEqual(RESULT); + + expect(mockedExecuteSQL).toHaveBeenCalledWith({ + credentials: props.source.credentials, + query: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,count(*) as value FROM __test__ GROUP BY _agg_year,_agg_month ORDER BY _agg_year,_agg_month`, + connection: props.source.connection, + opts: { + abortController: undefined + } + }); + }); + }); + + describe('week stepSize', () => { + const MOCK_API_RESULT = [ + { + ['_agg_' + GroupDateTypes.YEARS]: 1970, + ['_agg_' + GroupDateTypes.MONTHS]: 1, + ['_agg_' + GroupDateTypes.DAYS]: 1, + value: 2, + _grouped_count: 10 + }, + { + ['_agg_' + GroupDateTypes.YEARS]: 1970, + ['_agg_' + GroupDateTypes.MONTHS]: 1, + ['_agg_' + GroupDateTypes.DAYS]: 2, + value: 3, + _grouped_count: 20 + } + ]; + + const RESULTS = { + [AggregationTypes.AVG]: [{ name: -259200000, value: 2.6666666666666665 }], + [AggregationTypes.SUM]: [{ name: -259200000, value: 5 }], + [AggregationTypes.MIN]: [{ name: -259200000, value: 2 }], + [AggregationTypes.MAX]: [{ name: -259200000, value: 3 }], + [AggregationTypes.COUNT]: [{ name: -259200000, value: 5 }] + }; + + const RESULTS_QUERIES = { + [AggregationTypes.AVG]: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,extract(day from cast(date_column as timestamp)) as _agg_day,avg(opt_column) as value,count(*) as _grouped_count FROM __test__ GROUP BY _agg_year,_agg_month,_agg_day ORDER BY _agg_year,_agg_month,_agg_day`, + [AggregationTypes.SUM]: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,extract(day from cast(date_column as timestamp)) as _agg_day,sum(opt_column) as value FROM __test__ GROUP BY _agg_year,_agg_month,_agg_day ORDER BY _agg_year,_agg_month,_agg_day`, + [AggregationTypes.MIN]: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,extract(day from cast(date_column as timestamp)) as _agg_day,min(opt_column) as value FROM __test__ GROUP BY _agg_year,_agg_month,_agg_day ORDER BY _agg_year,_agg_month,_agg_day`, + [AggregationTypes.MAX]: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,extract(day from cast(date_column as timestamp)) as _agg_day,max(opt_column) as value FROM __test__ GROUP BY _agg_year,_agg_month,_agg_day ORDER BY _agg_year,_agg_month,_agg_day`, + [AggregationTypes.COUNT]: `SELECT extract(year from cast(date_column as timestamp)) as _agg_year,extract(month from cast(date_column as timestamp)) as _agg_month,extract(day from cast(date_column as timestamp)) as _agg_day,count(*) as value FROM __test__ GROUP BY _agg_year,_agg_month,_agg_day ORDER BY _agg_year,_agg_month,_agg_day` + }; + + test('should work correctly', () => { + mockedExecuteSQL.mockImplementation( + () => new Promise((resolve) => resolve(MOCK_API_RESULT)) + ); + + const props = { + ...DEFAULT_PROPS, + stepSize: GroupDateTypes.WEEKS + }; + + Object.keys(RESULTS).forEach((operation) => { + getTimeSeries({ ...props, operation }).then((data) => + expect(data).toEqual(RESULTS[operation]) + ); + + expect(mockedExecuteSQL).toHaveBeenCalledWith({ + credentials: props.source.credentials, + query: RESULTS_QUERIES[operation], + connection: props.source.connection, + opts: { + abortController: undefined + } + }); + }); + }); + }); + }); +}); diff --git a/packages/react-widgets/src/models/TimeSeriesModel.js b/packages/react-widgets/src/models/TimeSeriesModel.js index de2efe3b1..2304cb22e 100644 --- a/packages/react-widgets/src/models/TimeSeriesModel.js +++ b/packages/react-widgets/src/models/TimeSeriesModel.js @@ -1,22 +1,241 @@ +import { executeSQL } from '@carto/react-api/'; +import { + AggregationTypes, + getMonday, + GroupDateTypes, + groupValuesByDateColumn +} from '@carto/react-core/'; import { Methods, executeTask } from '@carto/react-workers'; +import { + formatOperationColumn, + formatTableNameWithFilters, + wrapModelCall +} from './utils'; -export const getTimeSeries = async (props) => { - const { - filters, - dataSource, - column, - operationColumn, - joinOperation, - operation, - stepSize - } = props; +export function getTimeSeries(props) { + return wrapModelCall(props, fromLocal, fromRemote); +} - return executeTask(dataSource, Methods.FEATURES_TIME_SERIES, { - filters, +// From local +function fromLocal({ + source, + column, + operationColumn, + joinOperation, + operation, + stepSize +}) { + return executeTask(source.id, Methods.FEATURES_TIME_SERIES, { + filters: source.filters, + filtersLogicalOperator: source.filtersLogicalOperator, column, stepSize, operationColumn: operationColumn || column, joinOperation, operation }); +} + +// From remote +const STEP_SIZE_TO_GROUP_QUERY_MAP = { + [GroupDateTypes.YEARS]: [GroupDateTypes.YEARS], + [GroupDateTypes.MONTHS]: [GroupDateTypes.YEARS, GroupDateTypes.MONTHS], + [GroupDateTypes.DAYS]: [ + GroupDateTypes.YEARS, + GroupDateTypes.MONTHS, + GroupDateTypes.DAYS + ], + [GroupDateTypes.HOURS]: [ + GroupDateTypes.YEARS, + GroupDateTypes.MONTHS, + GroupDateTypes.DAYS, + GroupDateTypes.HOURS + ], + [GroupDateTypes.MINUTES]: [ + GroupDateTypes.YEARS, + GroupDateTypes.MONTHS, + GroupDateTypes.DAYS, + GroupDateTypes.HOURS, + GroupDateTypes.MINUTES + ] +}; + +const FORMAT_NAME_BY_STEP_SIZE = { + [GroupDateTypes.YEARS]: (row) => + Date.UTC(row[stepSizeQueryColumn(GroupDateTypes.YEARS)], 0), + [GroupDateTypes.MONTHS]: (row) => + Date.UTC( + row[stepSizeQueryColumn(GroupDateTypes.YEARS)], + row[stepSizeQueryColumn(GroupDateTypes.MONTHS)] - 1 + ), + [GroupDateTypes.DAYS]: (row) => + Date.UTC( + row[stepSizeQueryColumn(GroupDateTypes.YEARS)], + row[stepSizeQueryColumn(GroupDateTypes.MONTHS)] - 1, + row[stepSizeQueryColumn(GroupDateTypes.DAYS)] + ), + [GroupDateTypes.HOURS]: (row) => + Date.UTC( + row[stepSizeQueryColumn(GroupDateTypes.YEARS)], + row[stepSizeQueryColumn(GroupDateTypes.MONTHS)] - 1, + row[stepSizeQueryColumn(GroupDateTypes.DAYS)], + row[stepSizeQueryColumn(GroupDateTypes.HOURS)] + ), + [GroupDateTypes.MINUTES]: (row) => + Date.UTC( + row[stepSizeQueryColumn(GroupDateTypes.YEARS)], + row[stepSizeQueryColumn(GroupDateTypes.MONTHS)] - 1, + row[stepSizeQueryColumn(GroupDateTypes.DAYS)], + row[stepSizeQueryColumn(GroupDateTypes.HOURS)], + row[stepSizeQueryColumn(GroupDateTypes.MINUTES)] + ) }; + +function groupWeeklyAverageValuesByDateColumn(data) { + const groups = data.reduce((acc, item) => { + const value = item.name; + const formattedValue = new Date(value); + const groupKey = getMonday(formattedValue); + + if (!isNaN(groupKey)) { + let groupedValues = acc.get(groupKey); + if (!groupedValues) { + groupedValues = []; + acc.set(groupKey, groupedValues); + } + + // In order to calculate the weighted average, we need to multiply the aggregated value by the weight that + // is the number of elements that were grouped (_grouped_count) + const weightedValue = item.value * item._grouped_count; + + const isValid = weightedValue !== null && weightedValue !== undefined; + + if (isValid) { + groupedValues.push([weightedValue, item._grouped_count]); + acc.set(groupKey, groupedValues); + } + } + + return acc; + }, new Map()); + + return [...groups.entries()].map(([name, value]) => { + const weightedValues = value.flat().filter((_, idx) => idx % 2 === 0); + const weights = value.flat().filter((_, idx) => idx % 2 !== 0); + + return { + name, + // Pondered average is the sum of the weighted values divided by the sum of the weights + value: sum(weightedValues) / sum(weights) + }; + }); +} + +// In order to keep compatibility with the different providers, we group the data in the query using +// multiple extract fns. After that, we must build the date using the resulting _agg_* columns +function formatRemoteData(data, props) { + const { stepSize, operation } = props; + + const dateFormatter = + FORMAT_NAME_BY_STEP_SIZE[ + stepSize === GroupDateTypes.WEEKS ? GroupDateTypes.DAYS : stepSize + ]; + + // For weeks, each provider behavies differently, so we need to group the data in the local + // For any other AggregationTypes it's easy, but AVG should be made using a weighted average + if (stepSize === GroupDateTypes.WEEKS && operation === AggregationTypes.AVG) { + const formattedData = data.map((row) => ({ + value: row.value, + name: dateFormatter(row), + _grouped_count: row._grouped_count + })); + + return groupWeeklyAverageValuesByDateColumn(formattedData); + } else { + const formattedData = data.map((row) => ({ + value: row.value, + name: dateFormatter(row) + })); + + if (stepSize === GroupDateTypes.WEEKS) { + return groupValuesByDateColumn({ + data: formattedData, + keysColumn: 'name', + valuesColumns: ['value'], + groupType: stepSize, + operation: operation === AggregationTypes.COUNT ? AggregationTypes.SUM : operation + }); + } + + return formattedData; + } +} + +async function fromRemote(props) { + const { source, abortController } = props; + const { credentials, connection } = source; + + const query = buildSqlQueryToGetTimeSeries(props); + + const data = await executeSQL({ + credentials, + query, + connection, + opts: { abortController } + }); + + return formatRemoteData(data, props); +} + +function buildSqlQueryToGetTimeSeries(props) { + const { column, operation, operationColumn, joinOperation, stepSize } = props; + + const isWeekly = stepSize === GroupDateTypes.WEEKS; + + const stepSizesToGroupBy = + STEP_SIZE_TO_GROUP_QUERY_MAP[ + // In weeks, group by days and then group by weeks in local + isWeekly ? GroupDateTypes.DAYS : stepSize + ]; + + if (!stepSizesToGroupBy) throw new Error(`${stepSize} not supported`); + + const selectDateClause = stepSizesToGroupBy + .map( + (step) => + `extract(${step} from cast(${column} as timestamp)) as ${stepSizeQueryColumn( + step + )}` + ) + .join(); + + const selectValueClause = `${operation}(${ + operation === AggregationTypes.COUNT + ? '*' + : formatOperationColumn(operationColumn || column, joinOperation) + }) as value`; + + const selectClause = [ + selectDateClause, + selectValueClause, + ...(isWeekly && operation === AggregationTypes.AVG + ? // _grouped_count is needed for weighted average + [`count(*) as _grouped_count`] + : []) + ]; + + const byColumns = stepSizesToGroupBy.map(stepSizeQueryColumn); + + const tableName = formatTableNameWithFilters(props); + + return `SELECT ${selectClause} FROM ${tableName} GROUP BY ${byColumns} ORDER BY ${byColumns}`.trim(); +} + +// Aux +function sum(data) { + return data.reduce((acc, item) => (acc += item), 0); +} + +function stepSizeQueryColumn(stepSize) { + return `_agg_${stepSize}`; +} diff --git a/packages/react-widgets/src/widgets/TimeSeriesWidget.js b/packages/react-widgets/src/widgets/TimeSeriesWidget.js index c532f5e01..51aae65bf 100644 --- a/packages/react-widgets/src/widgets/TimeSeriesWidget.js +++ b/packages/react-widgets/src/widgets/TimeSeriesWidget.js @@ -1,11 +1,7 @@ import React, { useCallback, useEffect, useState } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; +import { useDispatch } from 'react-redux'; import { getTimeSeries } from '../models'; -import { - addFilter, - removeFilter, - selectAreFeaturesReadyForSource -} from '@carto/react-redux'; +import { addFilter, removeFilter } from '@carto/react-redux'; import { TimeSeriesWidgetUI, WrapperWidgetUI, @@ -19,8 +15,8 @@ import { } from '@carto/react-core'; import { capitalize, Menu, MenuItem, SvgIcon, Typography } from '@material-ui/core'; import { PropTypes } from 'prop-types'; -import useSourceFilters from '../hooks/useSourceFilters'; import { columnAggregationOn } from './utils/propTypesFns'; +import useWidgetFetch from '../hooks/useWidgetFetch'; // Due to the widget groups the data by a certain stepSize, when filtering // the filter applied must be a range that represent the grouping range. @@ -54,6 +50,7 @@ const STEP_SIZE_RANGE_MAPPING = { * @param {string} [props.height] - Height of the chart. * @param {boolean} [props.showControls] - Enable/disable animation controls (play, pause, stop, speed). True by default. * @param {boolean} [props.animation] - Enable/disable widget animations on data updates. Enabled by default. + * @param {boolean} [props.global] - Enable/disable the viewport filtering in the data fetching. * @param {function} [props.onError] - Function to handle error messages from the widget. * @param {Object} [props.wrapperProps] - Extra props to pass to [WrapperWidgetUI](https://storybook-react.carto.com/?path=/docs/widgets-wrapperwidgetui--default) * @param {Object} [props.noDataAlertProps] - Extra props to pass to [NoDataAlert]() @@ -78,6 +75,7 @@ function TimeSeriesWidget({ joinOperation, operation, stepSizeOptions, + global, onError, wrapperProps, noDataAlertProps, @@ -102,14 +100,7 @@ function TimeSeriesWidget({ }) { const dispatch = useDispatch(); - const isSourceReady = useSelector((state) => - selectAreFeaturesReadyForSource(state, dataSource) - ); - const { filters } = useSourceFilters({ dataSource, id }); - - const [timeSeriesData, setTimeSeriesData] = useState([]); const [selectedStepSize, setSelectedStepSize] = useState(stepSize); - const [isLoading, setIsLoading] = useState(true); useEffect(() => { if (stepSize !== selectedStepSize) { @@ -119,43 +110,19 @@ function TimeSeriesWidget({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [stepSize]); - useEffect(() => { - setIsLoading(true); - - if (isSourceReady) { - getTimeSeries({ - filters, - dataSource, - column, - joinOperation, - stepSize: selectedStepSize, - operationColumn, - operation - }) - .then((data) => { - if (data) { - setIsLoading(false); - setTimeSeriesData(data); - } - }) - .catch((error) => { - setIsLoading(false); - if (onError) onError(error); - }); - } - }, [ + const { data = [], isLoading } = useWidgetFetch(getTimeSeries, { id, - filters, dataSource, - column, - selectedStepSize, - isSourceReady, - setIsLoading, - operationColumn, - joinOperation, - operation, + params: { + column, + joinOperation, + stepSize: selectedStepSize, + operationColumn, + operation + }, + global, onError - ]); + }); const handleTimeWindowUpdate = useCallback( (timeWindow) => { @@ -179,7 +146,7 @@ function TimeSeriesWidget({ const handleTimelineUpdate = useCallback( (timelinePosition) => { if (!isLoading) { - const { name: moment } = timeSeriesData[timelinePosition]; + const { name: moment } = data[timelinePosition]; dispatch( addFilter({ id: dataSource, @@ -201,7 +168,7 @@ function TimeSeriesWidget({ id, onTimelineUpdate, selectedStepSize, - timeSeriesData + data ] ); @@ -253,9 +220,9 @@ function TimeSeriesWidget({ : []) ]} > - {timeSeriesData.length || isLoading ? ( + {data.length || isLoading ? (