Allow server-side table widget (#798)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Not released
 
+- Allow server-side table widget without hard limit [#798](https://github.com/CartoDB/carto-react/pull/798)
+
 ## 2.2
 
 ### 2.2.15 (2023-11-15)

packages/react-widgets/__tests__/models/TableModel.test.js

@@ -1,4 +1,4 @@
-import { getTable, paginateTable } from '../../src/models/TableModel';
+import { getTable } from '../../src/models/TableModel';
 import { Methods, executeTask } from '@carto/react-workers';
 
 const RESULT = {
@@ -9,9 +9,9 @@ const RESULT = {
     { id: 40, city: 'Paris', value: 400 },
     { id: 50, city: 'London', value: 500 }
   ],
-  totalCount: 5,
-  hasData: true,
-  isDataComplete: true
+  metadata: {
+    total: 5
+  }
 };
 
 const mockedExecuteModel = jest.fn();
@@ -30,32 +30,6 @@ jest.mock('@carto/react-workers', () => ({
   }
 }));
 
-describe('paginateTable', () => {
-  const data = {
-    rows: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], // 11 must hidden
-    totalCount: 10 // not 11
-  };
-  const tests = [
-    [0, 5, [1, 2, 3, 4, 5], 2], // first 5 elems
-    [1, 5, [6, 7, 8, 9, 10], 2], // second 5 elems
-    [0, 7, [1, 2, 3, 4, 5, 6, 7], 2], // first 7 elems
-    [1, 7, [8, 9, 10], 2], // second 7 elems, but not 11 that is hidden
-    [2, 5, [], 2], // 11 is hidden
-    [3, 5, [], 2], // no data at all
-    [0, 10, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 1] // corner case, 1 page not 2
-  ];
-  it.each(tests)(
-    'should work as expected',
-    (page, rowsPerPage, expectedRows, expectedPages) => {
-      const expected = {
-        rows: expectedRows,
-        pages: expectedPages
-      };
-      expect(paginateTable(data, page, rowsPerPage)).toStrictEqual(expected);
-    }
-  );
-});
-
 describe('getTable', () => {
   const tableParams = {
     source: {
@@ -81,8 +55,7 @@ describe('getTable', () => {
             { id: 7, city: 'b', value: 1 }
           ],
           totalCount: 2,
-          hasData: true,
-          isDataComplete: true
+          hasData: true
         })
       );
       const data = await getTable(tableParams);
@@ -92,8 +65,7 @@ describe('getTable', () => {
           { id: 7, city: 'b', value: 1 }
         ],
         totalCount: 2,
-        hasData: true,
-        isDataComplete: true
+        hasData: true
       });
     });
 
@@ -115,7 +87,11 @@ describe('getTable', () => {
   describe('remote mode', () => {
     test('correctly returns data', async () => {
       const data = await getTable({ ...tableParams, remoteCalculation: true });
-      expect(data).toStrictEqual(RESULT);
+      expect(data).toStrictEqual({
+        rows: RESULT.rows,
+        totalCount: RESULT.metadata.total,
+        hasData: true
+      });
     });
   });
 });

packages/react-widgets/src/models/TableModel.js

@@ -2,56 +2,52 @@ import { _executeModel } from '@carto/react-api';
 import { Methods, executeTask } from '@carto/react-workers';
 import { normalizeObjectKeys, wrapModelCall } from './utils';
 
-// Make sure this is sync with the same constant in cloud-native/maps-api
-export const HARD_LIMIT = 100;
-
 export function getTable(props) {
   return wrapModelCall(props, fromLocal, fromRemote);
 }
 
 function fromLocal(props) {
   // Injecting sortByColumnType externally from metadata gives better results. It allows to avoid deriving type from row value itself (with potential null values)
-  const { source, sortBy, sortDirection, sortByColumnType } = props;
+  const { source, sortBy, sortDirection, sortByColumnType, page, rowsPerPage } = props;
 
   return executeTask(source.id, Methods.FEATURES_RAW, {
     filters: source.filters,
     filtersLogicalOperator: source.filtersLogicalOperator,
     sortBy,
     sortByDirection: sortDirection,
-    sortByColumnType
+    sortByColumnType,
+    page,
+    rowsPerPage
   });
 }
 
-export function paginateTable({ rows, totalCount }, page, rowsPerPage) {
-  const sliced = rows.slice(
-    Math.min(rowsPerPage * Math.max(0, page), totalCount),
-    Math.min(rowsPerPage * Math.max(1, page + 1), totalCount)
-  );
-  const pages = Math.ceil(totalCount / rowsPerPage);
-  return { rows: sliced, pages };
-}
-
 function formatResult(res) {
-  const hasData = res.length > 0;
-  // We can detect if the data is complete because we request HARD_LIMIT + 1
-  const isDataComplete = res.length <= HARD_LIMIT;
-  // The actual extra record is hidden from pagination logic
-  const totalCount = isDataComplete ? res.length : HARD_LIMIT;
-  return { rows: res, totalCount, hasData, isDataComplete };
+  const { rows, totalCount } = res;
+  const hasData = totalCount > 0;
+  return { rows, totalCount, hasData };
 }
 
 // From remote
 function fromRemote(props) {
   const { source, spatialFilter, abortController, ...params } = props;
-  const { columns, sortBy, sortDirection } = params;
+  const { columns, sortBy, sortDirection, page, rowsPerPage } = params;
 
   return _executeModel({
     model: 'table',
     source,
     spatialFilter,
-    params: { column: columns, sortBy, sortDirection, limit: HARD_LIMIT + 1 },
+    params: {
+      column: columns,
+      sortBy,
+      sortDirection,
+      limit: rowsPerPage,
+      offset: page * rowsPerPage
+    },
     opts: { abortController }
   })
-    .then((res) => normalizeObjectKeys(res.rows))
+    .then((res) => ({
+      rows: normalizeObjectKeys(res.rows),
+      totalCount: res.metadata.total
+    }))
     .then(formatResult);
 }

packages/react-widgets/src/models/index.js

@@ -4,4 +4,4 @@ export { getCategories } from './CategoryModel';
 export { geocodeStreetPoint } from './GeocodingModel';
 export { getScatter, HARD_LIMIT as SCATTER_PLOT_HARD_LIMIT } from './ScatterPlotModel';
 export { getTimeSeries } from './TimeSeriesModel';
-export { getTable, paginateTable, HARD_LIMIT as TABLE_HARD_LIMIT } from './TableModel';
+export { getTable } from './TableModel';

packages/react-widgets/src/widgets/TableWidget.js

@@ -1,7 +1,7 @@
 import React, { useState, useEffect } from 'react';
 import PropTypes from 'prop-types';
 import { WrapperWidgetUI, TableWidgetUI } from '@carto/react-ui';
-import { getTable, paginateTable, TABLE_HARD_LIMIT } from '../models';
+import { getTable } from '../models';
 import useWidgetFetch from '../hooks/useWidgetFetch';
 import WidgetWithAlert from './utils/WidgetWithAlert';
 import { _FeatureFlags, _hasFeatureFlag } from '@carto/react-core';
@@ -54,30 +54,29 @@ function TableWidget({
   const [sortDirection, setSortDirection] = useState('asc');
 
   const {
-    data = { rows: [], totalCount: 0, hasData: false, isDataComplete: true },
+    data = { rows: [], totalCount: 0, hasData: false },
     isLoading,
-    warning
+    warning,
+    remoteCalculation
   } = useWidgetFetch(getTable, {
     id,
     dataSource,
     params: {
       columns: columns.map((c) => c.field),
       sortBy,
       sortDirection,
-      sortByColumnType
+      sortByColumnType,
+      page,
+      rowsPerPage
     },
     global,
     onError,
     onStateChange,
     attemptRemoteCalculation: _hasFeatureFlag(_FeatureFlags.REMOTE_WIDGETS)
   });
 
-  const { totalCount, hasData, isDataComplete } = data;
-  const { rows, pages } = paginateTable(data, page, rowsPerPage);
-
-  const lastPageTooltip = isDataComplete
-    ? undefined
-    : `This view is limited to ${TABLE_HARD_LIMIT} rows`;
+  const { rows, totalCount, hasData } = data;
+  const pages = Math.ceil(totalCount / rowsPerPage);
 
   useEffect(() => {
     if (pageSize !== undefined) setRowsPerPage(pageSize);
@@ -106,6 +105,7 @@ function TableWidget({
         warning={warning}
         global={global}
         droppingFeaturesAlertProps={droppingFeaturesAlertProps}
+        showDroppingFeaturesAlert={!remoteCalculation}
         noDataAlertProps={noDataAlertProps}
         stableHeight={stableHeight}
       >
@@ -127,7 +127,6 @@ function TableWidget({
             height={height}
             dense={dense}
             isLoading={isLoading}
-            lastPageTooltip={lastPageTooltip}
           />
         )}
       </WidgetWithAlert>

packages/react-workers/__tests__/methods.test.js

@@ -78,7 +78,6 @@ describe('Worker Methods', () => {
       ).toEqual({
         totalCount: 6,
         hasData: true,
-        isDataComplete: true,
         rows: sampleGeoJson.features.map((f) => f.properties)
       });
     });
@@ -94,7 +93,6 @@ describe('Worker Methods', () => {
       ).toEqual({
         totalCount: 6,
         hasData: true,
-        isDataComplete: true,
         rows: sampleGeoJson.features
           .map((f) => f.properties)
           .sort((a, b) => b.size_m2 - a.size_m2)

packages/react-workers/src/workers/methods.js

@@ -226,7 +226,10 @@ export function getRawFeatures({
   filtersLogicalOperator,
   sortBy,
   sortByDirection = 'asc',
-  sortByColumnType
+  sortByColumnType,
+  // page and rowsPerPage are optional and only used for pagination
+  page = undefined,
+  rowsPerPage = undefined
 }) {
   let rows = [];
   let totalCount = 0;
@@ -243,7 +246,14 @@ export function getRawFeatures({
     hasData = true;
   }
 
-  return { rows, totalCount, hasData, isDataComplete: true };
+  if (page !== undefined && rowsPerPage !== undefined) {
+    rows = rows.slice(
+      Math.min(rowsPerPage * Math.max(0, page), totalCount),
+      Math.min(rowsPerPage * Math.max(1, page + 1), totalCount)
+    );
+  }
+
+  return { rows, totalCount, hasData };
 }
 
 function getFilteredFeatures(filters = {}, filtersLogicalOperator) {