#10770: Vector files import limits (#10772)

* #10770: Vector files import limits Description: - handle checking the file size for imported vector layers only - show a warning notification for each layer has size larger than the configurable limit if configured - add translations - add test units * #10770: Vector files import limits Description: - add enhancements to the code including some helpful comments * #10770: Vector files import limits Description: - rename the configurable prop name of vector max size to 'importedVectorFileMaxSizeInMB' instead of 'importedVectorFileSizeInMB' to be more meaningful * #10770: Vector files import limits Description: - replace using 'getConfigProp' for 'importedVectorFileMaxSizeInMB' with direct cfg to map import - modify files based on this change - put default max file size with 2 mega byte * #10770: Vector files import limits Description: - edit unit tests of processFiles due to changes to fix FE test failures * #10770: Vector files import limits Description: - update max file size to be 3 mb instead of only 2 * #10770: Vector files import limits Description: - replace fixed max file size in useFiles with the constant 'DEFAULT_VECTOR_FILE_MAX_SIZE_IN_MB' instead
geosolutions-it · Feb 20, 2025 · e2c8896 · e2c8896
1 parent 836ba6a
commit e2c8896
Show file tree

Hide file tree

Showing 14 changed files with 148 additions and 29 deletions.
diff --git a/web/client/components/import/ImportDragZone.jsx b/web/client/components/import/ImportDragZone.jsx
@@ -26,7 +26,9 @@ export default compose(
         ...props
     }) => <DragZone
         onClose={onClose}
-        onDrop={onDrop}
+        onDrop={(files) => {
+            return onDrop({ files, options: { importedVectorFileMaxSizeInMB: props.importedVectorFileMaxSizeInMB} });
+        }}
         onRef={onRef}
     >
         <Content {...props} />

diff --git a/web/client/components/import/dragZone/enhancers/__tests__/processFiles-test.jsx b/web/client/components/import/dragZone/enhancers/__tests__/processFiles-test.jsx
@@ -46,7 +46,7 @@ describe('processFiles enhancer', () => {
     it('processFiles read error', (done) => {
         const Sink = compose(
             processFiles,
-            mapPropsStream(props$ => props$.merge(props$.take(1).do(({ onDrop = () => { } }) => onDrop(["ABC"])).ignoreElements()))
+            mapPropsStream(props$ => props$.merge(props$.take(1).do(({ onDrop = () => { } }) =>onDrop({ files: ["ABC"], options: {} })).ignoreElements()))
         )(createSink( props => {
             expect(props).toBeTruthy();
             if (props.error) {
@@ -61,7 +61,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getShapeFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getShapeFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -77,7 +77,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getKmzFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getKmzFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -93,7 +93,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getGpxFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getGpxFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -109,7 +109,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getKmlFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getKmlFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -125,7 +125,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getGeoJsonFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getGeoJsonFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -141,7 +141,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getAnnotationGeoJsonFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getAnnotationGeoJsonFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             try {
                 expect(props).toBeTruthy();
@@ -163,7 +163,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getGeoJsonFile("file.geojson").map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getGeoJsonFile("file.geojson").map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -179,7 +179,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getMapFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getMapFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.files) {
@@ -196,7 +196,7 @@ describe('processFiles enhancer', () => {
             mapPropsStream(props$ => props$.merge(
                 props$
                     .take(1)
-                    .switchMap(({ onDrop = () => { } }) => getUnsupportedMapFile().map((file) => onDrop([file]))).ignoreElements()))
+                    .switchMap(({ onDrop = () => { } }) => getUnsupportedMapFile().map((file) => onDrop({ files: [file], options: {} }))).ignoreElements()))
         )(createSink(props => {
             expect(props).toBeTruthy();
             if (props.error) {

diff --git a/web/client/components/import/dragZone/enhancers/__tests__/useFiles-test.js b/web/client/components/import/dragZone/enhancers/__tests__/useFiles-test.js
@@ -184,6 +184,56 @@ describe('useFiles enhancer', () => {
         ReactDOM.render(<EnhancedSink layers={[layer]}
             setLayers={actions.setLayers} warning={handlers.warning} onClose={handlers.onClose} />, document.getElementById("container"));
 
+    });
+    it('useFiles rendering with layer having size exceed the max limit should call warnig()', (done) => {
+        const handlers = {
+            warning: () => {},
+            onClose: () => {},
+            loadAnnotations: () => {},
+            loadMap: () => {}
+        };
+        const warningSpy = expect.spyOn(handlers, 'warning');
+
+        const actions = {
+            setLayers: (layers) => {
+                expect(layers).toExist();
+                // length is 1 since just one layer is valid and the another is invalid for its size
+                expect(layers.length).toBe(1);
+                expect(warningSpy).toHaveBeenCalled();
+                done();
+            }
+        };
+
+        const sink = createSink( props => {
+            expect(props).toExist();
+            expect(props.layers).toExist();
+            expect(props.useFiles).toExist();
+            props.useFiles({layers: props.layers});
+
+        });
+        const EnhancedSink = useFiles(sink);
+
+        const layers = [{
+            type: 'vector', name: "FileName", hideLoading: true,
+            bbox: {crs: "EPSG:4326"},
+            "features": [{
+                "type": "Feature",
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [-20, 30]
+                },
+                "properties": {
+                    "prop0": "value0"
+                }
+            }]
+        }, {
+            type: 'vector', name: "FileName01", hideLoading: true,
+            exceedFileMaxSize: true,
+            "features": []
+        }];
+        ReactDOM.render(<EnhancedSink layers={layers}
+            setLayers={actions.setLayers} warning={handlers.warning} onClose={handlers.onClose} />, document.getElementById("container"));
+
     });
     it('useFiles rendering with new annotation layer', (done) => {
 

diff --git a/web/client/components/import/dragZone/enhancers/processFiles.jsx b/web/client/components/import/dragZone/enhancers/processFiles.jsx
@@ -25,7 +25,8 @@ import {
     readZip,
     recognizeExt,
     shpToGeoJSON,
-    readGeoJson
+    readGeoJson,
+    isFileSizeExceedMaxLimit
 } from '../../../../utils/FileUtils';
 import { geoJSONToLayer } from '../../../../utils/LayersUtils';
 
@@ -63,9 +64,19 @@ const checkFileType = (file) => {
  * Create a function that return a Promise for reading file. The Promise resolves with an array of (json)
  * @param {function} onWarnings callback in case of warnings to report
  */
-const readFile = (onWarnings) => (file) => {
+const readFile = ({onWarnings, options}) => (file) => {
     const ext = recognizeExt(file.name);
     const type = file.type || MIME_LOOKUPS[ext];
+    // Check the file size first before file conversion process to avoid this useless effort
+    const configurableFileSizeLimitInMB = options.importedVectorFileMaxSizeInMB;
+    const isVectorFile = type !== 'application/json';       // skip json as json is for map file
+    if (configurableFileSizeLimitInMB && isVectorFile) {
+        if (isFileSizeExceedMaxLimit(file, configurableFileSizeLimitInMB)) {
+            // add 'exceedFileMaxSize' and fileSizeLimitInMB into layer object to be used in useFiles
+            return [[{ "type": "FeatureCollection", features: [{}], "fileName": file.name, name: file.name, exceedFileMaxSize: true, fileSizeLimitInMB: configurableFileSizeLimitInMB }]];
+        }
+    }
+
     const projectionDefs = ConfigUtils.getConfigProp('projectionDefs') || [];
     const supportedProjections = (projectionDefs.length && projectionDefs.map(({code})  => code) || []).concat(["EPSG:4326", "EPSG:3857", "EPSG:900913"]);
     if (type === 'application/vnd.google-earth.kml+xml') {
@@ -151,18 +162,26 @@ export default compose(
             const { handler: onWarnings, stream: warnings$} = createEventHandler();
             return props$.combineLatest(
                 drop$.switchMap(
-                    files => Rx.Observable.from(files)
+                    ({files, options}) => Rx.Observable.from(files)
                         .flatMap(checkFileType) // check file types are allowed
-                        .flatMap(readFile(onWarnings)) // read files to convert to json
+                        .flatMap(readFile({onWarnings, options})) // read files to convert to json
                         .reduce((result, jsonObjects) => ({ // divide files by type
                             layers: (result.layers || [])
                                 .concat(
                                     jsonObjects.filter(json => isGeoJSON(json))
-                                        .map(json => (isAnnotation(json) ?
-                                            // annotation GeoJSON to layers
-                                            { name: "Annotations", features: importJSONToAnnotations(json), filename: json.filename} :
+                                        .map(json => {
+                                            const isLayerExceedsMaxFileSize = json?.exceedFileMaxSize || false;
+                                            const isAnnotationLayer = isAnnotation(json);
+                                            if (isAnnotationLayer) {
+                                                // annotation GeoJSON to layers
+                                                return { name: "Annotations", features: importJSONToAnnotations(json), filename: json.filename};
+                                            } else if (isLayerExceedsMaxFileSize) {
+                                            // layers with size exceeds the max size limit
+                                                return { ...json, filename: json.filename};
+                                            }
                                             // other GeoJSON to layers
-                                            {...geoJSONToLayer(json), filename: json.filename}))
+                                            return {...geoJSONToLayer(json), filename: json.filename};
+                                        })
                                 ),
                             maps: (result.maps || [])
                                 .concat(
@@ -193,5 +212,6 @@ export default compose(
                 })
             );
         }
+
     )
 );
diff --git a/web/client/components/import/dragZone/enhancers/useFiles.js b/web/client/components/import/dragZone/enhancers/useFiles.js
@@ -1,6 +1,7 @@
 
 import { compose, mapPropsStream, withHandlers } from 'recompose';
 import { checkIfLayerFitsExtentForProjection } from '../../../../utils/CoordinatesUtils';
+import { DEFAULT_VECTOR_FILE_MAX_SIZE_IN_MB } from '../../../../utils/FileUtils';
 
 /**
  * Enhancer for processing map configuration and layers object
@@ -39,9 +40,18 @@ export default compose(
                     } else {
                         let validLayers = [];
                         layers.forEach((layer) => {
-                            const valid = layer.type === "vector" ? checkIfLayerFitsExtentForProjection(layer) : true;
-                            if (valid) {
+                            const isFileSizeNotValid = !!layer?.exceedFileMaxSize;     // this check is for file size limit for vector layer
+                            const valid = layer.type === "vector" ? (checkIfLayerFitsExtentForProjection(layer) && !isFileSizeNotValid) : true;
+                            if (valid && !isFileSizeNotValid) {
                                 validLayers.push(layer);
+                            } else if (isFileSizeNotValid) {
+                                warning({
+                                    title: "notification.warning",
+                                    message: "mapImport.errors.exceedFileSizeLimit",
+                                    autoDismiss: 6,
+                                    position: "tc",
+                                    values: {filename: layer.name ?? " ", maxfilesize: layer?.fileSizeLimitInMB ?? DEFAULT_VECTOR_FILE_MAX_SIZE_IN_MB}
+                                });
                             } else {
                                 warning({
                                     title: "notification.warning",

diff --git a/web/client/plugins/MapImport.jsx b/web/client/plugins/MapImport.jsx
@@ -31,6 +31,7 @@ import { toggleControl } from '../actions/controls';
 import assign from 'object-assign';
 import { Glyphicon } from 'react-bootstrap';
 import { mapTypeSelector } from '../selectors/maptype';
+import { DEFAULT_VECTOR_FILE_MAX_SIZE_IN_MB } from '../utils/FileUtils';
 
 /**
  * Allows the user to import a file into current map.
@@ -46,13 +47,14 @@ import { mapTypeSelector } from '../selectors/maptype';
  * @memberof plugins
  * @name MapImport
  * @class
+ * @prop {number} cfg.importedVectorFileMaxSizeInMB it is the max allowable file size for import vectir layers in mega bytes
  */
 export default {
     MapImportPlugin: assign({loadPlugin: (resolve) => {
         import('./import/Import').then((importMod) => {
             const Import = importMod.default;
 
-            const ImportPlugin = connect((state) => (
+            const ImportPlugin = connect((state, ownProps) =>(
                 {
                     enabled: state.controls && state.controls.mapimport && state.controls.mapimport.enabled,
                     layers: state.mapimport && state.mapimport.layers || null,
@@ -62,7 +64,8 @@ export default {
                     errors: state.mapimport && state.mapimport.errors || null,
                     shapeStyle: state.style || {},
                     mapType: mapTypeSelector(state),
-                    annotationsLayer: annotationsLayerSelector(state)
+                    annotationsLayer: annotationsLayerSelector(state),
+                    importedVectorFileMaxSizeInMB: ownProps?.importedVectorFileMaxSizeInMB || DEFAULT_VECTOR_FILE_MAX_SIZE_IN_MB
                 }
             ), {
                 setLayers,

diff --git a/web/client/translations/data.de-DE.json b/web/client/translations/data.de-DE.json
@@ -1585,7 +1585,8 @@
                     "fileNotSupported": "Datei wird nicht unterstützt",
                     "unknownError": "Beim Import ist ein unbekannter Fehler aufgetreten",
                     "projectionNotSupported": "Die Projektion der Datei, die Sie importieren möchten, wird nicht unterstützt",
-                    "fileBeyondBoundaries": "Die Datei {Dateiname} kann nicht importiert werden, da sie nicht zu den Kartengrenzen passt"
+                    "fileBeyondBoundaries": "Die Datei {Dateiname} kann nicht importiert werden, da sie nicht zu den Kartengrenzen passt",
+                    "exceedFileSizeLimit": "Die Datei {filename}, die Sie importieren möchten, überschreitet die maximale Dateigröße von {maxfilesize} MB"
                 }
             },
         "mapExport": {

diff --git a/web/client/translations/data.en-US.json b/web/client/translations/data.en-US.json
@@ -1546,6 +1546,7 @@
                 "fileNotSupported": "File not supported",
                 "unknownError": "there was an unknown error during import",
                 "projectionNotSupported": "The projection of the file(s) you're trying to import is not supported",
+                "exceedFileSizeLimit": "The file {filename} you're trying to import is exceeded the max file size limit {maxfilesize} MB",
                 "fileBeyondBoundaries": "The file  {filename} cannot be imported because it does not fit the map boundaries"
             }
         },

diff --git a/web/client/translations/data.es-ES.json b/web/client/translations/data.es-ES.json
@@ -1547,7 +1547,8 @@
                 "fileNotSupported": "Archivo no compatible",
                 "unknownError": "hubo un error desconocido durante la importación",
                 "projectionNotSupported": "La proyección del archivo que está intentando importar no es compatible",
-                "fileBeyondBoundaries": "El archivo {filename} no se pudo importar porque no encaja dentro de los límites del mapa"
+                "fileBeyondBoundaries": "El archivo {filename} no se pudo importar porque no encaja dentro de los límites del mapa",
+                "exceedFileSizeLimit": "El archivo {filename} que estás intentando importar ha superado el límite máximo de tamaño de archivo {maxfilesize} MB"
             }
         },
         "mapExport": {

diff --git a/web/client/translations/data.fr-FR.json b/web/client/translations/data.fr-FR.json
@@ -1547,7 +1547,8 @@
                 "fileNotSupported": "Fichier non pris en charge",
                 "unknownError": "il y a eu une erreur inconnue lors de l'import",
                 "projectionNotSupported": "La projection du/des fichier(s) que vous essayez d'importer n'est pas prise en charge",
-                "fileBeyondBoundaries": "Le fichier {filename} ne peut pas être importé car il ne correspond pas aux limites de la carte"
+                "fileBeyondBoundaries": "Le fichier {filename} ne peut pas être importé car il ne correspond pas aux limites de la carte",
+                "exceedFileSizeLimit": "Le fichier {filename} que vous essayez d'importer dépasse la limite de taille de fichier maximale {maxfilesize} Mo"
             }
         },
         "mapExport": {

diff --git a/web/client/translations/data.is-IS.json b/web/client/translations/data.is-IS.json
@@ -1425,7 +1425,8 @@
         "fileNotSupported": "File not supported",
         "unknownError": "there was an unknown error during import",
         "projectionNotSupported": "The projection of the file(s) you're trying to import is not supported",
-        "fileBeyondBoundaries": "The file  {filename} cannot be imported because it does not fit the map boundaries"
+        "fileBeyondBoundaries": "The file  {filename} cannot be imported because it does not fit the map boundaries",
+        "exceedFileSizeLimit": "The file {filename} you're trying to import is exceeded the max file size limit {maxfilesize} MB"
       }
     },
     "mapExport": {

diff --git a/web/client/translations/data.it-IT.json b/web/client/translations/data.it-IT.json
@@ -1544,7 +1544,8 @@
                 "fileNotSupported": "File non supportato",
                 "unknownError": "si è verificato un errore sconosciuto durante l'importazione",
                 "projectionNotSupported": "La proiezione del file che stai tentando di importare non è supportata",
-                "fileBeyondBoundaries": "Il file {nomefile} non può essere importato perché non si adatta ai confini della mappa"
+                "fileBeyondBoundaries": "Il file {nomefile} non può essere importato perché non si adatta ai confini della mappa",
+                "exceedFileSizeLimit": "Il file {filename} che stai tentando di importare ha superato il limite massimo di dimensione file {maxfilesize} MB"
             }
         },
         "mapExport": {