diff --git a/functions/csvLoader/components/fileLoader.ts b/functions/csvLoader/components/fileLoader.ts index 6e3a111..2bea5e3 100644 --- a/functions/csvLoader/components/fileLoader.ts +++ b/functions/csvLoader/components/fileLoader.ts @@ -1,12 +1,12 @@ +import { Context } from '@azure/functions'; +import csv2json from 'csvtojson'; +import { ItemFromFile } from '../types'; + /** * Load data from file based on file format * @param {Buffer} incomingBlob - blob which triggered the function and is processed * @returns {Array} - array with data from file */ -import { Context } from '@azure/functions'; -import csv2json from 'csvtojson'; -import { ItemFromFile } from '../types'; - const fileLoader = async (incomingBlob: Buffer, context: Context): Promise> => { let returnJson; const stringFromBuffer = incomingBlob.toString('utf8'); diff --git a/initial-data-load/.gitignore b/initial-data-load/.gitignore index a4a1132..e297440 100644 --- a/initial-data-load/.gitignore +++ b/initial-data-load/.gitignore @@ -4,3 +4,8 @@ node_modules .vscode spinup.sh .DS_Store + +/data +!/data/Sample +!/data/testCountry +!/data/testCountry3 diff --git a/initial-data-load/package.json b/initial-data-load/package.json index 3ef9920..43977a0 100644 --- a/initial-data-load/package.json +++ b/initial-data-load/package.json @@ -8,6 +8,7 @@ "dependencies": { "@azure/storage-queue": "^12.2.0", "@hapi/joi": "^17.1.1", + "@turf/bbox": "^6.5.0", "axios": "^0.21.2", "azure-storage": "^2.10.3", "bottleneck": "^2.19.5", diff --git a/initial-data-load/src/mapLayers/db.js b/initial-data-load/src/mapLayers/db.js index c47e458..d2118d2 100644 --- a/initial-data-load/src/mapLayers/db.js +++ b/initial-data-load/src/mapLayers/db.js @@ -10,8 +10,10 @@ const createIndex = async () => { await mongoose.connection.db.collection('mapLayers').createIndex({ title: 1 }); }; -const createGeoDataIndex = async (collectionName) => { - await mongoose.connection.db.collection(collectionName).createIndex({ geometry: '2dsphere' }); +const createGeoDataIndex = async (collectionName, geoKey = 'geometry') => { + const indexKey = {}; + indexKey[geoKey] = '2dsphere'; + await mongoose.connection.db.collection(collectionName).createIndex(indexKey); }; const deleteAllFromCollection = async (collectionName) => { diff --git a/initial-data-load/src/mapLayers/geoFeaturesSchema.js b/initial-data-load/src/mapLayers/geoFeaturesSchema.js index 9a71872..f866632 100644 --- a/initial-data-load/src/mapLayers/geoFeaturesSchema.js +++ b/initial-data-load/src/mapLayers/geoFeaturesSchema.js @@ -8,10 +8,19 @@ const GeometrySchema = new mongoose.Schema( { _id: false }, ); +const BboxSchema = new mongoose.Schema( + { + type: String, + coordinates: Array, + }, + { _id: false }, +); + const GeoFeaturesSchema = new mongoose.Schema({ type: String, properties: mongoose.Schema.Types.Mixed, geometry: GeometrySchema, + bbox: BboxSchema, }); module.exports = { diff --git a/initial-data-load/src/mapLayers/layersGeoDataUpload.js b/initial-data-load/src/mapLayers/layersGeoDataUpload.js index 3e4bc26..53d43a2 100644 --- a/initial-data-load/src/mapLayers/layersGeoDataUpload.js +++ b/initial-data-load/src/mapLayers/layersGeoDataUpload.js @@ -2,6 +2,8 @@ const fs = require('fs'); const path = require('path'); const yaml = require('js-yaml'); const axios = require('axios'); +const bbox = require('@turf/bbox'); + const logger = require('../config/winston'); const { @@ -15,7 +17,9 @@ const { saveGeoJsonFromUrlSourceToStorage, saveGeoJsonFromFileToStorage } = requ const storeGeoDataToDb = async (fromFile, data, filePath) => { let geojsonData; + logger.info(`Clearing database collection ${data.collectionName}`); await deleteAllFromCollection(data.collectionName); + logger.info(`Loading data from file ${filePath}`); if (fromFile) { geojsonData = JSON.parse(fs.readFileSync(filePath, 'utf8')); } else { @@ -26,8 +30,41 @@ const storeGeoDataToDb = async (fromFile, data, filePath) => { } geojsonData = result.data; } - await createGeoDataIndex(data.collectionName); - await storeGeoFeaturesData(geojsonData.features, data.collectionName); + logger.info(`Creating index on collection...`); + await createGeoDataIndex(data.collectionName, 'bbox'); + logger.info(`Generating bounding boxes for features...`); + + const { features } = geojsonData; + if (!features || !features.length) { + logger.info(`No features in file ${filePath}`); + return; + } + const featuresWithBbox = features.map((feature) => { + if (feature.geometry.type === 'Point') { + return { ...feature, bbox: feature.geometry }; + } + const [minX, minY, maxX, maxY] = bbox.default({ + type: 'FeatureCollection', + name: 'Polygon', + features: [feature], + }); + const generatedBBox = [ + [ + [minX, minY], + [minX, maxY], + [maxX, maxY], + [maxX, minY], + [minX, minY], + ], + ]; + return { + ...feature, + bbox: { type: 'Polygon', coordinates: generatedBBox }, + }; + }); + + logger.info(`Storing features into database...`); + await storeGeoFeaturesData(featuresWithBbox, data.collectionName); }; const formatLayerGeoData = async (data, country) => { @@ -40,7 +77,7 @@ const formatLayerGeoData = async (data, country) => { if (data.geoDataUrl) { logger.info(`Downloading geojson ${data.geoDataUrl} for layer ${data.referenceId}...`); if (data.storeToDb) { - await storeGeoDataToDb(false, data, null); + await storeGeoDataToDb(false, data, data.geoDataUrl); url = data.apiUrl; logger.info(`Layer ${data.name} has new URL for geodata: ${url}`); } else { diff --git a/initial-data-load/yarn.lock b/initial-data-load/yarn.lock index d645ff4..d2473d2 100644 --- a/initial-data-load/yarn.lock +++ b/initial-data-load/yarn.lock @@ -651,6 +651,26 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@turf/bbox@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@turf/bbox/-/bbox-6.5.0.tgz#bec30a744019eae420dac9ea46fb75caa44d8dc5" + integrity sha512-RBbLaao5hXTYyyg577iuMtDB8ehxMlUqHEJiMs8jT1GHkFhr6sYre3lmLsPeYEi/ZKj5TP5tt7fkzNdJ4GIVyw== + dependencies: + "@turf/helpers" "^6.5.0" + "@turf/meta" "^6.5.0" + +"@turf/helpers@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e" + integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw== + +"@turf/meta@^6.5.0": + version "6.5.0" + resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-6.5.0.tgz#b725c3653c9f432133eaa04d3421f7e51e0418ca" + integrity sha512-RrArvtsV0vdsCBegoBtOalgdSOfkBrTJ07VkpiCnq/491W67hnMWmDu7e6Ztw0C3WldRYTXkg3SumfdzZxLBHA== + dependencies: + "@turf/helpers" "^6.5.0" + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7": version "7.1.16" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.16.tgz#bc12c74b7d65e82d29876b5d0baf5c625ac58702"