Bbox in geo collection

functions/csvLoader/components/fileLoader.ts

@@ -1,12 +1,12 @@
+import { Context } from '@azure/functions';
+import csv2json from 'csvtojson';
+import { ItemFromFile } from '../types';
+
 /**
  * Load data from file based on file format
  * @param  {Buffer} incomingBlob - blob which triggered the function and is processed
  * @returns {Array<ItemFromFile>} - array with data from file
  */
-import { Context } from '@azure/functions';
-import csv2json from 'csvtojson';
-import { ItemFromFile } from '../types';
-
 const fileLoader = async (incomingBlob: Buffer, context: Context): Promise<Array<ItemFromFile>> => {
   let returnJson;
   const stringFromBuffer = incomingBlob.toString('utf8');

initial-data-load/.gitignore

@@ -4,3 +4,8 @@ node_modules
 .vscode
 spinup.sh
 .DS_Store
+
+/data
+!/data/Sample
+!/data/testCountry
+!/data/testCountry3

initial-data-load/package.json

@@ -8,6 +8,7 @@
   "dependencies": {
     "@azure/storage-queue": "^12.2.0",
     "@hapi/joi": "^17.1.1",
+    "@turf/bbox": "^6.5.0",
     "axios": "^0.21.2",
     "azure-storage": "^2.10.3",
     "bottleneck": "^2.19.5",

initial-data-load/src/mapLayers/db.js

@@ -10,8 +10,10 @@ const createIndex = async () => {
   await mongoose.connection.db.collection('mapLayers').createIndex({ title: 1 });
 };
 
-const createGeoDataIndex = async (collectionName) => {
-  await mongoose.connection.db.collection(collectionName).createIndex({ geometry: '2dsphere' });
+const createGeoDataIndex = async (collectionName, geoKey = 'geometry') => {
+  const indexKey = {};
+  indexKey[geoKey] = '2dsphere';
+  await mongoose.connection.db.collection(collectionName).createIndex(indexKey);
 };
 
 const deleteAllFromCollection = async (collectionName) => {

initial-data-load/src/mapLayers/geoFeaturesSchema.js

@@ -8,10 +8,19 @@ const GeometrySchema = new mongoose.Schema(
   { _id: false },
 );
 
+const BboxSchema = new mongoose.Schema(
+  {
+    type: String,
+    coordinates: Array,
+  },
+  { _id: false },
+);
+
 const GeoFeaturesSchema = new mongoose.Schema({
   type: String,
   properties: mongoose.Schema.Types.Mixed,
   geometry: GeometrySchema,
+  bbox: BboxSchema,
 });
 
 module.exports = {

initial-data-load/src/mapLayers/layersGeoDataUpload.js

@@ -2,6 +2,8 @@ const fs = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
 const axios = require('axios');
+const bbox = require('@turf/bbox');
+
 const logger = require('../config/winston');
 
 const {
@@ -15,7 +17,9 @@ const { saveGeoJsonFromUrlSourceToStorage, saveGeoJsonFromFileToStorage } = requ
 
 const storeGeoDataToDb = async (fromFile, data, filePath) => {
   let geojsonData;
+  logger.info(`Clearing database collection ${data.collectionName}`);
   await deleteAllFromCollection(data.collectionName);
+  logger.info(`Loading data from file ${filePath}`);
   if (fromFile) {
     geojsonData = JSON.parse(fs.readFileSync(filePath, 'utf8'));
   } else {
@@ -26,8 +30,41 @@ const storeGeoDataToDb = async (fromFile, data, filePath) => {
     }
     geojsonData = result.data;
   }
-  await createGeoDataIndex(data.collectionName);
-  await storeGeoFeaturesData(geojsonData.features, data.collectionName);
+  logger.info(`Creating index on collection...`);
+  await createGeoDataIndex(data.collectionName, 'bbox');
+  logger.info(`Generating bounding boxes for features...`);
+
+  const { features } = geojsonData;
+  if (!features || !features.length) {
+    logger.info(`No features in file ${filePath}`);
+    return;
+  }
+  const featuresWithBbox = features.map((feature) => {
+    if (feature.geometry.type === 'Point') {
+      return { ...feature, bbox: feature.geometry };
+    }
+    const [minX, minY, maxX, maxY] = bbox.default({
+      type: 'FeatureCollection',
+      name: 'Polygon',
+      features: [feature],
+    });
+    const generatedBBox = [
+      [
+        [minX, minY],
+        [minX, maxY],
+        [maxX, maxY],
+        [maxX, minY],
+        [minX, minY],
+      ],
+    ];
+    return {
+      ...feature,
+      bbox: { type: 'Polygon', coordinates: generatedBBox },
+    };
+  });
+
+  logger.info(`Storing features into database...`);
+  await storeGeoFeaturesData(featuresWithBbox, data.collectionName);
 };
 
 const formatLayerGeoData = async (data, country) => {
@@ -40,7 +77,7 @@ const formatLayerGeoData = async (data, country) => {
   if (data.geoDataUrl) {
     logger.info(`Downloading geojson ${data.geoDataUrl} for layer ${data.referenceId}...`);
     if (data.storeToDb) {
-      await storeGeoDataToDb(false, data, null);
+      await storeGeoDataToDb(false, data, data.geoDataUrl);
       url = data.apiUrl;
       logger.info(`Layer ${data.name} has new URL for geodata: ${url}`);
     } else {

initial-data-load/yarn.lock

@@ -651,6 +651,26 @@
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
   integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==
 
+"@turf/bbox@^6.5.0":
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/@turf/bbox/-/bbox-6.5.0.tgz#bec30a744019eae420dac9ea46fb75caa44d8dc5"
+  integrity sha512-RBbLaao5hXTYyyg577iuMtDB8ehxMlUqHEJiMs8jT1GHkFhr6sYre3lmLsPeYEi/ZKj5TP5tt7fkzNdJ4GIVyw==
+  dependencies:
+    "@turf/helpers" "^6.5.0"
+    "@turf/meta" "^6.5.0"
+
+"@turf/helpers@^6.5.0":
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/@turf/helpers/-/helpers-6.5.0.tgz#f79af094bd6b8ce7ed2bd3e089a8493ee6cae82e"
+  integrity sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw==
+
+"@turf/meta@^6.5.0":
+  version "6.5.0"
+  resolved "https://registry.yarnpkg.com/@turf/meta/-/meta-6.5.0.tgz#b725c3653c9f432133eaa04d3421f7e51e0418ca"
+  integrity sha512-RrArvtsV0vdsCBegoBtOalgdSOfkBrTJ07VkpiCnq/491W67hnMWmDu7e6Ztw0C3WldRYTXkg3SumfdzZxLBHA==
+  dependencies:
+    "@turf/helpers" "^6.5.0"
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7":
   version "7.1.16"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.16.tgz#bc12c74b7d65e82d29876b5d0baf5c625ac58702"