From ec357d5d7f4902747c11cf9489613a06ec6ef48e Mon Sep 17 00:00:00 2001 From: Daniela Barborova Date: Tue, 9 Aug 2022 15:41:36 +0200 Subject: [PATCH] #140-geodata route for fetching values from geodata tables in db --- api/src/database/geodata.ts | 45 +++++++++++ .../database/mongoDb/models/geoDataModel.ts | 64 +++++++++++++++ api/src/database/postgis/filters.ts | 34 +++++++- .../database/postgis/models/geoDataModel.ts | 57 ++++++++++++++ .../postgis/models/pointAttributesModel.ts | 31 +------- api/src/openapi/apiSchema.yml | 78 +++++++++++++++++++ api/src/routes/geodata.ts | 69 ++++++++++++++++ api/src/routes/index.ts | 2 + .../src/database/postgis/index.ts | 7 +- 9 files changed, 356 insertions(+), 31 deletions(-) create mode 100644 api/src/database/geodata.ts create mode 100644 api/src/database/mongoDb/models/geoDataModel.ts create mode 100644 api/src/database/postgis/models/geoDataModel.ts create mode 100644 api/src/routes/geodata.ts diff --git a/api/src/database/geodata.ts b/api/src/database/geodata.ts new file mode 100644 index 0000000..e9bb75c --- /dev/null +++ b/api/src/database/geodata.ts @@ -0,0 +1,45 @@ +import config from '../config/config'; +import APIError from '../helpers/APIError'; + +import mongoDb from './mongoDb/models/geoDataModel'; +import postgis from './postgis/models/geoDataModel'; + +export const getGeoData = async (tableName: string, bottomLeft: string, topRight: string, proj: string) => { + if (config.postgresUser && config.postgresPassword && config.postgresDb) { + return postgis.getGeoData(tableName, bottomLeft, topRight, proj); + } + if (config.mongoUri) { + return mongoDb.getGeoData(tableName, bottomLeft, topRight); + } + throw new APIError('No credentials for database', 500, false, undefined); +}; + +export const getProperty = async (tableName, propertyName) => { + if (config.postgresUser && config.postgresPassword && config.postgresDb) { + return postgis.getProperty(tableName, propertyName); + } + if (config.mongoUri) { + return mongoDb.getProperty(tableName, propertyName); + } +}; + +export const getUniqueValuesForProperty = async (tableName, propertyName) => { + if (config.postgresUser && config.postgresPassword && config.postgresDb) { + return postgis.getUniqueValuesForProperty(tableName, propertyName); + } + if (config.mongoUri) { + return mongoDb.getUniqueValuesForProperty(tableName, propertyName); + } + throw new APIError('No credentials for database', 500, false, undefined); +}; + +export const getPropertySum = async (tableName, propertyName) => { + if (config.postgresUser && config.postgresPassword && config.postgresDb) { + return postgis.getPropertySum(tableName, propertyName); + } + if (config.mongoUri) { + return mongoDb.getPropertySum(tableName, propertyName); + } +}; + +export default { getGeoData, getProperty, getUniqueValuesForProperty, getPropertySum }; diff --git a/api/src/database/mongoDb/models/geoDataModel.ts b/api/src/database/mongoDb/models/geoDataModel.ts new file mode 100644 index 0000000..8a827d4 --- /dev/null +++ b/api/src/database/mongoDb/models/geoDataModel.ts @@ -0,0 +1,64 @@ +import mongoose from 'mongoose'; +import { filterCoordinates } from '../filters'; + +const getGeoData = async (tableName: string, bottomLeft: string, topRight: string) => { + const { connection } = mongoose; + const { db } = connection; + + let filter = {}; + if (bottomLeft && topRight) { + filter = { bbox: filterCoordinates(filter, bottomLeft, topRight) }; + } + + const features = await db + .collection(tableName) + .aggregate([ + { $match: filter }, + { + $project: { + _id: 0, + id: '$_id', + type: 1, + properties: 1, + geometry: 1, + }, + }, + ]) + .toArray(); + return features; +}; + +const getProperty = async (tableName: string, propertyName: string) => { + const { connection } = mongoose; + const { db } = connection; + + const data = await db + .collection(tableName) + .aggregate([{ $project: { _id: 0, id: `$_id`, value: `$properties.${propertyName}` } }]) + .toArray(); + return data; +}; + +const getUniqueValuesForProperty = async (tableName: string, propertyName: string) => { + const { connection } = mongoose; + const { db } = connection; + + const data = await db + .collection(tableName) + .aggregate([{ $group: { _id: `$properties.${propertyName}` } }]) + .toArray(); + return data.map((item) => item._id); +}; + +const getPropertySum = async (tableName: string, propertyName: string) => { + const { connection } = mongoose; + const { db } = connection; + + const data = await db + .collection(tableName) + .aggregate([{ $group: { _id: null, value: { $sum: `$properties.${propertyName}` } } }]) + .toArray(); + return data; +}; + +export default { getGeoData, getProperty, getUniqueValuesForProperty, getPropertySum }; diff --git a/api/src/database/postgis/filters.ts b/api/src/database/postgis/filters.ts index 375ae83..42c2e15 100644 --- a/api/src/database/postgis/filters.ts +++ b/api/src/database/postgis/filters.ts @@ -1,3 +1,6 @@ +import { getDb } from './index'; +import APIError from '../../helpers/APIError'; + /** * Return bounding box in GeoJSON format used for filtering * @param {string} bottomLeft - bottom left corner of boundingBox window, string 'lon,lat' @@ -23,4 +26,33 @@ export const getBoundingBox = (bottomLeft: string, topRight: string) => { return geometry; }; -export default { getBoundingBox }; +/** + * Connect to database table to verify that projection of geometry data from API is compatible with table. + * @param {string} proj - projection of data from API + * @param {string} tableName - name of table with geographical data + * @param {} db=getDb() - database connection or transaction + */ +export const getProjectionFilter = async (proj, tableName, db = getDb()) => { + // projection in query must correspond to SRID in geometry column + if (proj) { + const SRIDarr = await db.distinct(db.raw(`ST_SRID(${tableName}.geometry)`)).from(tableName); + + if (!SRIDarr.length) { + return; + } + const SRID = SRIDarr[0].st_srid; + const projNum = parseInt(proj.split(':')[1], 10); + + if (SRID === projNum) { + return projNum; + } + throw new APIError( + `Projection SRID ${projNum} doesn't correspond to geometry column SRID ${SRID}`, + 400, + true, + undefined, + ); + } +}; + +export default { getBoundingBox, getProjectionFilter }; diff --git a/api/src/database/postgis/models/geoDataModel.ts b/api/src/database/postgis/models/geoDataModel.ts new file mode 100644 index 0000000..5b0072e --- /dev/null +++ b/api/src/database/postgis/models/geoDataModel.ts @@ -0,0 +1,57 @@ +/* eslint-disable no-unused-vars */ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { getDb } from '../index'; +import { getBoundingBox, getProjectionFilter } from '../filters'; +import APIError from '../../../helpers/APIError'; + +const getGeoData = async (tableName: string, bottomLeft?: string, topRight?: string, proj?: string, db = getDb()) => { + const filter: { + geometry?: { + type: string, + coordinates: Array>>, + }, + proj?: number, + } = {}; + if (bottomLeft && topRight) { + filter.geometry = getBoundingBox(bottomLeft, topRight); + filter.proj = await getProjectionFilter(proj, tableName); + } + + const data = await db + .from(tableName) + .select('id', db.raw('ST_AsGeoJSON(geometry) as geometry'), 'properties') + .where((qb) => { + if (filter.geometry && filter.proj) { + qb.andWhere( + db.raw(`ST_Intersects(geometry, ST_SetSRID(ST_GeomFromGeoJSON(?), ${filter.proj}))`, [ + JSON.stringify(filter.geometry), + ]), + ); + } + }); + return data.map((item) => ({ ...item, geometry: JSON.parse(item.geometry) })); +}; + +const getProperty = async (tableName: string, propertyName: string, db = getDb()) => { + const data = await db.from(tableName).select('id', db.raw(`properties->>'${propertyName}' as value`)); + return data; +}; + +const getUniqueValuesForProperty = async (tableName: string, propertyName: string, db = getDb()) => { + const data = await db + .from(tableName) + .select(db.raw(`properties->>'${propertyName}' as property`)) + .distinct(db.raw(`properties->>'${propertyName}'`)); + return data.map((item) => item.property); +}; + +const getPropertySum = async (tableName: string, propertyName: string, db = getDb()) => { + try { + const data = await db.from(tableName).sum(db.raw(`properties->>'${propertyName}'`)); + return data; + } catch (error) { + throw new APIError(`Attempting to sum non-numeric values for property ${propertyName}.`, 500, false, error); + } +}; + +export default { getGeoData, getProperty, getUniqueValuesForProperty, getPropertySum }; diff --git a/api/src/database/postgis/models/pointAttributesModel.ts b/api/src/database/postgis/models/pointAttributesModel.ts index 9100c91..f2ef7f5 100644 --- a/api/src/database/postgis/models/pointAttributesModel.ts +++ b/api/src/database/postgis/models/pointAttributesModel.ts @@ -1,7 +1,7 @@ import { Knex } from 'knex'; import APIError from '../../../helpers/APIError'; import { POINT_ATTRIBUTES_TABLE } from '../constants'; -import { getBoundingBox } from '../filters'; +import { getBoundingBox, getProjectionFilter } from '../filters'; import { dateIsValidDatum } from '../../../helpers/utils'; import { getDb } from '../index'; import { PointAttributeFilter, PointAttribute } from '../../../types'; @@ -37,31 +37,6 @@ const getAttributesFilterConditions = (filter: PointAttributeFilter, qb: Knex.Qu } }; -const getProjectionFilter = async (proj, db = getDb()) => { - // projection in query must correspond to SRID in geometry column - if (proj) { - const SRIDarr = await db - .distinct(db.raw(`ST_SRID(${POINT_ATTRIBUTES_TABLE}.geometry)`)) - .from(POINT_ATTRIBUTES_TABLE); - - if (!SRIDarr.length) { - return; - } - const SRID = SRIDarr[0].st_srid; - const projNum = parseInt(proj.split(':')[1], 10); - - if (SRID === projNum) { - return projNum; - } - throw new APIError( - `Projection SRID ${projNum} doesn't correspond to geometry column SRID ${SRID}`, - 400, - true, - undefined, - ); - } -}; - /** * Compose filter from settings from query * @param {string} attributeId - id of point attribute @@ -88,7 +63,7 @@ const getFilteredPointAttributes = async ( } if (filter.geometry) { - filter.proj = await getProjectionFilter(proj); + filter.proj = await getProjectionFilter(proj, POINT_ATTRIBUTES_TABLE); } // date filters @@ -154,7 +129,7 @@ const getLastDatePointAttributes = async ( filter.attributeId = attributeId; } if (filter.geometry) { - filter.proj = await getProjectionFilter(proj); + filter.proj = await getProjectionFilter(proj, POINT_ATTRIBUTES_TABLE); } const lastDate = await db diff --git a/api/src/openapi/apiSchema.yml b/api/src/openapi/apiSchema.yml index 0f01c3e..b783b29 100644 --- a/api/src/openapi/apiSchema.yml +++ b/api/src/openapi/apiSchema.yml @@ -186,6 +186,84 @@ paths: "200": description: Successful response + /api/geodata/{tableName}: + get: + summary: Get geodata stored in database table + description: Get all geodata stored in database table + parameters: + - in: path + name: tableName + schema: + type: string + description: name of table + required: true + responses: + 200: + description: Successful response + + /api/geodata/{tableName}/properties/{propertyName}: + get: + summary: Get all values for one property + description: Get all values for one property + parameters: + - in: path + name: tableName + schema: + type: string + description: name of table + required: true + - in: path + name: propertyName + schema: + type: string + description: name of property + required: true + responses: + 200: + description: Successful response + + /api/geodata/{tableName}/uniqueProperties/{propertyName}: + get: + summary: Get all unique values for one property + description: Get all unique values for one property + parameters: + - in: path + name: tableName + schema: + type: string + description: name of table + required: true + - in: path + name: propertyName + schema: + type: string + description: name of property + required: true + responses: + 200: + description: Successfull response + + /api/geodata/{tableName}/properties/{propertyName}/sum: + get: + summary: Get sum of all values for property + description: Get sum of all values for property + parameters: + - in: path + name: tableName + schema: + type: string + description: name of table + required: true + - in: path + name: propertyName + schema: + type: string + description: name of property + required: true + responses: + 200: + description: Successful resposne + /api/staticLayers: get: summary: Gets static layers data diff --git a/api/src/routes/geodata.ts b/api/src/routes/geodata.ts new file mode 100644 index 0000000..2f919c5 --- /dev/null +++ b/api/src/routes/geodata.ts @@ -0,0 +1,69 @@ +import express from 'express'; +import swaggerValidation from '../config/swagger'; +import utils from '../helpers/utils'; +import { getGeoData, getProperty, getUniqueValuesForProperty, getPropertySum } from '../database/geodata'; + +const router = express.Router(); + +router.get( + '/:tableName', + swaggerValidation.validate, + utils.forwardError(async (req, res) => { + const { bottomLeft, topRight, proj } = req.query; + const { tableName } = req.params; + + const features = await getGeoData(tableName, bottomLeft, topRight, proj); + if (features && features.length) { + res.send({ + type: 'FeatureCollection', + name: tableName, + features, + }); + return; + } + res.send({ + type: 'FeatureCollection', + name: tableName, + features: [], + }); + }), +); + +router.get( + '/:tableName/properties/:propertyName', + swaggerValidation.validate, + utils.forwardError(async (req, res) => { + const { tableName, propertyName } = req.params; + + const data = await getProperty(tableName, propertyName); + + res.send(data); + }), +); + +router.get( + '/:tableName/uniqueProperties/:propertyName', + swaggerValidation.validate, + utils.forwardError(async (req, res) => { + const { tableName, propertyName } = req.params; + + let dataArray = []; + dataArray = await getUniqueValuesForProperty(tableName, propertyName); + + res.send(dataArray); + }), +); + +router.get( + '/:tableName/properties/:propertyName/sum', + swaggerValidation.validate, + utils.forwardError(async (req, res) => { + const { tableName, propertyName } = req.params; + + const data = await getPropertySum(tableName, propertyName); + + res.send(data); + }), +); + +export default router; diff --git a/api/src/routes/index.ts b/api/src/routes/index.ts index 40089d7..07bed74 100644 --- a/api/src/routes/index.ts +++ b/api/src/routes/index.ts @@ -2,6 +2,7 @@ import express from 'express'; import dataLayers from './dataLayers'; import attributes from './attributes'; import pointAttributes from './pointAttributes'; +import geodata from './geodata'; import uploads from './uploads'; import authorization from './authorization'; import config from './config'; @@ -12,6 +13,7 @@ router.use('/dataLayers', dataLayers); router.use('/staticLayers', dataLayers); router.use('/attributes', attributes); router.use('/pointAttributes', pointAttributes); +router.use('/geodata', geodata); router.use('/uploads', uploads); router.use('/authorization', authorization); router.use('/config', config); diff --git a/initial-data-load/src/database/postgis/index.ts b/initial-data-load/src/database/postgis/index.ts index e6fb278..9232d30 100644 --- a/initial-data-load/src/database/postgis/index.ts +++ b/initial-data-load/src/database/postgis/index.ts @@ -33,13 +33,16 @@ export const disconnect = async () => { export const checkIfTableExists = (tableName: string, db = getDb()) => db.schema.hasTable(tableName); -export const createGeometryTable = (tableName: string, db = getDb()) => - db.schema.createTable(tableName, (table) => { +export const createGeometryTable = async (tableName: string, db = getDb()) => { + await db.raw('create extension if not exists "uuid-ossp"'); + return db.schema.createTable(tableName, (table) => { + table.uuid('id', { primaryKey: true }).defaultTo(knex.raw('uuid_generate_v4()')); table.text('type'); table.json('properties'); table.geometry('geometry'); table.geometry('bbox'); }); +}; /** * Clear all rows from table