#112-Postgis-setup

.env.example

@@ -8,6 +8,12 @@ MONGO_URI=mongodb://oscar-db-service/oscar
 DB_NAME=oscar
 URL_FILE=ExampleDailyUrls.yml
 
+POSTGRES_USER=oscar
+POSTGRES_PASSWORD=example
+POSTGRES_DB=oscar
+PGADMIN_DEFAULT_EMAIL=admin@test.com
+PGADMIN_DEFAULT_PASSWORD=admin
+
 AUTHORIZE_TOKEN_ATTRIBUTE=false
 AUTHORIZATION_EXPECTED_TOKEN_ATTRIBUTE=
 AUTHORIZATION_EXPECTED_TOKEN_ATTRIBUTE_VALUE=

api/src/config/config.js

@@ -8,7 +8,7 @@ const envVarsSchema = Joi.object({
   NODE_ENV: Joi.string().allow('development', 'production', 'test', 'provision').default('development'),
   PORT: Joi.number().default(8080),
   LOG_LABEL: Joi.string().default('oscar-api'),
-  MONGO_URI: Joi.string(),
+  MONGO_URI: Joi.string().allow(''),
   DB_NAME: Joi.string(),
   AZURE_STORAGE_CONNECTION_STRING: Joi.string(),
   AZURE_STORAGE_LAYER_CONTAINER_NAME: Joi.string(),

doc/development/development.md

@@ -86,7 +86,7 @@ is a tool for defining and running multi-container Docker applications. With Doc
 
 1. Define your app’s environment with a Dockerfile
 2. Define the services that make up your app in docker-compose.yml
-3. Run `docker compose up` to start and run your app (that is done in ./start.sh script)
+3. Run `docker compose up` to start and run your app (that is done in ./start.sh script or in ./start_postgis.sh)
 
 We have more Docker Compose files in the Oscar application.
 
@@ -137,7 +137,7 @@ This project is based on the [create-react-app starter kit](https://github.com/f
 Locally the app runs in the Development Mode. In this mode performance is degraded but developer experience is best.
 
 - Run `yarn` to install all the packages
-- You can start only frontend with `yarn start` in ./frontend folder, but frontend is strongly data-based so for most functionality you need to run api also. This is done by simply starting all necessary services with `./start.sh`from root folder.
+- You can start only frontend with `yarn start` in ./frontend folder, but frontend is strongly data-based so for most functionality you need to run api also. This is done by simply starting all necessary services with `./start.sh` or `./start_postgis.sh` from root folder.
 
 Open [localhost on port 3000](http://localhost:3000) to view it in your browser. The page will reload when you make changes thanks to Hot reloading. You may also see any lint errors in the terminal. In Production mode code is transpiled to be compatible to older browser versions and the application is minimized, uglyfied and bundled into a single file for maximum performance. The production mode outputs static files which need to be hosted on a separate webserver.
 
@@ -166,6 +166,8 @@ Schema of endpoints can be found in [apiSchema.yml](../api/src/openapi/apiSchema
 
 ## Database
 
+Database used in local development is determined by running either `start.sh` script for running evironment with MongoDB or `start_postgis.sh` for running with Postgis.
+
 ### MongoDB
 
 For local development we are using [MongoDB](https://www.mongodb.com/). MongoDB is a general purpose distributed database that stores data in [JSON-like documents](https://docs.mongodb.com/manual/reference/bson-types/). Group of documents is called collection. MongoDb is running on localhost on port 27017. It is created and started as a Docker container when you use ./start.sh script to run the application. When you stop this script, database will be removed and all data deleted. For MongoDB administration we are using Mongo Express.
@@ -186,6 +188,19 @@ For easy access to database structure and quick editing of stored documents, we
 
 [Mongoose](https://mongoosejs.com/) is an Object Data Modeling (ODM) Node.js library that provides schema based translation of javascript objects to MongoDB documents and vice versa. We use it to communicate with MongoDB in our services api and initial-data-load.
 
+### Postgis
+
+For local development we also use dockerized [Postgis](https://postgis.net/) which is based on [Postgres](https://www.postgresql.org/) with added GIS pluggins. Locally it runs on port 5432. It is created and started as a Docker container when you use ./start_postgis.sh script to run the application. For Postgis administration we are using PgAdmin.
+
+### PgAdmin
+
+For easy access to database structure and quick editing of stored tables, we use web-based PostgreSQL [admin interface](https://www.pgadmin.org/) that is running on [localhost on port 8184](http://localhost:8184) after you start the application with ./start_postgis.sh script. You can use it to connect to multiple databases, view, add and delete databases etc.
+
+- Run `./start_postgis.sh` to start the app.
+- Log in using `admin@test.com` and `adminPass` defined in environmental variables.
+- In the admin you can see list of database servers.
+- Click on Test and under it click on oscar.
+
 ## Azure Blob Storage
 
 Azure Blob Storage is Microsoft's object storage solution for the cloud. Blob Storage is optimized for storing massive amounts of unstructured data, such as text or binary data. For more info check [documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-introduction#blob-storage). We are using it to store files of various types. For local development we are using its emulator called Azurite.
@@ -208,7 +223,7 @@ Azure Blob Storage is Microsoft's object storage solution for the cloud. Blob St
 
 Now you have successfully connected to the service in Storage account.
 
-1. In terminal start app with `./start.sh` a run `./runinitialload.sh` to have data in the Azurite.
+1. In terminal start app with `./start.sh` or `./start_postgis.sh` a run `./runinitialload.sh` to have data in the Azurite.
 2. Click on Toggle Explorer button in the left sidebar.
 3. In Microsoft Azure Storage Explorer click on Refresh All.
 4. Path to Blob Containers is Local & Attached -> Storage Accounts -> Display name.

doc/getting-started/run-application.md

@@ -37,6 +37,10 @@ The backend services and MongoDB database can be run in docker with frontend sta
 ```
 ./start.sh
 ```
+or
+```
+./start_postgis.sh
+```
 
 Use 'docker ps' to check on the running containers. You should see the following output:
 
@@ -49,7 +53,7 @@ e5f8694a9ae6   mongo:3.6.18-xenial                       "docker-entrypoint.s…
 237e52634948   mongo-express                             "tini -- /docker-ent…"   35 seconds ago   Up 30 seconds   0.0.0.0:8081->8081/tcp, :::8081->8081/tcp                               oscar_mongo-express_1
 ```
 
-After startup, there is a MongoDB explorer UI available here: http://localhost:8081/
+After startup, there is a MongoDB explorer UI available here: http://localhost:8081/ or PgAdmin http://localhost:8184/ where you can log in using `admin@test.com` and `adminPass` defined in environmental variables.
 Oscar application will start here: http://localhost:3000/
 
 ## Start the Application inside Minikube on Mac
@@ -82,6 +86,10 @@ Run the script below in new terminal tab
 cd ./minikube
 ./start.sh
 ```
+or
+```
+./start_postgis.sh
+```
 
 It will connect docker to the runtime in minikube and export minikube ip as env variable so it can be used by React proxy.
 
@@ -95,12 +103,22 @@ minikube ip
 ```
 and pasting provided ip in browser with port 8081
 
+Or if app is being run by `./start_postgis.sh` there is PgAdmin UI for Postgis database available by running
+```
+./pgadmin.sh
+```
+or by running command
+```
+minikube ip
+```
+and pasting provided ip in browser with port 8184, after that you can log in using `admin@test.com` and `adminPass` defined in environmental variables.
+
 - before running other services like initial-data-load in another terminal, docker needs to be connected to minikube by running
 ```
 eval $(minikube docker-env)
 ```
 
-- if React frontend will be started separately from `./start.sh` script minikube ip needs to be exported in that terminal window before running `yarn start`
+- if React frontend will be started separately from `./start.sh` or `./start_postgis.sh` script minikube ip needs to be exported in that terminal window before running `yarn start`
 ```
 export API_IP=$(minikube ip)
 cd ../frontend
@@ -117,7 +135,7 @@ yarn start
 minikube delete
 
 ## Unit testing services with minikube
-1. after starting app with `start.sh` in minikube folder
+1. after starting app with `start.sh` or `start_postgis.sh` in minikube folder
 2. open directory of service you want to test i.e. `api`
 3. run commands `eval $(minikube docker-env)` to connect to docker-env in minikube and `export MONGO_URI=mongodb://$(minikube ip):27017/testDb` to create new mongoDB in already running container
 4. run `yarn test` or `yarn test:coverage` or `yarn test:watch`

doc/services/initial-data-load.md

@@ -23,6 +23,6 @@ Initial data load service sets up database structure, indexes and storage struct
 - Geographical location data for map layer, that are stored in geojson format.
 - Data related to the point or area for map layer, that are stored in geojson or csv format.
 
-Once everything is set you can start the application with `start.sh` script, run this service (even more than once) with `runinitialload.sh` script and refresh web page in browser. You can see map layers in map view and graphs filled with data on the dashboard.
+Once everything is set you can start the application with `start.sh` script or `start_postgis.sh` for version with Postgis DB, run this service (even more than once) with `runinitialload.sh` script and refresh web page in browser. You can see map layers in map view and graphs filled with data on the dashboard.
 
 For a detailed description how to achieve this check [Run application with own data](../tutorials/run-application-with-own-data.md).

doc/tutorials/advanced/run-application-with-sample-data.md

@@ -27,7 +27,7 @@ Attributes folder has config file in javascript to easily add current dates in t
 If this is your first attempt to run the application, please consult [Getting started](../../getting-started/run-application.md) first. There are some necessary prerequisities.
 Now you are set and ready to run the scripts. As all files are already in place, all you need is run these two scripts:
 
-`./start.sh` will start the api, frontend and database. Once you can see the frontend, you can go to next script (don't worry about the empty menu and dashboard, it will be filled in next step).
+`./start.sh` or `./start_postgis.sh` will start the api, frontend and database. Once you can see the frontend, you can go to next script (don't worry about the empty menu and dashboard, it will be filled in next step).
 
 `./runinitialload.sh` Second script will run the initial-data-load service to fill application with data. Wait until this is finished (you will see `Successfully uploaded all initial data.` in console, where you run the script). Return to the browser and reload the page. Now you have the application set.
 

doc/tutorials/advanced/start-application-locally-for-development.md

@@ -33,6 +33,10 @@ This part is done in single start script:
 ```
 ./start.sh
 ```
+or
+```
+./start_postgis.sh
+```
 
 First, docker will fetch required images, build new images with our code and settings and create containers. The images are stored for later use, so next time you run this script, it will be faster.
 After all backend containers are started, it proceeds to frontend and opens the application directly in browser.

docker-compose.mongo.yml

@@ -0,0 +1,15 @@
+version: "3"
+services:
+  oscar-db-service:
+    image: mongo:3.6.18-xenial
+    hostname: oscar-db-service
+    ports:
+      - 27017:27017
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_SERVER: oscar-db-service

docker-compose.postgis.yml

@@ -0,0 +1,22 @@
+version: "3"
+services:
+  db-service:
+    image: postgis/postgis:13-3.2
+    hostname: db-service
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    ports:
+      - 5432:5432
+    restart: on-failure
+
+  db-ui:
+    build:
+      context: ./pgadmin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD}
+    ports:
+      - 8184:80
+    restart: on-failure

docker-compose.yml

@@ -33,20 +33,6 @@ services:
       AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;
       KOBO_CONNECTION_STRING: ${KOBO_CONNECTION_STRING}
 
-  oscar-db-service:
-    image: mongo:3.6.18-xenial
-    hostname: oscar-db-service
-    ports:
-      - 27017:27017
-
-  mongo-express:
-    image: mongo-express
-    restart: always
-    ports:
-      - 8081:8081
-    environment:
-      ME_CONFIG_MONGODB_SERVER: oscar-db-service
-
   azurite:
     image: mcr.microsoft.com/azure-storage/azurite
     hostname: azurite

minikube/.env.example

@@ -6,6 +6,13 @@ COUNTRY=Sample
 NEW_STORAGE_CONTAINERS=layer-geo-data,raw-data
 MONGO_URI=mongodb://oscar-db-service/oscar
 DB_NAME=oscar
+
+POSTGRES_USER=oscar
+POSTGRES_PASSWORD=example
+POSTGRES_DB=oscar
+PGADMIN_DEFAULT_EMAIL=admin@test.com
+PGADMIN_DEFAULT_PASSWORD=admin
+
 URL_FILE=ExampleDailyUrls.yml
 
 AUTHORIZE_TOKEN_ATTRIBUTE=false

minikube/docker-compose.mongo.yml

@@ -0,0 +1,15 @@
+version: "3"
+services:
+  oscar-db-service:
+    image: mongo:3.6.18-xenial
+    hostname: oscar-db-service
+    ports:
+      - 27017:27017
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_SERVER: oscar-db-service

minikube/docker-compose.postgis.yml

@@ -0,0 +1,22 @@
+version: "3"
+services:
+  db-service:
+    image: postgis/postgis:13-3.2
+    hostname: db-service
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    ports:
+      - 5432:5432
+    restart: on-failure
+
+  db-ui:
+    build:
+      context: ../pgadmin
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
+      PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD}
+    ports:
+      - 8184:80
+    restart: on-failure

minikube/docker-compose.yml

@@ -33,20 +33,6 @@ services:
       AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;
       KOBO_CONNECTION_STRING: ${KOBO_CONNECTION_STRING}
 
-  oscar-db-service:
-    image: mongo:3.6.18-xenial
-    hostname: oscar-db-service
-    ports:
-      - 27017:27017
-
-  mongo-express:
-    image: mongo-express
-    restart: always
-    ports:
-      - 8081:8081
-    environment:
-      ME_CONFIG_MONGODB_SERVER: oscar-db-service
-
   azurite:
     image: mcr.microsoft.com/azure-storage/azurite
     hostname: azurite

minikube/pgadmin.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+open http://$(minikube ip):8184

minikube/start.sh

@@ -10,7 +10,7 @@ eval $(minikube docker-env)
 export API_IP=$(minikube ip)
 
 set -e
-docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.proxy.yml up --build -d
+docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.mongo.yml -f docker-compose.proxy.yml up --build -docker
 
 # frontend
 cd ../frontend

minikube/start_postgis.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# generate env file if not exist
+[[ -e .env ]] || cp .env.example .env
+
+# Connect docker to the runtime in minikube:
+eval $(minikube docker-env)
+
+# export IP of minikube so React can setup correct proxy
+export API_IP=$(minikube ip)
+
+set -e
+docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.postgis.yml -f docker-compose.proxy.yml up --build -d
+
+# frontend
+cd ../frontend
+yarn start

pgadmin/Dockerfile

@@ -0,0 +1,3 @@
+FROM dpage/pgadmin4:6.4
+
+COPY --chown=pgadmin:pgadmin pgadmin4.db /var/lib/pgadmin/

pgadmin/README.md

@@ -0,0 +1,9 @@
+Contains configuration for the pgadmin tool used locally to view the postgresql data.
+
+The file is on git ignore with this command, as pgadmin changes it when loaded:
+
+```
+git update-index --skip-worktree pgadmin4.db
+```
+
+See https://stackoverflow.com/questions/4348590/how-can-i-make-git-ignore-future-revisions-to-a-file

start.sh

@@ -3,7 +3,7 @@
 
 #!/bin/bash
 set -e
-docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.proxy.yml up --build -d
+docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.mongo.yml -f docker-compose.proxy.yml up --build -docker
 
 # frontend
 cd ./frontend

start_postgis.sh

@@ -0,0 +1,10 @@
+# generate env file if not exist
+[[ -e .env ]] || cp .env.example .env
+
+#!/bin/bash
+set -e
+docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.postgis.yml -f docker-compose.proxy.yml up --build -d
+
+# frontend
+cd ./frontend
+yarn start