diff --git a/.env.example b/.env.example index 9967561..b6067d4 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,12 @@ MONGO_URI=mongodb://oscar-db-service/oscar DB_NAME=oscar URL_FILE=ExampleDailyUrls.yml +POSTGRES_USER=oscar +POSTGRES_PASSWORD=example +POSTGRES_DB=oscar +PGADMIN_DEFAULT_EMAIL=admin@test.com +PGADMIN_DEFAULT_PASSWORD=admin + AUTHORIZE_TOKEN_ATTRIBUTE=false AUTHORIZATION_EXPECTED_TOKEN_ATTRIBUTE= AUTHORIZATION_EXPECTED_TOKEN_ATTRIBUTE_VALUE= diff --git a/api/src/config/config.js b/api/src/config/config.js index 663650b..1502617 100644 --- a/api/src/config/config.js +++ b/api/src/config/config.js @@ -8,7 +8,7 @@ const envVarsSchema = Joi.object({ NODE_ENV: Joi.string().allow('development', 'production', 'test', 'provision').default('development'), PORT: Joi.number().default(8080), LOG_LABEL: Joi.string().default('oscar-api'), - MONGO_URI: Joi.string(), + MONGO_URI: Joi.string().allow(''), DB_NAME: Joi.string(), AZURE_STORAGE_CONNECTION_STRING: Joi.string(), AZURE_STORAGE_LAYER_CONTAINER_NAME: Joi.string(), diff --git a/doc/development/development.md b/doc/development/development.md index 1c68ade..b727505 100644 --- a/doc/development/development.md +++ b/doc/development/development.md @@ -86,7 +86,7 @@ is a tool for defining and running multi-container Docker applications. With Doc 1. Define your app’s environment with a Dockerfile 2. Define the services that make up your app in docker-compose.yml -3. Run `docker compose up` to start and run your app (that is done in ./start.sh script) +3. Run `docker compose up` to start and run your app (that is done in ./start.sh script or in ./start_postgis.sh) We have more Docker Compose files in the Oscar application. @@ -137,7 +137,7 @@ This project is based on the [create-react-app starter kit](https://github.com/f Locally the app runs in the Development Mode. In this mode performance is degraded but developer experience is best. - Run `yarn` to install all the packages -- You can start only frontend with `yarn start` in ./frontend folder, but frontend is strongly data-based so for most functionality you need to run api also. This is done by simply starting all necessary services with `./start.sh`from root folder. +- You can start only frontend with `yarn start` in ./frontend folder, but frontend is strongly data-based so for most functionality you need to run api also. This is done by simply starting all necessary services with `./start.sh` or `./start_postgis.sh` from root folder. Open [localhost on port 3000](http://localhost:3000) to view it in your browser. The page will reload when you make changes thanks to Hot reloading. You may also see any lint errors in the terminal. In Production mode code is transpiled to be compatible to older browser versions and the application is minimized, uglyfied and bundled into a single file for maximum performance. The production mode outputs static files which need to be hosted on a separate webserver. @@ -166,6 +166,8 @@ Schema of endpoints can be found in [apiSchema.yml](../api/src/openapi/apiSchema ## Database +Database used in local development is determined by running either `start.sh` script for running environment with MongoDB or `start_postgis.sh` for running with Postgis. + ### MongoDB For local development we are using [MongoDB](https://www.mongodb.com/). MongoDB is a general purpose distributed database that stores data in [JSON-like documents](https://docs.mongodb.com/manual/reference/bson-types/). Group of documents is called collection. MongoDb is running on localhost on port 27017. It is created and started as a Docker container when you use ./start.sh script to run the application. When you stop this script, database will be removed and all data deleted. For MongoDB administration we are using Mongo Express. @@ -186,6 +188,19 @@ For easy access to database structure and quick editing of stored documents, we [Mongoose](https://mongoosejs.com/) is an Object Data Modeling (ODM) Node.js library that provides schema based translation of javascript objects to MongoDB documents and vice versa. We use it to communicate with MongoDB in our services api and initial-data-load. +### Postgis + +For local development we also use dockerized [Postgis](https://postgis.net/) which is based on [Postgres](https://www.postgresql.org/) with added GIS pluggins. Locally it runs on port 5432. It is created and started as a Docker container when you use ./start_postgis.sh script to run the application. For Postgis administration we are using PgAdmin. + +### PgAdmin + +For easy access to database structure and quick editing of stored tables, we use web-based PostgreSQL [admin interface](https://www.pgadmin.org/) that is running on [localhost on port 8184](http://localhost:8184) after you start the application with ./start_postgis.sh script. You can use it to connect to multiple databases, view, add and delete databases etc. + +- Run `./start_postgis.sh` to start the app. +- Log in using `admin@test.com` and `adminPass` defined in environmental variables. +- In the admin you can see list of database servers. +- Click on Test and under it click on oscar. + ## Azure Blob Storage Azure Blob Storage is Microsoft's object storage solution for the cloud. Blob Storage is optimized for storing massive amounts of unstructured data, such as text or binary data. For more info check [documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-introduction#blob-storage). We are using it to store files of various types. For local development we are using its emulator called Azurite. @@ -208,7 +223,7 @@ Azure Blob Storage is Microsoft's object storage solution for the cloud. Blob St Now you have successfully connected to the service in Storage account. -1. In terminal start app with `./start.sh` a run `./runinitialload.sh` to have data in the Azurite. +1. In terminal start app with `./start.sh` or `./start_postgis.sh` a run `./runinitialload.sh` to have data in the Azurite. 2. Click on Toggle Explorer button in the left sidebar. 3. In Microsoft Azure Storage Explorer click on Refresh All. 4. Path to Blob Containers is Local & Attached -> Storage Accounts -> Display name. diff --git a/doc/getting-started/run-application.md b/doc/getting-started/run-application.md index 75e494d..8f1b9cf 100644 --- a/doc/getting-started/run-application.md +++ b/doc/getting-started/run-application.md @@ -37,6 +37,10 @@ The backend services and MongoDB database can be run in docker with frontend sta ``` ./start.sh ``` +or +``` +./start_postgis.sh +``` Use 'docker ps' to check on the running containers. You should see the following output: @@ -49,7 +53,7 @@ e5f8694a9ae6 mongo:3.6.18-xenial "docker-entrypoint.s… 237e52634948 mongo-express "tini -- /docker-ent…" 35 seconds ago Up 30 seconds 0.0.0.0:8081->8081/tcp, :::8081->8081/tcp oscar_mongo-express_1 ``` -After startup, there is a MongoDB explorer UI available here: http://localhost:8081/ +After startup, there is a MongoDB explorer UI available here: http://localhost:8081/ or PgAdmin http://localhost:8184/ where you can log in using `admin@test.com` and `adminPass` defined in environmental variables. Oscar application will start here: http://localhost:3000/ ## Start the Application inside Minikube on Mac @@ -82,6 +86,10 @@ Run the script below in new terminal tab cd ./minikube ./start.sh ``` +or +``` +./start_postgis.sh +``` It will connect docker to the runtime in minikube and export minikube ip as env variable so it can be used by React proxy. @@ -95,12 +103,22 @@ minikube ip ``` and pasting provided ip in browser with port 8081 +Or if app is being run by `./start_postgis.sh` there is PgAdmin UI for Postgis database available by running +``` +./pgadmin.sh +``` +or by running command +``` +minikube ip +``` +and pasting provided ip in browser with port 8184, after that you can log in using `admin@test.com` and `adminPass` defined in environmental variables. + - before running other services like initial-data-load in another terminal, docker needs to be connected to minikube by running ``` eval $(minikube docker-env) ``` -- if React frontend will be started separately from `./start.sh` script minikube ip needs to be exported in that terminal window before running `yarn start` +- if React frontend will be started separately from `./start.sh` or `./start_postgis.sh` script minikube ip needs to be exported in that terminal window before running `yarn start` ``` export API_IP=$(minikube ip) cd ../frontend @@ -117,7 +135,7 @@ yarn start minikube delete ## Unit testing services with minikube -1. after starting app with `start.sh` in minikube folder +1. after starting app with `start.sh` or `start_postgis.sh` in minikube folder 2. open directory of service you want to test i.e. `api` 3. run commands `eval $(minikube docker-env)` to connect to docker-env in minikube and `export MONGO_URI=mongodb://$(minikube ip):27017/testDb` to create new mongoDB in already running container 4. run `yarn test` or `yarn test:coverage` or `yarn test:watch` diff --git a/doc/services/initial-data-load.md b/doc/services/initial-data-load.md index bbfcd30..84b4aed 100644 --- a/doc/services/initial-data-load.md +++ b/doc/services/initial-data-load.md @@ -23,6 +23,6 @@ Initial data load service sets up database structure, indexes and storage struct - Geographical location data for map layer, that are stored in geojson format. - Data related to the point or area for map layer, that are stored in geojson or csv format. -Once everything is set you can start the application with `start.sh` script, run this service (even more than once) with `runinitialload.sh` script and refresh web page in browser. You can see map layers in map view and graphs filled with data on the dashboard. +Once everything is set you can start the application with `start.sh` script or `start_postgis.sh` for version with Postgis DB, run this service (even more than once) with `runinitialload.sh` script and refresh web page in browser. You can see map layers in map view and graphs filled with data on the dashboard. For a detailed description how to achieve this check [Run application with own data](../tutorials/run-application-with-own-data.md). diff --git a/doc/tutorials/advanced/run-application-with-sample-data.md b/doc/tutorials/advanced/run-application-with-sample-data.md index 48bd80c..aa1508b 100644 --- a/doc/tutorials/advanced/run-application-with-sample-data.md +++ b/doc/tutorials/advanced/run-application-with-sample-data.md @@ -27,7 +27,7 @@ Attributes folder has config file in javascript to easily add current dates in t If this is your first attempt to run the application, please consult [Getting started](../../getting-started/run-application.md) first. There are some necessary prerequisities. Now you are set and ready to run the scripts. As all files are already in place, all you need is run these two scripts: -`./start.sh` will start the api, frontend and database. Once you can see the frontend, you can go to next script (don't worry about the empty menu and dashboard, it will be filled in next step). +`./start.sh` or `./start_postgis.sh` will start the api, frontend and database. Once you can see the frontend, you can go to next script (don't worry about the empty menu and dashboard, it will be filled in next step). `./runinitialload.sh` Second script will run the initial-data-load service to fill application with data. Wait until this is finished (you will see `Successfully uploaded all initial data.` in console, where you run the script). Return to the browser and reload the page. Now you have the application set. diff --git a/doc/tutorials/advanced/start-application-locally-for-development.md b/doc/tutorials/advanced/start-application-locally-for-development.md index 8268469..190c12d 100644 --- a/doc/tutorials/advanced/start-application-locally-for-development.md +++ b/doc/tutorials/advanced/start-application-locally-for-development.md @@ -33,6 +33,10 @@ This part is done in single start script: ``` ./start.sh ``` +or +``` +./start_postgis.sh +``` First, docker will fetch required images, build new images with our code and settings and create containers. The images are stored for later use, so next time you run this script, it will be faster. After all backend containers are started, it proceeds to frontend and opens the application directly in browser. diff --git a/docker-compose.mongo.yml b/docker-compose.mongo.yml new file mode 100644 index 0000000..3b8d050 --- /dev/null +++ b/docker-compose.mongo.yml @@ -0,0 +1,15 @@ +version: "3" +services: + oscar-db-service: + image: mongo:3.6.18-xenial + hostname: oscar-db-service + ports: + - 27017:27017 + + mongo-express: + image: mongo-express + restart: always + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_SERVER: oscar-db-service diff --git a/docker-compose.postgis.yml b/docker-compose.postgis.yml new file mode 100644 index 0000000..ce30b01 --- /dev/null +++ b/docker-compose.postgis.yml @@ -0,0 +1,22 @@ +version: "3" +services: + db-service: + image: postgis/postgis:13-3.2 + hostname: db-service + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - 5432:5432 + restart: on-failure + + db-ui: + build: + context: ./pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD} + ports: + - 8184:80 + restart: on-failure diff --git a/docker-compose.yml b/docker-compose.yml index b130701..d4f7096 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,20 +33,6 @@ services: AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1; KOBO_CONNECTION_STRING: ${KOBO_CONNECTION_STRING} - oscar-db-service: - image: mongo:3.6.18-xenial - hostname: oscar-db-service - ports: - - 27017:27017 - - mongo-express: - image: mongo-express - restart: always - ports: - - 8081:8081 - environment: - ME_CONFIG_MONGODB_SERVER: oscar-db-service - azurite: image: mcr.microsoft.com/azure-storage/azurite hostname: azurite diff --git a/minikube/.env.example b/minikube/.env.example index 9967561..c18b7aa 100644 --- a/minikube/.env.example +++ b/minikube/.env.example @@ -6,6 +6,13 @@ COUNTRY=Sample NEW_STORAGE_CONTAINERS=layer-geo-data,raw-data MONGO_URI=mongodb://oscar-db-service/oscar DB_NAME=oscar + +POSTGRES_USER=oscar +POSTGRES_PASSWORD=example +POSTGRES_DB=oscar +PGADMIN_DEFAULT_EMAIL=admin@test.com +PGADMIN_DEFAULT_PASSWORD=admin + URL_FILE=ExampleDailyUrls.yml AUTHORIZE_TOKEN_ATTRIBUTE=false diff --git a/minikube/docker-compose.mongo.yml b/minikube/docker-compose.mongo.yml new file mode 100644 index 0000000..3b8d050 --- /dev/null +++ b/minikube/docker-compose.mongo.yml @@ -0,0 +1,15 @@ +version: "3" +services: + oscar-db-service: + image: mongo:3.6.18-xenial + hostname: oscar-db-service + ports: + - 27017:27017 + + mongo-express: + image: mongo-express + restart: always + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_SERVER: oscar-db-service diff --git a/minikube/docker-compose.postgis.yml b/minikube/docker-compose.postgis.yml new file mode 100644 index 0000000..3af1d35 --- /dev/null +++ b/minikube/docker-compose.postgis.yml @@ -0,0 +1,22 @@ +version: "3" +services: + db-service: + image: postgis/postgis:13-3.2 + hostname: db-service + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + ports: + - 5432:5432 + restart: on-failure + + db-ui: + build: + context: ../pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD} + ports: + - 8184:80 + restart: on-failure diff --git a/minikube/docker-compose.yml b/minikube/docker-compose.yml index e794630..5fd6114 100644 --- a/minikube/docker-compose.yml +++ b/minikube/docker-compose.yml @@ -33,20 +33,6 @@ services: AzureWebJobsStorage: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1; KOBO_CONNECTION_STRING: ${KOBO_CONNECTION_STRING} - oscar-db-service: - image: mongo:3.6.18-xenial - hostname: oscar-db-service - ports: - - 27017:27017 - - mongo-express: - image: mongo-express - restart: always - ports: - - 8081:8081 - environment: - ME_CONFIG_MONGODB_SERVER: oscar-db-service - azurite: image: mcr.microsoft.com/azure-storage/azurite hostname: azurite diff --git a/minikube/pgadmin.sh b/minikube/pgadmin.sh new file mode 100755 index 0000000..6a63d76 --- /dev/null +++ b/minikube/pgadmin.sh @@ -0,0 +1,2 @@ +#!/bin/bash +open http://$(minikube ip):8184 diff --git a/minikube/start.sh b/minikube/start.sh index 2c97862..29a52f8 100755 --- a/minikube/start.sh +++ b/minikube/start.sh @@ -10,7 +10,7 @@ eval $(minikube docker-env) export API_IP=$(minikube ip) set -e -docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.proxy.yml up --build -d +docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.mongo.yml -f docker-compose.proxy.yml up --build -docker # frontend cd ../frontend diff --git a/minikube/start_postgis.sh b/minikube/start_postgis.sh new file mode 100755 index 0000000..dd96eac --- /dev/null +++ b/minikube/start_postgis.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# generate env file if not exist +[[ -e .env ]] || cp .env.example .env + +# Connect docker to the runtime in minikube: +eval $(minikube docker-env) + +# export IP of minikube so React can setup correct proxy +export API_IP=$(minikube ip) + +set -e +docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.postgis.yml -f docker-compose.proxy.yml up --build -d + +# frontend +cd ../frontend +yarn start diff --git a/pgadmin/Dockerfile b/pgadmin/Dockerfile new file mode 100644 index 0000000..ad1bdbb --- /dev/null +++ b/pgadmin/Dockerfile @@ -0,0 +1,3 @@ +FROM dpage/pgadmin4:6.4 + +COPY --chown=pgadmin:pgadmin pgadmin4.db /var/lib/pgadmin/ diff --git a/pgadmin/README.md b/pgadmin/README.md new file mode 100644 index 0000000..31065d3 --- /dev/null +++ b/pgadmin/README.md @@ -0,0 +1,9 @@ +Contains configuration for the pgadmin tool used locally to view the postgresql data. + +The file is on git ignore with this command, as pgadmin changes it when loaded: + +``` +git update-index --skip-worktree pgadmin4.db +``` + +See https://stackoverflow.com/questions/4348590/how-can-i-make-git-ignore-future-revisions-to-a-file diff --git a/pgadmin/pgadmin4.db b/pgadmin/pgadmin4.db new file mode 100644 index 0000000..a021b11 Binary files /dev/null and b/pgadmin/pgadmin4.db differ diff --git a/start.sh b/start.sh index 1cb5d4a..af3c699 100755 --- a/start.sh +++ b/start.sh @@ -3,7 +3,7 @@ #!/bin/bash set -e -docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.proxy.yml up --build -d +docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.mongo.yml -f docker-compose.proxy.yml up --build -docker # frontend cd ./frontend diff --git a/start_postgis.sh b/start_postgis.sh new file mode 100755 index 0000000..e34ef4d --- /dev/null +++ b/start_postgis.sh @@ -0,0 +1,10 @@ +# generate env file if not exist +[[ -e .env ]] || cp .env.example .env + +#!/bin/bash +set -e +docker-compose down --remove-orphans && docker-compose -f docker-compose.yml -f docker-compose.postgis.yml -f docker-compose.proxy.yml up --build -d + +# frontend +cd ./frontend +yarn start