DDLS-379 add a read only user (#1703)

* DDLS-379 add a read only user
* dockerise sql run command

Makefile

@@ -197,5 +197,25 @@ resilience-tests: ##@resilience-tests Run resilience tests (requires app to be u
 	docker compose build orchestration
 	docker compose run -e LOG_AND_CONTINUE=true --remove-orphans orchestration sh tests/run-resilience-tests.sh
 
+sql-custom-command-insert: ##@sql-custom-command Run SQL insert custom command
+	docker compose -f docker-compose.commands.yml build sql-custom-command
+	docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) insert --sql_file=_run.sql --verification_sql_file=_verification.sql --expected_before=$(before) --expected_after=$(after)
+
+sql-custom-command-get: ##@sql-custom-command Run SQL get custom command
+	docker compose -f docker-compose.commands.yml build sql-custom-command
+	docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) get --query_id=$(id)
+
+sql-custom-command-sign-off: ##@sql-custom-command Run SQL sign off custom command
+	docker compose -f docker-compose.commands.yml build sql-custom-command
+	docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) sign_off --query_id=$(id)
+
+sql-custom-command-execute: ##@sql-custom-command Run SQL execute custom command
+	docker compose -f docker-compose.commands.yml build sql-custom-command
+	docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) execute --query_id=$(id)
+
+sql-custom-command-revoke: ##@sql-custom-command Run SQL revoke custom command
+	docker compose -f docker-compose.commands.yml build sql-custom-command
+	docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) revoke --query_id=$(id)
+
 set-feature-flag: ##@localstack Set a particular feature flags value e.g. set-feature-flag name=multi-accounts value=1
 	docker compose exec localstack awslocal ssm put-parameter --name "/local/flag/$(name)" --value "$(value)" --type String --overwrite

api/app/api.env

@@ -6,6 +6,7 @@ DATABASE_PASSWORD=api
 DATABASE_SSL=allow
 
 CUSTOM_SQL_DATABASE_PASSWORD=api
+READONLY_SQL_DATABASE_PASSWORD=api
 SESSION_PREFIX=dd_api
 
 SECRETS_ENDPOINT=http://localstack:4566

api/app/scripts/readonly_user_setup/_0_readonly_user_create.sql

@@ -0,0 +1,9 @@
+DO
+$$
+BEGIN
+  IF NOT EXISTS (SELECT * FROM pg_user WHERE usename = 'readonly_sql_user') THEN
+     CREATE USER readonly_sql_user WITH PASSWORD 'string_to_replace_with_real_password';
+  END IF;
+END
+$$
+;

api/app/scripts/readonly_user_setup/_1_readonly_user_grant_schema.sql

@@ -0,0 +1,11 @@
+DO $$
+DECLARE
+    schema_name_var text;
+BEGIN
+    FOR schema_name_var IN
+        SELECT schema_name FROM information_schema.schemata
+        WHERE schema_name NOT IN ('information_schema', 'pg_catalog')  -- Skip system schemas
+    LOOP
+        EXECUTE format('GRANT USAGE ON SCHEMA %I TO readonly_sql_user;', schema_name_var);
+    END LOOP;
+END $$;

api/app/scripts/readonly_user_setup/_2_readonly_user_grant_select.sql

@@ -0,0 +1,11 @@
+DO $$
+DECLARE
+    tbl record;
+BEGIN
+    FOR tbl IN
+        SELECT schemaname, tablename FROM pg_tables
+        WHERE schemaname NOT IN ('information_schema', 'pg_catalog')  -- Skip system schemas
+    LOOP
+        EXECUTE format('GRANT SELECT ON TABLE %I.%I TO readonly_sql_user;', tbl.schemaname, tbl.tablename);
+    END LOOP;
+END $$;

api/app/scripts/readonly_user_setup/_3_readonly_user_revoke_update_delete.sql

@@ -0,0 +1,11 @@
+DO $$
+DECLARE
+    tbl record;
+BEGIN
+    FOR tbl IN
+        SELECT schemaname, tablename FROM pg_tables
+        WHERE schemaname NOT IN ('information_schema', 'pg_catalog')  -- Skip system schemas
+    LOOP
+        EXECUTE format('REVOKE UPDATE, DELETE ON TABLE %I.%I FROM readonly_sql_user;', tbl.schemaname, tbl.tablename);
+    END LOOP;
+END $$;

api/app/scripts/reset_db_fixtures.sh

@@ -13,3 +13,4 @@ fi
 
 # Run the custom SQL query setup script
 ./scripts/setup_custom_sql_query.sh
+./scripts/setup_readonly_sql_query.sh

api/app/scripts/setup_custom_sql_query.sh

@@ -19,6 +19,12 @@ for sql_file in $(ls $SQL_DIR/*.sql | sort -V); do
     # Create a temporary file for the modified SQL
     temp_file=$(mktemp)
 
+    # Check if password is empty and exit if it is!
+    if [ -z "$CUSTOM_SQL_DATABASE_PASSWORD" ]; then
+        echo "CUSTOM_SQL_DATABASE_PASSWORD is empty. Exiting..."
+        exit 1
+    fi
+
     # Replace the placeholder string with the real password
     sed "s/string_to_replace_with_real_password/$CUSTOM_SQL_DATABASE_PASSWORD/g" "$sql_file" > "$temp_file"
 

api/app/scripts/setup_readonly_sql_query.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# Directory where SQL scripts are stored
+SQL_DIR="./scripts/readonly_user_setup"
+
+# Export the database password for psql command
+export PGPASSWORD="$DATABASE_PASSWORD"
+
+# Check if directory exists
+if [ ! -d "$SQL_DIR" ]; then
+    echo "Directory $SQL_DIR does not exist."
+    exit 1
+fi
+
+# Find all SQL files in the directory, sort them numerically, and loop through each one
+for sql_file in $(ls $SQL_DIR/*.sql | sort -V); do
+    echo "Running $sql_file ..."
+
+    # Create a temporary file for the modified SQL
+    temp_file=$(mktemp)
+
+    # Check if password is empty and exit if it is!
+    if [ -z "$READONLY_SQL_DATABASE_PASSWORD" ]; then
+        echo "READONLY_SQL_DATABASE_PASSWORD is empty. Exiting..."
+        exit 1
+    fi
+
+    # Replace the placeholder string with the real password
+    sed "s/string_to_replace_with_real_password/$READONLY_SQL_DATABASE_PASSWORD/g" "$sql_file" > "$temp_file"
+
+    # Run the modified SQL file
+    psql -h "$DATABASE_HOSTNAME" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -p "$DATABASE_PORT" -f "$temp_file"
+
+    # Check for errors
+    if [ $? -ne 0 ]; then
+        echo "Error occurred while executing $sql_file. Exiting."
+        rm "$temp_file" # Remove the temp file if an error occurs
+        exit 1
+    fi
+
+    # Remove the temporary file after successful execution
+    rm "$temp_file"
+done
+
+echo "All scripts executed successfully."
+
+# Unset the password environment variable
+unset PGPASSWORD

docker-compose.commands.yml

@@ -27,3 +27,13 @@ services:
     volumes:
       - "./zap/:/zap/wrk/"
     command: bash -c "zap.sh -cmd -addonupdate -addoninstall pscanrules -addoninstall pscanrulesBeta -addoninstall pscanrulesAlpha -addoninstall ascanrules -addoninstall ascanrulesBeta -addoninstall ascanrulesAlpha; zap.sh -cmd -autorun /zap/wrk/zap-digideps-front.yaml"
+
+  sql-custom-command:
+    build:
+      context: ./scripts/custom_sql_query
+      dockerfile: Dockerfile
+    environment:
+      AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY
+      AWS_SESSION_TOKEN: $AWS_SESSION_TOKEN
+      WORKSPACE: $WORKSPACE

lambdas/functions/custom_sql_query/Dockerfile

@@ -8,6 +8,7 @@ RUN apk update && apk add --no-cache \
     python3-dev \
     musl-dev
 RUN apk upgrade
+RUN pip install --upgrade pip setuptools wheel
 
 # Build image
 FROM python-alpine as build-image

lambdas/functions/custom_sql_query/requirements.txt

@@ -1,4 +1,4 @@
 awslambdaric~=2.2.1
 boto3~=1.35.20
 psycopg2~=2.9.9
-setuptools~=70.0.0
+setuptools~=75.2.0

scripts/custom_sql_query/Dockerfile

@@ -0,0 +1,36 @@
+# Define function directory
+ARG FUNCTION_DIR="/function"
+
+# ===== BASE IMAGE =====
+FROM python:3.12-alpine3.18 AS python-alpine
+RUN pip install --upgrade pip setuptools wheel
+RUN apk update && apk upgrade
+
+# ===== Build image =====
+FROM python-alpine as build-image
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+# Create function directory
+RUN mkdir -p ${FUNCTION_DIR}
+# Copy function code
+COPY run_custom_query.py ${FUNCTION_DIR}/run_custom_query.py
+COPY _verification.sql ${FUNCTION_DIR}/_verification.sql
+COPY _run.sql ${FUNCTION_DIR}/_run.sql
+
+COPY requirements.txt requirements.txt
+# Install the requirements
+RUN python -m pip install --upgrade pip
+RUN python -m pip install \
+        --target ${FUNCTION_DIR} \
+        --requirement requirements.txt
+
+# ===== FINAL IMAGE =====
+FROM python-alpine
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+# Copy in the build image dependencies
+COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR}
+
+ENTRYPOINT [ "python3", "/function/run_custom_query.py" ]

scripts/custom_sql_query/custom_sql_instructions.md

@@ -0,0 +1,19 @@
+## How to run custom queries
+
+You will need aws-vault and operator permissions.
+
+You can then perform the commands required using a docker wrapper container wrapped by a make file.
+
+Full details about how this works here: [Custom SQL Details](../../lambdas/functions/custom_sql_query/custom_sql_query.md)
+
+Remember to edit the SQL and validation scripts in this folder.
+
+Example make commands:
+
+```
+aws-vault exec identity -- make sql-custom-command-insert workspace=ddls1234000 before=1 after=0
+aws-vault exec identity -- make sql-custom-command-get workspace=ddls1234000 id=1
+aws-vault exec identity -- make sql-custom-command-sign-off workspace=ddls1234000 id=1
+aws-vault exec identity -- make sql-custom-command-execute workspace=ddls1234000 id=1
+aws-vault exec identity -- make sql-custom-command-revoke workspace=ddls1234000 id=1
+```

scripts/custom_sql_query/requirements.txt

@@ -0,0 +1,2 @@
+boto3~=1.35.20
+requests~=2.32.3

orchestration/custom_sql_query/run_custom_query.py → scripts/custom_sql_query/run_custom_query.py

@@ -58,7 +58,7 @@ def get_lambda_client(environment):
         return LocalLambdaClient()
     else:
         session = assume_operator(environment)
-        return session.client("lambda")
+        return session.client("lambda", region_name="eu-west-1")
 
 
 class LocalLambdaClient:

terraform/account/region/secrets.tf

@@ -16,7 +16,8 @@ module "environment_secrets" {
     "public-jwt-key-base64",
     "private-jwt-key-base64",
     "smoke-test-variables",
-    "custom-sql-db-password"
+    "custom-sql-db-password",
+    "readonly-sql-db-password"
   ]
   tags = var.default_tags
 }

terraform/environment/region/ecs_iam_execution.tf

@@ -135,7 +135,8 @@ data "aws_iam_policy_document" "execution_role_secrets_db" {
     resources = [
       data.aws_secretsmanager_secret.database_password.arn,
       data.aws_secretsmanager_secret.api_secret.arn,
-      data.aws_secretsmanager_secret.custom_sql_db_password.arn
+      data.aws_secretsmanager_secret.custom_sql_db_password.arn,
+      data.aws_secretsmanager_secret.readonly_sql_db_password.arn
     ]
     actions = ["secretsmanager:GetSecretValue"]
   }

terraform/environment/region/ecs_task_reset_database.tf

@@ -36,6 +36,10 @@ locals {
           name      = "CUSTOM_SQL_DATABASE_PASSWORD",
           valueFrom = data.aws_secretsmanager_secret.custom_sql_db_password.arn
         },
+        {
+          name      = "READONLY_SQL_DATABASE_PASSWORD",
+          valueFrom = data.aws_secretsmanager_secret.readonly_sql_db_password.arn
+        },
         {
           name      = "SECRET",
           valueFrom = data.aws_secretsmanager_secret.api_secret.arn

terraform/environment/region/secrets.tf

@@ -54,6 +54,10 @@ data "aws_secretsmanager_secret" "custom_sql_db_password" {
   name = join("/", compact([var.secrets_prefix, "custom-sql-db-password"]))
 }
 
+data "aws_secretsmanager_secret" "readonly_sql_db_password" {
+  name = join("/", compact([var.secrets_prefix, "readonly-sql-db-password"]))
+}
+
 data "aws_secretsmanager_secret" "anonymise-default-pw" {
   name = "anonymisation-default-user-pw"
 }