From 14080739d55189bbac248c38080aa87908a08870 Mon Sep 17 00:00:00 2001 From: Jim <58939809+jamesrwarren@users.noreply.github.com> Date: Thu, 24 Oct 2024 17:40:15 +0100 Subject: [PATCH] DDLS-379 add a read only user (#1703) * DDLS-379 add a read only user * dockerise sql run command --- Makefile | 20 ++++++++ api/app/api.env | 1 + .../_0_readonly_user_create.sql | 9 ++++ .../_1_readonly_user_grant_schema.sql | 11 +++++ .../_2_readonly_user_grant_select.sql | 11 +++++ .../_3_readonly_user_revoke_update_delete.sql | 11 +++++ api/app/scripts/reset_db_fixtures.sh | 1 + api/app/scripts/setup_custom_sql_query.sh | 6 +++ api/app/scripts/setup_readonly_sql_query.sh | 48 +++++++++++++++++++ docker-compose.commands.yml | 10 ++++ lambdas/functions/custom_sql_query/Dockerfile | 1 + .../custom_sql_query/requirements.txt | 2 +- .../custom_sql_query/outstanding_work.md | 6 --- scripts/custom_sql_query/Dockerfile | 36 ++++++++++++++ .../custom_sql_query/_run.sql | 0 .../custom_sql_query/_verification.sql | 0 .../custom_sql_instructions.md | 19 ++++++++ scripts/custom_sql_query/requirements.txt | 2 + .../custom_sql_query/run_custom_query.py | 2 +- terraform/account/region/secrets.tf | 3 +- .../environment/region/ecs_iam_execution.tf | 3 +- .../region/ecs_task_reset_database.tf | 4 ++ terraform/environment/region/secrets.tf | 4 ++ 23 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 api/app/scripts/readonly_user_setup/_0_readonly_user_create.sql create mode 100644 api/app/scripts/readonly_user_setup/_1_readonly_user_grant_schema.sql create mode 100644 api/app/scripts/readonly_user_setup/_2_readonly_user_grant_select.sql create mode 100644 api/app/scripts/readonly_user_setup/_3_readonly_user_revoke_update_delete.sql create mode 100755 api/app/scripts/setup_readonly_sql_query.sh delete mode 100644 orchestration/custom_sql_query/outstanding_work.md create mode 100644 scripts/custom_sql_query/Dockerfile rename {orchestration => scripts}/custom_sql_query/_run.sql (100%) rename {orchestration => scripts}/custom_sql_query/_verification.sql (100%) create mode 100644 scripts/custom_sql_query/custom_sql_instructions.md create mode 100644 scripts/custom_sql_query/requirements.txt rename {orchestration => scripts}/custom_sql_query/run_custom_query.py (99%) diff --git a/Makefile b/Makefile index c6a386aa67..1dda9f5ad0 100644 --- a/Makefile +++ b/Makefile @@ -197,5 +197,25 @@ resilience-tests: ##@resilience-tests Run resilience tests (requires app to be u docker compose build orchestration docker compose run -e LOG_AND_CONTINUE=true --remove-orphans orchestration sh tests/run-resilience-tests.sh +sql-custom-command-insert: ##@sql-custom-command Run SQL insert custom command + docker compose -f docker-compose.commands.yml build sql-custom-command + docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) insert --sql_file=_run.sql --verification_sql_file=_verification.sql --expected_before=$(before) --expected_after=$(after) + +sql-custom-command-get: ##@sql-custom-command Run SQL get custom command + docker compose -f docker-compose.commands.yml build sql-custom-command + docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) get --query_id=$(id) + +sql-custom-command-sign-off: ##@sql-custom-command Run SQL sign off custom command + docker compose -f docker-compose.commands.yml build sql-custom-command + docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) sign_off --query_id=$(id) + +sql-custom-command-execute: ##@sql-custom-command Run SQL execute custom command + docker compose -f docker-compose.commands.yml build sql-custom-command + docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) execute --query_id=$(id) + +sql-custom-command-revoke: ##@sql-custom-command Run SQL revoke custom command + docker compose -f docker-compose.commands.yml build sql-custom-command + docker compose -f docker-compose.commands.yml run --remove-orphans sql-custom-command $(workspace) revoke --query_id=$(id) + set-feature-flag: ##@localstack Set a particular feature flags value e.g. set-feature-flag name=multi-accounts value=1 docker compose exec localstack awslocal ssm put-parameter --name "/local/flag/$(name)" --value "$(value)" --type String --overwrite diff --git a/api/app/api.env b/api/app/api.env index 925338592e..ed23e15beb 100644 --- a/api/app/api.env +++ b/api/app/api.env @@ -6,6 +6,7 @@ DATABASE_PASSWORD=api DATABASE_SSL=allow CUSTOM_SQL_DATABASE_PASSWORD=api +READONLY_SQL_DATABASE_PASSWORD=api SESSION_PREFIX=dd_api SECRETS_ENDPOINT=http://localstack:4566 diff --git a/api/app/scripts/readonly_user_setup/_0_readonly_user_create.sql b/api/app/scripts/readonly_user_setup/_0_readonly_user_create.sql new file mode 100644 index 0000000000..8f7c336927 --- /dev/null +++ b/api/app/scripts/readonly_user_setup/_0_readonly_user_create.sql @@ -0,0 +1,9 @@ +DO +$$ +BEGIN + IF NOT EXISTS (SELECT * FROM pg_user WHERE usename = 'readonly_sql_user') THEN + CREATE USER readonly_sql_user WITH PASSWORD 'string_to_replace_with_real_password'; + END IF; +END +$$ +; diff --git a/api/app/scripts/readonly_user_setup/_1_readonly_user_grant_schema.sql b/api/app/scripts/readonly_user_setup/_1_readonly_user_grant_schema.sql new file mode 100644 index 0000000000..e758464e04 --- /dev/null +++ b/api/app/scripts/readonly_user_setup/_1_readonly_user_grant_schema.sql @@ -0,0 +1,11 @@ +DO $$ +DECLARE + schema_name_var text; +BEGIN + FOR schema_name_var IN + SELECT schema_name FROM information_schema.schemata + WHERE schema_name NOT IN ('information_schema', 'pg_catalog') -- Skip system schemas + LOOP + EXECUTE format('GRANT USAGE ON SCHEMA %I TO readonly_sql_user;', schema_name_var); + END LOOP; +END $$; diff --git a/api/app/scripts/readonly_user_setup/_2_readonly_user_grant_select.sql b/api/app/scripts/readonly_user_setup/_2_readonly_user_grant_select.sql new file mode 100644 index 0000000000..0d8575f110 --- /dev/null +++ b/api/app/scripts/readonly_user_setup/_2_readonly_user_grant_select.sql @@ -0,0 +1,11 @@ +DO $$ +DECLARE + tbl record; +BEGIN + FOR tbl IN + SELECT schemaname, tablename FROM pg_tables + WHERE schemaname NOT IN ('information_schema', 'pg_catalog') -- Skip system schemas + LOOP + EXECUTE format('GRANT SELECT ON TABLE %I.%I TO readonly_sql_user;', tbl.schemaname, tbl.tablename); + END LOOP; +END $$; diff --git a/api/app/scripts/readonly_user_setup/_3_readonly_user_revoke_update_delete.sql b/api/app/scripts/readonly_user_setup/_3_readonly_user_revoke_update_delete.sql new file mode 100644 index 0000000000..411007e9f3 --- /dev/null +++ b/api/app/scripts/readonly_user_setup/_3_readonly_user_revoke_update_delete.sql @@ -0,0 +1,11 @@ +DO $$ +DECLARE + tbl record; +BEGIN + FOR tbl IN + SELECT schemaname, tablename FROM pg_tables + WHERE schemaname NOT IN ('information_schema', 'pg_catalog') -- Skip system schemas + LOOP + EXECUTE format('REVOKE UPDATE, DELETE ON TABLE %I.%I FROM readonly_sql_user;', tbl.schemaname, tbl.tablename); + END LOOP; +END $$; diff --git a/api/app/scripts/reset_db_fixtures.sh b/api/app/scripts/reset_db_fixtures.sh index 9e123e8d88..103ff3dbb9 100644 --- a/api/app/scripts/reset_db_fixtures.sh +++ b/api/app/scripts/reset_db_fixtures.sh @@ -13,3 +13,4 @@ fi # Run the custom SQL query setup script ./scripts/setup_custom_sql_query.sh +./scripts/setup_readonly_sql_query.sh diff --git a/api/app/scripts/setup_custom_sql_query.sh b/api/app/scripts/setup_custom_sql_query.sh index 9ab3377abc..acaaacf560 100755 --- a/api/app/scripts/setup_custom_sql_query.sh +++ b/api/app/scripts/setup_custom_sql_query.sh @@ -19,6 +19,12 @@ for sql_file in $(ls $SQL_DIR/*.sql | sort -V); do # Create a temporary file for the modified SQL temp_file=$(mktemp) + # Check if password is empty and exit if it is! + if [ -z "$CUSTOM_SQL_DATABASE_PASSWORD" ]; then + echo "CUSTOM_SQL_DATABASE_PASSWORD is empty. Exiting..." + exit 1 + fi + # Replace the placeholder string with the real password sed "s/string_to_replace_with_real_password/$CUSTOM_SQL_DATABASE_PASSWORD/g" "$sql_file" > "$temp_file" diff --git a/api/app/scripts/setup_readonly_sql_query.sh b/api/app/scripts/setup_readonly_sql_query.sh new file mode 100755 index 0000000000..1a0ed0d6ee --- /dev/null +++ b/api/app/scripts/setup_readonly_sql_query.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +# Directory where SQL scripts are stored +SQL_DIR="./scripts/readonly_user_setup" + +# Export the database password for psql command +export PGPASSWORD="$DATABASE_PASSWORD" + +# Check if directory exists +if [ ! -d "$SQL_DIR" ]; then + echo "Directory $SQL_DIR does not exist." + exit 1 +fi + +# Find all SQL files in the directory, sort them numerically, and loop through each one +for sql_file in $(ls $SQL_DIR/*.sql | sort -V); do + echo "Running $sql_file ..." + + # Create a temporary file for the modified SQL + temp_file=$(mktemp) + + # Check if password is empty and exit if it is! + if [ -z "$READONLY_SQL_DATABASE_PASSWORD" ]; then + echo "READONLY_SQL_DATABASE_PASSWORD is empty. Exiting..." + exit 1 + fi + + # Replace the placeholder string with the real password + sed "s/string_to_replace_with_real_password/$READONLY_SQL_DATABASE_PASSWORD/g" "$sql_file" > "$temp_file" + + # Run the modified SQL file + psql -h "$DATABASE_HOSTNAME" -U "$DATABASE_USERNAME" -d "$DATABASE_NAME" -p "$DATABASE_PORT" -f "$temp_file" + + # Check for errors + if [ $? -ne 0 ]; then + echo "Error occurred while executing $sql_file. Exiting." + rm "$temp_file" # Remove the temp file if an error occurs + exit 1 + fi + + # Remove the temporary file after successful execution + rm "$temp_file" +done + +echo "All scripts executed successfully." + +# Unset the password environment variable +unset PGPASSWORD diff --git a/docker-compose.commands.yml b/docker-compose.commands.yml index 7998e5d4d1..c3049079d7 100644 --- a/docker-compose.commands.yml +++ b/docker-compose.commands.yml @@ -27,3 +27,13 @@ services: volumes: - "./zap/:/zap/wrk/" command: bash -c "zap.sh -cmd -addonupdate -addoninstall pscanrules -addoninstall pscanrulesBeta -addoninstall pscanrulesAlpha -addoninstall ascanrules -addoninstall ascanrulesBeta -addoninstall ascanrulesAlpha; zap.sh -cmd -autorun /zap/wrk/zap-digideps-front.yaml" + + sql-custom-command: + build: + context: ./scripts/custom_sql_query + dockerfile: Dockerfile + environment: + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN: $AWS_SESSION_TOKEN + WORKSPACE: $WORKSPACE diff --git a/lambdas/functions/custom_sql_query/Dockerfile b/lambdas/functions/custom_sql_query/Dockerfile index 0214f82756..7521b6b1ec 100644 --- a/lambdas/functions/custom_sql_query/Dockerfile +++ b/lambdas/functions/custom_sql_query/Dockerfile @@ -8,6 +8,7 @@ RUN apk update && apk add --no-cache \ python3-dev \ musl-dev RUN apk upgrade +RUN pip install --upgrade pip setuptools wheel # Build image FROM python-alpine as build-image diff --git a/lambdas/functions/custom_sql_query/requirements.txt b/lambdas/functions/custom_sql_query/requirements.txt index 3e4eac2a77..97d3f57936 100644 --- a/lambdas/functions/custom_sql_query/requirements.txt +++ b/lambdas/functions/custom_sql_query/requirements.txt @@ -1,4 +1,4 @@ awslambdaric~=2.2.1 boto3~=1.35.20 psycopg2~=2.9.9 -setuptools~=70.0.0 +setuptools~=75.2.0 diff --git a/orchestration/custom_sql_query/outstanding_work.md b/orchestration/custom_sql_query/outstanding_work.md deleted file mode 100644 index f04ca93e7b..0000000000 --- a/orchestration/custom_sql_query/outstanding_work.md +++ /dev/null @@ -1,6 +0,0 @@ -### Remaining to do list - -- add secret for custom_sql_user to cycle secrets -- add additional user auth -- improve error handling -- add unit tests! diff --git a/scripts/custom_sql_query/Dockerfile b/scripts/custom_sql_query/Dockerfile new file mode 100644 index 0000000000..a7b6850d19 --- /dev/null +++ b/scripts/custom_sql_query/Dockerfile @@ -0,0 +1,36 @@ +# Define function directory +ARG FUNCTION_DIR="/function" + +# ===== BASE IMAGE ===== +FROM python:3.12-alpine3.18 AS python-alpine +RUN pip install --upgrade pip setuptools wheel +RUN apk update && apk upgrade + +# ===== Build image ===== +FROM python-alpine as build-image +# Include global arg in this stage of the build +ARG FUNCTION_DIR +# Create function directory +RUN mkdir -p ${FUNCTION_DIR} +# Copy function code +COPY run_custom_query.py ${FUNCTION_DIR}/run_custom_query.py +COPY _verification.sql ${FUNCTION_DIR}/_verification.sql +COPY _run.sql ${FUNCTION_DIR}/_run.sql + +COPY requirements.txt requirements.txt +# Install the requirements +RUN python -m pip install --upgrade pip +RUN python -m pip install \ + --target ${FUNCTION_DIR} \ + --requirement requirements.txt + +# ===== FINAL IMAGE ===== +FROM python-alpine +# Include global arg in this stage of the build +ARG FUNCTION_DIR +# Set working directory to function root directory +WORKDIR ${FUNCTION_DIR} +# Copy in the build image dependencies +COPY --from=build-image ${FUNCTION_DIR} ${FUNCTION_DIR} + +ENTRYPOINT [ "python3", "/function/run_custom_query.py" ] diff --git a/orchestration/custom_sql_query/_run.sql b/scripts/custom_sql_query/_run.sql similarity index 100% rename from orchestration/custom_sql_query/_run.sql rename to scripts/custom_sql_query/_run.sql diff --git a/orchestration/custom_sql_query/_verification.sql b/scripts/custom_sql_query/_verification.sql similarity index 100% rename from orchestration/custom_sql_query/_verification.sql rename to scripts/custom_sql_query/_verification.sql diff --git a/scripts/custom_sql_query/custom_sql_instructions.md b/scripts/custom_sql_query/custom_sql_instructions.md new file mode 100644 index 0000000000..e262bd2a7a --- /dev/null +++ b/scripts/custom_sql_query/custom_sql_instructions.md @@ -0,0 +1,19 @@ +## How to run custom queries + +You will need aws-vault and operator permissions. + +You can then perform the commands required using a docker wrapper container wrapped by a make file. + +Full details about how this works here: [Custom SQL Details](../../lambdas/functions/custom_sql_query/custom_sql_query.md) + +Remember to edit the SQL and validation scripts in this folder. + +Example make commands: + +``` +aws-vault exec identity -- make sql-custom-command-insert workspace=ddls1234000 before=1 after=0 +aws-vault exec identity -- make sql-custom-command-get workspace=ddls1234000 id=1 +aws-vault exec identity -- make sql-custom-command-sign-off workspace=ddls1234000 id=1 +aws-vault exec identity -- make sql-custom-command-execute workspace=ddls1234000 id=1 +aws-vault exec identity -- make sql-custom-command-revoke workspace=ddls1234000 id=1 +``` diff --git a/scripts/custom_sql_query/requirements.txt b/scripts/custom_sql_query/requirements.txt new file mode 100644 index 0000000000..1dddd8fd1a --- /dev/null +++ b/scripts/custom_sql_query/requirements.txt @@ -0,0 +1,2 @@ +boto3~=1.35.20 +requests~=2.32.3 diff --git a/orchestration/custom_sql_query/run_custom_query.py b/scripts/custom_sql_query/run_custom_query.py similarity index 99% rename from orchestration/custom_sql_query/run_custom_query.py rename to scripts/custom_sql_query/run_custom_query.py index 8196dd33be..5e48322e93 100644 --- a/orchestration/custom_sql_query/run_custom_query.py +++ b/scripts/custom_sql_query/run_custom_query.py @@ -58,7 +58,7 @@ def get_lambda_client(environment): return LocalLambdaClient() else: session = assume_operator(environment) - return session.client("lambda") + return session.client("lambda", region_name="eu-west-1") class LocalLambdaClient: diff --git a/terraform/account/region/secrets.tf b/terraform/account/region/secrets.tf index c6e1f2fde0..9a4208832d 100644 --- a/terraform/account/region/secrets.tf +++ b/terraform/account/region/secrets.tf @@ -16,7 +16,8 @@ module "environment_secrets" { "public-jwt-key-base64", "private-jwt-key-base64", "smoke-test-variables", - "custom-sql-db-password" + "custom-sql-db-password", + "readonly-sql-db-password" ] tags = var.default_tags } diff --git a/terraform/environment/region/ecs_iam_execution.tf b/terraform/environment/region/ecs_iam_execution.tf index bafb2ec6dd..25cc474cb8 100644 --- a/terraform/environment/region/ecs_iam_execution.tf +++ b/terraform/environment/region/ecs_iam_execution.tf @@ -135,7 +135,8 @@ data "aws_iam_policy_document" "execution_role_secrets_db" { resources = [ data.aws_secretsmanager_secret.database_password.arn, data.aws_secretsmanager_secret.api_secret.arn, - data.aws_secretsmanager_secret.custom_sql_db_password.arn + data.aws_secretsmanager_secret.custom_sql_db_password.arn, + data.aws_secretsmanager_secret.readonly_sql_db_password.arn ] actions = ["secretsmanager:GetSecretValue"] } diff --git a/terraform/environment/region/ecs_task_reset_database.tf b/terraform/environment/region/ecs_task_reset_database.tf index 05d2be7ac3..eb02fa8028 100644 --- a/terraform/environment/region/ecs_task_reset_database.tf +++ b/terraform/environment/region/ecs_task_reset_database.tf @@ -36,6 +36,10 @@ locals { name = "CUSTOM_SQL_DATABASE_PASSWORD", valueFrom = data.aws_secretsmanager_secret.custom_sql_db_password.arn }, + { + name = "READONLY_SQL_DATABASE_PASSWORD", + valueFrom = data.aws_secretsmanager_secret.readonly_sql_db_password.arn + }, { name = "SECRET", valueFrom = data.aws_secretsmanager_secret.api_secret.arn diff --git a/terraform/environment/region/secrets.tf b/terraform/environment/region/secrets.tf index af2ed8bd66..c4943a2a19 100644 --- a/terraform/environment/region/secrets.tf +++ b/terraform/environment/region/secrets.tf @@ -54,6 +54,10 @@ data "aws_secretsmanager_secret" "custom_sql_db_password" { name = join("/", compact([var.secrets_prefix, "custom-sql-db-password"])) } +data "aws_secretsmanager_secret" "readonly_sql_db_password" { + name = join("/", compact([var.secrets_prefix, "readonly-sql-db-password"])) +} + data "aws_secretsmanager_secret" "anonymise-default-pw" { name = "anonymisation-default-user-pw" }