feat: Support custom CA roots (getsentry#1015)

Mount a certificate folder to local ca storage in containers,
and add update command to cron image's entrypoint.

Result of poking and prodding from getsentry/sentry#26851

.github/workflows/test.yml

@@ -15,7 +15,7 @@ defaults:
     shell: bash
 jobs:
   unit-test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     name: "unit tests"
     steps:
       - name: Checkout
@@ -26,8 +26,8 @@ jobs:
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
   integration-test:
-    runs-on: ubuntu-18.04
-    name: "test"
+    runs-on: ubuntu-20.04
+    name: "integration test"
     steps:
       - name: Pin docker-compose
         run: |
@@ -42,13 +42,16 @@ jobs:
       - name: Integration Test
         run: |
           echo "Testing initial install"
+          # Create ./certificates here because install.sh will create it with root:root
+          # and then run.sh (-> setup.sh) won't be able to write to it.
+          mkdir certificates
           ./install.sh
-          ./test.sh
+          ./_integration-test/run.sh
           echo "Testing in-place upgrade"
           # Also test plugin installation here
           echo "sentry-auth-oidc" >> sentry/requirements.txt
           ./install.sh --minimize-downtime
-          ./test.sh
+          ./_integration-test/run.sh
 
       - name: Inspect failure
         if: failure()

.gitignore

@@ -87,3 +87,10 @@ geoip/.geoipupdate.lock
 
 # wal2json download
 postgres/wal2json
+
+# custom certificate authorities
+certificates
+
+# integration testing
+_integration-test/custom-ca-roots/nginx/*
+sentry/test-custom-ca-roots.py

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details.
+
 ## 21.7.0
 
 - No documented changes.

_integration-test/custom-ca-roots/docker-compose.test.yml

@@ -0,0 +1,12 @@
+version: '3.4'
+services:
+  fixture-custom-ca-roots:
+    image: nginx:1.21.0-alpine
+    restart: unless-stopped
+    volumes:
+      - ./_integration-test/custom-ca-roots/nginx:/etc/nginx:ro
+    networks:
+      default:
+        aliases:
+          - self.test
+          - fail.test

_integration-test/custom-ca-roots/nginx/nginx.conf

@@ -0,0 +1,32 @@
+user nginx;
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+	worker_connections 1024;
+}
+
+http {
+  server {
+    listen               443 ssl;
+    server_name          "self.test";
+    ssl_certificate      "/etc/nginx/self.test.crt";
+    ssl_certificate_key  "/etc/nginx/self.test.key";
+    location / {
+      add_header Content-Type text/plain;
+      return 200 'ok';
+    }
+  }
+  server {
+    listen               443 ssl;
+    server_name          "fake.test";
+    ssl_certificate      "/etc/nginx/fake.test.crt";
+    ssl_certificate_key  "/etc/nginx/fake.test.key";
+    location / {
+      add_header Content-Type text/plain;
+      return 200 'bad';
+    }
+  }
+}

_integration-test/custom-ca-roots/setup.sh

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+set -e
+
+export COMPOSE_FILE="../docker-compose.yml:./custom-ca-roots/docker-compose.test.yml"
+
+TEST_NGINX_CONF_PATH="./custom-ca-roots/nginx"
+CUSTOM_CERTS_PATH="../certificates"
+
+# generate tightly constrained CA
+# NB: `-addext` requires LibreSSL 3.1.0+, or OpenSSL (brew install openssl)
+openssl req -x509 -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/ca.key \
+-sha256 -days 1 -out $TEST_NGINX_CONF_PATH/ca.crt -batch \
+-subj "/CN=TEST CA *DO NOT TRUST*" \
+-addext "keyUsage = critical, keyCertSign, cRLSign" \
+-addext "nameConstraints = critical, permitted;DNS:self.test"
+
+## Lines like the following are debug helpers ...
+# openssl x509 -in nginx/ca.crt -text -noout
+
+mkdir -p $CUSTOM_CERTS_PATH
+cp $TEST_NGINX_CONF_PATH/ca.crt $CUSTOM_CERTS_PATH/test-custom-ca-roots.crt
+
+# generate server certificate
+openssl req -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/self.test.key \
+-addext "subjectAltName=DNS:self.test" \
+-out $TEST_NGINX_CONF_PATH/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
+
+# openssl req -in nginx/self.test.req -text -noout
+
+openssl x509 -req -in $TEST_NGINX_CONF_PATH/self.test.req -CA $TEST_NGINX_CONF_PATH/ca.crt -CAkey $TEST_NGINX_CONF_PATH/ca.key \
+-extfile <(printf "subjectAltName=DNS:self.test") \
+-CAcreateserial -out $TEST_NGINX_CONF_PATH/self.test.crt -days 1 -sha256
+
+# openssl x509 -in nginx/self.test.crt -text -noout
+
+# sanity check that signed certificate passes OpenSSL's validation
+openssl verify -CAfile $TEST_NGINX_CONF_PATH/ca.crt $TEST_NGINX_CONF_PATH/self.test.crt
+
+# self signed certificate, for sanity check of not just accepting all certs
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/fake.test.key \
+-out $TEST_NGINX_CONF_PATH/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
+
+# openssl x509 -in nginx/fake.test.crt -text -noout
+
+cp ./custom-ca-roots/test.py ../sentry/test-custom-ca-roots.py
+
+$dc up -d fixture-custom-ca-roots

_integration-test/custom-ca-roots/teardown.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+$dc rm -s -f -v fixture-custom-ca-roots
+rm -f ../certificates/test-custom-ca-roots.crt ../sentry/test-custom-ca-roots.py
+unset COMPOSE_FILE

_integration-test/custom-ca-roots/test.py

@@ -0,0 +1,15 @@
+import unittest
+import requests
+
+
+class CustomCATests(unittest.TestCase):
+    def test_valid_self_signed(self):
+        self.assertEqual(requests.get("https://self.test").text, 'ok')
+
+    def test_invalid_self_signed(self):
+        with self.assertRaises(requests.exceptions.SSLError):
+            requests.get("https://fail.test")
+
+
+if __name__ == '__main__':
+    unittest.main()

test.sh → _integration-test/run.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-source "$(dirname $0)/install/_lib.sh" 
+source "$(dirname $0)/../install/_lib.sh"
 
 echo "${_group}Setting up variables and helpers ..."
 export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
@@ -42,6 +42,7 @@ echo 'SENTRY_BEACON=False' >> $SENTRY_CONFIG_PY
 $dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true
 $dc up -d
 printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done'
+echo ""
 echo "${_endgroup}"
 
 echo "${_group}Running tests ..."
@@ -99,7 +100,7 @@ export -f sentry_api_request get_csrf_token
 export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH
 printf "Getting the test event back"
 timeout 30 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
-echo "";
+echo " got it!";
 
 EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH")
 declare -a EVENT_TEST_STRINGS=(
@@ -119,3 +120,9 @@ echo "${_endgroup}"
 echo "${_group}Ensure cleanup crons are working ..."
 $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
+
+echo "${_group}Test custom CAs work ..."
+source ./custom-ca-roots/setup.sh
+$dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py
+source ./custom-ca-roots/teardown.sh
+echo "${_endgroup}"

cron/entrypoint.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then
+  update-ca-certificates
+fi
+
 # Prior art:
 # - https://git.io/fjNOg
 # - https://blog.knoldus.com/running-a-cron-job-in-docker-container/

docker-compose.yml

@@ -35,13 +35,22 @@ x-sentry-defaults: &sentry_defaults
     PYTHONUSERBASE: "/data/custom-packages"
     SENTRY_CONF: "/etc/sentry"
     SNUBA: "http://snuba-api:1218"
+    # Force everything to use the system CA bundle
+    # This is mostly needed to support installing custom CA certs
+    # This one is used by botocore
+    DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt"
+    # This one is used by requests
+    REQUESTS_CA_BUNDLE: *ca_bundle
+    # This one is used by grpc/google modules
+    GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle
     # Leaving the value empty to just pass whatever is set
     # on the host system (or in the .env file)
     SENTRY_EVENT_RETENTION_DAYS:
   volumes:
     - "sentry-data:/data"
     - "./sentry:/etc/sentry"
     - "./geoip:/geoip:ro"
+    - "./certificates:/usr/local/share/ca-certificates:ro"
 x-snuba-defaults: &snuba_defaults
   <<: *restart_policy
   depends_on:

install/_lib.sh

@@ -6,10 +6,10 @@ log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt"
 exec &> >(tee -a "$log_file")
 
 # Work from /install/ for install.sh, project root otherwise
-if [[ "$(basename $0)" = "install.sh" || "$(basename $0)" = "test.sh" ]]; then
+if [[ "$(basename $0)" = "install.sh"  ]]; then
   cd "$(dirname $0)/install/"
 else
-  cd "$(dirname $0)"  # assume we're a *-test.sh script
+  cd "$(dirname $0)"  # assume we're a test script or some such
 fi
 
 _ENV="$(realpath ../.env)"

sentry/entrypoint.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -e
 
+if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then
+  update-ca-certificates
+fi
+
 req_file="/etc/sentry/requirements.txt"
 plugins_dir="/data/custom-packages"
 checksum_file="$plugins_dir/.checksum"