diff --git a/.env b/.env index 3848c8c0eb..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -7,3 +7,4 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +WAL2JSON_VERSION=latest diff --git a/.gitignore b/.gitignore index 707622f425..8a169049fa 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock + +# wal2json download +postgres/wal2json diff --git a/docker-compose.yml b/docker-compose.yml index 5415f5d1e6..9a78d96930 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,10 +71,16 @@ services: postgres: <<: *restart_policy image: "postgres:9.6" + command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] environment: POSTGRES_HOST_AUTH_METHOD: "trust" + entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" + - type: bind + read_only: true + source: ./postgres/ + target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" diff --git a/install.sh b/install.sh index 171851b59d..3886463061 100755 --- a/install.sh +++ b/install.sh @@ -18,6 +18,7 @@ source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh +source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh new file mode 100644 index 0000000000..5ed58005e0 --- /dev/null +++ b/install/install-wal2json.sh @@ -0,0 +1,34 @@ +echo "${_group}Downloading and installing wal2json ..." + +FILE_TO_USE="../postgres/wal2json/wal2json.so" +ARCH=$(uname -m) +FILE_NAME="wal2json-Linux-$ARCH-glibc.so" + +DOCKER_CURL="docker run --rm curlimages/curl" + +if [[ $WAL2JSON_VERSION == "latest" ]]; then + VERSION=$( + $DOCKER_CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' + ) + + if [[ ! $VERSION ]]; then + echo "Cannot find wal2json latest version" + exit 1 + fi +else + VERSION=$WAL2JSON_VERSION +fi + +mkdir -p ../postgres/wal2json +if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$VERSION" + $DOCKER_CURL -L \ + "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ + > "../postgres/wal2json/$VERSION/$FILE_NAME" + + cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" +fi + +echo "${_endgroup}" diff --git a/postgres/init_hba.sh b/postgres/init_hba.sh new file mode 100755 index 0000000000..f4b332abfa --- /dev/null +++ b/postgres/init_hba.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Initializes the pg_hba file with access permissions to the replication +# slots. + +set -e + +{ echo "host replication all all trust"; } >> "$PGDATA/pg_hba.conf" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh new file mode 100755 index 0000000000..0b0d98a964 --- /dev/null +++ b/postgres/postgres-entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# This script replaces the default docker entrypoint for postgres in the +# development environment. +# Its job is to ensure postgres is properly configured to support the +# Change Data Capture pipeline (by setting access permissions and installing +# the replication plugin we use for CDC). Unfortunately the default +# Postgres image does not allow this level of configurability so we need +# to do it this way in order not to have to publish and maintain our own +# Postgres image. +# +# This then, at the end, transfers control to the default entrypoint. + +set -e + +prep_init_db() { + cp /opt/sentry/init_hba.sh /docker-entrypoint-initdb.d/init_hba.sh +} + +cdc_setup_hba_conf() { + # Ensure pg-hba is properly configured to allow connections + # to the replication slots. + + PG_HBA="$PGDATA/pg_hba.conf" + if [ ! -f "$PG_HBA" ]; then + echo "DB not initialized. Postgres will take care of pg_hba" + elif [ "$(grep -c -E "^host\s+replication" "$PGDATA"/pg_hba.conf)" != 0 ]; then + echo "Replication config already present in pg_hba. Not changing anything." + else + # Execute the same script we run on DB initialization + /opt/sentry/init_hba.sh + fi +} + +bind_wal2json() { + # Copy the file in the right place + cp /opt/sentry/wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so +} + +echo "Setting up Change Data Capture" + +prep_init_db +if [ "$1" = 'postgres' ]; then + cdc_setup_hba_conf + bind_wal2json +fi +exec /docker-entrypoint.sh "$@" diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4c8bb5a5d3..b5d4586c42 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -8,11 +8,14 @@ OLD_VERSION="$1" NEW_VERSION="$2" SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} +WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" +echo "New wal2json version: $WAL2JSON_VERSION"