From 237486e8602d3683cc528d67c7f1dbd970b9760a Mon Sep 17 00:00:00 2001 From: idanovinda Date: Wed, 16 Oct 2024 16:43:37 +0200 Subject: [PATCH 1/7] moving timescaledb to pkg instead of building --- postgres-appliance/Dockerfile | 4 ---- postgres-appliance/build_scripts/base.sh | 26 +++++++----------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index c2ca9338..db702eec 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,6 +1,5 @@ ARG BASE_IMAGE=ubuntu:22.04 ARG PGVERSION=17 -ARG TIMESCALEDB="2.15.3 2.17.0" ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= @@ -44,7 +43,6 @@ COPY build_scripts/base.sh /builddeps/ COPY --from=dependencies-builder /builddeps/*.deb /builddeps/ ARG PGVERSION -ARG TIMESCALEDB ARG TIMESCALEDB_APACHE_ONLY=true ARG TIMESCALEDB_TOOLKIT=true ARG COMPRESS @@ -92,7 +90,6 @@ FROM builder-${COMPRESS} LABEL maintainer="Team ACID @ Zalando " ARG PGVERSION -ARG TIMESCALEDB ARG DEMO ARG COMPRESS @@ -102,7 +99,6 @@ ENV LC_ALL=en_US.utf-8 \ PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \ PGHOME=/home/postgres \ RW_DIR=/run \ - TIMESCALEDB=$TIMESCALEDB \ DEMO=$DEMO ENV WALE_ENV_DIR=$RW_DIR/etc/wal-e.d/env \ diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 9b6f2845..69ea831d 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -56,7 +56,6 @@ curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT. curl -sL "https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git -git clone https://github.com/timescale/timescaledb.git apt-get install -y \ postgresql-common \ @@ -70,6 +69,11 @@ apt-get install -y \ # forbid creation of a main cluster when package is installed sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf +DISTRIB_CODENAME=$(sed -n 's/DISTRIB_CODENAME=//p' /etc/lsb-release) +# add TimescaleDB repository +echo "deb [signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${DISTRIB_CODENAME} main" | tee /etc/apt/sources.list.d/timescaledb.list +curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor | tee /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg > /dev/null + for version in $DEB_PG_SUPPORTED_VERSIONS; do sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list apt-get update @@ -96,7 +100,8 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-${version}-repack" "postgresql-${version}-wal2json" "postgresql-${version}-pllua" - "postgresql-${version}-pgvector") + "postgresql-${version}-pgvector" + "timescaledb-2-postgresql-${version}") if [ "$version" != "17" ]; then EXTRAS+=("postgresql-${version}-decoderbufs") @@ -121,23 +126,6 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do # Install 3rd party stuff - # use subshell to avoid having to cd back (SC2103) - ( - cd timescaledb - for v in $TIMESCALEDB; do - git checkout "$v" - sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt - if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ - -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \ - -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then - make -C build install - strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so - fi - git reset --hard - git clean -f -d - done - ) - if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then __versionCodename=$(sed Date: Wed, 16 Oct 2024 16:56:45 +0200 Subject: [PATCH 2/7] use the same gpg key location and name --- delivery.yaml | 1 - postgres-appliance/build_scripts/base.sh | 10 +++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 286fe4e2..4c21d3d8 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -31,7 +31,6 @@ pipeline: --build-arg PGVERSION="$PGVERSION" \ --build-arg BASE_IMAGE="$BASE_IMAGE" \ --build-arg PGOLDVERSIONS="14 15 16" \ - --build-arg TIMESCALEDB="2.17.0" \ -t "$ECR_TEST_IMAGE" \ --push . cdp-promote-image "$ECR_TEST_IMAGE" diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 69ea831d..91215771 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -69,10 +69,10 @@ apt-get install -y \ # forbid creation of a main cluster when package is installed sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf -DISTRIB_CODENAME=$(sed -n 's/DISTRIB_CODENAME=//p' /etc/lsb-release) # add TimescaleDB repository -echo "deb [signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${DISTRIB_CODENAME} main" | tee /etc/apt/sources.list.d/timescaledb.list -curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor | tee /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg > /dev/null +__versionCodename=$(sed /usr/share/keyrings/timescale_E7391C94080429FF.gpg for version in $DEB_PG_SUPPORTED_VERSIONS; do sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list @@ -127,10 +127,6 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do # Install 3rd party stuff if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then - __versionCodename=$(sed /usr/share/keyrings/timescale_E7391C94080429FF.gpg - apt-get update if [ "$(apt-cache search --names-only "^timescaledb-toolkit-postgresql-${version}$" | wc -l)" -eq 1 ]; then apt-get install "timescaledb-toolkit-postgresql-$version" From 58878dabf04bf05a177b806be0093930ab3e0147 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Thu, 17 Oct 2024 11:53:45 +0200 Subject: [PATCH 3/7] give option for oss only timescaledb --- postgres-appliance/build_scripts/base.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 91215771..97c54cbf 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -100,8 +100,7 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-${version}-repack" "postgresql-${version}-wal2json" "postgresql-${version}-pllua" - "postgresql-${version}-pgvector" - "timescaledb-2-postgresql-${version}") + "postgresql-${version}-pgvector") if [ "$version" != "17" ]; then EXTRAS+=("postgresql-${version}-decoderbufs") @@ -111,6 +110,12 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do EXTRAS+=("postgresql-plperl-${version}") fi + if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then + EXTRAS+=("timescaledb-2-oss-postgresql-${version}") + else + EXTRAS+=("timescaledb-2-postgresql-${version}") + fi + fi # Install PostgreSQL binaries, contrib, plproxy and multiple pl's From e0e71792120e7f1f88ae5aed706fc8f8fa2228c2 Mon Sep 17 00:00:00 2001 From: idanovinda Date: Thu, 17 Oct 2024 12:48:16 +0200 Subject: [PATCH 4/7] change gpg key and always install timescale --- postgres-appliance/build_scripts/base.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 97c54cbf..ab9522c6 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -70,9 +70,9 @@ apt-get install -y \ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf # add TimescaleDB repository -__versionCodename=$(sed /usr/share/keyrings/timescale_E7391C94080429FF.gpg +DISTRIB_CODENAME=$(sed /dev/null for version in $DEB_PG_SUPPORTED_VERSIONS; do sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list @@ -110,12 +110,12 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do EXTRAS+=("postgresql-plperl-${version}") fi - if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then - EXTRAS+=("timescaledb-2-oss-postgresql-${version}") - else - EXTRAS+=("timescaledb-2-postgresql-${version}") - fi + fi + if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then + EXTRAS+=("timescaledb-2-oss-postgresql-${version}") + else + EXTRAS+=("timescaledb-2-postgresql-${version}") fi # Install PostgreSQL binaries, contrib, plproxy and multiple pl's From fcfe2503506ec4cfcc1442e4902139711854f03a Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 4 Nov 2024 13:33:04 +0100 Subject: [PATCH 5/7] fix removal of keyring --- postgres-appliance/build_scripts/base.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index ab9522c6..dc92a653 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -140,7 +140,7 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do fi rm /etc/apt/sources.list.d/timescaledb.list - rm /usr/share/keyrings/timescale_E7391C94080429FF.gpg + rm /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg fi EXTRA_EXTENSIONS=() From 2ed670488187609d858a729efa6de7bb15f46dbc Mon Sep 17 00:00:00 2001 From: idanovinda Date: Mon, 4 Nov 2024 16:48:12 +0100 Subject: [PATCH 6/7] add timescale in every iteration --- postgres-appliance/build_scripts/base.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index dc92a653..4a217859 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -69,13 +69,14 @@ apt-get install -y \ # forbid creation of a main cluster when package is installed sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf -# add TimescaleDB repository -DISTRIB_CODENAME=$(sed /dev/null - for version in $DEB_PG_SUPPORTED_VERSIONS; do sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list + + # add TimescaleDB repository + DISTRIB_CODENAME=$(sed /dev/null + apt-get update if [ "$DEMO" != "true" ]; then From 18fec4f752919cdd556adea8a07d20d6ac3e888a Mon Sep 17 00:00:00 2001 From: idanovinda Date: Tue, 5 Nov 2024 09:56:14 +0100 Subject: [PATCH 7/7] move downloading timescale to prepare.sh file --- postgres-appliance/build_scripts/base.sh | 9 --------- postgres-appliance/build_scripts/prepare.sh | 4 ++++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 4a217859..201b0776 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -71,12 +71,6 @@ sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/create for version in $DEB_PG_SUPPORTED_VERSIONS; do sed -i "s/ main.*$/ main $version/g" /etc/apt/sources.list.d/pgdg.list - - # add TimescaleDB repository - DISTRIB_CODENAME=$(sed /dev/null - apt-get update if [ "$DEMO" != "true" ]; then @@ -139,9 +133,6 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do else echo "Skipping timescaledb-toolkit-postgresql-$version as it's not found in the repository" fi - - rm /etc/apt/sources.list.d/timescaledb.list - rm /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg fi EXTRA_EXTENSIONS=() diff --git a/postgres-appliance/build_scripts/prepare.sh b/postgres-appliance/build_scripts/prepare.sh index 50f32db8..66c2a2cb 100644 --- a/postgres-appliance/build_scripts/prepare.sh +++ b/postgres-appliance/build_scripts/prepare.sh @@ -40,6 +40,10 @@ for t in deb deb-src; do done curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg +# add TimescaleDB repository +echo "deb [signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${DISTRIB_CODENAME} main" | tee /etc/apt/sources.list.d/timescaledb.list +curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor | tee /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg > /dev/null + # Clean up apt-get purge -y libcap2-bin apt-get autoremove -y