diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 84585035d..3bdb4acfe 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=ubuntu:22.04 -ARG PGVERSION=16 -ARG TIMESCALEDB="2.11.2 2.14.2" +ARG PGVERSION=17 +ARG TIMESCALEDB="2.14.2" ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= @@ -55,14 +55,15 @@ ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" # Install PostgreSQL, extensions and contribs ENV POSTGIS_VERSION=3.4 \ - BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \ - PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \ - PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \ + BG_MON_COMMIT=7f5887218790b263fe3f42f85f4ddc9c8400b154 \ + PG_AUTH_MON_COMMIT=fe099eef7662cbc85b0b79191f47f52f1e96b779 \ + PG_MON_COMMIT=ead1de70794ed62ca1e34d4022f6165ff36e9a91 \ SET_USER=REL4_0_1 \ + SET_USER_COMMT=7f063e5d6e9e03847cf96c40b6ff62583ef457d5 \ PLPROFILER=REL4_2_4 \ PG_PROFILE=4.6 \ PAM_OAUTH2=v1.0.1 \ - PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 + PG_PERMISSIONS_COMMIT=f4b7c18676fa64236a1c8e28d34a35764e4a70e2 WORKDIR /builddeps RUN bash base.sh diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index e0d555d8a..f27d45408 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -55,7 +55,8 @@ curl -sL "https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz" | curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz -git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git +curl -sL "https://github.com/pgaudit/set_user/archive/$SET_USER_COMMT.tar.gz" | tar xz +# git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git # pg17 support is not released git clone https://github.com/timescale/timescaledb.git apt-get install -y \ @@ -82,22 +83,25 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-${version}-hll" "postgresql-${version}-hypopg" "postgresql-${version}-plproxy" - "postgresql-${version}-partman" "postgresql-${version}-pgaudit" "postgresql-${version}-pldebugger" - "postgresql-${version}-pglogical" - "postgresql-${version}-pglogical-ticker" "postgresql-${version}-plpgsql-check" - "postgresql-${version}-pg-checksums" - "postgresql-${version}-pgl-ddl-deploy" "postgresql-${version}-pgq-node" - "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}" - "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts" - "postgresql-${version}-repack" "postgresql-${version}-wal2json" - "postgresql-${version}-decoderbufs" "postgresql-${version}-pllua" "postgresql-${version}-pgvector") + + if [ "$version" != "17" ]; then + EXTRAS+=("postgresql-${version}-partman" + "postgresql-${version}-pglogical" + "postgresql-${version}-pglogical-ticker" + "postgresql-${version}-pg-checksums" + "postgresql-${version}-pgl-ddl-deploy" + "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}" + "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts" + "postgresql-${version}-repack" + "postgresql-${version}-decoderbufs") + fi if [ "$WITH_PERL" = "true" ]; then EXTRAS+=("postgresql-plperl-${version}") @@ -113,26 +117,31 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-plpython3-${version}" \ "postgresql-server-dev-${version}" \ "postgresql-${version}-pgq3" \ - "postgresql-${version}-pg-stat-kcache" \ "${EXTRAS[@]}" + + if [ "$version" != "17" ]; then + apt-get install --allow-downgrades -y "postgresql-${version}-pg-stat-kcache" + fi # Install 3rd party stuff # use subshell to avoid having to cd back (SC2103) ( cd timescaledb - for v in $TIMESCALEDB; do - git checkout "$v" - sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt - if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ - -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \ - -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then - make -C build install - strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so - fi - git reset --hard - git clean -f -d - done + if [ "$version" != "17" ]; then + for v in $TIMESCALEDB; do + git checkout "$v" + sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt + if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ + -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \ + -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then + make -C build install + strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so + fi + git reset --hard + git clean -f -d + done + fi ) if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then @@ -153,12 +162,15 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do EXTRA_EXTENSIONS=() if [ "$DEMO" != "true" ]; then - EXTRA_EXTENSIONS+=("plprofiler" "pg_mon-${PG_MON_COMMIT}") + EXTRA_EXTENSIONS+=("pg_mon-${PG_MON_COMMIT}") + if [ "$version" != "17" ]; then + EXTRA_EXTENSIONS+=("plprofiler") + fi fi for n in bg_mon-${BG_MON_COMMIT} \ pg_auth_mon-${PG_AUTH_MON_COMMIT} \ - set_user \ + set_user-${SET_USER_COMMT} \ pg_permissions-${PG_PERMISSIONS_COMMIT} \ pg_profile-${PG_PROFILE} \ "${EXTRA_EXTENSIONS[@]}"; do @@ -177,6 +189,9 @@ done if [ "$DEMO" != "true" ]; then for version in $DEB_PG_SUPPORTED_VERSIONS; do + if [ "$version" = "17" ]; then + continue + fi # create postgis symlinks to make it possible to perform update ln -s "postgis-${POSTGIS_VERSION%.*}.so" "/usr/lib/postgresql/${version}/lib/postgis-2.5.so" done @@ -258,6 +273,9 @@ if [ "$DEMO" != "true" ]; then # relink files with the same name and content across different major versions started=0 for v2 in $(find /usr/share/postgresql -type d -mindepth 1 -maxdepth 1 | sort -Vr); do + if [ "${v2##*/}" = "17" ]; then + continue + fi if [ "$v1" = "$v2" ]; then started=1 elif [ $started = 1 ]; then diff --git a/postgres-appliance/scripts/spilo_commons.py b/postgres-appliance/scripts/spilo_commons.py index d80a4dae9..77e702779 100644 --- a/postgres-appliance/scripts/spilo_commons.py +++ b/postgres-appliance/scripts/spilo_commons.py @@ -13,12 +13,12 @@ # (min_version, max_version, shared_preload_libraries, extwlist.extensions) extensions = { 'timescaledb': (9.6, 16, True, True), - 'pg_cron': (9.5, 16, True, False), + 'pg_cron': (9.5, 17, True, False), 'pg_stat_kcache': (9.4, 16, True, False), 'pg_partman': (9.4, 16, False, True) } if os.environ.get('ENABLE_PG_MON') == 'true': - extensions['pg_mon'] = (11, 16, True, False) + extensions['pg_mon'] = (11, 17, True, False) def adjust_extensions(old, version, extwlist=False): diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index 31886dba2..f0399ecb2 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -51,7 +51,7 @@ services: postgresql: parameters: shared_buffers: 32MB - PGVERSION: '12' + PGVERSION: '13' # Just to test upgrade with clone. Without CLONE_SCOPE they don't work CLONE_WAL_S3_BUCKET: *bucket CLONE_AWS_ACCESS_KEY_ID: *access_key diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index 478c44640..e17fb8c03 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -124,15 +124,15 @@ function drop_timescaledb() { } function test_inplace_upgrade_wrong_version() { - docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' + docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' } function test_inplace_upgrade_wrong_capacity() { - docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' + docker_exec "$1" "PGVERSION=14 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' } -function test_successful_inplace_upgrade_to_13() { - docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 3" +function test_successful_inplace_upgrade_to_14() { + docker_exec "$1" "PGVERSION=14 $UPGRADE_SCRIPT 3" } function test_envdir_suffix() { @@ -147,15 +147,7 @@ function test_envdir_updated_to_x() { } function test_failed_inplace_upgrade_big_replication_lag() { - ! test_successful_inplace_upgrade_to_13 "$1" -} - -function test_successful_inplace_upgrade_to_13() { - docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 3" -} - -function test_successful_inplace_upgrade_to_14() { - docker_exec "$1" "PGVERSION=14 $UPGRADE_SCRIPT 3" + ! test_successful_inplace_upgrade_to_14 "$1" } function test_successful_inplace_upgrade_to_15() { @@ -166,8 +158,12 @@ function test_successful_inplace_upgrade_to_16() { docker_exec "$1" "PGVERSION=16 $UPGRADE_SCRIPT 3" } -function test_pg_upgrade_to_16_check_failed() { - ! test_successful_inplace_upgrade_to_16 "$1" +function test_successful_inplace_upgrade_to_17() { + docker_exec "$1" "PGVERSION=17 $UPGRADE_SCRIPT 3" +} + +function test_pg_upgrade_to_17_check_failed() { + ! test_successful_inplace_upgrade_to_17 "$1" } function start_clone_with_wale_upgrade_container() { @@ -175,7 +171,7 @@ function start_clone_with_wale_upgrade_container() { docker-compose run \ -e SCOPE=upgrade \ - -e PGVERSION=13 \ + -e PGVERSION=14 \ -e CLONE_SCOPE=demo \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ @@ -188,24 +184,24 @@ function start_clone_with_wale_upgrade_replica_container() { start_clone_with_wale_upgrade_container 2 } -function start_clone_with_wale_upgrade_to_16_container() { +function start_clone_with_wale_upgrade_to_17_container() { docker-compose run \ -e SCOPE=upgrade3 \ - -e PGVERSION=16 \ + -e PGVERSION=17 \ -e CLONE_SCOPE=demo \ - -e CLONE_PGVERSION=12 \ + -e CLONE_PGVERSION=13 \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ --name "${PREFIX}upgrade4" \ -d "spilo3" } -function start_clone_with_wale_16_container() { +function start_clone_with_wale_17_container() { docker-compose run \ -e SCOPE=clone16 \ - -e PGVERSION=16 \ + -e PGVERSION=17 \ -e CLONE_SCOPE=upgrade3 \ - -e CLONE_PGVERSION=16 \ + -e CLONE_PGVERSION=17 \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_hour)" \ --name "${PREFIX}clone16" \ @@ -216,7 +212,7 @@ function start_clone_with_basebackup_upgrade_container() { local container=$1 docker-compose run \ -e SCOPE=upgrade2 \ - -e PGVERSION=14 \ + -e PGVERSION=15 \ -e CLONE_SCOPE=upgrade \ -e CLONE_METHOD=CLONE_WITH_BASEBACKUP \ -e CLONE_HOST="$(docker_exec "$container" "hostname --ip-address")" \ @@ -242,17 +238,17 @@ function verify_archive_mode_is_on() { } -# TEST SUITE 1 - In-place major upgrade 12->13->...->16 -# TEST SUITE 2 - Major upgrade 12->16 after wal-e clone (with CLONE_PGVERSION set) -# TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+) -# TEST SUITE 4 - Major upgrade 12->13 after wal-e clone (no CLONE_PGVERSION) +# TEST SUITE 1 - In-place major upgrade 15->16->...->17 +# TEST SUITE 2 - Major upgrade 13->17 after wal-e clone (with CLONE_PGVERSION set) +# TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (14+) +# TEST SUITE 4 - Major upgrade 15->16 after wal-e clone (no CLONE_PGVERSION) # TEST SUITE 5 - Replica bootstrap with wal-e -# TEST SUITE 6 - Major upgrade 13->14 after clone with basebackup +# TEST SUITE 6 - Major upgrade 15->16 after clone with basebackup function test_spilo() { # TEST SUITE 1 local container=$1 - run_test test_envdir_suffix "$container" 12 + run_test test_envdir_suffix "$container" 13 log_info "[TS1] Testing wrong upgrade setups" run_test test_inplace_upgrade_wrong_version "$container" @@ -269,66 +265,66 @@ function test_spilo() { # TEST SUITE 2 local upgrade3_container - upgrade3_container=$(start_clone_with_wale_upgrade_to_16_container) # SCOPE=upgrade3 PGVERSION=16 CLONE: _SCOPE=demo _PGVERSION=12 _TARGET_TIME= - log_info "[TS2] Started $upgrade3_container for testing major upgrade 12->16 after clone with wal-e" + upgrade3_container=$(start_clone_with_wale_upgrade_to_17_container) # SCOPE=upgrade3 PGVERSION=17 CLONE: _SCOPE=demo _PGVERSION=13 _TARGET_TIME= + log_info "[TS2] Started $upgrade3_container for testing major upgrade 13->17 after clone with wal-e" # TEST SUITE 4 local upgrade_container - upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=13 CLONE: _SCOPE=demo _TARGET_TIME= - log_info "[TS4] Started $upgrade_container for testing major upgrade 12->13 after clone with wal-e" + upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=14 CLONE: _SCOPE=demo _TARGET_TIME= + log_info "[TS4] Started $upgrade_container for testing major upgrade 13->14 after clone with wal-e" # TEST SUITE 1 # wait clone to finish and prevent timescale installation gets cloned find_leader "$upgrade3_container" find_leader "$upgrade_container" - create_timescaledb "$container" # we don't install it at the beginning, as we do 12->16 in a clone + create_timescaledb "$container" # we don't install it at the beginning, as we do 13->17 in a clone - log_info "[TS1] Testing in-place major upgrade 12->13" + log_info "[TS1] Testing in-place major upgrade 13->14" wait_zero_lag "$container" - run_test test_successful_inplace_upgrade_to_13 "$container" + run_test test_successful_inplace_upgrade_to_14 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 13 + run_test test_envdir_updated_to_x 14 # TEST SUITE 2 - log_info "[TS2] Testing in-place major upgrade 12->16 after wal-e clone" - run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 12 16 + log_info "[TS2] Testing in-place major upgrade 13->17 after wal-e clone" + run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 13 17 run_test verify_archive_mode_is_on "$upgrade3_container" wait_backup "$upgrade3_container" # TEST SUITE 3 - local clone16_container - clone16_container=$(start_clone_with_wale_16_container) # SCOPE=clone16 CLONE: _SCOPE=upgrade3 _PGVERSION=16 _TARGET_TIME= - log_info "[TS3] Started $clone16_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" + local clone17_container + clone17_container=$(start_clone_with_wale_17_container) # SCOPE=clone17 CLONE: _SCOPE=upgrade3 _PGVERSION=17 _TARGET_TIME= + log_info "[TS3] Started $clone17_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 14+" # TEST SUITE 1 - log_info "[TS1] Testing in-place major upgrade 13->14" - run_test test_successful_inplace_upgrade_to_14 "$container" + log_info "[TS1] Testing in-place major upgrade 14->15" + run_test test_successful_inplace_upgrade_to_15 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 14 + run_test test_envdir_updated_to_x 15 # TEST SUITE 3 find_leader "$clone16_container" - run_test verify_archive_mode_is_on "$clone16_container" + run_test verify_archive_mode_is_on "$clone17_container" # TEST SUITE 1 wait_backup "$container" - log_info "[TS1] Testing in-place major upgrade to 14->15" - run_test test_successful_inplace_upgrade_to_15 "$container" + log_info "[TS1] Testing in-place major upgrade to 15->16" + run_test test_successful_inplace_upgrade_to_16 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 15 + run_test test_envdir_updated_to_x 16 # TEST SUITE 4 - log_info "[TS4] Testing in-place major upgrade 12->13 after clone with wal-e" - run_test verify_clone_upgrade "$upgrade_container" "wal-e" 12 13 + log_info "[TS4] Testing in-place major upgrade 13->14 after clone with wal-e" + run_test verify_clone_upgrade "$upgrade_container" "wal-e" 13 14 run_test verify_archive_mode_is_on "$upgrade_container" wait_backup "$upgrade_container" @@ -342,20 +338,20 @@ function test_spilo() { # TEST SUITE 6 local basebackup_container - basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=14 CLONE: _SCOPE=upgrade - log_info "[TS6] Started $basebackup_container for testing major upgrade 13->14 after clone with basebackup" + basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=15 CLONE: _SCOPE=upgrade + log_info "[TS6] Started $basebackup_container for testing major upgrade 14->15 after clone with basebackup" # TEST SUITE 1 - # run_test test_pg_upgrade_to_16_check_failed "$container" # pg_upgrade --check complains about timescaledb + run_test test_pg_upgrade_to_17_check_failed "$container" # pg_upgrade --check complains about timescaledb wait_backup "$container" # drop_timescaledb "$container" - log_info "[TS1] Testing in-place major upgrade 15->16" - run_test test_successful_inplace_upgrade_to_16 "$container" + log_info "[TS1] Testing in-place major upgrade 16->17" + run_test test_successful_inplace_upgrade_to_17 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 16 + run_test test_envdir_updated_to_x 17 # TEST SUITE 5 @@ -364,8 +360,8 @@ function test_spilo() { # TEST SUITE 6 - log_info "[TS6] Testing in-place major upgrade 13->14 after clone with basebackup" - run_test verify_clone_upgrade "$basebackup_container" "basebackup" 13 14 + log_info "[TS6] Testing in-place major upgrade 14->15 after clone with basebackup" + run_test verify_clone_upgrade "$basebackup_container" "basebackup" 14 15 run_test verify_archive_mode_is_on "$basebackup_container" }