From 83e90a8150d0b5005dc96c0daf4e0b94a6d3907f Mon Sep 17 00:00:00 2001 From: unidevel Date: Fri, 7 Mar 2025 07:49:18 +0000 Subject: [PATCH 1/2] Refactor presto release actions --- ...ble-release.yml => presto-release-cut.yml} | 11 +- .github/workflows/presto-release-notes.yml | 42 +++++++ .../presto-release-publish-image.yml | 108 +++++++++++++++++ .../presto-release-publish-maven.yml | 111 +++++++++++++++++ .../presto-release-publish-native-image.yml | 114 ++++++++++++++++++ .../workflows/presto-release-tag-finalize.yml | 50 ++++++++ 6 files changed, 432 insertions(+), 4 deletions(-) rename .github/workflows/{presto-stable-release.yml => presto-release-cut.yml} (87%) create mode 100644 .github/workflows/presto-release-notes.yml create mode 100644 .github/workflows/presto-release-publish-image.yml create mode 100644 .github/workflows/presto-release-publish-maven.yml create mode 100644 .github/workflows/presto-release-publish-native-image.yml create mode 100644 .github/workflows/presto-release-tag-finalize.yml diff --git a/.github/workflows/presto-stable-release.yml b/.github/workflows/presto-release-cut.yml similarity index 87% rename from .github/workflows/presto-stable-release.yml rename to .github/workflows/presto-release-cut.yml index d5ad83adecc12..24412c54d609a 100644 --- a/.github/workflows/presto-stable-release.yml +++ b/.github/workflows/presto-release-cut.yml @@ -1,11 +1,10 @@ -name: Presto Stable Release Workflow +name: Presto Stable - Cut Release Branch on: workflow_dispatch: jobs: - presto-release: - name: Presto Stable Release Workflow + cut-stable-release: runs-on: ubuntu-latest environment: release @@ -60,11 +59,15 @@ jobs: -DautoVersionSubmodules \ -DdevelopmentVersion=${{ env.PRESTO_RELEASE_VERSION }} \ -DreleaseVersion=${{ env.PRESTO_RELEASE_VERSION }} + grep -m 1 "" pom.xml + git log --pretty="format:%ce: %s" -5 git push --follow-tags origin master - name: Push release branch env: PRESTO_RELEASE_VERSION: ${{ steps.get-version.outputs.PRESTO_RELEASE_VERSION }} run: | - git checkout -b release-${{ env.PRESTO_RELEASE_VERSION }} + git checkout ${{ env.PRESTO_RELEASE_VERSION }} + git switch -c release-${{ env.PRESTO_RELEASE_VERSION }} + git log --pretty="format:%ce: %s" -3 git push origin release-${{ env.PRESTO_RELEASE_VERSION }} diff --git a/.github/workflows/presto-release-notes.yml b/.github/workflows/presto-release-notes.yml new file mode 100644 index 0000000000000..014756944ada0 --- /dev/null +++ b/.github/workflows/presto-release-notes.yml @@ -0,0 +1,42 @@ +name: Presto Stable - Generate Release Notes + +on: + workflow_dispatch: + +env: + UPSTREAM: prestodb/presto + +jobs: + release-notes: + runs-on: ubuntu-latest + + steps: + - name: Check for master branch + if: ${{ github.ref != 'refs/heads/master' }} + run: echo "Invalid branch. This action can only be run on the master branch." && exit 1 + + - name: Check for personal repository + if: ${{ github.repository == env.UPSTREAM }} + run: echo "This action can only be run on personal repository, please clone ${{ env.UPSTREAM }}, and run this action in your own repo" && exit 1 + + - name: Checkout presto source + uses: actions/checkout@v4 + with: + ref: master + show-progress: false + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Add git upstream + run: | + git remote add upstream https://github.com/${{ env.UPSTREAM }}.git + git fetch upstream --tags + git remote -v + + - name: Create release notes pull request + run: | + ./src/release/release-notes.sh ${{ github.repository_owner }} ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/presto-release-publish-image.yml b/.github/workflows/presto-release-publish-image.yml new file mode 100644 index 0000000000000..8c919a4b0a5ad --- /dev/null +++ b/.github/workflows/presto-release-publish-image.yml @@ -0,0 +1,108 @@ +name: Presto Stable - Publish Docker Images + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 0.291)' + required: true + publish_as_latest: + description: 'Also publish as latest version' + type: boolean + default: true + required: false + +env: + VERSION: ${{ github.event.inputs.version }} + DOCKER_REPO: ${{ github.repository }} + +jobs: + maven-build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.VERSION }} + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Maven build + run: mvn clean install -DskipTests + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: presto-artifacts-${{ env.VERSION }} + retention-days: 1 + path: | + presto-server/target/presto-server-*.tar.gz + presto-cli/target/presto-cli-*-executable.jar + + docker-publish: + needs: [maven-build] + runs-on: ubuntu-latest + environment: release + permissions: + packages: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.VERSION }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-artifacts-${{ env.VERSION }} + path: ./ + + - name: Login to dockerhub + uses: docker/login-action@v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to gitHub container registry + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.PRESTODB_CI_TOKEN }} + + - name: Set up qemu + uses: docker/setup-qemu-action@v3 + + - name: Set up docker buildx + uses: docker/setup-buildx-action@v3.9.0 + + - name: Create and use builder + run: | + docker buildx create --name container --use + docker buildx inspect --bootstrap + + - name: Move artifacts to docker directory + run: | + mv ./presto-server/target/presto-server-*.tar.gz docker/ + mv ./presto-cli/target/presto-cli-*-executable.jar docker/ + + - name: Build docker image and publish + uses: docker/build-push-action@v6 + with: + context: docker + platforms: linux/amd64,linux/arm64,linux/ppc64le + file: docker/Dockerfile + push: true + build-args: | + PRESTO_VERSION=${{ env.VERSION }} + JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0 + tags: | + ${{ env.DOCKER_REPO }}:${{ env.VERSION }} + ${{ github.event.inputs.publish_as_latest == 'true' && format('{0}:latest', env.DOCKER_REPO) || '' }} + ghcr.io/${{ github.repository }}:${{ env.VERSION }} + ${{ github.event.inputs.publish_as_latest == 'true' && format('ghcr.io/{0}:latest', github.repository) || '' }} diff --git a/.github/workflows/presto-release-publish-maven.yml b/.github/workflows/presto-release-publish-maven.yml new file mode 100644 index 0000000000000..083c8b556a128 --- /dev/null +++ b/.github/workflows/presto-release-publish-maven.yml @@ -0,0 +1,111 @@ +name: Presto Stable - Publish Maven Artifacts + +on: + workflow_dispatch: + inputs: + RELEASE_BRANCH: + description: 'Release branch (e.g., release-0.290)' + required: true + RELEASE_VERSION: + description: 'Release version (e.g., 0.290)' + required: true + +jobs: + publish-stable-release: + runs-on: ubuntu-latest + environment: release + timeout-minutes: 300 # 5 hours + + env: + NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + + steps: + - name: Setup JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y build-essential git gpg python3 python3-venv + + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.RELEASE_BRANCH }} + token: ${{ secrets.PRESTODB_CI_TOKEN }} + fetch-depth: 0 + fetch-tags: true + + - name: Configure Git + run: | + git config --global user.email "ci@lists.prestodb.io" + git config --global user.name "prestodb-ci" + git checkout ${{ github.event.inputs.RELEASE_VERSION }} + git log --pretty="format:%ce: %s" -5 + + - name: Import GPG key + run: | + echo "${{ secrets.GPG_SECRET }}" > ${{ github.workspace }}/secret-key.gpg + chmod 600 ${{ github.workspace }}/secret-key.gpg + gpg --import --batch ${{ github.workspace }}/secret-key.gpg + rm -f ${{ github.workspace }}/secret-key.gpg + gpg --list-secret-keys + echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf + env: + GPG_TTY: $(tty) + + - name: Create Maven Settings + run: | + cat > ${{ github.workspace }}/settings.xml << 'EOL' + + + + sonatype-nexus-snapshots + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + sonatype.snapshots + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + ossrh + ${env.NEXUS_USERNAME} + ${env.NEXUS_PASSWORD} + + + + + nexus + + + + + + nexus + + + EOL + + - name: Release Maven Artifacts + run: | + unset MAVEN_CONFIG + ./mvnw -s ${{ github.workspace }}/settings.xml -V -B -U -e -T1C deploy \ + -Dgpg.passphrase="${{ secrets.GPG_PASSPHRASE }}" \ + -Dmaven.wagon.http.retryHandler.count=8 \ + -DskipTests \ + -DstagingProfileId=28a0d8c4350ed \ + -DkeepStagingRepositoryOnFailure=true \ + -DkeepStagingRepositoryOnCloseRuleFailure=true \ + -DautoReleaseAfterClose=true \ + -DstagingProgressTimeoutMinutes=60 \ + -Poss-release \ + -Pdeploy-to-ossrh \ + -pl '!presto-test-coverage' + env: + GPG_TTY: $(tty) diff --git a/.github/workflows/presto-release-publish-native-image.yml b/.github/workflows/presto-release-publish-native-image.yml new file mode 100644 index 0000000000000..82ab593514dd6 --- /dev/null +++ b/.github/workflows/presto-release-publish-native-image.yml @@ -0,0 +1,114 @@ +name: Presto Stable - Publish Native Docker Image + +on: + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g., 0.291)' + required: true + publish_as_latest: + description: 'Also publish as latest version' + type: boolean + default: true + required: false + dependency_image: + description: 'Dependency image(e.g., prestodb/presto-native-dependency:0.290-20241014120930-e1fc090)' + required: false + default: '' + +env: + VERSION: ${{ github.event.inputs.version }} + ORG_NAME: ${{ github.repository_owner }} + IMAGE_NAME: presto-native + +jobs: + build-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + environment: release + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ env.VERSION }} + submodules: true + + - name: Initialize Prestissimo submodules + run: | + cd presto-native-execution && make submodules + echo "COMMIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Login to DockerHub + uses: docker/login-action@v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set dependency image tag + run: | + if [[ -n "${{ github.event.inputs.dependency_image }}" ]]; then + echo "DEPENDENCY_IMAGE=${{ github.event.inputs.dependency_image }}" >> $GITHUB_ENV + else + echo "DEPENDENCY_IMAGE=ghcr.io/${{ github.repository_owner }}/presto-native-dependency:${{ env.VERSION }}-${{ env.COMMIT_SHA }}" >> $GITHUB_ENV + fi + + - name: Build Dependency Image + working-directory: presto-native-execution + run: | + if docker pull ${{ env.DEPENDENCY_IMAGE }}; then + echo "Using dependency image ${{ env.DEPENDENCY_IMAGE }}" + docker tag ${{ env.DEPENDENCY_IMAGE }} presto/prestissimo-dependency:centos9 + else + echo "Building new depedency image" + docker compose build centos-native-dependency + docker tag presto/prestissimo-dependency:centos9 ghcr.io/${{ github.repository_owner }}/presto-native-dependency:${{ env.VERSION }}-${{ env.COMMIT_SHA }} + docker push ghcr.io/${{ github.repository_owner }}/presto-native-dependency:${{ env.VERSION }}-${{ env.COMMIT_SHA }} + fi + docker images + + - name: Build Runtime Image + working-directory: presto-native-execution + run: | + if docker pull ghcr.io/${{ github.repository_owner }}/presto-native:${{ env.VERSION }}-${{ env.COMMIT_SHA }}; then + docker tag ghcr.io/${{ github.repository_owner }}/presto-native:${{ env.VERSION }}-${{ env.COMMIT_SHA }} docker.io/presto/prestissimo-runtime:centos9 + else + docker compose build centos-native-runtime + docker tag presto/prestissimo-runtime:centos9 ghcr.io/${{ github.repository_owner }}/presto-native:${{ env.VERSION }}-${{ env.COMMIT_SHA }} + docker push ghcr.io/${{ github.repository_owner }}/presto-native:${{ env.VERSION }}-${{ env.COMMIT_SHA }} + fi + + - name: Add release tag + working-directory: presto-native-execution + run: | + docker tag presto/prestissimo-runtime:centos9 ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} + if [[ "${{ github.event.inputs.publish_as_latest }}" == "true" ]]; then + docker tag presto/prestissimo-runtime:centos9 ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest + fi + + - name: Push to DockerHub + run: | + docker push ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} + if [[ "${{ github.event.inputs.publish_as_latest }}" == "true" ]]; then + docker tag ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest + docker push ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:latest + fi + + - name: Tag and push to GitHub Packages + run: | + docker tag ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} + docker push ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} + if [[ "${{ github.event.inputs.publish_as_latest }}" == "true" ]]; then + docker tag ${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }} ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest + docker push ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest + fi \ No newline at end of file diff --git a/.github/workflows/presto-release-tag-finalize.yml b/.github/workflows/presto-release-tag-finalize.yml new file mode 100644 index 0000000000000..ce3a9615fd9ac --- /dev/null +++ b/.github/workflows/presto-release-tag-finalize.yml @@ -0,0 +1,50 @@ +name: Presto Stable - Tag Finalize + +on: + workflow_dispatch: + inputs: + release-tag: + description: 'Release tag (e.g., 0.291)' + required: true + release-notes-commit: + description: 'Commit hash containing release notes' + required: true + +env: + RELEASE_BRANCH: release-${{ github.event.inputs.release-tag }} + RELEASE_TAG: ${{ github.event.inputs.release-tag }} + RELEASE_NOTES_COMMIT: ${{ github.event.inputs.release-notes-commit }} + +jobs: + finalize-release: + runs-on: ubuntu-latest + environment: release + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ env.RELEASE_BRANCH }} + token: ${{ secrets.PRESTODB_CI_TOKEN }} + fetch-depth: 0 + fetch-tags: true + show-progress: false + + - name: Configure Git + run: | + git config --global user.email "ci@lists.prestodb.io" + git config --global user.name "prestodb-ci" + git config pull.rebase false + + - name: Cherry-pick release notes + run: | + git cherry-pick ${{ env.RELEASE_NOTES_COMMIT }} + + - name: Delete existing release tag + run: | + git push origin :${{ env.RELEASE_TAG }} || true + git tag -d ${{ env.RELEASE_TAG }} || true + + - name: Create new release tag + run: | + git tag -a ${{ env.RELEASE_TAG }} -m "release ${{ env.RELEASE_TAG }}" + git push origin ${{ env.RELEASE_BRANCH }} --tags \ No newline at end of file From 4812f01c6cfda406e65505df322ddb75ef48107c Mon Sep 17 00:00:00 2001 From: unidevel Date: Fri, 7 Mar 2025 20:54:06 +0000 Subject: [PATCH 2/2] Add release notes for 0.291 --- presto-docs/src/main/sphinx/release.rst | 1 + .../src/main/sphinx/release/release-0.291.rst | 207 +++++++++--------- 2 files changed, 101 insertions(+), 107 deletions(-) diff --git a/presto-docs/src/main/sphinx/release.rst b/presto-docs/src/main/sphinx/release.rst index 5db6f9e7ab17f..ed64295584af3 100644 --- a/presto-docs/src/main/sphinx/release.rst +++ b/presto-docs/src/main/sphinx/release.rst @@ -5,6 +5,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release/release-0.291 Release-0.291 [2025-01-27] Release-0.290 [2024-11-01] Release-0.289 [2024-08-23] diff --git a/presto-docs/src/main/sphinx/release/release-0.291.rst b/presto-docs/src/main/sphinx/release/release-0.291.rst index 7b954ae36c084..08dfb2fcd9f47 100644 --- a/presto-docs/src/main/sphinx/release/release-0.291.rst +++ b/presto-docs/src/main/sphinx/release/release-0.291.rst @@ -4,146 +4,139 @@ Release 0.291 **Highlights** ============== -* Add :doc:`/sql/alter-table` SET PROPERTIES statement. `#21495 `_ -* Add catalog and schema level access checks in :doc:`/sql/use` statement. `#24182 `_ -* Add single worker execution. To improve latency of tiny queries running on a large cluster, we introduce single worker execution mode: query will only use one node to execute and plan would be optimized accordingly. This feature can be turned on by the configuration property ``single-node-execution-enabled`` or the session property ``single_node_execution_enabled``. `#24172 `_ -* Add support for the histogram statistic type. `#22365 `_ -* Add support for ``ALTER VIEW RENAME TO`` operation, including the necessary infrastructure for connector implementations. `#23749 `_ -* Improve scheduling for CTE materialization: Now, only the stages containing CTE table scans that reference CTE table write stages are blocked till the write is complete, instead of the entire query being blocked as was the case previously. This is controlled by the session property ``enhanced_cte_scheduling_enabled`` (on by default). `#24108 `_ -* Add support for time type partitioning in the ORC file format for Iceberg. `#24091 `_ -* Enable ``scale-writers`` by default. `#24107 `_ -* Improve UUID comparisons so they conform to `IETF RFC 4122 `_. `#23847 `_ **Details** =========== General Changes _______________ -* Fix behavior of :func:`width_bucket(x, bins) -> bigint` which previously treated all ``null`` elements in bins as ``0``. The function throws an error if it finds a ``null`` or non-finite element in ``bins``. `#24103 `_ -* Fix bug in ``strrpos`` function for multibyte characters. `#24226 `_ -* Improve Request Headers in the Authentication Filter Class. `#23380 `_ -* Improve coordinator task management performance. `#24369 `_ -* Improve efficiency of coordinator when running a large number of tasks. `#24288 `_ -* Improve scheduling for CTE materialization: Now, only the stages containing CTE table scans that reference CTE table write stages are blocked till the write is complete, instead of the entire query being blocked as was the case previously. This is controlled by the session property ``enhanced_cte_scheduling_enabled`` (on by default). `#24108 `_ -* Improve security of internal HTTP endpoints by sanitizing headers before they are used. `#24004 `_ -* Add :doc:`/sql/alter-table` SET PROPERTIES statement. `#21495 `_ -* Add :func:`google_polyline_decode` function to convert Google polyline to Presto ST_Geometry types. `#23999 `_ -* Add :func:`google_polyline_encode` function to convert Presto ST_Geometry to Google polyline types. `#23999 `_ -* Add ``ClientRequestFilter.java`` interface in Presto-spi: :doc:`/develop/client-request-filter`. `#23380 `_ -* Add a configuration property ``plan-checker.config-dir`` to set the configuration directory for PlanCheckerProvider configurations. `#23955 `_ -* Add a session property ``include_values_node_in_connector_optimizer`` to enable connector optimizer optimize plans with values node. `#24227 `_ -* Add ``native_enforce_join_build_input_partition`` session property to not enforce input partition for join build. `#24163 `_ -* Add catalog and schema level access checks in :doc:`/sql/use` statement. `#24182 `_ -* Add delete node in subfield pruning optimizer. `#24206 `_ -* Add single worker execution. To improve latency of tiny queries running on a large cluster, we introduce single worker execution mode: query will only use one node to execute and plan would be optimized accordingly. This feature can be turned on by the configuration property ``single-node-execution-enabled`` or the session property ``single_node_execution_enabled``. `#24172 `_ -* Add support for ORC metadata cache invalidation based on file modification time. `#24346 `_ -* Add support for ``ALTER VIEW RENAME TO`` operation, including the necessary infrastructure for connector implementations. `#23749 `_ -* Add support for the histogram statistic type. `#22365 `_ -* Enable ``scale-writers`` by default. `#24107 `_ -* Update usage of MD5 to SHA256. `#23903 `_ -* Improve UUID comparisons so they conform to `IETF RFC 4122 `_. `#23847 `_ - -Prestissimo (native Execution) Changes -______________________________________ -* Improve partitioned remote exchanges for wide data sets (more than 500 columns) to use row wise encoding. `#23929 `_ -* Add native plan checker to the native sidecar plugin and native endpoint for Velox plan conversion. `#23596 `_ -* Add session properties ``native_spill_prefixsort_enabled``, ``native_prefixsort_normalized_key_max_bytes``, and ``native_prefixsort_min_rows``. `#24043 `_ -* Add support for automatic scaling of writer threads for partitioned tables. Can be enabled with the ``native_execution_scale_partitioned_writer_threads_enabled`` session property. Native execution only. `#24155 `_ -* Remove the ``experimental.table-writer-merge-operator-enabled`` configuration property and the ``table_writer_merge_operator_enabled`` session property. `#24145 `_ -* Remove deprecated ``native_query_trace_task_reg_exp session`` property from Prestissimo. `#24270 `_ -* Add utilizing environment variables as stand in for configuration values. The environment variable value is retrieved and used for the configuration option. `#23880 `_ -* Add session property ``native_query_trace_fragment_id`` and ``native_query_trace_shard_id`` for easier trace control. `#24209 `_ -* Add session property: ``native_table_scan_scaled_processing_enabled``. `#24284 `_ -* Add session property: ``native_table_scan_scale_up_memory_usage_ratio``. `#24284 `_ -* Add session property ``native_scaled_writer_rebalance_max_memory_usage_ratio``. `#24261 `_ -* Add session property ``native_scaled_writer_max_partitions_per_writer``. `#24261 `_ -* Add session property ``native_scaled_writer_min_partition_processed_bytes_rebalance_threshold``. `#24261 `_ -* Add session property ``native_scaled_writer_min_processed_bytes_rebalance_threshold``. `#24261 `_ +* Fix behavior of :func:`width_bucket(x, bins) -> bigint` which previously treated all ``null`` elements in bins as ``0``. Now the function will throw an error if it finds a ``null`` or non-finite element in ``bins``.. :pr:`24103`. `#24103 `_ +* Fix bug in strrpos function for multibyte characters. `#24226 `_ +* Improve Request Headers in the Authentication Filter Class. :pr:`23380`. `#23380 `_ +* Improve coordinator task management performance. :pr:`24369`. `#24369 `_ +* Improve efficiency of coordinator when running a large number of tasks. :pr:`24288`. `#24288 `_ +* Improve scheduling for CTE materialization: Now, only the stages containing CTE table scans that reference CTE table write stages are blocked till the write is complete, instead of the entire query being blocked as was the case previously. This is controlled by the session property ``enhanced_cte_scheduling_enabled`` (on by default) :pr:`24108`. `#24108 `_ +* Add :doc:`/sql/alter-table` SET PROPERTIES statement :pr:`21495`. `#21495 `_ +* Add :func:`google_polyline_decode` function to convert Google polyline to Presto ST_Geometry types. :pr:`23999`. `#23999 `_ +* Add :func:`google_polyline_encode` function to convert Presto ST_Geometry to Google polyline types. :pr:`23999`. `#23999 `_ +* Add Delete TableWriter TableFinish node to SPI :pr:`24088`. `#24088 `_ +* Add SemiJoin Join TableWriter Delete TableFinish node to connector optimizer :pr:`24154`. `#24154 `_ +* Add ``ClientRequestFilter.java`` interface in Presto-spi. :pr:`23380`. `#23380 `_ +* Add ``query-data-cache-enabled-default`` configuration property to align C++ cache behavior with Java. Set it to ``true``(default) for current C++ behavior or to ``false`` to match Java's cache logic. :pr:`24076`. `#24076 `_ +* Add a configuration property ``plan-checker.config-dir`` to set the configuration directory for PlanCheckerProvider configurations. :pr:`23955`. `#23955 `_ +* Add a session property `include_values_node_in_connector_optimizer` to enable connector optimizer optimize plans with values node :pr:`12345`. `#24227 `_ +* Add an optional input distribution constraint to DeleteNode :pr:`24104`. `#24104 `_ +* Add an session property to not enforce input partition for join build :pr:`24163`. `#24163 `_ +* Add catalog and schema level access checks in :doc:`/sql/use` statement. :pr:`24182`. `#24182 `_ +* Add delete node in subfield pruning optimizer :pr:`24206`. `#24206 `_ +* Add single worker execution. To improve latency of tiny queries running on a large cluster, we introduce single worker execution mode: query will only use one node to execute and plan would be optimized accordingly. This feature can be turned on by config `single-node-execution-enabled` or session property `single_node_execution_enabled`.:pr:`24172`. `#24172 `_ +* Add support for ORC metadata cache invalidation based on file modification time. :pr:`24346`. `#24346 `_ +* Add support for ``ALTER VIEW RENAME TO`` operation, including the necessary infrastructure for connector implementations. :pr:`23749`. `#23749 `_ +* Add support for ``BigInt`` data type in the SQL Client on Presto UI on supported browsers. See `compatibility `_ for the supported browsers. :pr:`24336`. `#24336 `_ +* Add support for the histogram statistic type. :pr:`22365`. `#22365 `_ +* Add support for time type partitioning in the ORC file format for Iceberg. :pr:`24091`. `#24091 `_ +* Add testing for partitioning using time type in Iceberg. :pr:`24091`. `#24091 `_ +* Added catalog and schema level access checks in USE statement :pr:`23882 `. `#23882 `_ +* Remove ``query-data-cache-enabled-default`` configuration property, which is no longer needed as per-split fine-grained cache control has been introduced. :pr:`24372 `. `#24372 `_ +* Remove the ``experimental.table-writer-merge-operator-enabled` config property and the ``table_writer_merge_operator_enabled`` session property :pr:`12345`. `#24145 `_ +* Deprecated native_query_trace_task_reg_exp session property from Prestissimo:pr:`24270`. `#24270 `_ +* Enable ``scale-writers`` by default. :pr:`24107`. `#24107 `_ +* Support automatic scaling of writer threads for partitioned tables. Can be enabled with the `native_execution_scale_partitioned_writer_threads_enabled` session property. Native execution only. :pr:`24155`. `#24155 `_ +* Throw exception on invalid http headers in async page transport servlet. :pr:`24004`. `#24004 `_ +* Update usage of MD5 to SHA256 :pr:`23903`. `#23903 `_ +* Upgrade avro to version 1.11.4 in response to `CVE-2024-47561 `_. :pr:`23868`. `#23943 `_ +* Upgrade grpc dependencies to version 1.68.0 in response to `CVE-2022-25647 ` :pr:`24051`. `#24051 `_ +* Upgrade gson from version 2.8.9 to v2.11.0 pr: `24247`. `#24247 `_ +* Upgrade gson to version 2.8.9 in response to `CVE-2022-25647 ` :pr:`24051`. `#24051 `_ +* Upgrade jetty dependencies to version 9.4.56.v20240826 in response to `CVE-2024-8184 `_. :pr:`24184`. `#24184 `_ +* Upgrade netty dependencies to version 4.1.115.Final in response to `CVE-2024-47535 `_. :pr:`24137`. `#24137 `_ Security Changes ________________ -* Fix security vulnerability in presto-pinot-toolkit and presto-product-tests in response to `CVE-2020-0287 `_. `#24249 `_ -* Fix security vulnerability in swagger-ui jar in response to `CVE-2018-25031 `_. `#24153 `_ -* Fix security vulnerability in swagger-ui jar in response to `CVE-2018-25031 `_. `#24199 `_ -* Improve pbkdf2 hashing using SHA-256 cipher in response to `CWE-759 `_. `#24132 `_ -* Add re2j regex matching in QueryStateInfoResource to protect from ReDoS attacks in response to `CVE-2024-45296 `_. `#24048 `_ -* Add security-related headers to the static resources serving from the Presto UI, including: ``Content-Security-Policy``, ``X-Content-Type-Options``. See reference docs `Content-Security-Policy `_ and `X-Content-Type-Options `_. `#24272 `_ -* Add support for pluggable Custom Presto Authenticators. `#24111 `_ -* Replace `alluxio-shaded-client` with `alluxio-core` libraries in response to `CVE-2023-44981 `_. `#24231 `_ -* Upgrade avro to version 1.11.4 in response to `CVE-2024-47561 `_. `#23943 `_ -* Upgrade grpc dependencies to version 1.68.0 in response to `CVE-2022-25647 `_. `#24051 `_ -* Upgrade gson from version 2.8.9 to v2.11.0. `#24247 `_ -* Upgrade gson to version 2.8.9 in response to `CVE-2022-25647 `_. `#24051 `_ -* Upgrade jetty dependencies to version 9.4.56.v20240826 in response to `CVE-2024-8184 `_. `#24184 `_ -* Upgrade netty dependencies to version 4.1.115.Final in response to `CVE-2024-47535 `_. `#24137 `_ -* Upgrade druid-processing to 30.0.1 in response to `CVE-2024-45384 `_ and `CVE-2024-45537 `_. `#23949 `_ -* Upgrade lucene-core to 8.10.0 in response to `WS-2021-0646 `_. `#24168 `_ -* Upgrade lucene-queryparser to 8.10.0 in response to `WS-2021-0646 `_. `#24168 `_ -* Upgrade lucene-analyzer-common to 8.10.0 in response to `WS-2021-0646 `_. `#24168 `_ -* Update hibernate-validator to 6.2.0.Final in response to `CVE-2023-1932 `_ and `CVE-2020-10693 `_. `#24177 `_ -* Upgrade Jackson dependencies to 2.15.4 in response to `PRISMA-2023-0067 `_. `#23753 `_ -* Upgrade snakeyaml to 2.0 in response to `CVE-2022-1471 `_ and `CVE-2022-25857 `_ and `CVE-2017-18640 `_ and `CVE-2022-38752 `_ and `CVE-2022-38751 `_ and `CVE-2022-38750 `_ and `CVE-2022-38749 `_ and `CVE-2022-41854 `_. `#24099 `_ -* Upgrade nanoid to 3.3.8 in response to `CVE-2024-55565 `_. `#24273 `_ - - -BigQuery Connector Changes +* Fix `CVE-2015-3192 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2015-5211 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2016-1000027 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2018-1199 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2018-1272 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2020-5421 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2021-0170 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2021-22060 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2021-22096 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2021-22096 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2022-22965 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2022-22970 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2022-22970 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2022-27772 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2023-20883 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-22243 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-22259 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-22262 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-38809 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-6763 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-6763 `_. :pr:`24112`. `#24112 `_ +* Fix `CVE-2024-8184 `_. :pr:`24112`. `#24112 `_ +* Fix security vulnerability in presto-pinot-toolkit and presto-product-tests in response to `CVE-2020-0287 `_. :pr:`24249`. `#24249 `_ +* Fix security vulnerability in swagger-ui jar in response to 'CVE-2018-25031 ' . :pr:`24153`. `#24153 `_ +* Fix security vulnerability in swagger-ui jar in response to 'CVE-2018-25031 ' . :pr:`24199`. `#24199 `_ +* Improve pbkdf2 hashing using SHA-256 cipher in response to `CWE-759 `_. :pr:`24132`. `#24132 `_ +* Add re2j regex matching in QueryStateInfoResource to protect from ReDoS attacks in response to `CVE-2024-45296 `_. :pr:`24048`. `#24048 `_ +* Add security-related headers to the static resources serving from the Presto UI, including: ``Content-Security-Policy``, ``X-Content-Type-Options``. See reference docs `Content-Security-Policy `_ and `X-Content-Type-Options `_. :pr:`24272`. `#24272 `_ +* Add support for pluggable Custom Presto Authenticators :pr:`#24111`. `#24111 `_ +* Replace `alluxio-shaded-client` with `alluxio-core` libraries libraries in response to `CVE-2023-44981 `_. :pr:`24231`. `#24231 `_ +* Upgrade druid-processing to 30.0.1 in response to `CVE-2024-45384 `_ and `CVE-2024-45537 `_. :pr:`23949`. `#23949 `_ + +Bigquery Connector Changes __________________________ -* Fix ``SELECT`` statements failing with ``NoClassDefFoundError: io/grpc/CallCredentials2``. `#23957 `_ +* Fix for BigQuery ``SELECT``. :pr:`23957`. `#23957 `_ Cassandra Connector Changes ___________________________ -* Improve cryptographic protocol in response to `Weak SSL/TLS protocols should not be used `_. `#24436 `_ - -Clickhouse Connector Changes -____________________________ -* Add ``DateTime64`` type support. `#24344 `_ +* Improve cryptographic protocol in response to `java:S4423 `_. :pr:`24436`. `#24436 `_ Delta Connector Changes _______________________ -* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. `#23401 `_ +* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. :pr:`23401`. `#23401 `_ Hive Connector Changes ______________________ -* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. `#23401 `_ +* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. :pr:`23401`. `#23401 `_ Hudi Connector Changes ______________________ -* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. `#23401 `_ +* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. :pr:`23401`. `#23401 `_ Iceberg Connector Changes _________________________ -* Improve performance of scan planning in Iceberg Connector. `#24376 `_ -* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. `#23401 `_ -* Add configuration property ``iceberg.rest.nested.namespace.enabled`` to support nested namespaces in Iceberg's REST Catalog. Defaults to ``true``. `#24083 `_ -* Add support for ``ALTER VIEW RENAME TO``. `#23749 `_ -* Add support of ``view`` for Iceberg connector when configured with ``REST`` and ``NESSIE``. `#23793 `_ -* Add support of specifying table location on creation for Iceberg connector when configured with ``REST`` and ``NESSIE``. `#24218 `_ -* Remove in-memory hive metastore cache in Iceberg connector. `#24326 `_ -* Add support for time type partitioning in the ORC file format for Iceberg. `#24091 `_ -* Add testing for partitioning using time type in Iceberg. `#24091 `_ +* Add ``catalog.system.invalidate_metastore_cache`` procedure to invalidate all, or portions of, the metastore cache. :pr:`23401`. `#23401 `_ +* Add configuration property ``iceberg.rest.nested.namespace.enabled`` to support nested namespaces in Iceberg's REST Catalog. Defaults to ``true``. :pr:`24083`. `#24083 `_ +* Add support for ``ALTER VIEW RENAME TO``. :pr:`23749`. `#23749 `_ +* Add support of ``view`` for Iceberg connector when configured with ``REST`` and ``NESSIE``. :pr:`23793`. `#23793 `_ +* Add support of specifying table location on creation for Iceberg connector when configured with ``REST`` and ``NESSIE``. :pr:`24218`. `#24218 `_ +* Remove in-memory hive metastore cache in Iceberg connector. :pr:`24326`. `#24326 `_ +* Optimize redundant getTable calls in Iceberg Connector :pr:`24376 `. `#24376 `_ Memory Connector Changes ________________________ -* Add support for ``ALTER VIEW RENAME TO``. `#23749 `_ +* Add support for ``ALTER VIEW RENAME TO``. :pr:`23749`. `#23749 `_ -MongoDB Connector Changes +Mongodb Connector Changes _________________________ -* Add steps to connect to MongoDB cluster with TLS CA File to :doc:`/connector/mongodb`. `#24352 `_ +* Add steps to connect to MongoDB cluster with TLS CA File to :doc:`/connector/mongodb`. :pr:`24352`. `#24352 `_ SPI Changes ___________ -* Improve ExpressionOptimizer#optimize method with a variable resolver to return ``RowExpression``. `#24287 `_ -* Add WindowNode, JoinNode, SemiJoinNode, MergeJoinNode, and SpatialJoinNode to the SPI. `#23976 `_ -* Add Delete, TableWriter, and TableFinish node to SPI. `#24088 `_ -* Add SemiJoin, Join, TableWriter, Delete, and TableFinish node to connector optimizer. `#24154 `_ -* Add a partition by attribute to specify the scope of sort node. `#24095 `_ -* Remove ConnectorJoinNode from the SPI. JoinNode can now be used instead. `#23976 `_ -* Remove experimental getPreferredShuffleLayout methods from the connector metadata in favor of existing `getNewTableLayout`, `getInsertLayout` methods `#24106 `_ -* Modify the signature of ``PlanCheckerProviderFactory.create`` to pass in a map of configuration properties and replace ``SimplePlanFragmentSerde`` with a ``PlanCheckerProviderContext``. `#23955 `_ - -UI Changes -__________ -* Add support for ``BigInt`` data type in the SQL Client on Presto UI on supported browsers. See `compatibility `_ for the supported browsers. `#24336 `_ +* Improve ExpressionOptimizer#optimize method with a variable resolver to return ``RowExpression``. :pr:`24287`. `#24287 `_ +* Add WindowNode, JoinNode, SemiJoinNode, MergeJoinNode, and SpatialJoinNode to the SPI. :pr:`23976`. `#23976 `_ +* Add a partition by attribute to specify the scope of sort node. :pr:`24095`. `#24095 `_ +* Remove ConnectorJoinNode from the SPI. JoinNode can now be used instead. :pr:`23976`. `#23976 `_ +* Remove experimental getPreferredShuffleLayout methods from the connector metadata in favor of existing `getNewTableLayout`, `getInsertLayout` methods :pr:`12345`. `#24106 `_ +* Modify the signature of ``PlanCheckerProviderFactory.create`` to pass in a map of configuration properties and replace ``SimplePlanFragmentSerde`` with a ``PlanCheckerProviderContext``. :pr:`23955`. `#23955 `_ + +Prestissimo (native Execution) Changes +______________________________________ +* Improve partitioned remote exchanges for wide data sets (more than 500 columns) to use row wise encoding. :pr:`23929 `. `#23929 `_ +* Add native plan checker to the native sidecar plugin and native endpoint for Velox plan conversion. :pr:`23596`. `#23596 `_ +* Add session properties ``native_spill_prefixsort_enabled``, ``native_prefixsort_normalized_key_max_bytes ``, and ``native_prefixsort_min_rows ``. :pr:`24043`. `#24043 `_ **Credits** ===========