From fe028eae3d1156f6ceaaafaa32ec38b0947052c2 Mon Sep 17 00:00:00 2001 From: Rodrigo Bozzolo Date: Fri, 19 Apr 2024 15:36:37 -0700 Subject: [PATCH] [#30789] Add support for Flink 1.18 --- .../beam_PostCommit_Java_Tpcds_Flink.yml | 2 +- ..._PostCommit_Java_ValidatesRunner_Flink.yml | 2 +- CHANGES.md | 2 +- gradle.properties | 2 +- release/build.gradle.kts | 2 +- runners/flink/1.18/build.gradle | 25 +++++++++++++++ .../1.18/job-server-container/build.gradle | 26 ++++++++++++++++ runners/flink/1.18/job-server/build.gradle | 31 +++++++++++++++++++ runners/flink/flink_runner.gradle | 2 +- sdks/go/examples/stringsplit/stringsplit.go | 2 +- sdks/typescript/package-lock.json | 4 +-- settings.gradle.kts | 4 +++ 12 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 runners/flink/1.18/build.gradle create mode 100644 runners/flink/1.18/job-server-container/build.gradle create mode 100644 runners/flink/1.18/job-server/build.gradle diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml index b6d27fd3377f..2818f69507af 100644 --- a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml +++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml @@ -101,5 +101,5 @@ jobs: with: gradle-command: :sdks:java:testing:tpcds:run arguments: | - -Ptpcds.runner=:runners:flink:1.17 \ + -Ptpcds.runner=:runners:flink:1.18 \ "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \ diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml index 00828674f653..e47e3a35b539 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml @@ -78,7 +78,7 @@ jobs: - name: run validatesRunner script uses: ./.github/actions/gradle-command-self-hosted-action with: - gradle-command: :runners:flink:1.17:validatesRunner + gradle-command: :runners:flink:1.18:validatesRunner - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/CHANGES.md b/CHANGES.md index 5fc2509ade8f..4d8cd76dfaa2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -57,7 +57,7 @@ ## Highlights -* Added FlinkRunner for Flink 1.17, removed support for Flink 1.12 and 1.13. Previous version of Pipeline running on Flink 1.16 and below can be upgraded to 1.17, if the Pipeline is first updated to Beam 2.56.0 with the same Flink version. After Pipeline runs with Beam 2.56.0, it should be possible to upgrade to FlinkRunner with Flink 1.17. ([#29939](https://github.com/apache/beam/issues/29939)) +* Added FlinkRunner for Flink 1.17 and 1.18, removed support for Flink 1.12 and 1.13. Previous version of Pipeline running on Flink 1.16 and below can be upgraded to 1.17 or 1.18, if the Pipeline is first updated to Beam 2.56.0 with the same Flink version. After Pipeline runs with Beam 2.56.0, it should be possible to upgrade to FlinkRunner with Flink 1.17 or 1.18. ([#29939](https://github.com/apache/beam/issues/29939)) ## I/Os diff --git a/gradle.properties b/gradle.properties index 0d33c30d3f78..8c41a9f86c8f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -39,6 +39,6 @@ docker_image_default_repo_root=apache docker_image_default_repo_prefix=beam_ # supported flink versions -flink_versions=1.14,1.15,1.16,1.17 +flink_versions=1.14,1.15,1.16,1.17,1.18 # supported python versions python_versions=3.8,3.9,3.10,3.11 diff --git a/release/build.gradle.kts b/release/build.gradle.kts index 4e4586f666f0..ca9982806b9f 100644 --- a/release/build.gradle.kts +++ b/release/build.gradle.kts @@ -39,7 +39,7 @@ task("runJavaExamplesValidationTask") { dependsOn(":runners:direct-java:runQuickstartJavaDirect") dependsOn(":runners:google-cloud-dataflow-java:runQuickstartJavaDataflow") dependsOn(":runners:spark:3:runQuickstartJavaSpark") - dependsOn(":runners:flink:1.17:runQuickstartJavaFlinkLocal") + dependsOn(":runners:flink:1.18:runQuickstartJavaFlinkLocal") dependsOn(":runners:direct-java:runMobileGamingJavaDirect") dependsOn(":runners:google-cloud-dataflow-java:runMobileGamingJavaDataflow") dependsOn(":runners:twister2:runQuickstartJavaTwister2") diff --git a/runners/flink/1.18/build.gradle b/runners/flink/1.18/build.gradle new file mode 100644 index 000000000000..ab6e6b63b773 --- /dev/null +++ b/runners/flink/1.18/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +project.ext { + flink_major = '1.18' + flink_version = '1.18.0' +} + +// Load the main build script which contains all build logic. +apply from: "../flink_runner.gradle" diff --git a/runners/flink/1.18/job-server-container/build.gradle b/runners/flink/1.18/job-server-container/build.gradle new file mode 100644 index 000000000000..afdb68a0fc91 --- /dev/null +++ b/runners/flink/1.18/job-server-container/build.gradle @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +def basePath = '../../job-server-container' + +project.ext { + resource_path = basePath +} + +// Load the main build script which contains all build logic. +apply from: "$basePath/flink_job_server_container.gradle" diff --git a/runners/flink/1.18/job-server/build.gradle b/runners/flink/1.18/job-server/build.gradle new file mode 100644 index 000000000000..e70fdcc0c581 --- /dev/null +++ b/runners/flink/1.18/job-server/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +def basePath = '../../job-server' + +project.ext { + // Look for the source code in the parent module + main_source_dirs = ["$basePath/src/main/java"] + test_source_dirs = ["$basePath/src/test/java"] + main_resources_dirs = ["$basePath/src/main/resources"] + test_resources_dirs = ["$basePath/src/test/resources"] + archives_base_name = 'beam-runners-flink-1.18-job-server' +} + +// Load the main build script which contains all build logic. +apply from: "$basePath/flink_job_server.gradle" diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index 6849334aea01..708af8e9b5c8 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -373,7 +373,7 @@ tasks.register("validatesRunnerSickbay", Test) { } } -// Generates :runners:flink:1.17:runQuickstartJavaFlinkLocal +// Generates :runners:flink:1.18:runQuickstartJavaFlinkLocal createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'FlinkLocal') tasks.register("examplesIntegrationTest", Test) { diff --git a/sdks/go/examples/stringsplit/stringsplit.go b/sdks/go/examples/stringsplit/stringsplit.go index 28d79815607f..266cdd99fb37 100644 --- a/sdks/go/examples/stringsplit/stringsplit.go +++ b/sdks/go/examples/stringsplit/stringsplit.go @@ -21,7 +21,7 @@ // 1. From a command line, navigate to the top-level beam/ directory and run // the Flink job server: // -// ./gradlew :runners:flink:1.17:job-server:runShadow -Djob-host=localhost -Dflink-master=local +// ./gradlew :runners:flink:1.18:job-server:runShadow -Djob-host=localhost -Dflink-master=local // // 2. The job server is ready to receive jobs once it outputs a log like the // following: `JobService started on localhost:8099`. Take note of the endpoint diff --git a/sdks/typescript/package-lock.json b/sdks/typescript/package-lock.json index 1eb77b748005..7d016128b0ad 100644 --- a/sdks/typescript/package-lock.json +++ b/sdks/typescript/package-lock.json @@ -1,12 +1,12 @@ { "name": "apache-beam", - "version": "2.54.0-SNAPSHOT", + "version": "2.57.0-SNAPSHOT", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "apache-beam", - "version": "2.54.0-SNAPSHOT", + "version": "2.57.0-SNAPSHOT", "dependencies": { "@google-cloud/pubsub": "^2.19.4", "@grpc/grpc-js": "~1.4.6", diff --git a/settings.gradle.kts b/settings.gradle.kts index 8c86a8a5e550..fc42e24d314c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -137,6 +137,10 @@ include(":runners:flink:1.16:job-server-container") include(":runners:flink:1.17") include(":runners:flink:1.17:job-server") include(":runners:flink:1.17:job-server-container") +// Flink 1.18 +include(":runners:flink:1.18") +include(":runners:flink:1.18:job-server") +include(":runners:flink:1.18:job-server-container") /* End Flink Runner related settings */ include(":runners:twister2") include(":runners:google-cloud-dataflow-java")