From 788f8626c46e0570e1e23d1c540e724e9b2ea83e Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Wed, 22 Jan 2025 13:27:59 -0500 Subject: [PATCH 1/3] #380 - Added configuration settings to prevent freeze of debezium loop. --- sink-connector-lightweight/docker/config.yml | 7 +++++++ sink-connector-lightweight/docker/config_postgres.yml | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sink-connector-lightweight/docker/config.yml b/sink-connector-lightweight/docker/config.yml index 554b2a339..a9738f1aa 100644 --- a/sink-connector-lightweight/docker/config.yml +++ b/sink-connector-lightweight/docker/config.yml @@ -183,3 +183,10 @@ metrics.enable: "true" # to reduce slow startup when replicating dbs with large number of tables #schema.history.internal.store.only.captured.tables.ddl: "true" #schema.history.internal.store.only.captured.databases.ddl: "true" + +# Required for fixing bug with freeze in debezium loop. +use.nongraceful.disconnect: "true" +database.keep.alive.interval.`ms: "30000" #Send keepalive every 30 seconds +database.connection.reconnect.backoff.ms: "1000" +database.connection.reconnect.backoff.max.ms: "10000" +database.ssl.mode: "disabled" \ No newline at end of file diff --git a/sink-connector-lightweight/docker/config_postgres.yml b/sink-connector-lightweight/docker/config_postgres.yml index e3ec71e05..d5178c43b 100644 --- a/sink-connector-lightweight/docker/config_postgres.yml +++ b/sink-connector-lightweight/docker/config_postgres.yml @@ -125,4 +125,12 @@ SELECT toDateTime(fromUnixTimestamp(toInt32(JSONExtractUInt(offset_val, 'ts_usec') / 1000000)), 'UTC') AS utc_time, fromUnixTimestamp(toInt32(JSONExtractUInt(offset_val, 'ts_usec') / 1000000)) AS local_time FROM altinity_sink_connector.replica_source_info -SETTINGS final = 1" \ No newline at end of file +SETTINGS final = 1" + + +# Required for fixing bug with freeze in debezium loop. +use.nongraceful.disconnect: "true" +database.keep.alive.interval.`ms: "30000" #Send keepalive every 30 seconds +database.connection.reconnect.backoff.ms: "1000" +database.connection.reconnect.backoff.max.ms: "10000" +database.ssl.mode: "disabled" \ No newline at end of file From ae3c4500c42d851b39d2666250b8bdf0410ecb42 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 23 Jan 2025 14:33:01 -0500 Subject: [PATCH 2/3] Updated version of upload-artifact to v4. --- .github/workflows/docker-build.yml | 8 ++++---- .github/workflows/testflows-sink-connector-kafka.yml | 2 +- .../workflows/testflows-sink-connector-lightweight.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6c8ea19d1..811276000 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -57,7 +57,7 @@ jobs: run: mvn -B package install -DskipTests=true --file pom.xml - name: Upload Maven build (Kafka) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: clickhouse-sink-connector-kafka-${{ env.IMAGE_TAG }}.jar path: sink-connector/target/clickhouse-kafka-sink-connector-${{ env.PROJECT_VERSION }}.jar @@ -69,7 +69,7 @@ jobs: docker save altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-kafka | gzip > clickhouse-sink-connector_${{ env.IMAGE_TAG }}-kafka.tar.gz - name: Upload Docker tar (Kafka) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} with: name: clickhouse-sink-connector_${{ env.IMAGE_TAG }}-kafka.tar.gz @@ -88,7 +88,7 @@ jobs: run: mvn -B package -DskipTests=true --file pom.xml - name: Upload Maven build (Lightweight) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: clickhouse-sink-connector-lt-${{ env.IMAGE_TAG }}.jar path: sink-connector-lightweight/target/clickhouse-debezium-embedded-${{ env.PROJECT_VERSION }}.jar @@ -103,7 +103,7 @@ jobs: docker save altinityinfra/clickhouse-sink-connector:${{ env.IMAGE_TAG }}-lt | gzip > clickhouse-sink-connector_${{ env.IMAGE_TAG }}-lt.tar.gz - name: Upload Docker tar (Lightweight) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} with: name: clickhouse-sink-connector_${{ env.IMAGE_TAG }}-lt.tar.gz diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index aa09a9a16..0e38b07b6 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -135,7 +135,7 @@ jobs: AWS_DEFAULT_REGION: 'eu-west-2' run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink/ --recursive --exclude "*" --include "*.log" --include "*.html" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: testflows-sink-connector-kafka-artefacts diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index c0da3df16..7cc06bc35 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -136,7 +136,7 @@ jobs: AWS_DEFAULT_REGION: 'eu-west-2' run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: testflows-sink-connector-lightweight-artefacts @@ -214,7 +214,7 @@ jobs: AWS_DEFAULT_REGION: 'eu-west-2' run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink_lw/ --recursive --exclude "*" --include "*.log" --include "*.html" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: testflows-sink-connector-lightweight-replicated-artefacts From 4932b76fc86964eaaaaa5047eae7112b9ce2ac96 Mon Sep 17 00:00:00 2001 From: Kanthi Subramanian Date: Thu, 23 Jan 2025 14:51:02 -0500 Subject: [PATCH 3/3] Updated version of download-artifact to v4. --- .github/workflows/publish.yml | 2 +- .github/workflows/testflows-sink-connector-kafka.yml | 2 +- .github/workflows/testflows-sink-connector-lightweight.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b25c83ac0..db6cc1bd8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -71,7 +71,7 @@ jobs: body_path: ${{ github.workspace }}/release-notes/${{ env.release_tag }}.md - name: Download jar files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 # - name: Zip jar files # run: | diff --git a/.github/workflows/testflows-sink-connector-kafka.yml b/.github/workflows/testflows-sink-connector-kafka.yml index 0e38b07b6..43ad67171 100644 --- a/.github/workflows/testflows-sink-connector-kafka.yml +++ b/.github/workflows/testflows-sink-connector-kafka.yml @@ -72,7 +72,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} with: name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-kafka.tar.gz diff --git a/.github/workflows/testflows-sink-connector-lightweight.yml b/.github/workflows/testflows-sink-connector-lightweight.yml index 7cc06bc35..63510d560 100644 --- a/.github/workflows/testflows-sink-connector-lightweight.yml +++ b/.github/workflows/testflows-sink-connector-lightweight.yml @@ -73,7 +73,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} with: name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz @@ -151,7 +151,7 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ github.event.pull_request.head.repo.full_name != 'Altinity/clickhouse-sink-connector' && github.event_name != 'workflow_dispatch' }} with: name: clickhouse-sink-connector_${{ github.event.number }}-${{ github.sha }}-lt.tar.gz