From 10e319e6ccdb69f360801af6b33d5c6aaebec17f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Mon, 3 Feb 2025 11:16:30 +0100 Subject: [PATCH 1/5] tests(kafka-connect): fixes integration tests setup --- .../tests/integration/kafka-connect/docker-compose.override.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 85b6a310a06d2..7757a0a6b59ef 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -37,6 +37,8 @@ services: # confluent-hub install --no-prompt confluentinc/kafka-connect-s3:10.5.1 # + mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib + # curl -k -SL "https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.27.tar.gz" \ | tar -xzf - -C /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib \ --strip-components=1 mysql-connector-java-8.0.27/mysql-connector-java-8.0.27.jar From 49dc86a9ba8c7d74c160ec47a6b6095ed41ba6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Mon, 3 Feb 2025 14:28:03 +0100 Subject: [PATCH 2/5] confluent install from ziip --- .../kafka-connect/docker-compose.override.yml | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 7757a0a6b59ef..534b245785d4e 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -23,19 +23,33 @@ services: - | echo "Installing Connector" # - confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.2.5 + #confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.2.5 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-jdbc/versions/10.2.5/confluentinc-kafka-connect-jdbc-10.2.5.zip + confluent-hub install --no-prompt confluentinc-kafka-connect-jdbc-10.2.5.zip # - confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.1 + #confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.1 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/connect-transforms/versions/1.4.1/confluentinc-connect-transforms-1.4.1.zip + confluent-hub install --no-prompt confluentinc-connect-transforms-1.4.1.zip # - confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.6.0 + #confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.6.0 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-datagen/versions/0.6.0/confluentinc-kafka-connect-datagen-0.6.0.zip + confluent-hub install --no-prompt confluentinc-kafka-connect-datagen-0.6.0.zip # - confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.7.0 + #confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.7.0 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/debezium/debezium-connector-mysql/versions/1.7.0/debezium-debezium-connector-mysql-1.7.0.zip + confluent-hub install --no-prompt debezium-debezium-connector-mysql-1.7.0.zip # - confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 + #confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/wepay/kafka-connect-bigquery/versions/1.6.8/wepay-kafka-connect-bigquery-1.6.8.zip + confluent-hub install --no-prompt wepay-kafka-connect-bigquery-1.6.8.zip # - confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 + #confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/mongodb/kafka-connect-mongodb/versions/1.10.1/mongodb-kafka-connect-mongodb-1.10.1.zip + confluent-hub install --no-prompt mongodb-kafka-connect-mongodb-1.10.1.zip # - confluent-hub install --no-prompt confluentinc/kafka-connect-s3:10.5.1 + #confluent-hub install --no-prompt confluentinc/kafka-connect-s3:10.5.1 + wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-s3/versions/10.5.1/confluentinc-kafka-connect-s3-10.5.1.zip + confluent-hub install --no-prompt confluentinc-kafka-connect-s3-10.5.1.zip # mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib # From 91ff618fe078daec8ac638194510b114c096fb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Mon, 3 Feb 2025 18:27:13 +0100 Subject: [PATCH 3/5] confluent connect plugin install --- .../kafka-connect/docker-compose.override.yml | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 534b245785d4e..259101d6f9f01 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -21,35 +21,28 @@ services: - bash - -c - | - echo "Installing Connector" + echo "Install confluent CLI" + # https://docs.confluent.io/confluent-cli/current/install.html#tarball-or-zip-installation + wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_arm64.tar.gz + mkdir -p /tmp/confluent-cli + tar -xvf confluent_linux_arm64.tar.gz -C /tmp/confluent-cli + export PATH=/tmp/confluent-cli/confluent:$PATH + echo "Done!" + echo "Installing Connectors" # - #confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.2.5 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-jdbc/versions/10.2.5/confluentinc-kafka-connect-jdbc-10.2.5.zip - confluent-hub install --no-prompt confluentinc-kafka-connect-jdbc-10.2.5.zip + confluent connect plugin install --force confluentinc/kafka-connect-jdbc:10.2.5 # - #confluent-hub install --no-prompt confluentinc/connect-transforms:1.4.1 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/connect-transforms/versions/1.4.1/confluentinc-connect-transforms-1.4.1.zip - confluent-hub install --no-prompt confluentinc-connect-transforms-1.4.1.zip + confluent connect plugin install --force confluentinc/connect-transforms:1.4.1 # - #confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:0.6.0 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-datagen/versions/0.6.0/confluentinc-kafka-connect-datagen-0.6.0.zip - confluent-hub install --no-prompt confluentinc-kafka-connect-datagen-0.6.0.zip + confluent connect plugin install --force confluentinc/kafka-connect-datagen:0.6.0 # - #confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.7.0 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/debezium/debezium-connector-mysql/versions/1.7.0/debezium-debezium-connector-mysql-1.7.0.zip - confluent-hub install --no-prompt debezium-debezium-connector-mysql-1.7.0.zip + confluent connect plugin install --force debezium/debezium-connector-mysql:1.7.0 # - #confluent-hub install --no-prompt wepay/kafka-connect-bigquery:1.6.8 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/wepay/kafka-connect-bigquery/versions/1.6.8/wepay-kafka-connect-bigquery-1.6.8.zip - confluent-hub install --no-prompt wepay-kafka-connect-bigquery-1.6.8.zip + confluent connect plugin install --force wepay/kafka-connect-bigquery:1.6.8 # - #confluent-hub install --no-prompt mongodb/kafka-connect-mongodb:1.10.1 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/mongodb/kafka-connect-mongodb/versions/1.10.1/mongodb-kafka-connect-mongodb-1.10.1.zip - confluent-hub install --no-prompt mongodb-kafka-connect-mongodb-1.10.1.zip + confluent connect plugin install --force mongodb/kafka-connect-mongodb:1.10.1 # - #confluent-hub install --no-prompt confluentinc/kafka-connect-s3:10.5.1 - wget https://d2p6pa21dvn84.cloudfront.net/api/plugins/confluentinc/kafka-connect-s3/versions/10.5.1/confluentinc-kafka-connect-s3-10.5.1.zip - confluent-hub install --no-prompt confluentinc-kafka-connect-s3-10.5.1.zip + confluent connect plugin install --force confluentinc/kafka-connect-s3:10.5.1 # mkdir -p /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib # From c1f17605fec12349fc1c68b9a540fdef3e72424d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Tue, 4 Feb 2025 07:58:41 +0100 Subject: [PATCH 4/5] Update docker-compose.override.yml --- .../tests/integration/kafka-connect/docker-compose.override.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 259101d6f9f01..282ea411d6ed3 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -23,7 +23,7 @@ services: - | echo "Install confluent CLI" # https://docs.confluent.io/confluent-cli/current/install.html#tarball-or-zip-installation - wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_arm64.tar.gz + wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_amd64.tar.gz mkdir -p /tmp/confluent-cli tar -xvf confluent_linux_arm64.tar.gz -C /tmp/confluent-cli export PATH=/tmp/confluent-cli/confluent:$PATH From 02aa1ae0517142726912e84a6be4a170d582a542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Tue, 4 Feb 2025 08:44:14 +0100 Subject: [PATCH 5/5] Update docker-compose.override.yml --- .../tests/integration/kafka-connect/docker-compose.override.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml index 282ea411d6ed3..5811346dce687 100644 --- a/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml +++ b/metadata-ingestion/tests/integration/kafka-connect/docker-compose.override.yml @@ -25,7 +25,7 @@ services: # https://docs.confluent.io/confluent-cli/current/install.html#tarball-or-zip-installation wget https://packages.confluent.io/confluent-cli/archives/4.16.0/confluent_linux_amd64.tar.gz mkdir -p /tmp/confluent-cli - tar -xvf confluent_linux_arm64.tar.gz -C /tmp/confluent-cli + tar -xvf confluent_linux_amd64.tar.gz -C /tmp/confluent-cli export PATH=/tmp/confluent-cli/confluent:$PATH echo "Done!" echo "Installing Connectors"