Skip to content

Commit

Permalink
Update E2E docker images (#13905)
Browse files Browse the repository at this point in the history
* update docker images

* revert image version 0.11.0.1

* newline

* fix create topics for v1.1

* fix create topics for v2.3

* increase timeout

* pin version 3.3.2
  • Loading branch information
fanny-jiang authored Feb 9, 2023
1 parent 0ac805a commit e644556
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 31 deletions.
15 changes: 8 additions & 7 deletions kafka_consumer/hatch.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
[[envs.default.matrix]]
python = ["2.7", "3.8"]
# Note: 0.9 is for testing legacy pre-0.10.2 implementation
version = ["0.9"]
version = ["0.9", "3.3"]
offset-storage = ["kafka", "zk"]

[[envs.default.matrix]]
Expand All @@ -12,16 +12,17 @@ version = ["0.11", "1.1", "2.3"]
offset-storage = ["kafka", "zk"]

[envs.default.overrides]
matrix.version.e2e-env = { value = true, if = ["0.9"] }
matrix.version.e2e-env = { value = true, if = ["0.9", "3.3"] }
matrix.version.env-vars = [
{ key = "KAFKA_VERSION", value = "0.9.0.1-1", if = ["0.9"] },
{ key = "KAFKA_VERSION", value = "0.11.0.1", if = ["0.11"] },
{ key = "KAFKA_VERSION", value = "1.1.0", if = ["1.1"] },
{ key = "KAFKA_VERSION", value = "2.12-2.3.0", if = ["2.3"] },
{ key = "KAFKA_VERSION", value = "1.1.1", if = ["1.1"] },
{ key = "KAFKA_VERSION", value = "2.3.1", if = ["2.3"] },
{ key = "KAFKA_VERSION", value = "3.3.2", if = ["3.3"] },
# Using Kafka 0.9 + with ZK 3.4 due to flakiness of Kafka 0.9 + ZK 3.6
{ key = "ZK_VERSION", value = "3.4.11", if = ["0.9"] },
{ key = "ZK_VERSION", value = "3.6.0", if = ["0.11", "1.1", "2.3"] },
{ key = "USE_MULTIPLE_BROKERS", value = "true", if = ["1.1", "2.3"] },
{ key = "ZK_VERSION", value = "3.6.4", if = ["0.11", "1.1", "2.3", "3.3"] },
{ key = "USE_MULTIPLE_BROKERS", value = "true", if = ["1.1", "2.3", "3.3"] },
# Can't have a working `docker compose` for these versions
{ key = "USE_MULTIPLE_BROKERS", value = "false", if = ["0.9", "0.11"] },
]
Expand All @@ -40,4 +41,4 @@ e2e-env = false
KAFKA_OFFSETS_STORAGE = "kafka"
KAFKA_VERSION = "latest"
USE_MULTIPLE_BROKERS = "true"
ZK_VERSION = "3.6.0"
ZK_VERSION = "3.6.4"
3 changes: 2 additions & 1 deletion kafka_consumer/tests/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,14 @@
DOCKER_IMAGE_PATH = os.path.join(HERE, 'docker', 'multiple-brokers.yaml')
else:
DOCKER_IMAGE_PATH = os.path.join(HERE, 'docker', 'single-broker.yaml')
KAFKA_VERSION = os.environ.get('KAFKA_VERSION')


def is_supported(flavor):
"""
Returns whether the current CI configuration is supported
"""
if not os.environ.get('KAFKA_VERSION'):
if not KAFKA_VERSION:
return False

if flavor != os.environ.get('KAFKA_OFFSETS_STORAGE'):
Expand Down
17 changes: 14 additions & 3 deletions kafka_consumer/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
import pytest
from datadog_test_libs.utils.mock_dns import mock_local
from kafka import KafkaConsumer
from packaging.version import parse as parse_version

from datadog_checks.dev import WaitFor, docker_run

from .common import DOCKER_IMAGE_PATH, HOST_IP, KAFKA_CONNECT_STR, PARTITIONS, TOPICS, ZK_CONNECT_STR
from .common import DOCKER_IMAGE_PATH, HOST_IP, KAFKA_CONNECT_STR, KAFKA_VERSION, PARTITIONS, TOPICS, ZK_CONNECT_STR
from .runners import KConsumer, Producer, ZKConsumer


Expand Down Expand Up @@ -53,10 +54,14 @@ def dd_environment(mock_local_kafka_hosts_dns, e2e_instance):
# Advertising the hostname doesn't work on docker:dind so we manually
# resolve the IP address. This seems to also work outside docker:dind
# so we got that goin for us.
'KAFKA_HOST': HOST_IP
'KAFKA_HOST': HOST_IP,
'BOOTSTRAP_SERVER_FLAG': _get_bootstrap_server_flag(),
},
):
yield e2e_instance, E2E_METADATA
yield {
'instances': [e2e_instance],
'init_config': {'zk_timeout': 30, 'kafka_timeout': 30},
}, E2E_METADATA


E2E_METADATA = {
Expand Down Expand Up @@ -117,3 +122,9 @@ def e2e_instance(kafka_instance, zk_instance):
return kafka_instance
elif flavor == 'zookeeper':
return zk_instance


def _get_bootstrap_server_flag():
if KAFKA_VERSION != 'latest' and parse_version(KAFKA_VERSION) < parse_version('3.0'):
return '--zookeeper zookeeper:2181'
return '--bootstrap-server kafka1:19092'
47 changes: 27 additions & 20 deletions kafka_consumer/tests/docker/multiple-brokers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,47 @@ version: '3.5'

services:
zookeeper:
image: zookeeper:${ZK_VERSION}
image: docker.io/bitnami/zookeeper:${ZK_VERSION}
ports:
- 2181:2181
environment:
- ALLOW_ANONYMOUS_LOGIN=yes

kafka1:
image: "wurstmeister/kafka:${KAFKA_VERSION}"
hostname: kafka1
image: docker.io/bitnami/kafka:${KAFKA_VERSION}
ports:
- 9092:9092
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_CREATE_TOPICS: "marvel:2:1,dc:2:1"
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092
KAFKA_ADVERTISED_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT

KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_CFG_LISTENERS: INTERNAL://:19092,EXTERNAL://:9092
KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://:19092,EXTERNAL://127.0.0.1:9092
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
ALLOW_PLAINTEXT_LISTENER: "true"
depends_on:
- zookeeper

kafka2:
image: "wurstmeister/kafka:${KAFKA_VERSION}"
hostname: kafka2
image: docker.io/bitnami/kafka:${KAFKA_VERSION}
ports:
- 9093:9093
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_LISTENERS: INTERNAL://:19093,EXTERNAL://:9093
KAFKA_ADVERTISED_LISTENERS: INTERNAL://:19093,EXTERNAL://:9093
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
KAFKA_CFG_BROKER_ID: 2
KAFKA_CFG_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL
KAFKA_CFG_LISTENERS: INTERNAL://:19093,EXTERNAL://:9093
KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL://:19093,EXTERNAL://127.0.0.1:9093
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
ALLOW_PLAINTEXT_LISTENER: "true"

depends_on:
- zookeeper

kafka-init:
image: docker.io/bitnami/kafka:${KAFKA_VERSION}
command: >
sh -c "sleep 10 &&./opt/bitnami/kafka/bin/kafka-topics.sh ${BOOTSTRAP_SERVER_FLAG} --create --topic marvel --if-not-exists --partitions 2 --replication-factor 1 &&
./opt/bitnami/kafka/bin/kafka-topics.sh ${BOOTSTRAP_SERVER_FLAG} --create --topic dc --if-not-exists --partitions 2 --replication-factor 1"
depends_on:
- kafka1

0 comments on commit e644556

Please sign in to comment.