From 7155d51d84883025253f297233f95ba13683de33 Mon Sep 17 00:00:00 2001
From: Dominic Evans <dominic.evans@uk.ibm.com>
Date: Thu, 26 Oct 2023 10:24:18 +0100
Subject: [PATCH] chore(ci): add kafka 3.6.0 to FVT and versions
- add V3_6_0_0 to Sarama
- bump default dockerfile KAFKA_VERSION to 3.6.0
- pin ubi-minimal version and digest and include in dependabot
- pin apidiff cmd in that workflow
- fix some linter warnings in Dockerfile
- add USER 65534:65534 to Dockerfile for nobody user
Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>
---
.github/dependabot.yml | 10 +++++++++
.github/workflows/apidiff.yml | 2 +-
.github/workflows/fvt-main.yml | 4 ++--
.github/workflows/fvt-pr.yml | 4 ++--
.github/workflows/fvt.yml | 3 ++-
Dockerfile.kafka | 27 +++++++++++++----------
docker-compose.yml | 40 +++++++++++++++++-----------------
entrypoint.sh | 19 +++++++++-------
utils.go | 6 +++--
9 files changed, 67 insertions(+), 48 deletions(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index b87f347ea..b3737c5ca 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -1,6 +1,15 @@
---
version: 2
updates:
+ - package-ecosystem: docker
+ directory: "/Dockerfile.*"
+ schedule:
+ interval: daily
+ labels:
+ - "ignore-for-release"
+ commit-message:
+ prefix: chore(ci)
+
- package-ecosystem: github-actions
directory: /
open-pull-requests-limit: 2
@@ -10,6 +19,7 @@ updates:
- "ignore-for-release"
commit-message:
prefix: chore(ci)
+
- package-ecosystem: gomod
directory: /
open-pull-requests-limit: 5
diff --git a/.github/workflows/apidiff.yml b/.github/workflows/apidiff.yml
index fe6fa330f..b41c10a06 100644
--- a/.github/workflows/apidiff.yml
+++ b/.github/workflows/apidiff.yml
@@ -27,7 +27,7 @@ jobs:
- name: Add GOBIN to PATH
run: echo "$(go env GOPATH)/bin" >>$GITHUB_PATH
- name: Install apidiff cmd
- run: go install golang.org/x/exp/cmd/apidiff@latest
+ run: go install golang.org/x/exp/cmd/apidiff@v0.0.0-20231006140011-7918f672742d
- name: Checkout base code
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
diff --git a/.github/workflows/fvt-main.yml b/.github/workflows/fvt-main.yml
index c17982b56..e1be6a84b 100644
--- a/.github/workflows/fvt-main.yml
+++ b/.github/workflows/fvt-main.yml
@@ -17,7 +17,7 @@ jobs:
fail-fast: false
matrix:
go-version: [1.21.x]
- kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.5.1]
+ kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.0]
include:
- kafka-version: 1.0.2
scala-version: 2.11
@@ -33,7 +33,7 @@ jobs:
scala-version: 2.12
- kafka-version: 3.3.2
scala-version: 2.13
- - kafka-version: 3.5.1
+ - kafka-version: 3.6.0
scala-version: 2.13
uses: ./.github/workflows/fvt.yml
with:
diff --git a/.github/workflows/fvt-pr.yml b/.github/workflows/fvt-pr.yml
index 023c5b269..13e55ea14 100644
--- a/.github/workflows/fvt-pr.yml
+++ b/.github/workflows/fvt-pr.yml
@@ -16,13 +16,13 @@ jobs:
fail-fast: false
matrix:
go-version: [1.21.x]
- kafka-version: [1.0.2, 2.6.2, 3.5.1]
+ kafka-version: [1.0.2, 2.6.2, 3.6.0]
include:
- kafka-version: 1.0.2
scala-version: 2.11
- kafka-version: 2.6.2
scala-version: 2.12
- - kafka-version: 3.5.1
+ - kafka-version: 3.6.0
scala-version: 2.13
uses: ./.github/workflows/fvt.yml
with:
diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml
index 4a570bfc0..f9d0afb50 100644
--- a/.github/workflows/fvt.yml
+++ b/.github/workflows/fvt.yml
@@ -9,7 +9,7 @@ on:
kafka-version:
required: false
type: string
- default: 3.5.1
+ default: 3.6.0
scala-version:
required: false
type: string
@@ -38,6 +38,7 @@ jobs:
builder: ${{ steps.buildx.outputs.name }}
files: docker-compose.yml
load: true
+ targets: kafka-1
set: |
*.cache-from=type=gha,scope=${{ github.workflow }}
*.cache-to=type=gha,scope=${{ github.workflow }},mode=max
diff --git a/Dockerfile.kafka b/Dockerfile.kafka
index b11d899b9..186c2eb18 100644
--- a/Dockerfile.kafka
+++ b/Dockerfile.kafka
@@ -1,26 +1,27 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8@sha256:b93deceb59a58588d5b16429fc47f98920f84740a1f2ed6454e33275f0701b59
USER root
-RUN microdnf update \
- && microdnf install curl gzip java-11-openjdk-headless tar tzdata-java \
- && microdnf reinstall tzdata \
- && microdnf clean all
+RUN microdnf update -y \
+ && microdnf install -y curl gzip java-11-openjdk-headless tar tzdata-java \
+ && microdnf reinstall -y tzdata \
+ && microdnf clean all
ENV JAVA_HOME=/usr/lib/jvm/jre-11
# https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html
# Ensure Java doesn't cache any dns results
RUN cd /etc/java/java-11-openjdk/*/conf/security \
- && sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
- && echo 'networkaddress.cache.ttl=0' >> java.security \
- && echo 'networkaddress.cache.negative.ttl=0' >> java.security
+ && sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
+ && echo 'networkaddress.cache.ttl=0' >> java.security \
+ && echo 'networkaddress.cache.negative.ttl=0' >> java.security
ARG SCALA_VERSION="2.13"
-ARG KAFKA_VERSION="3.5.1"
+ARG KAFKA_VERSION="3.6.0"
-# https://github.com/apache/kafka/blob/53eeaad946cd053e9eb1a762972d4efeacb8e4fc/tests/docker/Dockerfile#L65-L69
+# https://github.com/apache/kafka/blob/9989b68d0d38c8f1357f78bf9d53a58c1476188d/tests/docker/Dockerfile#L46-L72
ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \
&& chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \
&& curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}"
@@ -28,8 +29,8 @@ RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \
# older kafka versions depend upon jaxb-api being bundled with the JDK, but it
# was removed from Java 11 so work around that by including it in the kafka
# libs dir regardless
-RUN cd /tmp \
- && curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
+WORKDIR /tmp
+RUN curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
&& for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
&& rm -f jaxb-api-2.3.0.jar
@@ -41,4 +42,6 @@ RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -
COPY entrypoint.sh /
+USER 65534:65534
+
ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker-compose.yml b/docker-compose.yml
index 4fdf1a517..e916416d5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -38,18 +38,18 @@ services:
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
kafka-1:
hostname: 'kafka-1'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
build:
context: .
dockerfile: Dockerfile.kafka
args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
SCALA_VERSION: ${SCALA_VERSION:-2.13}
healthcheck:
test:
[
'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-1:9091',
]
@@ -64,7 +64,7 @@ services:
- toxiproxy
restart: always
environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
@@ -83,18 +83,18 @@ services:
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-2:
hostname: 'kafka-2'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
build:
context: .
dockerfile: Dockerfile.kafka
args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
SCALA_VERSION: ${SCALA_VERSION:-2.13}
healthcheck:
test:
[
'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-2:9091',
]
@@ -109,7 +109,7 @@ services:
- toxiproxy
restart: always
environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
@@ -128,18 +128,18 @@ services:
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-3:
hostname: 'kafka-3'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
build:
context: .
dockerfile: Dockerfile.kafka
args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
SCALA_VERSION: ${SCALA_VERSION:-2.13}
healthcheck:
test:
[
'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-3:9091',
]
@@ -154,7 +154,7 @@ services:
- toxiproxy
restart: always
environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
@@ -173,18 +173,18 @@ services:
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-4:
hostname: 'kafka-4'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
build:
context: .
dockerfile: Dockerfile.kafka
args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
SCALA_VERSION: ${SCALA_VERSION:-2.13}
healthcheck:
test:
[
'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-4:9091',
]
@@ -199,7 +199,7 @@ services:
- toxiproxy
restart: always
environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
@@ -218,18 +218,18 @@ services:
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-5:
hostname: 'kafka-5'
- image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
+ image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
build:
context: .
dockerfile: Dockerfile.kafka
args:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
SCALA_VERSION: ${SCALA_VERSION:-2.13}
healthcheck:
test:
[
'CMD',
- '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
+ '/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-5:9091',
]
@@ -244,7 +244,7 @@ services:
- toxiproxy
restart: always
environment:
- KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+ KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
diff --git a/entrypoint.sh b/entrypoint.sh
index 7f405f92d..9fe9a44b1 100755
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,6 +1,9 @@
#!/bin/bash
-KAFKA_VERSION="${KAFKA_VERSION:-3.5.1}"
+set -eu
+set -o pipefail
+
+KAFKA_VERSION="${KAFKA_VERSION:-3.6.0}"
KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
if [ ! -d "${KAFKA_HOME}" ]; then
@@ -10,19 +13,19 @@ fi
cd "${KAFKA_HOME}" || exit 1
-# discard all empty/commented lines
-sed -e '/^#/d' -e '/^$/d' -i".orig" config/server.properties
+# discard all empty/commented lines from default config and copy to /tmp
+sed -e '/^#/d' -e '/^$/d' config/server.properties >/tmp/server.properties
-echo "########################################################################" >>config/server.properties
+echo "########################################################################" >>/tmp/server.properties
# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
for var in "${!KAFKA_CFG_@}"; do
key="$(echo "$var" | sed -e 's/^KAFKA_CFG_//g' -e 's/_/\./g' -e 's/.*/\L&/')"
- sed -e '/^'$key'/d' -i"" config/server.properties
+ sed -e '/^'$key'/d' -i"" /tmp/server.properties
value="${!var}"
- echo "$key=$value" >>config/server.properties
+ echo "$key=$value" >>/tmp/server.properties
done
-sort config/server.properties
+sort /tmp/server.properties
-exec bin/kafka-server-start.sh config/server.properties
+exec bin/kafka-server-start.sh /tmp/server.properties
diff --git a/utils.go b/utils.go
index fe5f0a52f..feadc0065 100644
--- a/utils.go
+++ b/utils.go
@@ -198,6 +198,7 @@ var (
V3_4_1_0 = newKafkaVersion(3, 4, 1, 0)
V3_5_0_0 = newKafkaVersion(3, 5, 0, 0)
V3_5_1_0 = newKafkaVersion(3, 5, 1, 0)
+ V3_6_0_0 = newKafkaVersion(3, 6, 0, 0)
SupportedVersions = []KafkaVersion{
V0_8_2_0,
@@ -258,9 +259,10 @@ var (
V3_4_1_0,
V3_5_0_0,
V3_5_1_0,
+ V3_6_0_0,
}
MinVersion = V0_8_2_0
- MaxVersion = V3_5_1_0
+ MaxVersion = V3_6_0_0
DefaultVersion = V2_1_0_0
// reduced set of protocol versions to matrix test
@@ -275,8 +277,8 @@ var (
V2_6_2_0,
V2_8_2_0,
V3_1_2_0,
- V3_2_3_0,
V3_3_2_0,
+ V3_6_0_0,
}
)