Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Kafka CI #863

Merged
merged 23 commits into from
Jul 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions .github/containers/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,22 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
zlib1g-dev && \
rm -rf /var/lib/apt/lists/*

# Build librdkafka from source
ARG LIBRDKAFKA_VERSION=2.1.1
RUN cd /tmp && \
wget https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.zip -O ./librdkafka.zip && \
unzip ./librdkafka.zip && \
rm ./librdkafka.zip && \
cd ./librdkafka-${LIBRDKAFKA_VERSION} && \
./configure && \
make all install && \
cd /tmp && \
rm -rf ./librdkafka-${LIBRDKAFKA_VERSION}

# Setup ODBC config
RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \
sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \
sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini
RUN sed -i 's|Driver=psqlodbca.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so|g' /etc/odbcinst.ini && \
sed -i 's|Driver=psqlodbcw.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so|g' /etc/odbcinst.ini && \
sed -i 's|Setup=libodbcpsqlS.so|Setup=/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so|g' /etc/odbcinst.ini

# Set the locale
RUN locale-gen --no-purge en_US.UTF-8
Expand Down
6 changes: 5 additions & 1 deletion .github/containers/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ default: test
.PHONY: build
build:
@# Perform a shortened build for testing
@docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local
@docker build $(MAKEFILE_DIR) \
-t ghcr.io/newrelic/newrelic-python-agent-ci:local \
--build-arg='PYTHON_VERSIONS=3.10 2.7'

.PHONY: test
test: build
Expand All @@ -38,7 +40,9 @@ run: build
@docker run --rm -it \
--mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \
--workdir=/home/github/python-agent \
--add-host=host.docker.internal:host-gateway \
-e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \
-e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \
-e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \
-e GITHUB_ACTIONS="true" \
ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash
7 changes: 2 additions & 5 deletions .github/containers/install-python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

SCRIPT_DIR=$(dirname "$0")
PIP_REQUIREMENTS=$(cat /requirements.txt)
set -eo pipefail

main() {
# Coerce space separated string to array
Expand Down Expand Up @@ -50,7 +47,7 @@ main() {
pyenv global ${PYENV_VERSIONS[@]}

# Install dependencies for main python installation
pyenv exec pip install --upgrade $PIP_REQUIREMENTS
pyenv exec pip install --upgrade -r /requirements.txt
}

main
2 changes: 1 addition & 1 deletion .github/scripts/retry.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,4 @@ for i in $(seq 1 $retries); do
done

# Exit with status code of wrapped command
exit $?
exit $result
176 changes: 83 additions & 93 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- elasticsearchserver08
- gearman
- grpc
#- kafka
- kafka
- memcached
- mongodb
- mssql
Expand Down Expand Up @@ -119,7 +119,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -164,7 +164,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -209,7 +209,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -269,7 +269,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -332,7 +332,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -395,7 +395,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -453,7 +453,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -513,7 +513,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -571,7 +571,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -619,85 +619,75 @@ jobs:
path: ./**/.coverage.*
retention-days: 1

# kafka:
# env:
# TOTAL_GROUPS: 4

# strategy:
# fail-fast: false
# matrix:
# group-number: [1, 2, 3, 4]

# runs-on: ubuntu-20.04
# container:
# image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
# options: >-
# --add-host=host.docker.internal:host-gateway
# timeout-minutes: 30

# services:
# zookeeper:
# image: bitnami/zookeeper:3.7
# env:
# ALLOW_ANONYMOUS_LOGIN: yes

# ports:
# - 2181:2181

# kafka:
# image: bitnami/kafka:3.2
# ports:
# - 8080:8080
# - 8081:8081
# env:
# ALLOW_PLAINTEXT_LISTENER: yes
# KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
# KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
# KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081
# KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081,
# KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT
# KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2

# steps:
# - uses: actions/checkout@v3

# - name: Fetch git tags
# run: |
# git config --global --add safe.directory "$GITHUB_WORKSPACE"
# git fetch --tags origin

# # Special case packages
# - name: Install librdkafka-dev
# run: |
# # Use lsb-release to find the codename of Ubuntu to use to install the correct library name
# sudo apt-get update
# sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
# sudo apt-get install -y wget gnupg2 software-properties-common
# sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add -
# sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main"
# sudo apt-get update
# sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2)

# - name: Get Environments
# id: get-envs
# run: |
# echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
# env:
# GROUP_NUMBER: ${{ matrix.group-number }}

# - name: Test
# run: |
# tox -vv -e ${{ steps.get-envs.outputs.envs }}
# env:
# TOX_PARALLEL_NO_SPINNER: 1
# PY_COLORS: 0

# - name: Upload Coverage Artifacts
# uses: actions/upload-artifact@v3
# with:
# name: coverage-${{ github.job }}-${{ strategy.job-index }}
# path: ./**/.coverage.*
# retention-days: 1
kafka:
env:
TOTAL_GROUPS: 4

strategy:
fail-fast: false
matrix:
group-number: [1, 2, 3, 4]

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30

services:
zookeeper:
image: bitnami/zookeeper:3.7
env:
ALLOW_ANONYMOUS_LOGIN: yes

ports:
- 2181:2181

kafka:
image: bitnami/kafka:3.2
ports:
- 8080:8080
- 8082:8082
- 8083:8083
env:
KAFKA_ENABLE_KRAFT: no
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
KAFKA_CFG_LISTENERS: L1://:8082,L2://:8083,L3://:8080
KAFKA_CFG_ADVERTISED_LISTENERS: L1://host.docker.internal:8082,L2://host.docker.internal:8083,L3://kafka:8080
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT,L3:PLAINTEXT
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L3

steps:
- uses: actions/checkout@v3

- name: Fetch git tags
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git fetch --tags origin

- name: Get Environments
id: get-envs
run: |
echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT
env:
GROUP_NUMBER: ${{ matrix.group-number }}

- name: Test
run: |
tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
env:
TOX_PARALLEL_NO_SPINNER: 1
PY_COLORS: 0

- name: Upload Coverage Artifacts
uses: actions/upload-artifact@v3
with:
name: coverage-${{ github.job }}-${{ strategy.job-index }}
path: ./**/.coverage.*
retention-days: 1

mongodb:
env:
Expand All @@ -710,7 +700,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -768,7 +758,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -828,7 +818,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down Expand Up @@ -889,7 +879,7 @@ jobs:

runs-on: ubuntu-20.04
container:
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28
image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-52dab3b91d91005b8ef657fba3fe08a5fbfa23b1
options: >-
--add-host=host.docker.internal:host-gateway
timeout-minutes: 30
Expand Down
3 changes: 0 additions & 3 deletions codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,5 @@ ignore:
- "newrelic/hooks/database_oursql.py"
- "newrelic/hooks/database_psycopg2ct.py"
- "newrelic/hooks/datastore_umemcache.py"
# Temporarily disable kafka
- "newrelic/hooks/messagebroker_kafkapython.py"
- "newrelic/hooks/messagebroker_confluentkafka.py"
- "newrelic/admin/*"
- "newrelic/console.py"
Loading