From 90d663ad01677e9bb02f6820f023d901ed8c2b5a Mon Sep 17 00:00:00 2001 From: David Cermak Date: Fri, 20 Dec 2024 18:12:57 +0100 Subject: [PATCH 1/3] feat(mosq): Add IDF MQTT stress tests to mosquitto CI --- .github/workflows/mosq__build.yml | 80 +++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/.github/workflows/mosq__build.yml b/.github/workflows/mosq__build.yml index b4dddb25ec..9e69a1ce24 100644 --- a/.github/workflows/mosq__build.yml +++ b/.github/workflows/mosq__build.yml @@ -101,3 +101,83 @@ jobs: exit 1 fi echo "Versions are consistent: $CONFIG_VERSION" + + build_idf_tests_with_mosq: + name: Build IDF tests + strategy: + matrix: + idf_ver: ["latest"] + idf_target: ["esp32"] + test: [ { app: publish, path: "tools/test_apps/protocols/mqtt/publish_connect_test" }] + runs-on: ubuntu-20.04 + container: espressif/idf:${{ matrix.idf_ver }} + env: + TARGET_TEST_DIR: build_esp32_local_broker + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }} + shell: bash + run: | + . ${IDF_PATH}/export.sh + pip install idf-component-manager idf-build-apps --upgrade + export OVERRIDE_PATH=`pwd`/components/mosquitto + echo ${OVERRIDE_PATH} + sed -i '/espressif\/mosquitto:/a \ \ \ \ override_path: "${OVERRIDE_PATH}"' ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml + cat ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml + export PEDANTIC_FLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function" + export EXTRA_CFLAGS="${PEDANTIC_FLAGS} -Wstrict-prototypes" + export EXTRA_CXXFLAGS="${PEDANTIC_FLAGS}" + cd ${IDF_PATH}/${{matrix.test.path}} + sed -i 's/4096, /5\*1024, /' main/publish_connect_test.c + cat main/publish_connect_test.c + idf-build-apps find --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR} + idf-build-apps build --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR} + ${GITHUB_WORKSPACE}/ci/clean_build_artifacts.sh `pwd`/${TARGET_TEST_DIR} + sed '/@pytest.mark.parametrize.*config.*/{ + s/@pytest.mark.parametrize.*config.*local_broker.*/@pytest.mark.protocols/ + t + d + }' pytest_mqtt_publish_app.py > ${TARGET_TEST_DIR}/pytest_local_mosq.py + cat ${TARGET_TEST_DIR}/pytest_local_mosq.py + zip -qur ${GITHUB_WORKSPACE}/artifacts.zip ${TARGET_TEST_DIR} + - uses: actions/upload-artifact@v4 + with: + name: mosq_publish_esp32_${{ matrix.idf_ver }} + path: artifacts.zip + if-no-files-found: error + + test_idf_ci_with_mosq: + # Skip running on forks since it won't have access to secrets + if: | + github.repository == 'espressif/esp-protocols' && + ( contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push' ) + name: Mosquitto IDF target tests + needs: build_idf_tests_with_mosq + strategy: + matrix: + idf_ver: ["latest"] + runs-on: + - self-hosted + - ESP32-ETHERNET-KIT + env: + TEST_DIR: examples + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: mosq_publish_esp32_${{ matrix.idf_ver }} + path: ${{ env.TEST_DIR }}/ci/ + - name: Run Test + working-directory: ${{ env.TEST_DIR }} + run: | + python -m pip install pytest-embedded-serial-esp pytest-embedded-idf pytest-rerunfailures pytest-timeout pytest-ignore-test-results "paho-mqtt<2" + unzip ci/artifacts.zip -d ci + for dir in `ls -d ci/build_*`; do + rm -rf build sdkconfig.defaults + mv $dir build + mv build/*.py . + python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32 -m protocols + done From dbd164dd91d73f9e66d1fb16f3756e55a21c6703 Mon Sep 17 00:00:00 2001 From: David Cermak Date: Tue, 7 Jan 2025 08:52:38 +0100 Subject: [PATCH 2/3] fix(mosq): Add a note about stack size --- components/mosquitto/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/mosquitto/README.md b/components/mosquitto/README.md index a6408db29a..3f8041982b 100644 --- a/components/mosquitto/README.md +++ b/components/mosquitto/README.md @@ -20,7 +20,7 @@ mosq_broker_run(&config); ## Memory Footprint Considerations -The broker primarily uses the heap for internal data, with minimal use of static/BSS memory. It consumes approximately 60 kB of program memory. +The broker primarily uses the heap for internal data, with minimal use of static/BSS memory. It consumes approximately 60 kB of program memory and minimum 5kB of stack size. - **Initial Memory Usage**: ~2 kB of heap on startup - **Per Client Memory Usage**: ~4 kB of heap for each connected client From 9162de1150d1ed8966647c90d59dafbd5e2d7f3d Mon Sep 17 00:00:00 2001 From: David Cermak Date: Wed, 8 Jan 2025 11:29:48 +0100 Subject: [PATCH 3/3] fix(mosq): Remove temp modification of IDF files --- .github/workflows/mosq__build.yml | 20 ++++----- components/mosquitto/test/README.md | 5 +++ .../mosquitto/test/replace_decorators.py | 42 +++++++++++++++++++ 3 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 components/mosquitto/test/README.md create mode 100644 components/mosquitto/test/replace_decorators.py diff --git a/.github/workflows/mosq__build.yml b/.github/workflows/mosq__build.yml index 9e69a1ce24..5a12bc30f5 100644 --- a/.github/workflows/mosq__build.yml +++ b/.github/workflows/mosq__build.yml @@ -123,25 +123,18 @@ jobs: run: | . ${IDF_PATH}/export.sh pip install idf-component-manager idf-build-apps --upgrade - export OVERRIDE_PATH=`pwd`/components/mosquitto - echo ${OVERRIDE_PATH} - sed -i '/espressif\/mosquitto:/a \ \ \ \ override_path: "${OVERRIDE_PATH}"' ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml - cat ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml + export MOSQUITTO_PATH=`pwd`/components/mosquitto + # to use the actual version of mosquitto + sed -i '/espressif\/mosquitto:/a \ \ \ \ override_path: "${MOSQUITTO_PATH}"' ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml export PEDANTIC_FLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function" export EXTRA_CFLAGS="${PEDANTIC_FLAGS} -Wstrict-prototypes" export EXTRA_CXXFLAGS="${PEDANTIC_FLAGS}" cd ${IDF_PATH}/${{matrix.test.path}} - sed -i 's/4096, /5\*1024, /' main/publish_connect_test.c - cat main/publish_connect_test.c idf-build-apps find --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR} idf-build-apps build --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR} ${GITHUB_WORKSPACE}/ci/clean_build_artifacts.sh `pwd`/${TARGET_TEST_DIR} - sed '/@pytest.mark.parametrize.*config.*/{ - s/@pytest.mark.parametrize.*config.*local_broker.*/@pytest.mark.protocols/ - t - d - }' pytest_mqtt_publish_app.py > ${TARGET_TEST_DIR}/pytest_local_mosq.py - cat ${TARGET_TEST_DIR}/pytest_local_mosq.py + # to replace mqtt test configs with specific mosquitto markers + python ${MOSQUITTO_PATH}/test/replace_decorators.py pytest_mqtt_publish_app.py ${TARGET_TEST_DIR}/pytest_mosquitto.py zip -qur ${GITHUB_WORKSPACE}/artifacts.zip ${TARGET_TEST_DIR} - uses: actions/upload-artifact@v4 with: @@ -179,5 +172,6 @@ jobs: rm -rf build sdkconfig.defaults mv $dir build mv build/*.py . - python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32 -m protocols + # Run only "test_mosquitto" marked tests + python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32 -m test_mosquitto done diff --git a/components/mosquitto/test/README.md b/components/mosquitto/test/README.md new file mode 100644 index 0000000000..eef6b4ea51 --- /dev/null +++ b/components/mosquitto/test/README.md @@ -0,0 +1,5 @@ +# ESP32 mosquitto tests + +Mosquitto component doesn't have any tests yet, but we upcycle IDF mqtt tests and run them with the current version of mosquitto. +For that we need to update the IDF test project's `idf_component.yml` file to reference this actual version of mosquitto. +We also need to update some pytest decorators to run only relevant test cases. See the [replacement](./replace_decorators.py) script. diff --git a/components/mosquitto/test/replace_decorators.py b/components/mosquitto/test/replace_decorators.py new file mode 100644 index 0000000000..6c842bedd4 --- /dev/null +++ b/components/mosquitto/test/replace_decorators.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Unlicense OR CC0-1.0 + +# This script replaces the `@pytest` decorators in the test files +# based on the value of the `config` parameter in the `@pytest` decorator. +# to reuse mqtt test cases for mosquitto broker. + +import re +import sys + + +def replace_decorators(in_file: str, out_file: str) -> None: + with open(in_file, 'r') as file: + content = file.read() + + # we replace config decorators to differentiate between local mosquitto based tests + pattern = r"@pytest\.mark\.parametrize\(\s*'config'\s*,\s*\[\s*'(.*?)'\s*\]\s*,.*\)" + + def replacement(match): + config_value = match.group(1) + if config_value == 'local_broker': + return '@pytest.mark.test_mosquitto' + else: + return '@pytest.mark.test_mqtt' + + # Replace occurrences + updated_content = re.sub(pattern, replacement, content) + + with open(out_file, 'w') as file: + file.write(updated_content) + + +# Main function to handle arguments +if __name__ == '__main__': + if len(sys.argv) != 3: + print('Usage: python replace_decorators.py ') + sys.exit(1) + + in_file = sys.argv[1] + out_file = sys.argv[2] + replace_decorators(in_file, out_file) + print(f'Replacements completed')