From 7067ed0aaeb572cc3c70272beeeec2da1008e724 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Sat, 24 Aug 2024 12:21:21 +0100 Subject: [PATCH] Adopt tox workflow using coactions/dynamic-matrix --- .github/workflows/jira_ci.yml | 15 ++- .github/workflows/tox.yml | 206 ++++++++++++++++++++++++++++++++++ tox.ini | 5 +- 3 files changed, 217 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/tox.yml diff --git a/.github/workflows/jira_ci.yml b/.github/workflows/jira_ci.yml index a793b12d7..56724efff 100644 --- a/.github/workflows/jira_ci.yml +++ b/.github/workflows/jira_ci.yml @@ -1,17 +1,16 @@ name: ci +# runs only after tox workflow finished succesfully on: - # Trigger the workflow on push or pull request, - # but only for the main branch - push: - branches: - - main - pull_request: - branches: - - main + workflow_run: + workflows: [tox] + branches: [main] + types: + - completed jobs: server: + if: ${{ github.event.workflow_run.conclusion == 'success' }} uses: pycontribs/jira/.github/workflows/jira_server_ci.yml@main cloud: diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml new file mode 100644 index 000000000..5740d2089 --- /dev/null +++ b/.github/workflows/tox.yml @@ -0,0 +1,206 @@ +--- +name: tox +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 # tox, pytest + PY_COLORS: 1 + +jobs: + prepare: + name: prepare + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Determine matrix + id: generate_matrix + uses: coactions/dynamic-matrix@main # v3 + with: + min_python: "3.9" + max_python: "3.12" + default_python: "3.9" + # We run sanity with 3 different versions of ansible as the result + # can be different. Each of them is testing with all supported + # python versions, regardless the tox pyXY name. + other_names: | + lint + docs + pkg + # ^ arm64 runner is using py311 for matching python version used in AAP 2.5 + platforms: linux,macos,linux-arm64:ubuntu-24.04-arm64-2core + skip_explode: "1" + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os || 'ubuntu-24.04' }} + continue-on-error: ${{ contains(matrix.name, 'integration') && true || false }} + needs: + - prepare + defaults: + run: + shell: ${{ matrix.shell || 'bash'}} + working-directory: ansible_collections/ansible/eda + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.matrix) }} + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # needed by setuptools-scm + path: ansible_collections/ansible/eda + submodules: true + + # - name: Install package dependencies (ubuntu) + # if: ${{ contains(matrix.os, 'ubuntu') }} + # run: | + # sudo apt remove -y docker-compose + # sudo apt-get update -y + # sudo apt-get --assume-yes --no-install-recommends install -y apt-transport-https curl libsystemd0 libsystemd-dev pkg-config + # sudo add-apt-repository ppa:deadsnakes/ppa + # curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + # echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + # sudo apt update -y + # sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin shellcheck + # # Do not install docker-compose-plugin because it would v1 (broken due to not working with newer requests library) + # sudo systemctl enable --now docker + # sudo curl -sL "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + # sudo chmod +x /usr/local/bin/docker-compose + + - name: Set pre-commit cache + uses: actions/cache@v4 + if: ${{ contains(matrix.name, 'lint') }} + with: + path: | + ~/.cache/pre-commit + key: pre-commit-${{ matrix.name }}-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Set up Python ${{ matrix.python_version || '3.10' }} + uses: actions/setup-python@v5 + with: + cache: pip + python-version: ${{ matrix.python_version || '3.10' }} + cache-dependency-path: "*requirements*.txt" + + - name: Install tox + run: | + python3 -m pip install --upgrade pip wheel tox + + - run: ${{ matrix.command }} + + - run: ${{ matrix.command2 }} + if: ${{ matrix.command2 }} + + - run: ${{ matrix.command3 }} + if: ${{ matrix.command3 }} + + - run: ${{ matrix.command4 }} + if: ${{ matrix.command4 }} + + - run: ${{ matrix.command5 }} + if: ${{ matrix.command5 }} + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.name }}.zip + if-no-files-found: error + path: | + ansible_collections/ansible/eda/.tox/**/log/ + ansible_collections/ansible/eda/.tox/**/coverage.xml + ansible_collections/ansible/eda/tests/output/reports/coverage.xml + ansible_collections/ansible/eda/tests/output/junit/*.xml + + - name: Report failure if git reports dirty status + run: | + if [[ -n $(git status -s) ]]; then + # shellcheck disable=SC2016 + echo -n '::error file=git-status::' + printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY" + exit 99 + fi + # https://github.com/actions/toolkit/issues/193 + check: + if: always() + environment: check + permissions: + id-token: write + checks: read + + needs: + - build + + runs-on: ubuntu-24.04 + + steps: + # checkout needed for codecov action which needs codecov.yml file + - uses: actions/checkout@v4 + + - name: Set up Python # likely needed for coverage + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pip3 install 'coverage>=7.5.1' + + - name: Merge logs into a single archive + uses: actions/upload-artifact/merge@v4 + with: + name: logs.zip + pattern: logs-*.zip + # artifacts like py312.zip and py312-macos do have overlapping files + separate-directories: true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: logs.zip + path: . + + - name: Check for expected number of coverage reports + run: .github/check-coverage.sh + + # Single uploads inside check job for codecov to allow use to retry + # it when it fails without running tests again. Fails often enough! + - name: Upload junit xml reports + # PRs from forks might not have access to the secret + if: env.CODECOV_TOKEN + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN || env.CODECOV_TOKEN }} + uses: codecov/test-results-action@v1 + with: + name: ${{ matrix.name }} + files: "*/tests/output/junit/*.xml" + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage data + uses: codecov/codecov-action@v4 + with: + name: ${{ matrix.name }} + # verbose: true # optional (default = false) + fail_ci_if_error: true + use_oidc: true # cspell:ignore oidc + files: "*/tests/output/reports/coverage.xml" + + # - name: Check codecov.io status + # if: github.event_name == 'pull_request' + # uses: coactions/codecov-status@main + + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + - name: Delete Merged Artifacts + uses: actions/upload-artifact/merge@v4 + with: + delete-merged: true diff --git a/tox.ini b/tox.ini index b398da265..8df5886e0 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,9 @@ requires = # tox-extra # tox-pyenv envlist = + lint + pkg + docs py311 py310 py39 @@ -99,7 +102,7 @@ commands = 'import pathlib; '\ 'docs_dir = pathlib.Path(r"{toxworkdir}") / "docs_out"; index_file = docs_dir / "index.html"; print(f"\nDocumentation available under `file://\{index_file\}`\n\nTo serve docs, use `python3 -m http.server --directory \{docs_dir\} 0`\n")' -[testenv:packaging] +[testenv:pkg] basepython = python3 description = Build package, verify metadata, install package and assert behavior when ansible is missing.