From 4b09221d0009a4ee605792bf71e65054c499563c Mon Sep 17 00:00:00 2001 From: docktermj Date: Mon, 6 May 2024 18:01:44 -0400 Subject: [PATCH] #19 Update workflows --- .github/workflows/bandit.yaml | 29 +++++++++++ .github/workflows/black.yaml | 34 ++++++++++++ .../create-sphinx-documentation.yaml | 10 +++- .../dependabot-approve-and-merge.yaml | 7 +-- .github/workflows/dependency-scan.yaml | 52 +++++++++++++++++++ .github/workflows/flake8.yaml | 16 +++--- .github/workflows/isort.yaml | 8 ++- .github/workflows/mypy.yaml | 35 +++++++++++++ .github/workflows/pylint.yaml | 8 ++- .github/workflows/pytest-darwin.yaml.disabled | 35 ++++++++++++- .github/workflows/pytest-linux.yaml | 40 +++++++++++++- .github/workflows/pytest-windows.yaml | 38 ++++++++++++-- .github/workflows/python-code-quality.yaml | 44 ---------------- .github/workflows/unittest-linux.yaml | 34 ------------ 14 files changed, 289 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/bandit.yaml create mode 100644 .github/workflows/black.yaml create mode 100644 .github/workflows/dependency-scan.yaml create mode 100644 .github/workflows/mypy.yaml delete mode 100644 .github/workflows/python-code-quality.yaml delete mode 100644 .github/workflows/unittest-linux.yaml diff --git a/.github/workflows/bandit.yaml b/.github/workflows/bandit.yaml new file mode 100644 index 0000000..524ffb2 --- /dev/null +++ b/.github/workflows/bandit.yaml @@ -0,0 +1,29 @@ +name: bandit + +on: + pull_request: + branches: [main] + paths-ignore: + - ".github/**" + +permissions: + contents: read + pull-requests: write + +jobs: + bandit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + steps: + - uses: actions/checkout@v4 + - name: Run Bandit Scan + uses: lukehinds/bandit-action@new-action + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + path: "examples src" + recursive: "true" diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..fb43d63 --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,34 @@ +name: black + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + black: + name: black Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: update pip + run: python3 -m pip install --upgrade pip + + - name: pip install python packages for testing + # run: pip install black pylint mypy + run: pip install black + + - name: black testing + run: | + # shellcheck disable=SC2046 + black --diff --check $(git ls-files '*.py' ':!:docs/source/*') diff --git a/.github/workflows/create-sphinx-documentation.yaml b/.github/workflows/create-sphinx-documentation.yaml index 0a2ad28..93c668e 100644 --- a/.github/workflows/create-sphinx-documentation.yaml +++ b/.github/workflows/create-sphinx-documentation.yaml @@ -1,6 +1,12 @@ name: create sphinx documentation -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [main] + paths: + - 'src' + - 'examples' + workflow_dispatch: permissions: contents: write @@ -18,7 +24,7 @@ jobs: run: | pip install sphinx sphinx_rtd_theme myst_parser autodocsumm sphinx-toolbox - - name: sphinx build + - name: Sphinx build run: | sphinx-build docs/source _build diff --git a/.github/workflows/dependabot-approve-and-merge.yaml b/.github/workflows/dependabot-approve-and-merge.yaml index 15a5bf9..f86f170 100644 --- a/.github/workflows/dependabot-approve-and-merge.yaml +++ b/.github/workflows/dependabot-approve-and-merge.yaml @@ -4,11 +4,12 @@ on: pull_request: branches: [main] +permissions: + contents: write + pull-requests: write + jobs: dependabot-approve-and-merge: - permissions: - contents: write - pull-requests: write secrets: SENZING_GITHUB_CODEOWNER_PR_RW_TOKEN: ${{ secrets.SENZING_GITHUB_CODEOWNER_PR_RW_TOKEN }} uses: senzing-factory/build-resources/.github/workflows/dependabot-approve-and-merge.yaml@v1 diff --git a/.github/workflows/dependency-scan.yaml b/.github/workflows/dependency-scan.yaml new file mode 100644 index 0000000..d2257c9 --- /dev/null +++ b/.github/workflows/dependency-scan.yaml @@ -0,0 +1,52 @@ +name: dependency scan + +on: + pull_request: + branches: [main] + paths-ignore: + - '.github/**' + +env: + DEP_PATH: requirements.txt + +permissions: + contents: read + +jobs: + safety: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + steps: + - name: checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Security vulnerabilities scan + uses: aufdenpunkt/python-safety-check@v1.0.5 + with: + safety_args: '-i 62044' + + pip-audit: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + steps: + - name: checkout repository + uses: actions/checkout@v4 + + - name: pip install + run: python -m pip install . + + - uses: pypa/gh-action-pip-audit@v1.0.8 + with: + inputs: requirements.txt diff --git a/.github/workflows/flake8.yaml b/.github/workflows/flake8.yaml index 6b87b29..52423e7 100644 --- a/.github/workflows/flake8.yaml +++ b/.github/workflows/flake8.yaml @@ -1,19 +1,23 @@ name: flake8 -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: flake8: - name: "flake8 Python ${{ matrix.python-version }}" + name: flake8 Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -23,6 +27,6 @@ jobs: - name: flake8 Lint uses: py-actions/flake8@v2 with: - max-line-length: "88" - path: "src/senzing_abstract" - plugins: "flake8-black" + max-line-length: 88 + path: src/senzing_abstract + plugins: flake8-black diff --git a/.github/workflows/isort.yaml b/.github/workflows/isort.yaml index 38a954e..de9a576 100644 --- a/.github/workflows/isort.yaml +++ b/.github/workflows/isort.yaml @@ -1,6 +1,9 @@ name: isort -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: isort: @@ -11,5 +14,6 @@ jobs: - uses: isort/isort-action@v1 with: - requirements-files: "requirements.txt" + configuration: + requirements-files: requirements.txt sort-paths: "src/senzing_abstract examples tests" diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml new file mode 100644 index 0000000..3ae0b15 --- /dev/null +++ b/.github/workflows/mypy.yaml @@ -0,0 +1,35 @@ +name: mypy + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + mypy: + name: mypy Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: update pip + run: python3 -m pip install --upgrade pip + + - name: pip install mypy + # run: pip install black pylint mypy + run: pip install mypy pytest orjson + + - name: mypy testing + #run: mypy --strict $(git ls-files '*.py' ':!:docs/source/*') + run: | + # shellcheck disable=SC2046 + mypy --strict $(git ls-files '*.py' ':!:docs/source/*' ':!:tools/*') diff --git a/.github/workflows/pylint.yaml b/.github/workflows/pylint.yaml index 5f96c66..96fb93d 100644 --- a/.github/workflows/pylint.yaml +++ b/.github/workflows/pylint.yaml @@ -1,13 +1,17 @@ name: pylint -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: pylint: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pytest-darwin.yaml.disabled b/.github/workflows/pytest-darwin.yaml.disabled index 308af2e..d6752c7 100644 --- a/.github/workflows/pytest-darwin.yaml.disabled +++ b/.github/workflows/pytest-darwin.yaml.disabled @@ -1,16 +1,22 @@ name: pytest darwin -on: [push] +# TODO: Once arm64 senzing binaries are available, rename file to "pytest-darwin.yaml" + +on: [pull_request, workflow_dispatch] env: PYTHONPATH: /Users/runner/work/sz-sdk-python-abstract/sz-sdk-python-abstract/src SENZING_TOOLS_ENABLE_ALL: true +permissions: + contents: read + jobs: pytest-darwin: name: "pytest OS: ${{ matrix.os }}; Python ${{ matrix.python-version }}" runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [macos-latest] # python-version: ["3.8", "3.9", "3.10", "3.11"] @@ -21,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -33,3 +39,28 @@ jobs: - name: run pytest on tests run: pytest tests/ --verbose --capture=no --cov=src/senzing_abstract + + # - name: Run pytest on examples + # run: | + # export DYLD_LIBRARY_PATH=/opt/senzing/g2/lib:/opt/senzing/g2/lib/macos + # pytest examples/ --verbose --capture=no + + - name: rename coverage file + env: + COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" + run: | + mv .coverage "$COVERAGE_FILE" + + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.${{ matrix.python-version }} + + coverage: + name: coverage + needs: pytest-darwin + permissions: + pull-requests: write + contents: write + uses: senzing-factory/build-resources/.github/workflows/python-coverage-comment.yaml@v1 diff --git a/.github/workflows/pytest-linux.yaml b/.github/workflows/pytest-linux.yaml index a26a909..b5a83fc 100644 --- a/.github/workflows/pytest-linux.yaml +++ b/.github/workflows/pytest-linux.yaml @@ -1,15 +1,19 @@ name: pytest linux -on: [push] +on: [push, pull_request] env: PYTHONPATH: /home/runner/work/sz-sdk-python-abstract/sz-sdk-python-abstract/src +permissions: + contents: read + jobs: pytest-linux: name: "pytest OS: ${{ matrix.os }}; Python ${{ matrix.python-version }}" runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest] # python-version: ["3.8", "3.9", "3.10", "3.11"] @@ -31,4 +35,36 @@ jobs: python -m pip install psutil pytest pytest-cov pytest-schema - name: run pytest on tests - run: pytest tests/ --verbose --capture=no --cov=src/senzing_abstract + run: pytest tests/ --verbose --capture=no --cov=src/senzing_abstract --cov-append + + - name: run unittest on examples + run: | + python3 -m unittest \ + examples/szconfig/*.py \ + examples/szconfigmanager/*.py \ + examples/szdiagnostic/*.py \ + examples/szengine/*.py \ + examples/szproduct/*.py + + # - name: Run pytest on examples + # run: pytest examples/ --verbose --capture=no + + - name: rename coverage file + env: + COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" + run: | + mv .coverage "$COVERAGE_FILE" + + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.${{ matrix.python-version }} + + coverage: + name: coverage + needs: pytest-linux + permissions: + pull-requests: write + contents: write + uses: senzing-factory/build-resources/.github/workflows/python-coverage-comment.yaml@v1 diff --git a/.github/workflows/pytest-windows.yaml b/.github/workflows/pytest-windows.yaml index 3cac941..98da40e 100644 --- a/.github/workflows/pytest-windows.yaml +++ b/.github/workflows/pytest-windows.yaml @@ -1,19 +1,23 @@ name: pytest windows -on: [push] +on: [pull_request, workflow_dispatch] env: PYTHONPATH: /Users/runner/work/sz-sdk-python-abstract/sz-sdk-python-abstract/src +permissions: + contents: read + jobs: pytest-windows: name: "pytest OS: ${{ matrix.os }}; Python ${{ matrix.python-version }}" runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + os: [windows-latest] # python-version: ["3.8", "3.9", "3.10", "3.11"] python-version: ["3.8", "3.11"] - os: [windows-latest] steps: - name: checkout repository @@ -24,5 +28,31 @@ jobs: with: python-version: ${{ matrix.python-version }} - # - name: run pytest on tests - # run: pytest tests/ --verbose --capture=no --cov=src/senzing_abstract + - name: install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + pip install psutil pytest pytest-cov pytest-schema + + - name: run pytest on tests + run: pytest tests/ --verbose --capture=no --cov=src/senzing_abstract + + - name: rename coverage file + env: + COVERAGE_FILE: ".coverage.${{ matrix.python-version }}" + run: | + Rename-Item -Path.coverage -NewName "$env:COVERAGE_FILE" + + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.${{ matrix.python-version }} + + coverage: + name: coverage + needs: pytest-windows + permissions: + pull-requests: write + contents: write + uses: senzing-factory/build-resources/.github/workflows/python-coverage-comment.yaml@v1 diff --git a/.github/workflows/python-code-quality.yaml b/.github/workflows/python-code-quality.yaml deleted file mode 100644 index 5b65a70..0000000 --- a/.github/workflows/python-code-quality.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: python code quality - -on: [push, pull_request] - -jobs: - python-code-quality: - name: 'code-quality Python ${{ matrix.python-version }}' - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.11'] - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --requirement requirements.txt - python -m pip install black flake8 mypy psutil pylint pytest pytest_schema types-psutil types-protobuf - - - name: black testing - run: | - # shellcheck disable=SC2046 - black --diff --check $(git ls-files '*.py' ':!:docs/source/*') - - - name: pylint testing - run: | - # shellcheck disable=SC2046 - pylint $(git ls-files '*.py' ':!:docs/source/*') - - - name: flake8 testing - run: | - # shellcheck disable=SC2046 - flake8 $(git ls-files '*.py' ':!:docs/source/*') - - - name: mypy testing - run: | - # shellcheck disable=SC2046 - mypy --follow-imports skip --strict $(git ls-files '*.py' ':!:docs/source/*') diff --git a/.github/workflows/unittest-linux.yaml b/.github/workflows/unittest-linux.yaml deleted file mode 100644 index d0ee88d..0000000 --- a/.github/workflows/unittest-linux.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: unittest linux - -on: [push] - -env: - PYTHONPATH: /home/runner/work/sz-sdk-python-abstract/sz-sdk-python-abstract/src - -jobs: - pytest-linux: - name: "pytest OS: ${{ matrix.os }}; Python ${{ matrix.python-version }}" - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - # python-version: ["3.8", "3.9", "3.10", "3.11"] - python-version: ["3.8", "3.11"] - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --requirement requirements.txt - # python -m pip install psutil pytest pytest-cov pytest-schema - - - name: run pytest on tests - run: python3 -m unittest examples/szconfig/*.py examples/szconfigmanager/*.py examples/szdiagnostic/*.py examples/szengine/*.py examples/szproduct/*.py