Support Python 3.11, 3.12 and 3.13 (#205)

Update pytest config: Replace all ignored warnings with "error" to make all warnings errors. Validate input Python version. This is validated only when the `ci-cd` package is installed and used, and is validated against the Python versions it supports. Update dependencies according to Python version. Add typing-extensions as an explicit dependency. Add pyupgrade as a pre-commit hook with --py37-plus arg. Run and upgrade all files accordingly. Configure mypy to check typing expecting Python 3.7. Use env var straight up in bash sections. Instead of using env vars as ${{ env.XXX }}, use them as ${XXX} if found in a bash "run" value segment. Add support for 3.11 and 3.12 Py versions. Add Python 3.13 support (alpha.1 release is out). Use instead allow-prereleases: true for setup-python action. Set default python language in pre-commit.
SINTEF · Nov 22, 2023 · dc78405 · dc78405
1 parent bb54428
commit dc78405
Show file tree

Hide file tree

Showing 25 changed files with 238 additions and 128 deletions.
diff --git a/.github/utils/run_hooks.py b/.github/utils/run_hooks.py
@@ -5,7 +5,6 @@
 """
 from __future__ import annotations
 
-# import platform
 import subprocess  # nosec
 import sys
 

diff --git a/.github/workflows/_local_ci_tests.yml b/.github/workflows/_local_ci_tests.yml
@@ -40,7 +40,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         os: ["ubuntu-latest", "windows-latest"]
 
     steps:
@@ -51,6 +51,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version}}
+        allow-prereleases: true
 
     - name: Install Python dependencies
       run: |
@@ -76,7 +77,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.10"]
+        # Use lowest and highest supported (stable) Python version
+        python-version: ["3.7", "3.12"]
         os: ["ubuntu-latest", "windows-latest"]
 
     steps:

diff --git a/.github/workflows/cd_release.yml b/.github/workflows/cd_release.yml
@@ -158,6 +158,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Validate inputs
+      run: |
+        if [[ ! "${{ inputs.python_version_build}}" =~ ^3\.([7-9]|1[0-3])(\..*)?$ ]]; then
+          echo "Python version '${{ inputs.python_version_build}}' is not supported."
+          echo "Supported versions are: 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13."
+          exit 1
+        fi
+
     - name: Checkout ${{ github.repository }}
       uses: actions/checkout@v4
       with:
@@ -167,6 +175,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_build }}"
+        allow-prereleases: true
 
     - name: Install Python dependencies
       run: |
@@ -199,7 +208,7 @@ jobs:
     - name: Update changelog
       uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
       with:
-        args: --user "${{ github.repository_owner }}" ${{ env.CHANGELOG_PROJECT }} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.release_branch }}" ${{ env.CHANGELOG_EXCLUDE_TAGS_REGEX }} ${{ env.CHANGELOG_EXCLUDE_LABELS }}
+        args: --user "${{ github.repository_owner }}" ${CHANGELOG_PROJECT} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.release_branch }}" ${CHANGELOG_EXCLUDE_TAGS_REGEX} ${CHANGELOG_EXCLUDE_LABELS}
 
     - name: Set up git user
       run: |
@@ -287,7 +296,7 @@ jobs:
     - name: Create release-specific changelog
       uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
       with:
-        args: --user "${{ github.repository_owner }}" ${{ env.CHANGELOG_PROJECT }} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.release_branch }}" --since-tag "${{ env.PREVIOUS_VERSION }}" --output release_changelog.md --usernames-as-github-logins ${{ env.CHANGELOG_EXCLUDE_TAGS_REGEX }} ${{ env.CHANGELOG_EXCLUDE_LABELS }}
+        args: --user "${{ github.repository_owner }}" ${CHANGELOG_PROJECT} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.release_branch }}" --since-tag "${PREVIOUS_VERSION}" --output release_changelog.md --usernames-as-github-logins ${CHANGELOG_EXCLUDE_TAGS_REGEX} ${CHANGELOG_EXCLUDE_LABELS}
 
     - name: Append changelog to release body
       run: |
@@ -341,6 +350,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_docs }}"
+        allow-prereleases: true
 
     - name: Install system dependencies
       if: inputs.system_dependencies != ''
@@ -367,6 +377,20 @@ jobs:
         pip install -U setuptools wheel
         pip install ${EDITABLE}.${{ inputs.doc_extras || inputs.install_extras }}
 
+        INSTALLED_PACKAGES=$(pip freeze)
+        if [ "${{ inputs.docs_framework }}" == "mkdocs" ]; then
+          if [[ $(echo -e "${INSTALLED_PACKAGES}" | grep -ciE '(mkdocs|mike)==') -eq 2 ]]; then
+            echo "Missing one or more of the following packages: mkdocs, mike"
+            exit 1
+        elif [ "${{ inputs.docs_framework }}" == "sphinx" ]; then
+          if [[ $(echo -e "${INSTALLED_PACKAGES}" | grep -ci 'sphinx==') -eq 1 ]]; then
+            echo "Missing one or more of the following packages: sphinx"
+            exit 1
+        else
+          echo "Unknown framework: ${{ inputs.docs_framework }}"
+          exit 1
+        fi
+
     - name: Set up git user
       run: |
         git config --global user.name "${{ inputs.git_username }}"

diff --git a/.github/workflows/ci_cd_updated_default_branch.yml b/.github/workflows/ci_cd_updated_default_branch.yml
@@ -164,15 +164,6 @@ jobs:
       UPDATE_DEFAULT_BRANCH: false
 
     steps:
-    - name: Check input
-      run: |
-        valid_frameworks=("mkdocs sphinx")
-        if [[ ! " ${valid_frameworks[*]} " =~ " ${{ inputs.docs_framework}} " ]]; then
-            echo "The input '${{ inputs.docs_framework}}' is not supported."
-            echo "Valid inputs are: ${valid_frameworks[*]}"
-            exit 1
-        fi
-
     - name: Release check
       run: |
         COMMIT_MSG="$(gh api /repos/${{ github.repository}}/commits/${{ inputs.default_repo_branch }} --jq '.commit.message')"
@@ -186,6 +177,22 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
 
+    - name: Validate inputs
+      if: env.RELEASE_RUN == 'false'
+      run: |
+        valid_frameworks=("mkdocs sphinx")
+        if [[ ! " ${valid_frameworks[*]} " =~ " ${{ inputs.docs_framework}} " ]]; then
+            echo "The input '${{ inputs.docs_framework}}' is not supported."
+            echo "Valid inputs are: ${valid_frameworks[*]}"
+            exit 1
+        fi
+
+        if [[ ! "${{ inputs.python_version}}" =~ ^3\.([7-9]|1[0-3])(\..*)?$ ]]; then
+          echo "Python version '${{ inputs.python_version}}' is not supported."
+          echo "Supported versions are: 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13."
+          exit 1
+        fi
+
     - name: Checkout ${{ github.repository }}
       if: env.RELEASE_RUN == 'false'
       uses: actions/checkout@v4
@@ -197,6 +204,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version }}"
+        allow-prereleases: true
 
     - name: Install system dependencies
       if: env.RELEASE_RUN == 'false' && inputs.system_dependencies != ''
@@ -225,6 +233,20 @@ jobs:
         pip install ${EDITABLE}.${{ inputs.doc_extras }}
         pip install git+https://github.com/SINTEF/ci-cd.git@v2.6.0
 
+        INSTALLED_PACKAGES=$(pip freeze)
+        if [ "${{ inputs.docs_framework }}" == "mkdocs" ]; then
+          if [[ $(echo -e "${INSTALLED_PACKAGES}" | grep -ciE '(mkdocs|mike)==') -eq 2 ]]; then
+            echo "Missing one or more of the following packages: mkdocs, mike"
+            exit 1
+        elif [ "${{ inputs.docs_framework }}" == "sphinx" ]; then
+          if [[ $(echo -e "${INSTALLED_PACKAGES}" | grep -ci 'sphinx==') -eq 1 ]]; then
+            echo "Missing one or more of the following packages: sphinx"
+            exit 1
+        else
+          echo "Unknown framework: ${{ inputs.docs_framework }}"
+          exit 1
+        fi
+
     - name: Set up git user
       if: env.RELEASE_RUN == 'false'
       run: |
@@ -357,7 +379,7 @@ jobs:
       if: env.RELEASE_RUN == 'false'
       uses: docker://githubchangeloggenerator/github-changelog-generator:1.16.2
       with:
-        args: --user "${{ github.repository_owner }}" ${{ env.CHANGELOG_PROJECT }} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.default_repo_branch }}" --future-release "Unreleased changes" ${{ env.CHANGELOG_EXCLUDE_TAGS_REGEX }} ${{ env.CHANGELOG_EXCLUDE_LABELS }}
+        args: --user "${{ github.repository_owner }}" ${CHANGELOG_PROJECT} --token "${{ secrets.PAT || secrets.GITHUB_TOKEN }}" --release-branch "${{ inputs.default_repo_branch }}" --future-release "Unreleased changes" ${CHANGELOG_EXCLUDE_TAGS_REGEX} ${CHANGELOG_EXCLUDE_LABELS}
 
     - name: Build (& deploy MkDocs) documentation
       if: env.RELEASE_RUN == 'false' && ( ! inputs.test )
@@ -377,7 +399,7 @@ jobs:
 
         if [ "${{ inputs.test }}" == "true" ]; then
           if [ "${{ inputs.docs_framework }}" == "mkdocs" ]; then
-            echo "Will here deploy documentation using 'mike' called 'latest' with alias '${{ inputs.release_branch }}'"
+            echo "Will here deploy documentation using 'mike' called 'latest' with alias '${{ inputs.default_repo_branch }}'"
           elif [ "${{ inputs.docs_framework }}" == "sphinx" ]; then
             echo "Will here deploy documentation using 'sphinx-build'."
             echo "sphinx-build options: ${SPHINX_OPTIONS[@]}"
@@ -448,7 +470,7 @@ jobs:
         git fetch origin
 
         LATEST_PR_BODY="$(gh api /repos/${{ github.repository}}/pulls -X GET -f state=closed -f per_page=1 -f sort=updated -f direction=desc --jq '.[].body')"
-        cat ${{ env.PR_BODY_FILE }} | head -8 > .tmp_file.txt
+        cat ${PR_BODY_FILE} | head -8 > .tmp_file.txt
         if [ -z "$(printf '%s\n' "${LATEST_PR_BODY}" | head -8 | diff - .tmp_file.txt --strip-trailing-cr)" ]; then
           echo "The dependencies have just been updated! Reset to ${{ inputs.default_repo_branch }}."
           git reset --hard origin/${{ inputs.default_repo_branch }}
@@ -459,8 +481,8 @@ jobs:
           echo "FORCE_PUSH=no" >> $GITHUB_ENV
         fi
 
-        if [ "${{ env.REMOVE_PR_BODY_FILE }}" == "true" ] && [ -f "${{ env.PR_BODY_FILE }}" ]; then
-          rm -f "${{ env.PR_BODY_FILE }}"
+        if [ "${REMOVE_PR_BODY_FILE}" == "true" ] && [ -f "${PR_BODY_FILE}" ]; then
+          rm -f "${PR_BODY_FILE}"
         fi
         if [ -f ".tmp_file.txt" ]; then rm -f .tmp_file.txt; fi
       env:

diff --git a/.github/workflows/ci_check_pyproject_dependencies.yml b/.github/workflows/ci_check_pyproject_dependencies.yml
@@ -67,6 +67,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Validate inputs
+      run: |
+        if [[ ! "${{ inputs.python_version}}" =~ ^3\.([7-9]|1[0-3])(\..*)?$ ]]; then
+          echo "Python version '${{ inputs.python_version}}' is not supported."
+          echo "Supported versions are: 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13."
+          exit 1
+        fi
+
     - name: Checkout permanent dependencies branch in ${{ github.repository }}
       uses: actions/checkout@v4
       with:
@@ -77,6 +85,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version }}"
+        allow-prereleases: true
 
     - name: Install Python dependencies
       run: |
@@ -167,7 +176,7 @@ jobs:
 
     - name: Remove temporary file
       if: env.REMOVE_PR_BODY_FILE == 'true'
-      run: rm -f ${{ env.PR_BODY_FILE }}
+      run: rm -f ${PR_BODY_FILE}
 
     - name: Create PR
       if: env.UPDATE_DEPS == 'true'

diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml
@@ -213,6 +213,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_pre-commit }}"
+        allow-prereleases: true
 
     - name: Install Python dependencies
       run: |
@@ -238,6 +239,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_pylint_safety }}"
+        allow-prereleases: true
 
     - name: Install dependencies
       run: |
@@ -306,6 +308,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_package }}"
+        allow-prereleases: true
 
     - name: Install Python dependencies
       run: |
@@ -345,6 +348,14 @@ jobs:
           exit 1
         fi
 
+    - name: Validate inputs
+      run: |
+        if [[ "${framework}" == "mkdocs" && ! "${{ inputs.python_version_docs}}" =~ ^3\.([7-9]|1[0-3])(\..*)?$ ]]; then
+          echo "Python version '${{ inputs.python_version_docs}}' is not supported."
+          echo "Supported versions are: 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13."
+          exit 1
+        fi
+
     - name: Checkout ${{ github.repository }}
       uses: actions/checkout@v4
       with:
@@ -354,6 +365,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version_docs }}"
+        allow-prereleases: true
 
     - name: Install system dependencies
       if: inputs.system_dependencies != ''
@@ -454,12 +466,12 @@ jobs:
         # Set STRICT option
 
         if [ "${{ inputs.warnings_as_errors }}" == "true" ]; then
-          if [ "${{ env.framework }}" == "mkdocs" ]; then
+          if [ "${framework}" == "mkdocs" ]; then
             STRICT="--strict"
-          elif [ "${{ env.framework }}" == "sphinx" ]; then
+          elif [ "${framework}" == "sphinx" ]; then
             STRICT="-W"
           else
-            echo "Unknown framework: ${{ env.framework }}"
+            echo "Unknown framework: ${framework}"
             exit 1
           fi
 
@@ -470,10 +482,10 @@ jobs:
 
         # Run build command
 
-        if [ "${{ env.framework }}" == "mkdocs" ]; then
+        if [ "${framework}" == "mkdocs" ]; then
           mkdocs build ${STRICT}
 
-        elif [ "${{ env.framework }}" == "sphinx" ]; then
+        elif [ "${framework}" == "sphinx" ]; then
           if [[ "${{ inputs.sphinx-build_options }}" =~ \n ]]; then
             # Expected to be a multi-line string
             SPHINX_OPTIONS=()
@@ -490,6 +502,6 @@ jobs:
             ${{ inputs.docs_folder }} ${{ inputs.build_target_folder }}
 
         else
-          echo "Unknown framework: ${{ env.framework }}"
+          echo "Unknown framework: ${framework}"
           exit 1
         fi
diff --git a/.github/workflows/ci_update_dependencies.yml b/.github/workflows/ci_update_dependencies.yml
@@ -87,6 +87,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: "${{ inputs.python_version }}"
+        allow-prereleases: true
 
     - name: Install `pre-commit` and dependencies
       if: inputs.update_pre-commit
@@ -114,14 +115,14 @@ jobs:
       if: inputs.update_pre-commit
       continue-on-error: true  # Still create the PR if this step fails
       run: |
-        if [ "${{ env.UPDATED_PRE_COMMIT_HOOKS }}" == "true" ]; then
+        if [ "${UPDATED_PRE_COMMIT_HOOKS}" == "true" ]; then
           SKIP=${{ inputs.skip_pre-commit_hooks }} pre-commit run --all-files
         fi
 
     - name: Possibly commit changes and updates
       if: inputs.update_pre-commit
       run: |
-        if [ "${{ env.UPDATED_PRE_COMMIT_HOOKS }}" == "true" ]; then
+        if [ "${UPDATED_PRE_COMMIT_HOOKS}" == "true" ]; then
           git commit -am "Update \`pre-commit\` hooks"
         fi
 
@@ -148,7 +149,7 @@ jobs:
 
     - name: Remove temporary file
       if: env.REMOVE_PR_BODY_FILE == 'true'
-      run: rm -f ${{ env.PR_BODY_FILE }}
+      run: rm -f ${PR_BODY_FILE}
 
     - name: Create PR
       id: cpr

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,8 @@
 # To install the git pre-commit hook run:
 #   pre-commit install
-# To update the pre-commit hooks run:
-#   pre-commit autoupdate
+default_language_version:
+  python: python3.9
+
 repos:
   # pre-commit-hooks supplies a multitude of small hooks
   # To get an overview of them all as well as the ones used here, please see
@@ -24,7 +25,21 @@ repos:
     rev: 5.12.0
     hooks:
     - id: isort
-      args: ["--profile", "black", "--filter-files", "--skip-gitignore"]
+      args:
+      - "--profile=black"
+      - "--filter-files"
+      - "--skip-gitignore"
+      - "--add-import=from __future__ import annotations"
+
+  # pyupgrade is a tool to automatically upgrade Python syntax for newer versions
+  # It works on files in-place
+  - repo: https://github.com/asottile/pyupgrade
+    # Latest version for Python 3.7: 3.3.2
+    # Latest version for Python 3.8: 3.8.0
+    rev: v3.15.0
+    hooks:
+    - id: pyupgrade
+      args: ["--py37-plus"]
 
   # Black is a code style and formatter
   # It works on files in-place