Add official support for python 3.10 and 3.11 (#6292)

StackStorm · Jan 10, 2025 · ae50851 · ae50851
2 parents 983f63f + 8f2fc19
commit ae50851
Show file tree

Hide file tree

Showing 20 changed files with 2,583 additions and 646 deletions.
diff --git a/.github/actions/setup-python/action.yaml b/.github/actions/setup-python/action.yaml
@@ -4,7 +4,7 @@ description:
   Light wrapper around the actions/setup-python and actions/cache actions
   to maintain the input vars in only one place for all workflows.
 
-input:
+inputs:
   python-version:
     description: Which version of python to install.
     required: true

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -11,7 +11,7 @@ on:
       # also version tags, which include bugfix releases (eg: v3.4.0)
       - v[0-9]+.[0-9]+.[0-9]+
   pull_request:
-    type: [opened, reopened, edited]
+    types: [opened, reopened, synchronize]
     branches:
       # Only for PRs targeting those branches
       - master
@@ -44,7 +44,7 @@ jobs:
     # NOTE: We always want to run job on master since we run some additional checks there (code
     # coverage, etc)
     # if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/master' }}
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+    name: '${{ matrix.make.name }} - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -53,26 +53,18 @@ jobs:
         # each job runs subset of tests.
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
-        include:
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+          - {version-short: '3.10', version: '3.10.15'}
+          - {version-short: '3.11', version: '3.11.10'}
+        make:
           - name: 'Lint Checks (black, flake8, etc.)'
             task: 'ci-checks'
-            python-version-short: '3.8'
-            python-version: '3.8.10'
           - name: 'Compile (pip deps, pylint, etc.)'
             task: 'ci-compile'
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Lint Checks (black, flake8, etc.)'
-            task: 'ci-checks'
-            python-version-short: '3.9'
-            python-version: '3.9.14'
-          - name: 'Compile (pip deps, pylint, etc.)'
-            task: 'ci-compile'
-            python-version-short: '3.9'
-            python-version: '3.9.14'
-
     env:
-      TASK: '${{ matrix.task }}'
+      TASK: '${{ matrix.make.task }}'
       COLUMNS: '120'
       PYLINT_CONCURRENCY: '6'
     steps:
@@ -81,10 +73,10 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+      - name: 'Set up Python (${{ matrix.python.version }}) and Cache Deps'
         uses: ./.github/actions/setup-python
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
       - name: Install virtualenv
@@ -109,15 +101,14 @@ jobs:
 
   self-check:
     needs: pre_job
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+    # FIXME: dropping the repetition in this name requires changing required checks on GitHub
+    name: 'Self-check on Python ${{ matrix.python.version-short }} - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: 'Self-check on Python 3.8'
-            python-version-short: '3.8'
-            python-version: '3.8.14'
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
     services:
       mongo:
         image: mongo:7.0
@@ -163,10 +154,10 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+      - name: 'Set up Python (${{ matrix.python.version }}) and Cache Deps'
         uses: ./.github/actions/setup-python
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
       - name: Install virtualenv
@@ -238,7 +229,7 @@ jobs:
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
-          name: logs-py${{ matrix.python-version }}
+          name: logs-py${{ matrix.python.version }}
           path: logs.tar.gz
           retention-days: 7
 
@@ -248,38 +239,29 @@ jobs:
     # coverage, etc)
     # NB: disabled. See TODO above pre_job
     # if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/master' }}
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+    name: '${{ matrix.make.name }} - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
         # each job runs subset of tests.
-        include:
-          - name: 'Unit Tests (chunk 1)'
-            task: 'ci-unit'
-            shard: {k: 0, n: 2}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Unit Tests (chunk 2)'
-            task: 'ci-unit'
-            shard: {k: 1, n: 2}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+          - {version-short: '3.10', version: '3.10.15'}
+          - {version-short: '3.11', version: '3.11.10'}
+        make:
           - name: 'Unit Tests (chunk 1)'
             task: 'ci-unit'
             shard: {k: 0, n: 2}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
           - name: 'Unit Tests (chunk 2)'
             task: 'ci-unit'
             shard: {k: 1, n: 2}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
+
           # This job is slow so we only run in on a daily basis
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
-          #   python-version: '3.6.13'
           #   shard: {k: 0, n: 1}
     services:
       mongo:
@@ -310,10 +292,10 @@ jobs:
           - 15672:15672/tcp # Management: HTTP, CLI
 
     env:
-      TASK: '${{ matrix.task }}'
-      PYTHON_VERSION_SHORT: '${{ matrix.python-version-short }}'
-      NODE_TOTAL: '${{ matrix.shard.n }}'
-      NODE_INDEX: '${{ matrix.shard.k }}'
+      TASK: '${{ matrix.make.task }}'
+      PYTHON_VERSION_SHORT: '${{ matrix.python.version-short }}'
+      NODE_TOTAL: '${{ matrix.make.shard.n }}'
+      NODE_INDEX: '${{ matrix.make.shard.k }}'
 
       # We need to explicitly specify terminal width otherwise some CLI tests fail on container
       # environments where small terminal size is used.
@@ -339,10 +321,10 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+      - name: 'Set up Python (${{ matrix.python.version }}) and Cache Deps'
         uses: ./.github/actions/setup-python
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
       - name: Install virtualenv
@@ -400,46 +382,30 @@ jobs:
     # NOTE: We always want to run job on master since we run some additional checks there (code
     # coverage, etc)
     # if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/master' }}
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+    name: '${{ matrix.make.name }} - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         # NOTE: To speed the CI run, we split unit and integration tests into multiple jobs where
         # each job runs subset of tests.
-        include:
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+          - {version-short: '3.10', version: '3.10.15'}
+          - {version-short: '3.11', version: '3.11.10'}
+        make:
           # We run pack tests here since they rely on some integration tests set
           # up (aka stanley user being present, etc.)
           - name: 'Pack Tests'
             task: 'ci-packs-tests'
             shard: {k: 0, n: 1}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Integration Tests (chunk 1)'
-            task: 'ci-integration'
-            shard: {k: 0, n: 2}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Integration Tests (chunk 2)'
-            task: 'ci-integration'
-            shard: {k: 1, n: 2}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            shard: {k: 0, n: 1}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
           - name: 'Integration Tests (chunk 1)'
             task: 'ci-integration'
             shard: {k: 0, n: 2}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
           - name: 'Integration Tests (chunk 2)'
             task: 'ci-integration'
             shard: {k: 1, n: 2}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
     services:
       mongo:
         image: mongo:7.0
@@ -491,10 +457,10 @@ jobs:
           - 6379:6379/tcp
 
     env:
-      TASK: '${{ matrix.task }}'
-      PYTHON_VERSION_SHORT: '${{ matrix.python-version-short }}'
-      NODE_TOTAL: '${{ matrix.shard.n }}'
-      NODE_INDEX: '${{ matrix.shard.k }}'
+      TASK: '${{ matrix.make.task }}'
+      PYTHON_VERSION_SHORT: '${{ matrix.python.version-short }}'
+      NODE_TOTAL: '${{ matrix.make.shard.n }}'
+      NODE_INDEX: '${{ matrix.make.shard.k }}'
 
       # We need to explicitly specify terminal width otherwise some CLI tests fail on container
       # environments where small terminal size is used.
@@ -521,10 +487,10 @@ jobs:
       - name: Custom Environment Setup
         run: |
           ./scripts/github/setup-environment.sh
-      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+      - name: 'Set up Python (${{ matrix.python.version }}) and Cache Deps'
         uses: ./.github/actions/setup-python
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
       - name: Install virtualenv
@@ -580,7 +546,7 @@ jobs:
         if: ${{ failure() && env.TASK == 'ci-integration' }}
         uses: actions/upload-artifact@v4
         with:
-          name: logs-py${{ matrix.python-version }}-pytest-${{ matrix.shard.k }}
+          name: logs-py${{ matrix.python.version }}-pytest-${{ matrix.make.shard.k }}
           path: logs.tar.gz
           retention-days: 7
 

diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -13,7 +13,7 @@ on:
       # also version tags, which include bugfix releases (eg: v3.4.0)
       - v[0-9]+.[0-9]+.[0-9]+
   pull_request:
-    type: [opened, reopened, edited]
+    types: [opened, reopened, synchronize]
     branches:
       # Only for PRs targeting those branches
       - master

diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml
@@ -5,6 +5,16 @@ name: Micro Benchmarks
 on:
   schedule:
     - cron:  '30 3 * * *'
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches:
+      # Only for PRs targeting those branches
+      - master
+      - v[0-9]+.[0-9]+
+    paths:
+      # Only for PRs that touch the benchmarks
+      - .github/workflows/microbenchmarks.yaml
+      - st2common/benchmarks/**
 
 jobs:
   # Special job which automatically cancels old runs for the same branch, prevents runs for the
@@ -26,24 +36,22 @@ jobs:
     # NOTE: We always want to run job on master since we run some additional checks there (code
     # coverage, etc)
     if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref == 'refs/heads/master' }}
-    name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}'
+    name: '${{ matrix.make.name }} - Python ${{ matrix.python.version-short }}'
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
       matrix:
         # NOTE: We need to use full Python version as part of Python deps cache key otherwise
         # setup virtualenv step will fail.
-        include:
+        python:
+          - {version-short: '3.8', version: '3.8.10'}
+          - {version-short: '3.9', version: '3.9.14'}
+          - {version-short: '3.10', version: '3.10.15'}
+          - {version-short: '3.11', version: '3.11.10'}
+        make:
           - name: 'Microbenchmarks'
             task: 'micro-benchmarks'
             shard: {k: 0, n: 1}
-            python-version-short: '3.8'
-            python-version: '3.8.10'
-          - name: 'Microbenchmarks'
-            task: 'micro-benchmarks'
-            shard: {k: 0, n: 1}
-            python-version-short: '3.9'
-            python-version: '3.9.14'
     services:
       mongo:
         image: mongo:7.0
@@ -60,10 +68,10 @@ jobs:
           - 15672:15672/tcp # Management: HTTP, CLI
 
     env:
-      TASK: '${{ matrix.task }}'
+      TASK: '${{ matrix.make.task }}'
 
-      NODE_TOTAL: '${{ matrix.shard.n }}'
-      NODE_INDEX: '${{ matrix.shard.k }}'
+      NODE_TOTAL: '${{ matrix.make.shard.n }}'
+      NODE_INDEX: '${{ matrix.make.shard.k }}'
 
       COLUMNS: '120'
       ST2_CI: 'true'
@@ -73,10 +81,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps'
+      - name: 'Set up Python (${{ matrix.python.version }}) and Cache Deps'
         uses: ./.github/actions/setup-python
         with:
-          python-version: '${{ matrix.python-version }}'
+          python-version: '${{ matrix.python.version }}'
       - name: Cache and Install APT Dependencies
         uses: ./.github/actions/apt-packages
       - name: Install virtualenv
@@ -96,7 +104,7 @@ jobs:
       - name: Upload Histograms
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark_histograms-py${{ matrix.python-version }}
+          name: benchmark_histograms-py${{ matrix.python.version }}
           path: benchmark_histograms/
           retention-days: 30