Merge V25.2.1 into main

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = 
+ignore =
         # E*** / W*** pycodestyle codes
         # whitespace before ','
         E203,

.github/ISSUE_TEMPLATE/jira_service_desk.md

@@ -1,5 +1,5 @@
 ---
-name: Please file issues in Jira Service Desk 
+name: Please file issues in Jira Service Desk
 about: https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87
 title: ''
 labels: ''
@@ -10,4 +10,4 @@ assignees: ''
 **Describe the bug**
 Please file issues or requests for help with schematic in [Jira Service Desk](https://sagebionetworks.jira.com/servicedesk/customer/portal/5/group/8/create/87).
 
-These issues are not monitored. 
+These issues are not monitored.

.github/workflows/api_test.yml

@@ -43,7 +43,7 @@ jobs:
             | python3 - --version ${{ env.POETRY_VERSION }};
           poetry config virtualenvs.create true;
           poetry config virtualenvs.in-project true;
-          
+
       #----------------------------------------------
       # install dependencies and root project
       #----------------------------------------------

.github/workflows/docker_build.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-          
+
       - name: Set env variable for version tag
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
 
@@ -54,4 +54,3 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             TAG=${{ env.RELEASE_VERSION }}
-

.github/workflows/pdoc.yml

@@ -12,12 +12,12 @@ permissions:
   contents: read
 
 concurrency:
-  # cancel the current running workflow from the same branch, PR when a new workflow is triggered 
+  # cancel the current running workflow from the same branch, PR when a new workflow is triggered
   # when the trigger is not a PR but a push, it will use the commit sha to generate the concurrency group
   # {{ github.workflow }}: the workflow name is used to generate the concurrency group. This allows you to have more than one workflows
-  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag 
-  # {{ github.event.pull_request.number}}: get PR number 
-  # {{ github.sha }}: full commit sha 
+  # {{ github.ref_type }}: the type of Git ref object created in the repository. Can be either branch or tag
+  # {{ github.event.pull_request.number}}: get PR number
+  # {{ github.sha }}: full commit sha
   # credit: https://github.com/Sage-Bionetworks-Workflows/sagetasks/blob/main/.github/workflows/ci.yml
   group: >-
     ${{ github.workflow }}-${{ github.ref_type }}-
@@ -32,7 +32,7 @@ jobs:
 
     steps:
       #----------------------------------------------
-      #       check-out repo and set-up python     
+      #       check-out repo and set-up python
       #----------------------------------------------
       - name: Check out repository
         uses: actions/checkout@v4
@@ -41,16 +41,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      
+
       #----------------------------------------------
-      #          install & configure poetry         
+      #          install & configure poetry
       #----------------------------------------------
       - name: Install Poetry
         run: |
           curl -sSL https://install.python-poetry.org \
             | python3 - --version ${{ env.POETRY_VERSION }};
-          poetry config virtualenvs.create true; 
-          poetry config virtualenvs.in-project true; 
+          poetry config virtualenvs.create true;
+          poetry config virtualenvs.in-project true;
 
       #----------------------------------------------
       # install dependencies and root project

.github/workflows/publish.yml

@@ -1,107 +1,92 @@
 name: Publish to PyPI
 on:
-  push:
-    tags:
-      - 'v[1-9][0-9].[0-9]+.[0-9]+'
-    branches: [main]
+  release:
+    types: [published]
 
 jobs:
-  pypi_release:
+  build:
     runs-on: ubuntu-22.04
     env:
-      POETRY_VERSION:  1.3.0
       PYTHON_VERSION: "3.10"
-    if: github.event_name == 'push' && contains(github.ref, 'refs/tags')
+    outputs:
+      sdist-package-name: ${{ steps.build-package.outputs.sdist-package-name }}
+      bdist-package-name: ${{ steps.build-package.outputs.bdist-package-name }}
     steps:
-      #----------------------------------------------
-      #       check-out repo and set-up python     
-      #----------------------------------------------
-      - name: Check out repository
-        uses: actions/checkout@v4
-
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v5
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-
-      #----------------------------------------------
-      #          install & configure poetry
-      #----------------------------------------------
-      - name: Install Poetry
+      - id: build-package
         run: |
-          curl -sSL https://install.python-poetry.org \
-            | python3 - --version ${{ env.POETRY_VERSION }};
-          poetry config virtualenvs.create true;
-          poetry config virtualenvs.in-project true;
+          python3 -m pip install --upgrade pip
+          python3 -m pip install setuptools
+          python3 -m pip install wheel
+          python3 -m pip install build
+          python3 -m pip install .
+          python3 -m build
 
-      #----------------------------------------------
-      # install dependencies and root project
-      #----------------------------------------------
-      - name: Install dependencies and root project
-        run: poetry install --no-interaction --all-extras
-
-      #----------------------------------------------
-      #    get current pushed tag
-      #----------------------------------------------      
-      - name: Show GitHub ref
-        run: echo "$GITHUB_REF"
+          VERSION=${{ github.event.release.tag_name }}
+          SDIST_PACKAGE_NAME="schematicpy-${VERSION}.tar.gz"
+          BDIST_PACKAGE_NAME="schematicpy-${VERSION}-py3-none-any.whl"
+          RELEASE_URL_PREFIX="https://uploads.github.com/repos/${{ github.event.repository.full_name }}/releases/${{ github.event.release.id }}/assets?name="
 
-      - name: Get current pushed tag
-        run:  |
-          echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
-          echo "$RELEASE_VERSION"
-
-      #----------------------------------------------
-      #    override version tag 
-      #----------------------------------------------      
-      - name: Override version tag
-        run: poetry run python3 override_version.py
-        shell: sh
+          echo "sdist-package-name=$SDIST_PACKAGE_NAME" >> $GITHUB_OUTPUT
+          echo "bdist-package-name=$BDIST_PACKAGE_NAME" >> $GITHUB_OUTPUT
 
-      #----------------------------------------------
-      #    publish to testpypi
-      #----------------------------------------------      
-      # - run: poetry config repositories.testpypi https://test.pypi.org/legacy/
-      # - run: poetry config pypi-token.testpypi ${{ secrets.TWINE_TEST_TOKEN }}
-      # - name: Publish package to test Pypi
-      #   run: poetry publish -vvvv --build -r testpypi
+          echo "sdist-release-url=${RELEASE_URL_PREFIX}${SDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT
+          echo "bdist-release-url=${RELEASE_URL_PREFIX}${BDIST_PACKAGE_NAME}" >> $GITHUB_OUTPUT
 
-      #----------------------------------------------
-      #    check tag
-      #----------------------------------------------      
-      - name: Check Tag
-        id: check-tag
-        run: |
-          if [[ ${{ github.event.ref }} =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-              echo ::set-output name=match::true
-          fi
-      #----------------------------------------------
-      #    publish to pypi
-      #----------------------------------------------  
-      - name: Publish package to Pypi
-        id: publish-to-pypi
-        if: steps.check-tag.outputs.match == 'true'
-        env:
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-          PYPI_USERNAME: __token__
-        run: |
-          poetry publish --build --username $PYPI_USERNAME --password $PYPI_TOKEN
+      - name: upload-sdist-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build-package.outputs.sdist-package-name }}
+          path: dist/
+          if-no-files-found: error
 
-      #----------------------------------------------
-      #    post a message to slack
-      #----------------------------------------------  
+      - name: upload-bdist-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.build-package.outputs.bdist-package-name }}
+          path: dist/
+          if-no-files-found: error
 
-      - name: Post to a Slack channel
-        if: steps.publish-to-pypi.outcome == 'success'
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.event.release.prerelease == true && 'testpypi' || 'pypi' }}
+      url: ${{ github.event.release.prerelease == true && 'https://test.pypi.org/p/schematicpy' || 'https://pypi.org/p/schematicpy' }}
+    permissions:
+      id-token: write
+    steps:
+      - name: download-sdist
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build.outputs.sdist-package-name }}
+          path: dist
+
+      - name: download-bdist
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.build.outputs.bdist-package-name }}
+          path: dist
+
+      - name: deploy-to-pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: ${{ github.event.release.prerelease == true && 'https://test.pypi.org/legacy/' || '' }}
+
+  slack-notification:
+    needs: publish
+    if: ${{ github.event.release.prerelease != true }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: post-slack-notification
         id: slack
         uses: slackapi/slack-github-action@v1.23.0
-        with:
-          # Slack channel id, channel name, or user id to post message.
-          # See also: https://api.slack.com/methods/chat.postMessage#channels
-          # You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs.
-          # ibc-fair-data channel and data-curator-schematic channel
-          channel-id: 'C050YD75QRL,C01ANC02U59'
-          # For posting a simple plain text message
-          slack-message: "Schematic has just been released. Check out new version: ${{ github.ref_name }}"
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        with:
+          # Post to the `fair-data-tools` slack channel
+          channel-id: 'C01ANC02U59'
+          slack-message: "A new version of Schematic has been released. Check out the new version: ${{ github.ref_name }}"

.github/workflows/scan_repo.yml

@@ -6,7 +6,7 @@ on:
     branches:
       - develop
   pull_request:
-  workflow_dispatch: 
+  workflow_dispatch:
 
 jobs:
   trivy:

.github/workflows/test.yml

@@ -87,11 +87,7 @@ jobs:
       #----------------------------------------------
       #             check formatting
       #----------------------------------------------
-      - name: Code formatting with black
-        run: |
-          # ran only on certain files for now
-          # add here when checked
-          poetry run black schematic tests schematic_api --check
+      - uses: pre-commit/action@v3.0.1
 
       #----------------------------------------------
       #             type checking/enforcement
@@ -121,37 +117,26 @@ jobs:
           SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
           SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
         run: >
-          poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov 
+          poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov
           --cov-report=xml:coverage.xml --cov=schematic/ --reruns 4 -n 8 tests/unit;
 
       #----------------------------------------------
       #              run integration test suite
       #----------------------------------------------
 
-      - name: Retrieve telemetry access token from IDP
-        if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
-        id: retrieve-telemetry-access-token
-        run: |
-          response=$(curl --request POST \
-            --url ${{ vars.TELEMETRY_AUTH_CLIENT_URL }} \
-            --header 'content-type: application/json' \
-            --data '{"client_id":"${{ vars.TELEMETRY_AUTH_CLIENT_ID }}","client_secret":"${{ secrets.TELEMETRY_AUTH_CLIENT_SECRET }}","audience":"${{ vars.TELEMETRY_AUTH_AUDIENCE }}","grant_type":"client_credentials"}')
-          access_token=$(echo $response | jq -r .access_token)
-          echo "::add-mask::$access_token"
-          echo "TELEMETRY_ACCESS_TOKEN=$access_token" >> "$GITHUB_OUTPUT"
       - name: Run integration tests
         if: ${{ contains(fromJSON('["3.10"]'), matrix.python-version) }}
         env:
           SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
           SERVICE_ACCOUNT_CREDS: ${{ secrets.SERVICE_ACCOUNT_CREDS }}
-          OTEL_EXPORTER_OTLP_HEADERS: "Authorization=Bearer ${{ steps.retrieve-telemetry-access-token.outputs.TELEMETRY_ACCESS_TOKEN }}"
+          OTEL_EXPORTER_OTLP_HEADERS: "signoz-ingestion-key=${{ secrets.TELEMETRY_API_TOKEN }}"
           DEPLOYMENT_ENVIRONMENT: ${{ vars.DEPLOYMENT_ENVIRONMENT }}
           OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OTEL_EXPORTER_OTLP_ENDPOINT }}
           TRACING_EXPORT_FORMAT: ${{ vars.TRACING_EXPORT_FORMAT }}
           LOGGING_EXPORT_FORMAT: ${{ vars.LOGGING_EXPORT_FORMAT }}
           TRACING_SERVICE_NAME: ${{ vars.TRACING_SERVICE_NAME }}
           LOGGING_SERVICE_NAME: ${{ vars.LOGGING_SERVICE_NAME }}
-          SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }} 
+          SERVICE_INSTANCE_ID: ${{ github.head_ref || github.ref_name }}
         run: >
           poetry run pytest --durations=0 --cov-append --cov-report=term --cov-report=html:htmlcov --cov-report=xml:coverage.xml --cov=schematic/
           -m "not (rule_benchmark or single_process_execution)" --reruns 4 -n 8 --ignore=tests/unit
@@ -213,12 +198,8 @@ jobs:
         uses: andstor/file-existence-action@v3
         with:
           files: "coverage.xml"
-        # This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
-      - name: Override Coverage Source Path for Sonar
-        if: steps.check_coverage_xml.outputs.files_exists == 'true'
-        run: sed -i "s/<source>\/home\/runner\/work\/schematic\/schematic\/schematic<\/source>/<source>\/github\/workspace\/schematic<\/source>/g" coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
+        uses: SonarSource/sonarqube-scan-action@v4.1.0
         if: ${{ always() }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -180,4 +180,4 @@ manifests/*
 https:*
 
 # schematic config file
-config.yml
+config.yml