Skip to content

Commit

Permalink
merge develop
Browse files Browse the repository at this point in the history
  • Loading branch information
kprokofi committed Mar 19, 2024
2 parents 509a226 + c76fb06 commit 5640921
Show file tree
Hide file tree
Showing 300 changed files with 8,859 additions and 2,196 deletions.
24 changes: 0 additions & 24 deletions .ci/build-deps.txt

This file was deleted.

1 change: 1 addition & 0 deletions .ci/build-deps.in → .ci/publish-deps.in
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
build==1.1.1
twine==5.0.0
320 changes: 320 additions & 0 deletions .ci/publish-deps.txt

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,8 @@ updates:
directory: /
schedule:
interval: weekly

- package-ecosystem: pip
directory: /for_developers/regression_test
schedule:
interval: weekly
1 change: 0 additions & 1 deletion .github/workflows/code_scan.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ on:
push:
branches:
- "releases/*"
- "v2"
schedule:
# every UTC 6PM from Mon to Fri
- cron: "0 18 * * 1-5"
Expand Down
76 changes: 76 additions & 0 deletions .github/workflows/codeql.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
push:
branches:
- develop
- releases
pull_request:
types:
- opened
- reopened
- synchronize
schedule:
- cron: "0 0 * * 0"

permissions:
contents: read

jobs:
analyze:
name: Analyze
# Runner size impacts CodeQL analysis time. To learn more, please see:
# - https://gh.io/recommended-hardware-resources-for-running-codeql
# - https://gh.io/supported-runners-and-hardware-resources
# - https://gh.io/using-larger-runners
# Consider using larger runners for possible analysis time improvements.
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
# required for all workflows
security-events: write

# only required for workflows in private repositories
actions: read
contents: read

strategy:
fail-fast: false
matrix:
language: ["python"]
# CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
# Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
# Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.

# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@47b3d888fe66b639e431abf22ebca059152f1eea # v3.24.5
with:
category: "/language:${{matrix.language}}"
78 changes: 78 additions & 0 deletions .github/workflows/docs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: Build Docs for the latest

on:
workflow_dispatch: # run on request (no need for PR)
push:
branches:
- develop

# Declare default permissions as read only.
permissions: read-all

jobs:
Build-Docs:
runs-on: ubuntu-20.04
permissions:
contents: write
steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.10"
- name: Install tox
run: python -m pip install --require-hashes --no-deps -r .ci/tox-deps.txt
- name: Build-Docs
run: tox -e build-doc
- name: Create gh-pages branch
run: |
if [[ ${{github.event_name}} == 'workflow_dispatch' ]]; then
echo RELEASE_VERSION="test_build" >> $GITHUB_ENV
else
echo RELEASE_VERSION=${GITHUB_REF#refs/*/} >> $GITHUB_ENV
fi
echo SOURCE_NAME=${GITHUB_REF#refs/*/} >> $GITHUB_OUTPUT
echo SOURCE_BRANCH=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT
echo SOURCE_TAG=${GITHUB_REF#refs/tags/} >> $GITHUB_OUTPUT
existed_in_remote=$(git ls-remote --heads origin gh-pages)
if [[ -z ${existed_in_remote} ]]; then
echo "Creating gh-pages branch"
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
git checkout --orphan gh-pages
git reset --hard
touch .nojekyll
git add .nojekyll
git commit -m "Initializing gh-pages branch"
git push origin gh-pages
git checkout ${{steps.branch_name.outputs.SOURCE_NAME}}
echo "Created gh-pages branch"
else
echo "Branch gh-pages already exists"
fi
- name: Commit docs to gh-pages branch
run: |
git fetch
git checkout gh-pages
mkdir -p /tmp/docs_build
cp -r docs/build/html/* /tmp/docs_build/
rm -rf ${{ env.RELEASE_VERSION }}/*
echo '<html><head><meta http-equiv="refresh" content="0; url=stable/" /></head></html>' > index.html
mkdir -p ${{ env.RELEASE_VERSION }}
cp -r /tmp/docs_build/* ./${{ env.RELEASE_VERSION }}
rm -rf /tmp/docs_build
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
if [[ ${{ env.RELEASE_VERSION }} != 'test_build' ]]; then
ln -sfn ${{ env.RELEASE_VERSION }} latest
fi
git add ./latest ${{ env.RELEASE_VERSION }}
git commit -m "Update documentation" -a || true
- name: Push changes
uses: ad-m/github-push-action@fcea09907c44d7a7a3331c9c04080d55d87c95fe # master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: gh-pages
18 changes: 13 additions & 5 deletions .github/workflows/perf_benchmark.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,17 @@ on:
- default # speed, balance, accuracy models only
- all # default + other models
default: default
data-size:
data-group:
type: choice
description: Dataset size to run benchmark
description: Data group to run benchmark
options:
- small
- medium
- large
- all
default: all
num-repeat:
description: Overrides default per-data-size number of repeat setting
description: Overrides default per-data-group number of repeat setting
default: 0
num-epoch:
description: Overrides default per-model number of epoch setting
Expand All @@ -43,6 +43,13 @@ on:
type: string
description: Root directory containing validation data in CI server.
default: "/home/validation/data/v2/"
otx-ref:
type: string
description: |
Target OTX ref (tag / branch name / commit hash) on main repo to test. Defaults to the current branch.
`pip install otx[full]@https://github.com/openvinotoolkit/training_extensions.git@{otx_ref}` will be executed before run,
and reverted after run. Works only for v2.x assuming CLI compatibility.
default: __CURRENT_BRANCH_COMMIT__

# Declare default permissions as read only.
permissions: read-all
Expand Down Expand Up @@ -82,13 +89,14 @@ jobs:
tox -vv -e perf-benchmark -- tests/perf/test_${{ matrix.task }}.py ${{ inputs.pytest-args }}
--model-category ${{ inputs.model-category }}
--data-root ${{ inputs.data-root }}
--data-size ${{ inputs.data-size }}
--data-group ${{ inputs.data-group }}
--num-repeat ${{ inputs.num-repeat }}
--num-epoch ${{ inputs.num-epoch }}
--eval-upto ${{ inputs.eval-upto }}
--summary-csv .tox/perf-benchmark-${{ matrix.task-short }}.csv
--summary-csv .tox/perf-benchmark-summary.csv
--mlflow-tracking-uri ${{ vars.MLFLOW_TRACKING_SERVER_URI }}
--user-name ${{ github.triggering_actor }}
--otx-ref ${{ inputs.otx-ref }}
- name: Upload test results
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/pre_merge.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
name: V2 PR Checks
name: PR Checks

on:
push:
branches:
- v2
- develop
- releases/**
pull_request:
branches:
- v2
types:
- opened
- reopened
Expand Down Expand Up @@ -94,6 +93,7 @@ jobs:
- task: "instance_segmentation"
- task: "semantic_segmentation"
- task: "visual_prompting"
- task: "anomaly"
name: Integration-Test-${{ matrix.task }}-py310
# This is what will cancel the job concurrency
concurrency:
Expand Down
6 changes: 2 additions & 4 deletions .github/workflows/publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
with:
python-version: "3.10"
- name: Install pypa/build
run: python -m pip install --require-hashes --no-deps -r .ci/build-deps.txt
run: python -m pip install --require-hashes --no-deps -r .ci/publish-deps.txt
- name: Build sdist
run: python -m build --sdist
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
Expand All @@ -52,12 +52,10 @@ jobs:
- name: Download artifacts
uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4.1.3
with:
# unpacks default artifact into dist/
# if `name: artifact` is omitted, the action will create extra parent dir
path: dist
pattern: artifact-*
merge-multiple: true
# to determine where to publish the source distribution to PyPI or TestPyPI
# to determine where to publish the package distribution to PyPI or TestPyPI
- name: Check tag
id: check-tag
uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2
Expand Down
91 changes: 91 additions & 0 deletions .github/workflows/publish_internal.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
name: Build and upload to internal PyPI

on:
workflow_dispatch: # run on request (no need for PR)

# Declare default permissions as read only.
permissions: read-all

jobs:
build_wheels:
name: Build wheels
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Build wheels
uses: pypa/cibuildwheel@0ecddd92b62987d7a2ae8911f4bb8ec9e2e4496a # v2.13.1
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: artifact-wheels
path: ./wheelhouse/*.whl

build_sdist:
name: Build source distribution
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Python 3.10
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.10"
- name: Install pypa/build
run: python -m pip install --require-hashes --no-deps -r .ci/publish-deps.txt
- name: Build sdist
run: python -m build --sdist
- uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: artifact-sdist
path: dist/*.tar.gz

publish_package:
name: Publish package
needs: [build_wheels, build_sdist]
environment: pypi
runs-on: [self-hosted, linux, x64, dev]
permissions:
packages: write
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Set up Python
uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
with:
python-version: "3.10"
- name: Install dependencies
run: python -m pip install --require-hashes --no-deps -r .ci/publish-deps.txt
- name: Download artifacts
uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85 # v4.1.3
with:
path: dist
pattern: artifact-*
merge-multiple: true
# to determine where to publish the package distribution to PyPI or TestPyPI
- name: Check tag
id: check-tag
uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2
with:
text: ${{ github.ref }}
regex: '^refs/heads/releases/[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+rc[0-9]+|rc[0-9]+)?$'
- name: Check dist contents
run: twine check dist/*
- name: Publish package dist to internal PyPI
if: ${{ steps.check-tag.outputs.match != '' }}
run: |
export no_proxy=${{ secrets.PYPI_HOST }}
export REPOSITORY_URL=http://${{ secrets.PYPI_HOST }}:${{ secrets.PYPI_PORT }}
twine upload --verbose --repository-url $REPOSITORY_URL dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }}
- name: Publish package distributions to TestPyPI
if: ${{ steps.check-tag.outputs.match == '' }}
run: |
export REPOSITORY_URL=https://test.pypi.org/legacy/
twine upload --verbose --repository-url $REPOSITORY_URL dist/* -u __token__ -p ${{ secrets.TEST_PYPI_API_TOKEN }}
- name: Clean up dist
if: ${{ always() }}
run: |
if OUTPUT=$(ls | grep -c dist)
then
echo "Cleaning up dist directory"
rm -r dist
fi
Loading

0 comments on commit 5640921

Please sign in to comment.