sync with upstream / dependencies, ci (#77)

* copying some root files

* copying rust module

* copying c api

* updating requirements, pyproject.toml, setup.py

* copying .ci and most of .github

* linter fixes

* downgrading numpy because cvat has numpy~=1.22.2

* removing not yet present jsons from MANIFEST.in

* linter fixes

* removing tritonclient requirement, because it has setuptools as dependency, which is considered unsafe by pip-compile-multi in cvat

* Apply suggestions from code review

Co-authored-by: Maxim Zhiltsov <zhiltsov.max35@gmail.com>

* linters fix

* removing openvino plugin dependencies from requirements-core.txt

* returned mac tests & switched pr check to python 3.9

* Update .github/workflows/linter.yml

---------

Co-authored-by: Maxim Zhiltsov <zhiltsov.max35@gmail.com>

.ci/trivy-csv.tmpl

@@ -0,0 +1,10 @@
+{{ range . }}
+Trivy Vulnerability Scan Results ({{ .Target }})
+VulnerabilityID,Severity,CVSS Score,Title,Library,Vulnerable Version,Fixed Version,Information URL,Triage Information
+{{ range .Vulnerabilities }}{{ .VulnerabilityID }},{{ .Severity }},{{ range $key, $value := .CVSS }}{{ if (eq $key "nvd") }}{{ .V3Score }}{{ end }}{{ end }},"{{ .Title }}","{{ .PkgName }}","{{ .InstalledVersion }}","{{ .FixedVersion }}",{{ .PrimaryURL }}
+{{ end }}
+Trivy Dependency Scan Results ({{ .Target }})
+ID,Name,Version,Notes
+{{ range .Packages }}{{ .ID }},{{ .Name }},{{ .Version }}
+{{ end }}
+{{ end }}

.ci/trivy-csv.yaml

@@ -0,0 +1,16 @@
+ignore-policy: ""
+ignorefile: .trivyignore
+insecure: false
+scan:
+  scanners:
+    - vuln
+    - secret
+  slow: false
+severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
+vulnerability:
+  ignore-unfixed: false
+format: template
+template: "@.ci/trivy-csv.tmpl"
+output: trivy-results-full.csv
+list-all-pkgs: true
+debug: true

.ci/trivy-json.yaml

@@ -0,0 +1,6 @@
+ignore-policy: ""
+ignorefile: .trivyignore
+format: spdx-json
+output: trivy-results-prod.spdx.json
+list-all-pkgs: true
+debug: true

.gitattributes

@@ -1,6 +1,7 @@
 *               text=auto whitespace=trailing-space,space-before-tab,-indent-with-non-tab,tab-in-indent,tabwidth=4
 
 .git*           text export-ignore
+Makefile        text whitespace=-tab-in-indent
 
 *.txt           text
 *.htm           text
@@ -26,3 +27,5 @@ LICENSE         text
 *.gif           binary
 *.ttf           binary
 *.pdf           binary
+
+notebooks/**    linguist-vendored

.github/codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        # adjust accordingly based on how flaky your tests are
+        # this allows a 2% drop from the previous base commit coverage
+        threshold: 2%

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: daily
+
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: daily

.github/pull_request_template.md

@@ -1,6 +1,7 @@
 <!-- Contributing guide: https://github.com/cvat-ai/datumaro#contributing -->
 
 ### Summary
+
 <!--
 Resolves #111 and #222.
 Depends on #1000 (for series of dependent commits).
@@ -18,21 +19,19 @@ not fully covered by unit tests or manual testing can be complicated. -->
 
 ### Checklist
 <!-- Put an 'x' in all the boxes that apply -->
-- [ ] I submit my changes into the `develop` branch
-- [ ] I have added description of my changes into [CHANGELOG](https://github.com/cvat-ai/datumaro/blob/develop/CHANGELOG.md)
+- [ ] I have added unit tests to cover my changes.​
+- [ ] I have added integration tests to cover my changes.​
+- [ ] I have added the description of my changes into [CHANGELOG](https://github.com/cvat-ai/datumaro/blob/develop/CHANGELOG.md).​
 - [ ] I have updated the [documentation](https://github.com/cvat-ai/datumaro/tree/develop/docs) accordingly
-- [ ] I have added tests to cover my changes
-- [ ] I have [linked related issues](https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)
 
 ### License
 
-- [ ] I submit _my code changes_ under the same [MIT License](
-  https://github.com/cvat-ai/datumaro/blob/develop/LICENSE) that covers the project.
+- [ ] I submit _my code changes_ under the same [MIT License](https://github.com/cvat-ai/datumaro/blob/develop/LICENSE) that covers the project.
   Feel free to contact the maintainers if that's a concern.
-- [ ] I have updated the license header for each file (see an example below)
+- [ ] I have updated the license header for each file (see an example below).
 
 ```python
-# Copyright (C) 2022 CVAT.ai Corporation
+# Copyright (C) 2025 CVAT.ai Corporation
 #
 # SPDX-License-Identifier: MIT
 ```

.github/workflows/code_scan.yml

@@ -0,0 +1,77 @@
+name: Code Scanning
+
+on:
+  workflow_dispatch: # run on request (no need for PR)
+  push:
+    branches: [ "develop", "releases/*" ]
+  schedule:
+    # every UTC 6PM from Mon to Fri
+    - cron: "0 18 * * 1-5"
+
+jobs:
+  Trivy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: python -m pip install pip-tools
+      - name: Freeze dependencies
+        run: |
+          mkdir -p .ci/base
+          pip-compile -o .ci/base/requirements.txt requirements.txt
+          mkdir -p .ci/dev/tests
+          pip-compile -o .ci/dev/tests/requirements.txt tests/requirements.txt
+          mkdir -p .ci/dev/docker/segment-anything/
+          pip-compile -o .ci/dev/docker/segment-anything/requirements.txt docker/segment-anything/requirements.txt
+          mkdir -p .ci/base/docs
+          pip-compile -o .ci/base/docs/requirements.txt docs/requirements.txt
+      - name: Run Trivy Scan (full, csv)
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # 0.29.0
+        with:
+          trivy-config: ".ci/trivy-csv.yaml"
+          scan-type: 'fs'
+          scan-ref: ".ci/"
+          scanners: vuln,secret
+      - name: Run Trivy Scan (prod, spdx.json)
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # 0.29.0
+        with:
+          trivy-config: ".ci/trivy-json.yaml"
+          scan-type: 'fs'
+          scan-ref: ".ci/base"
+      - name: Upload Trivy results
+        uses: actions/upload-artifact@v4
+        with:
+          name: trivy-results-prod-json
+          path: '${{ github.workspace }}/trivy-results-*'
+  Bandit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: python -m pip install tox
+      - name: Bandit Scanning
+        run: tox -e bandit-scan
+      - name: Upload Bandit artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: bandit-report
+          path: .tox/bandit-report.txt
+        # Use always() to always run this step to publish scan results when there are test failures
+        if: ${{ always() }}
+  call-notify-to-teams:
+    needs: [Trivy, Bandit]
+    if: |
+      always() &&
+      contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/notify_teams.yml
+    secrets: inherit

.github/workflows/health_check.yml

@@ -1,29 +1,50 @@
 name: Build
 on:
+  workflow_dispatch: # for testing
   push:
     branches:
       - 'develop'
+
+# This is what will cancel the workflow concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  coverage_tests_on_python:
+  check_integrity:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        include:
+          - python-version: "3.9"
+            tox-env-py: "39"
+          - python-version: "3.10"
+            tox-env-py: "310"
+          - python-version: "3.11"
+            tox-env-py: "311"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - name: Installing Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
       - name: Installing python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Installing dependencies
         run: |
-          pip install -e '.[default,tf,tfds]' pytest pytest-cov
+          python -m pip install tox
       - name: Code instrumentation
         run: |
-          pytest -v --cov --cov-report xml:coverage.xml
-          datum -h
-      - name: Sending coverage results
-        if: matrix.python-version == '3.8'
-        run: |
-          bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r coverage.xml -t ${{ secrets.CODACY_PROJECT_TOKEN }}
+          tox -e tests-py${{ matrix.tox-env-py }}-lin
+  call-notify-to-teams:
+    needs: [check_integrity]
+    if: |
+      always() &&
+      contains(needs.*.result, 'failure')
+    uses: ./.github/workflows/notify_teams.yml
+    secrets: inherit

.github/workflows/linter.yml

@@ -1,74 +1,13 @@
 name: Linter
 on: pull_request
 jobs:
-  Bandit:
-    runs-on: ubuntu-latest
+  pre-commit:
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
-
-      - name: Run checks
-        run: |
-          pip install --user -r <(grep "^bandit" ./requirements.txt)
-          echo "Bandit version: "`bandit --version | head -1`
-          bandit -r ./
-  isort:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Run checks
-        run: |
-          pip install --user -r <(grep "^isort" ./requirements.txt)
-          isort --check --diff .
-  black:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Run checks
-        run: |
-          pip install --user -r <(grep "^black" ./requirements.txt)
-          black --check --diff --color .
-  Pylint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Run checks
-        run: |
-          pip install --user -r <(grep "^pylint" ./requirements.txt)
-          echo "Pylint version: "`pylint --version | head -1`
-          git ls-files -z '*.py' | xargs -0 pylint
-  remark:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+          python-version: "3.9"
+      - uses: pre-commit/action@v3.0.1
         with:
-            node-version: 12
-
-      - name: Run checks
-        run: |
-          npm ci
-          echo "remark version: "`npx remark --version`
-          npx remark --frail .
-  whitespace:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check whitespace
-        run: |
-          empty_tree="$(git hash-object -t tree --stdin < /dev/null)"
-          git diff --check "$empty_tree"
+          extra_args: --all-files