Merge branch 'release-1.3.0'

.coveragerc

@@ -5,6 +5,7 @@ branch = true
 source =
     cvat/apps/
     utils/cli/
+    utils/dataset_manifest
 
 omit =
     cvat/settings/*

.eslintrc.js

@@ -1,4 +1,4 @@
-// Copyright (C) 2018-2020 Intel Corporation
+// Copyright (C) 2018-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -16,8 +16,8 @@ module.exports = {
     extends: ['eslint:recommended', 'prettier'],
     rules: {
         'header/header': [2, 'line', [{
-            pattern: ' {1}Copyright \\(C\\) (?:20\\d{2}-)?2020 Intel Corporation',
-            template: ' Copyright (C) 2020 Intel Corporation'
+            pattern: ' {1}Copyright \\(C\\) (?:20\\d{2}-)?2021 Intel Corporation',
+            template: ' Copyright (C) 2021 Intel Corporation'
         }, '', ' SPDX-License-Identifier: MIT']],
     },
 };

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 <!---
-Copyright (C) 2020 Intel Corporation
+Copyright (C) 2020-2021 Intel Corporation
 
 SPDX-License-Identifier: MIT
 -->
@@ -45,7 +45,7 @@ If you're unsure about any of these, don't hesitate to ask. We're here to help!
 - [ ] I have updated the license header for each file (see an example below)
 
 ```python
-# Copyright (C) 2020 Intel Corporation
+# Copyright (C) 2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 ```

.github/workflows/bandit.yml

@@ -0,0 +1,41 @@
+name: Linter
+on: pull_request
+jobs:
+  Bandit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run checks
+        run: |
+          URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
+          PR_FILES=$(curl -s -X GET -G $URL | jq -r '.[] | select(.status != "removed") | .filename')
+          for files in $PR_FILES; do
+            extension="${files##*.}"
+            if [[ $extension == 'py' ]]; then
+              changed_files_bandit+=" ${files}"
+            fi
+          done
+
+          if [[ ! -z ${changed_files_bandit} ]]; then
+            sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv
+            python3 -m venv .env
+            . .env/bin/activate
+            pip install -U pip wheel setuptools
+            pip install bandit
+            mkdir -p bandit_report
+
+            echo "Bandit version: "`bandit --version | head -1`
+            echo "The files will be checked: "`echo ${changed_files_bandit}`
+            bandit ${changed_files_bandit} --exclude '**/tests/**' -a file --ini ./.bandit -f html -o ./bandit_report/bandit_checks.html
+            deactivate
+          else
+            echo "No files with the \"py\" extension found"
+          fi
+
+      - name: Upload artifacts
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: bandit_report
+          path: bandit_report

.github/workflows/eslint.yml

@@ -0,0 +1,42 @@
+name: Linter
+on: pull_request
+jobs:
+  ESLint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+
+      - name: Run checks
+        run: |
+          URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
+          PR_FILES=$(curl -s -X GET -G $URL | jq -r '.[] | select(.status != "removed") | .filename')
+          for files in $PR_FILES; do
+            extension="${files##*.}"
+            if [[ $extension == 'js' || $extension == 'ts' || $extension == 'jsx' || $extension == 'tsx' ]]; then
+              changed_files_eslint+=" ${files}"
+            fi
+          done
+
+          if [[ ! -z ${changed_files_eslint} ]]; then
+            for package_files in `find -maxdepth 2 -name "package.json" -type f`; do
+              cd $(dirname $package_files) && npm ci && cd ${{ github.workspace }}
+            done
+            npm install eslint-detailed-reporter --save-dev
+            mkdir -p eslint_report
+
+            echo "ESLint version: "`npx eslint --version`
+            echo "The files will be checked: "`echo ${changed_files_eslint}`
+            npx eslint ${changed_files_eslint} -f node_modules/eslint-detailed-reporter/lib/detailed.js -o ./eslint_report/eslint_checks.html
+          else
+            echo "No files with the \"js|ts|jsx|tsx\" extension found"
+          fi
+
+      - name: Upload artifacts
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: eslint_report
+          path: eslint_report

.github/workflows/main.yml

@@ -0,0 +1,57 @@
+name: CI
+on:
+  push:
+    branches:
+      - 'master'
+      - 'develop'
+  pull_request:
+    branches:
+      - '*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+      - name: Build CVAT
+        env:
+          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
+          CONTAINER_COVERAGE_DATA_DIR: '/coverage_data'
+          DJANGO_SU_NAME: 'admin'
+          DJANGO_SU_EMAIL: 'admin@localhost.company'
+          DJANGO_SU_PASSWORD: '12qwaszx'
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
+          docker-compose up -d
+          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+      - name: Code instrumentation
+        run: |
+          npm ci
+          npm run coverage
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
+      - name: End-to-end testing
+        run: |
+          cd ./tests
+          npm ci
+          npx cypress run --headless --browser chrome
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots
+      - name: Collect coverage data
+        env:
+          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
+          CONTAINER_COVERAGE_DATA_DIR: "/coverage_data"
+          COVERALLS_SERVICE_NAME: github
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./tests/.nyc_output ./
+          npx nyc report --reporter=text-lcov >> ${HOST_COVERAGE_DATA_DIR}/lcov.info
+          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd ${CONTAINER_COVERAGE_DATA_DIR} && coveralls-lcov -v -n lcov.info > ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json'
+          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.git . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/.coverage . && ln -s ${CONTAINER_COVERAGE_DATA_DIR}/coverage.json . && coveralls --merge=coverage.json'

.github/workflows/publish_docker_images.yml

@@ -0,0 +1,61 @@
+name: Publish Docker images
+on:
+  release:
+    types: [published]
+
+jobs:
+  build_and_push_to_registry:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+
+      - name: Build images
+        run: |
+          CLAM_AV=yes INSTALL_SOURCES=yes docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+
+      - name: Run unit tests
+        env:
+          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
+          CONTAINER_COVERAGE_DATA_DIR: '/coverage_data'
+          DJANGO_SU_NAME: 'admin'
+          DJANGO_SU_EMAIL: 'admin@localhost.company'
+          DJANGO_SU_PASSWORD: '12qwaszx'
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test'
+          docker-compose up -d
+          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+
+      - name: Run end-to-end tests
+        run: |
+          cd ./tests
+          npm ci
+          npm run cypress:run:chrome
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push to Docker Hub
+        env:
+          DOCKERHUB_WORKSPACE: 'openvino'
+          SERVER_IMAGE_REPO: 'cvat_server'
+          UI_IMAGE_REPO: 'cvat_ui'
+        run: |
+          docker tag "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest"
+
+          docker tag "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest"

.github/workflows/pylint.yml

@@ -0,0 +1,45 @@
+name: Linter
+on: pull_request
+jobs:
+  PyLint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Run checks
+        run: |
+          URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
+          PR_FILES=$(curl -s -X GET -G $URL | jq -r '.[] | select(.status != "removed") | .filename')
+          for files in $PR_FILES; do
+            extension="${files##*.}"
+            if [[ $extension == 'py' ]]; then
+              changed_files_pylint+=" ${files}"
+            fi
+          done
+
+          if [[ ! -z ${changed_files_pylint} ]]; then
+            sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv
+            python3 -m venv .env
+            . .env/bin/activate
+            pip install -U pip wheel setuptools
+            pip install pylint-json2html
+            pip install $(egrep "pylint.*" ./cvat/requirements/development.txt)
+            pip install $(egrep "Django.*" ./cvat/requirements/base.txt)
+            mkdir -p pylint_report
+
+            echo "Pylint version: "`pylint --version | head -1`
+            echo "The files will be checked: "`echo ${changed_files_pylint}`
+            pylint ${changed_files_pylint} --output-format=json > ./pylint_report/pylint_checks.json || exit_code=`echo $?` || true
+            pylint-json2html -o ./pylint_report/pylint_checks.html ./pylint_report/pylint_checks.json
+            deactivate
+            exit ${exit_code}
+          else
+            echo "No files with the \"py\" extension found"
+          fi
+
+      - name: Upload artifacts
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: pylint_report
+          path: pylint_report

.github/workflows/schedule.yml

@@ -0,0 +1,34 @@
+name: CI-nightly
+on:
+  schedule:
+    - cron: '0 22 * * *'
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+      - name: Build CVAT
+        env:
+          DJANGO_SU_NAME: "admin"
+          DJANGO_SU_EMAIL: "admin@localhost.company"
+          DJANGO_SU_PASSWORD: "12qwaszx"
+          API_ABOUT_PAGE: "localhost:8080/api/v1/server/about"
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f ./tests/docker-compose.email.yml up -d --build
+          /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
+          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+      - name: End-to-end testing
+        run: |
+          cd ./tests
+          npm ci
+          npm run cypress:run:firefox
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots

.remarkrc.js

@@ -3,7 +3,7 @@ exports.settings = { bullet: '*', paddedTable: false };
 exports.plugins = [
     'remark-preset-lint-recommended',
     'remark-preset-lint-consistent',
-    ['remark-preset-lint-markdown-style-guide', 'mixed'],
+    ['remark-lint-list-item-indent', 'space'],
     ['remark-lint-no-dead-urls', { skipOffline: true }],
     ['remark-lint-maximum-line-length', 120],
     ['remark-lint-maximum-heading-length', 120],