Merge branch 'develop' into ay/add-icdar

.github/workflows/main.yml

@@ -38,6 +38,12 @@ jobs:
           cd ./tests
           npm ci
           npx cypress run --headless --browser chrome
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots
       - name: Collect coverage data
         env:
           HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}

.github/workflows/publish_docker_images.yml

@@ -33,7 +33,14 @@ jobs:
         run: |
           cd ./tests
           npm ci
-          npx cypress run --headless --browser chrome
+          npm run cypress:run:chrome
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots
+
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:

.github/workflows/schedule.yml

@@ -26,3 +26,9 @@ jobs:
           cd ./tests
           npm ci
           npm run cypress:run:firefox
+      - name: Uploading cypress screenshots as an artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress_screenshots
+          path: ${{ github.workspace }}/tests/cypress/screenshots

CHANGELOG.md

@@ -67,6 +67,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - An error about track shapes outside of the task frames during export (<https://github.com/openvinotoolkit/cvat/pull/2890>)
 - Fixed project search field updating (<https://github.com/openvinotoolkit/cvat/pull/2901>)
 - Fixed export error when invalid polygons are present in overlapping frames (<https://github.com/openvinotoolkit/cvat/pull/2852>)
+- Fixed image quality option for tasks created from images (<https://github.com/openvinotoolkit/cvat/pull/2963>)
 
 ### Security
 

cvat-data/package.json

@@ -21,7 +21,7 @@
     "worker-loader": "^2.0.0"
   },
   "dependencies": {
-    "async-mutex": "^0.3.0",
+    "async-mutex": "^0.3.1",
     "jszip": "3.6.0"
   },
   "scripts": {

cvat/apps/engine/media_extractors.py

@@ -373,20 +373,37 @@ def save_as_chunk(self, images, chunk_path):
         return image_sizes
 
 class Mpeg4ChunkWriter(IChunkWriter):
-    def __init__(self, _):
-        super().__init__(17)
+    def __init__(self, quality=67):
+        # translate inversed range [1:100] to [0:51]
+        quality = round(51 * (100 - quality) / 99)
+        super().__init__(quality)
         self._output_fps = 25
-
-    @staticmethod
-    def _create_av_container(path, w, h, rate, options, f='mp4'):
+        try:
+            codec = av.codec.Codec('libopenh264', 'w')
+            self._codec_name = codec.name
+            self._codec_opts = {
+                'profile': 'constrained_baseline',
+                'qmin': str(self._image_quality),
+                'qmax': str(self._image_quality),
+                'rc_mode': 'buffer',
+            }
+        except av.codec.codec.UnknownCodecError:
+            codec = av.codec.Codec('libx264', 'w')
+            self._codec_name = codec.name
+            self._codec_opts = {
+                "crf": str(self._image_quality),
+                "preset": "ultrafast",
+            }
+
+    def _create_av_container(self, path, w, h, rate, options, f='mp4'):
             # x264 requires width and height must be divisible by 2 for yuv420p
             if h % 2:
                 h += 1
             if w % 2:
                 w += 1
 
             container = av.open(path, 'w',format=f)
-            video_stream = container.add_stream('libopenh264', rate=rate)
+            video_stream = container.add_stream(self._codec_name, rate=rate)
             video_stream.pix_fmt = "yuv420p"
             video_stream.width = w
             video_stream.height = h
@@ -406,12 +423,7 @@ def save_as_chunk(self, images, chunk_path):
             w=input_w,
             h=input_h,
             rate=self._output_fps,
-            options={
-                'profile': 'constrained_baseline',
-                'qmin': str(self._image_quality),
-                'qmax': str(self._image_quality),
-                'rc_mode': 'buffer',
-            },
+            options=self._codec_opts,
         )
 
         self._encode_images(images, output_container, output_v_stream)
@@ -434,10 +446,15 @@ def _encode_images(images, container, stream):
 
 class Mpeg4CompressedChunkWriter(Mpeg4ChunkWriter):
     def __init__(self, quality):
-        # translate inversed range [1:100] to [0:51]
-        self._image_quality = round(51 * (100 - quality) / 99)
-        self._output_fps = 25
-
+        super().__init__(quality)
+        if self._codec_name == 'libx264':
+            self._codec_opts = {
+                'profile': 'baseline',
+                'coder': '0',
+                'crf': str(self._image_quality),
+                'wpredp': '0',
+                'flags': '-loop',
+            }
 
     def save_as_chunk(self, images, chunk_path):
         if not images:
@@ -458,12 +475,7 @@ def save_as_chunk(self, images, chunk_path):
             w=output_w,
             h=output_h,
             rate=self._output_fps,
-            options={
-                'profile': 'constrained_baseline',
-                'qmin': str(self._image_quality),
-                'qmax': str(self._image_quality),
-                'rc_mode': 'buffer',
-            },
+            options=self._codec_opts,
         )
 
         self._encode_images(images, output_container, output_v_stream)

cvat/apps/engine/task.py

@@ -10,9 +10,9 @@
 import rq
 import shutil
 from traceback import print_exception
-from urllib import error as urlerror
 from urllib import parse as urlparse
 from urllib import request as urlrequest
+import requests
 
 from cvat.apps.engine.media_extractors import get_mime, MEDIA_TYPES, Mpeg4ChunkWriter, ZipChunkWriter, Mpeg4CompressedChunkWriter, ZipCompressedChunkWriter, ValidateDimension
 from cvat.apps.engine.models import DataChoice, StorageMethodChoice, StorageChoice, RelatedFile
@@ -195,20 +195,16 @@ def _download_data(urls, upload_dir):
         job.meta['status'] = '{} is being downloaded..'.format(url)
         job.save_meta()
 
-        req = urlrequest.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
-        try:
-            with urlrequest.urlopen(req) as fp, open(os.path.join(upload_dir, name), 'wb') as tfp:
-                while True:
-                    block = fp.read(8192)
-                    if not block:
-                        break
-                    tfp.write(block)
-        except urlerror.HTTPError as err:
-            raise Exception("Failed to download " + url + ". " + str(err.code) + ' - ' + err.reason)
-        except urlerror.URLError as err:
-            raise Exception("Invalid URL: " + url + ". " + err.reason)
+        response = requests.get(url, stream=True)
+        if response.status_code == 200:
+            response.raw.decode_content = True
+            with open(os.path.join(upload_dir, name), 'wb') as output_file:
+                shutil.copyfileobj(response.raw, output_file)
+        else:
+            raise Exception("Failed to download " + url)
 
         local_files[name] = True
+
     return list(local_files.keys())
 
 @transaction.atomic
@@ -298,13 +294,20 @@ def update_progress(progress):
         update_progress.call_counter = (update_progress.call_counter + 1) % len(progress_animation)
 
     compressed_chunk_writer_class = Mpeg4CompressedChunkWriter if db_data.compressed_chunk_type == DataChoice.VIDEO else ZipCompressedChunkWriter
-    original_chunk_writer_class = Mpeg4ChunkWriter if db_data.original_chunk_type == DataChoice.VIDEO else ZipChunkWriter
+    if db_data.original_chunk_type == DataChoice.VIDEO:
+        original_chunk_writer_class = Mpeg4ChunkWriter
+        # Let's use QP=17 (that is 67 for 0-100 range) for the original chunks, which should be visually lossless or nearly so.
+        # A lower value will significantly increase the chunk size with a slight increase of quality.
+        original_quality = 67
+    else:
+        original_chunk_writer_class = ZipChunkWriter
+        original_quality = 100
 
     kwargs = {}
     if validate_dimension.dimension == DimensionType.DIM_3D:
         kwargs["dimension"] = validate_dimension.dimension
     compressed_chunk_writer = compressed_chunk_writer_class(db_data.image_quality, **kwargs)
-    original_chunk_writer = original_chunk_writer_class(100)
+    original_chunk_writer = original_chunk_writer_class(original_quality)
 
     # calculate chunk size if it isn't specified
     if db_data.chunk_size is None:

tests/cypress.json

@@ -15,6 +15,7 @@
     "actions_tasks_objects/**/*",
     "actions_users/**/*",
     "actions_projects/**/*",
+    "canvas3d_functionality/*",
     "remove_users_tasks_projects.js"
   ]
 }

tests/cypress/integration/actions_projects/registration_involved/base_actions_project_task_user.js

@@ -6,15 +6,6 @@
 
 import { projectName } from '../../../support/const_project';
 
-const randomString = (isPassword) => {
-    let result = '';
-    const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
-    for (let i = 0; i <= 8; i++) {
-        result += characters.charAt(Math.floor(Math.random() * characters.length));
-    }
-    return isPassword ? `${result}${Math.floor(Math.random() * 10)}` : result;
-};
-
 context('Base actions on the project', () => {
     const labelName = `Base label for ${projectName}`;
     const taskName = {
@@ -45,11 +36,11 @@ context('Base actions on the project', () => {
     const newLabelName2 = `Second label ${projectName}`;
     const newLabelName3 = `Third label ${projectName}`;
     const newLabelName4 = `Fourth label ${projectName}`;
-    const firstName = `${randomString()}`;
-    const lastName = `${randomString()}`;
-    const userName = `${randomString()}`;
+    const firstName = 'Seconduser fm';
+    const lastName = 'Seconduser lm';
+    const userName = 'Seconduser';
     const emailAddr = `${userName}@local.local`;
-    const password = `${randomString(true)}`;
+    const password = 'GDrb41RguF!';
     let projectID = '';
 
     function getProjectID(projectName) {
@@ -65,6 +56,10 @@ context('Base actions on the project', () => {
         cy.openProject(projectName);
     });
 
+    after(() => {
+        cy.deletingRegisteredUsers([userName]);
+    });
+
     describe(`Testing "Base actions on the project"`, () => {
         it('Add some labels to project.', () => {
             cy.addNewLabel(newLabelName1);
@@ -125,7 +120,7 @@ context('Base actions on the project', () => {
             cy.userRegistration(firstName, lastName, userName, emailAddr, password);
             cy.goToProjectsList();
             // tries to create project
-            const failProjectName = `${randomString()}`;
+            const failProjectName = 'failProject';
             cy.createProjects(failProjectName, labelName, attrName, textDefaultValue, null, 'fail');
             cy.closeNotification('.cvat-notification-notice-create-project-failed');
             cy.goToProjectsList();
@@ -159,6 +154,7 @@ context('Base actions on the project', () => {
             cy.goToTaskList();
             cy.contains('strong', taskName.firstTask).should('not.exist');
             cy.contains('strong', taskName.secondTask).should('not.exist');
+            cy.logout();
         });
     });
 });