Avoid fetching a list of shapes/tags from db, optimized fetching trac…

…ks (#7852)

<!-- Raise an issue to propose your change
(https://github.com/cvat-ai/cvat/issues).
It helps to avoid duplication of efforts from multiple independent
contributors.
Discuss your ideas with maintainers to be sure that changes will be
approved and merged.
Read the [Contribution guide](https://docs.cvat.ai/docs/contributing/).
-->

<!-- Provide a general summary of your changes in the Title above -->

### Motivation and context
<!-- Why is this change required? What problem does it solve? If it
fixes an open
issue, please link to the issue here. Describe your changes in detail,
add
screenshots. -->

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->

### Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply.
If an item isn't applicable for some reason, then ~~explicitly
strikethrough~~ the whole
line. If you don't do that, GitHub will show incorrect progress for the
pull request.
If you're unsure about any of these, don't hesitate to ask. We're here
to help! -->
- [x] I submit my changes into the `develop` branch
- [ ] I have created a changelog fragment <!-- see top comment in
CHANGELOG.md -->
- [ ] I have updated the documentation accordingly
- [ ] I have added tests to cover my changes
- [ ] I have linked related issues (see [GitHub docs](

https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))
- [x] I have increased versions of npm packages if it is necessary

([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning),

[cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning),

[cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning)
and

[cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning))

### License

- [x] I submit _my code changes_ under the same [MIT License](
https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the
project.
  Feel free to contact the maintainers if that's a concern.


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit


- **Refactor**
- Updated the method for counting objects in analytics reports to
improve accuracy.
- Made internal methods for initializing tags, shapes, and tracks
publicly accessible, enhancing external usability.

- **Bug Fixes**
	- Fixed import paths for better module integration and reliability.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

cvat/apps/analytics_report/report/primary_metrics/annotation_speed.py

@@ -7,7 +7,6 @@
 
 from dateutil import parser
 
-import cvat.apps.dataset_manager as dm
 from cvat.apps.analytics_report.models import (
     BinaryOperatorType,
     GranularityChoice,
@@ -18,6 +17,7 @@
     DataExtractorBase,
     PrimaryMetricBase,
 )
+from cvat.apps.dataset_manager.task import merge_table_rows
 from cvat.apps.engine.models import SourceType
 
 
@@ -70,31 +70,59 @@ class JobAnnotationSpeed(PrimaryMetricBase):
     ]
 
     def calculate(self):
-        def get_tags_count(annotations):
-            return sum(1 for t in annotations["tags"] if t["source"] != SourceType.FILE)
+        def get_tags_count():
+            return self._db_obj.labeledimage_set.exclude(source=SourceType.FILE).count()
+
+        def get_shapes_count():
+            return (
+                self._db_obj.labeledshape_set.filter(parent=None)
+                .exclude(source=SourceType.FILE)
+                .count()
+            )
 
-        def get_shapes_count(annotations):
-            return sum(1 for s in annotations["shapes"] if s["source"] != SourceType.FILE)
+        def get_track_count():
+            db_tracks = (
+                self._db_obj.labeledtrack_set.filter(parent=None)
+                .exclude(source=SourceType.FILE)
+                .values(
+                    "id",
+                    "source",
+                    "trackedshape__id",
+                    "trackedshape__frame",
+                    "trackedshape__outside",
+                )
+                .order_by("id", "trackedshape__frame")
+                .iterator(chunk_size=2000)
+            )
+
+            db_tracks = merge_table_rows(
+                rows=db_tracks,
+                keys_for_merge={
+                    "shapes": [
+                        "trackedshape__id",
+                        "trackedshape__frame",
+                        "trackedshape__outside",
+                    ],
+                },
+                field_id="id",
+            )
 
-        def get_track_count(annotations):
             count = 0
-            for track in annotations["tracks"]:
-                if track["source"] == SourceType.FILE:
-                    continue
+            for track in db_tracks:
                 if len(track["shapes"]) == 1:
                     count += self._db_obj.segment.stop_frame - track["shapes"][0]["frame"] + 1
+
                 for prev_shape, cur_shape in zip(track["shapes"], track["shapes"][1:]):
                     if prev_shape["outside"] is not True:
                         count += cur_shape["frame"] - prev_shape["frame"]
 
             return count
 
         # Calculate object count
-        annotations = dm.task.get_job_data(self._db_obj.id)
         object_count = 0
-        object_count += get_tags_count(annotations)
-        object_count += get_shapes_count(annotations)
-        object_count += get_track_count(annotations)
+        object_count += get_tags_count()
+        object_count += get_shapes_count()
+        object_count += get_track_count()
 
         start_datetime = self._db_obj.created_date
         timestamp = self._db_obj.updated_date

cvat/apps/dataset_manager/task.py

@@ -48,7 +48,7 @@ def values(cls):
     def __str__(self):
         return self.value
 
-def _merge_table_rows(rows, keys_for_merge, field_id):
+def merge_table_rows(rows, keys_for_merge, field_id):
     # It is necessary to keep a stable order of original rows
     # (e.g. for tracked boxes). Otherwise prev_box.frame can be bigger
     # than next_box.frame.
@@ -506,7 +506,7 @@ def _init_tags_from_db(self):
             'labeledimageattributeval__id',
         ).order_by('frame').iterator(chunk_size=2000)
 
-        db_tags = _merge_table_rows(
+        db_tags = merge_table_rows(
             rows=db_tags,
             keys_for_merge={
                 "labeledimageattributeval_set": [
@@ -546,7 +546,7 @@ def _init_shapes_from_db(self):
             'labeledshapeattributeval__id',
         ).order_by('frame').iterator(chunk_size=2000)
 
-        db_shapes = _merge_table_rows(
+        db_shapes = merge_table_rows(
             rows=db_shapes,
             keys_for_merge={
                 'labeledshapeattributeval_set': [
@@ -604,7 +604,7 @@ def _init_tracks_from_db(self):
             "trackedshape__trackedshapeattributeval__id",
         ).order_by('id', 'trackedshape__frame').iterator(chunk_size=2000)
 
-        db_tracks = _merge_table_rows(
+        db_tracks = merge_table_rows(
             rows=db_tracks,
             keys_for_merge={
                 "labeledtrackattributeval_set": [
@@ -632,7 +632,7 @@ def _init_tracks_from_db(self):
         tracks = {}
         elements = {}
         for db_track in db_tracks:
-            db_track["trackedshape_set"] = _merge_table_rows(db_track["trackedshape_set"], {
+            db_track["trackedshape_set"] = merge_table_rows(db_track["trackedshape_set"], {
                 'trackedshapeattributeval_set': [
                     'trackedshapeattributeval__value',
                     'trackedshapeattributeval__spec_id',

cvat/apps/engine/migrations/0017_db_redesign_20190221.py

@@ -4,7 +4,7 @@
 from django.db import migrations, models
 import django.db.models.deletion
 from django.conf import settings
-from cvat.apps.dataset_manager.task import _merge_table_rows
+from cvat.apps.dataset_manager.task import merge_table_rows as _merge_table_rows
 
 # some modified functions to transfer annotation
 def _bulk_create(db_model, db_alias, objects, flt_param):