From 9605ac932c8089da8b230e6d54b06f6479f59c04 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 14:07:48 +0200 Subject: [PATCH 01/18] Add a quality setting for avoiding using bbox in point group matching --- ...image_space_for_point_group_comparisons.py | 18 +++++++++++++ cvat/apps/quality_control/models.py | 2 ++ cvat/apps/quality_control/quality_reports.py | 27 +++++++++++++++---- cvat/apps/quality_control/serializers.py | 9 +++++++ 4 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py diff --git a/cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py b/cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py new file mode 100644 index 000000000000..d2b306aa0c9d --- /dev/null +++ b/cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-11-01 11:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0003_qualityreport_assignee_last_updated_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="qualitysettings", + name="use_image_space_for_point_group_comparisons", + field=models.BooleanField(default=False), + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index 37f0f1f9612d..76c441c66bd5 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -205,6 +205,8 @@ class QualitySettings(models.Model): low_overlap_threshold = models.FloatField() + use_image_space_for_point_group_comparisons = models.BooleanField(default=False) + compare_line_orientation = models.BooleanField() line_orientation_threshold = models.FloatField() diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 437cdb72615f..606c0133bf82 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -187,6 +187,9 @@ class ComparisonParameters(_Serializable): oks_sigma: float = 0.09 "Like IoU threshold, but for points, % of the bbox area to match a pair of points" + use_image_space_for_point_group_comparisons: bool = False + "Compare points in the image space, instead of using the point group bbox" + line_thickness: float = 0.01 "Thickness of polylines, relatively to the (image area) ^ 0.5" @@ -955,6 +958,7 @@ def __init__( # https://cocodataset.org/#keypoints-eval # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L523 oks_sigma: float = 0.09, + use_image_space_for_point_group_comparisons: bool = False, compare_line_orientation: bool = False, line_torso_radius: float = 0.01, panoptic_comparison: bool = False, @@ -968,6 +972,11 @@ def __init__( self.oks_sigma = oks_sigma "% of the shape area" + self.use_image_space_for_point_group_comparisons = ( + use_image_space_for_point_group_comparisons + ) + "Do not infer bbox for point group comparison, use the image space" + self.compare_line_orientation = compare_line_orientation "Whether lines are oriented or not" @@ -1293,13 +1302,18 @@ def _distance(a: dm.Points, b: dm.Points) -> float: else: # Complex case: multiple points, grouped points, points with a bbox # Try to align points and then return the metric - # match them in their bbox space - if dm.ops.bbox_iou(a_bbox, b_bbox) <= 0: - return 0 + if self.use_image_space_for_point_group_comparisons: + scale = img_h * img_w + else: + # match points in their bbox space + + if dm.ops.bbox_iou(a_bbox, b_bbox) <= 0: + # this early exit may not work for points forming an axis-aligned line + return 0 - bbox = dm.ops.mean_bbox([a_bbox, b_bbox]) - scale = bbox[2] * bbox[3] + bbox = dm.ops.mean_bbox([a_bbox, b_bbox]) + scale = bbox[2] * bbox[3] a_points = np.reshape(a.points, (-1, 2)) b_points = np.reshape(b.points, (-1, 2)) @@ -1525,6 +1539,9 @@ def __init__(self, categories: dm.CategoriesInfo, *, settings: ComparisonParamet panoptic_comparison=settings.panoptic_comparison, iou_threshold=settings.iou_threshold, oks_sigma=settings.oks_sigma, + use_image_space_for_point_group_comparisons=( + settings.use_image_space_for_point_group_comparisons + ), line_torso_radius=settings.line_thickness, compare_line_orientation=False, # should not be taken from outside, handled differently ) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index fe6b372d9cb4..e12b7c7056e3 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -81,6 +81,7 @@ class Meta: "max_validations_per_job", "iou_threshold", "oks_sigma", + "use_image_space_for_point_group_comparisons", "line_thickness", "low_overlap_threshold", "compare_line_orientation", @@ -98,6 +99,9 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} + extra_kwargs.setdefault("use_image_space_for_point_group_comparisons", {}).setdefault( + "default", False + ) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -119,6 +123,11 @@ class Meta: where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval """, + "use_image_space_for_point_group_comparisons": """ + Compare point groups in the image space, instead of using the point group bbox. + Useful if point groups may not represent a single object or grouped boxes + do not represent object boundaries. + """, "line_thickness": """ Thickness of polylines, relatively to the (image area) ^ 0.5. The distance to the boundary around the GT line, From 7bd4d87dc8f6310697639956a093ac438f196f65 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 14:38:58 +0200 Subject: [PATCH 02/18] Update ui --- cvat-core/src/quality-settings.ts | 11 +++++++ cvat-core/src/server-response-types.ts | 1 + .../quality-control/quality-control-page.tsx | 1 + .../task-quality/quality-settings-form.tsx | 29 +++++++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index c5e3ea6974c2..bd1e9d5d09f2 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -22,6 +22,7 @@ export default class QualitySettings { #task: number; #iouThreshold: number; #oksSigma: number; + #useImageSpaceForPointGroupComparisons: boolean; #lineThickness: number; #lowOverlapThreshold: number; #orientedLines: boolean; @@ -42,6 +43,7 @@ export default class QualitySettings { this.#maxValidationsPerJob = initialData.max_validations_per_job; this.#iouThreshold = initialData.iou_threshold; this.#oksSigma = initialData.oks_sigma; + this.#useImageSpaceForPointGroupComparisons = initialData.use_image_space_for_point_group_comparisons; this.#lineThickness = initialData.line_thickness; this.#lowOverlapThreshold = initialData.low_overlap_threshold; this.#orientedLines = initialData.compare_line_orientation; @@ -79,6 +81,14 @@ export default class QualitySettings { this.#oksSigma = newVal; } + get useImageSpaceForPointGroupComparisons(): boolean { + return this.#useImageSpaceForPointGroupComparisons; + } + + set useImageSpaceForPointGroupComparisons(newVal: boolean) { + this.#useImageSpaceForPointGroupComparisons = newVal; + } + get lineThickness(): number { return this.#lineThickness; } @@ -197,6 +207,7 @@ export default class QualitySettings { const result: SerializedQualitySettingsData = { iou_threshold: this.#iouThreshold, oks_sigma: this.#oksSigma, + use_image_space_for_point_group_comparisons: this.#useImageSpaceForPointGroupComparisons, line_thickness: this.#lineThickness, low_overlap_threshold: this.#lowOverlapThreshold, compare_line_orientation: this.#orientedLines, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 4bf7a482bccb..58b950c670b1 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -247,6 +247,7 @@ export interface SerializedQualitySettingsData { max_validations_per_job?: number; iou_threshold?: number; oks_sigma?: number; + use_image_space_for_point_group_comparisons?: boolean; line_thickness?: number; low_overlap_threshold?: number; compare_line_orientation?: boolean; diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 09f7cfe5e4d8..cc716aa201eb 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -225,6 +225,7 @@ function QualityControlPage(): JSX.Element { settings.compareAttributes = values.compareAttributes; settings.oksSigma = values.oksSigma / 100; + settings.useImageSpaceForPointGroupComparisons = values.useImageSpaceForPointGroupComparisons; settings.lineThickness = values.lineThickness / 100; settings.lineOrientationThreshold = values.lineOrientationThreshold / 100; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index f1746fe675ac..3e165def9bbf 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -35,6 +35,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element compareAttributes: settings.compareAttributes, oksSigma: settings.oksSigma * 100, + useImageSpaceForPointGroupComparisons: settings.useImageSpaceForPointGroupComparisons, lineThickness: settings.lineThickness * 100, lineOrientationThreshold: settings.lineOrientationThreshold * 100, @@ -89,6 +90,10 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element makeTooltipFragment('Object Keypoint Similarity (OKS)', settings.descriptions.oksSigma), ); + const pointTooltip = makeTooltip( + makeTooltipFragment('Use image space', settings.descriptions.useImageSpaceForPointGroupComparisons), + ); + const linesTooltip = makeTooltip( <> {makeTooltipFragment('Line thickness', settings.descriptions.lineThickness)} @@ -249,6 +254,30 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element + + + Point Comparison + + + + + + + + + + Use image space + + + + + Line Comparison From bd12c4cdc8771b99187bda42aa0451d928c879ae Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 14:39:20 +0200 Subject: [PATCH 03/18] Update changelog --- ...101_140759_mzhiltso_compare_point_groups_in_image_space.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/20241101_140759_mzhiltso_compare_point_groups_in_image_space.md diff --git a/changelog.d/20241101_140759_mzhiltso_compare_point_groups_in_image_space.md b/changelog.d/20241101_140759_mzhiltso_compare_point_groups_in_image_space.md new file mode 100644 index 000000000000..833f8dcdcb7a --- /dev/null +++ b/changelog.d/20241101_140759_mzhiltso_compare_point_groups_in_image_space.md @@ -0,0 +1,4 @@ +### Added + +- A quality setting to compare point groups without using bbox + () From 1f5347758e3b1da5b117e9549ef52665ea37817b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 15:28:08 +0200 Subject: [PATCH 04/18] Update schema --- cvat/schema.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cvat/schema.yml b/cvat/schema.yml index 4bda991cbb12..45f42362ca55 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9661,6 +9661,13 @@ components: The percent of the bbox area, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval + use_image_space_for_point_group_comparisons: + type: boolean + default: false + description: | + Compare point groups in the image space, instead of using the point group bbox. + Useful if point groups may not represent a single object or grouped boxes + do not represent object boundaries. line_thickness: type: number format: double @@ -10141,6 +10148,13 @@ components: The percent of the bbox area, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval + use_image_space_for_point_group_comparisons: + type: boolean + default: false + description: | + Compare point groups in the image space, instead of using the point group bbox. + Useful if point groups may not represent a single object or grouped boxes + do not represent object boundaries. line_thickness: type: number format: double From ccd3f8cfa281250222568c936cda1be34a4f76d9 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 17:45:36 +0200 Subject: [PATCH 05/18] Use shorter name --- cvat-core/src/quality-settings.ts | 14 +++++++------- cvat-core/src/server-response-types.ts | 2 +- .../quality-control/quality-control-page.tsx | 2 +- .../task-quality/quality-settings-form.tsx | 8 ++++---- ...ualitysettings_use_bbox_size_for_points.py} | 7 ++++--- cvat/apps/quality_control/models.py | 2 +- cvat/apps/quality_control/quality_reports.py | 18 +++++++----------- cvat/apps/quality_control/serializers.py | 13 ++++++------- cvat/schema.yml | 18 ++++++++++-------- 9 files changed, 41 insertions(+), 43 deletions(-) rename cvat/apps/quality_control/migrations/{0004_qualitysettings_use_image_space_for_point_group_comparisons.py => 0004_qualitysettings_use_bbox_size_for_points.py} (62%) diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index bd1e9d5d09f2..76b06c63f26c 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -22,7 +22,7 @@ export default class QualitySettings { #task: number; #iouThreshold: number; #oksSigma: number; - #useImageSpaceForPointGroupComparisons: boolean; + #useBboxSizeForPoints: boolean; #lineThickness: number; #lowOverlapThreshold: number; #orientedLines: boolean; @@ -43,7 +43,7 @@ export default class QualitySettings { this.#maxValidationsPerJob = initialData.max_validations_per_job; this.#iouThreshold = initialData.iou_threshold; this.#oksSigma = initialData.oks_sigma; - this.#useImageSpaceForPointGroupComparisons = initialData.use_image_space_for_point_group_comparisons; + this.#useBboxSizeForPoints = initialData.use_bbox_size_for_points; this.#lineThickness = initialData.line_thickness; this.#lowOverlapThreshold = initialData.low_overlap_threshold; this.#orientedLines = initialData.compare_line_orientation; @@ -81,12 +81,12 @@ export default class QualitySettings { this.#oksSigma = newVal; } - get useImageSpaceForPointGroupComparisons(): boolean { - return this.#useImageSpaceForPointGroupComparisons; + get useBboxSizeForPoints(): boolean { + return this.#useBboxSizeForPoints; } - set useImageSpaceForPointGroupComparisons(newVal: boolean) { - this.#useImageSpaceForPointGroupComparisons = newVal; + set useBboxSizeForPoints(newVal: boolean) { + this.#useBboxSizeForPoints = newVal; } get lineThickness(): number { @@ -207,7 +207,7 @@ export default class QualitySettings { const result: SerializedQualitySettingsData = { iou_threshold: this.#iouThreshold, oks_sigma: this.#oksSigma, - use_image_space_for_point_group_comparisons: this.#useImageSpaceForPointGroupComparisons, + use_bbox_size_for_points: this.#useBboxSizeForPoints, line_thickness: this.#lineThickness, low_overlap_threshold: this.#lowOverlapThreshold, compare_line_orientation: this.#orientedLines, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 53438ed65d21..8b334819705b 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -247,7 +247,7 @@ export interface SerializedQualitySettingsData { max_validations_per_job?: number; iou_threshold?: number; oks_sigma?: number; - use_image_space_for_point_group_comparisons?: boolean; + use_bbox_size_for_points?: boolean; line_thickness?: number; low_overlap_threshold?: number; compare_line_orientation?: boolean; diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index cc716aa201eb..5d27f58ac0f0 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -225,7 +225,7 @@ function QualityControlPage(): JSX.Element { settings.compareAttributes = values.compareAttributes; settings.oksSigma = values.oksSigma / 100; - settings.useImageSpaceForPointGroupComparisons = values.useImageSpaceForPointGroupComparisons; + settings.useBboxSizeForPoints = values.useBboxSizeForPoints; settings.lineThickness = values.lineThickness / 100; settings.lineOrientationThreshold = values.lineOrientationThreshold / 100; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 3e165def9bbf..151f70afca49 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -35,7 +35,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element compareAttributes: settings.compareAttributes, oksSigma: settings.oksSigma * 100, - useImageSpaceForPointGroupComparisons: settings.useImageSpaceForPointGroupComparisons, + useBboxSizeForPoints: settings.useBboxSizeForPoints, lineThickness: settings.lineThickness * 100, lineOrientationThreshold: settings.lineOrientationThreshold * 100, @@ -91,7 +91,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element ); const pointTooltip = makeTooltip( - makeTooltipFragment('Use image space', settings.descriptions.useImageSpaceForPointGroupComparisons), + makeTooltipFragment('Use image space', settings.descriptions.useBboxSizeForPoints), ); const linesTooltip = makeTooltip( @@ -267,12 +267,12 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element - Use image space + Use bbox size diff --git a/cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py b/cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py similarity index 62% rename from cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py rename to cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py index d2b306aa0c9d..9325a21a3f2a 100644 --- a/cvat/apps/quality_control/migrations/0004_qualitysettings_use_image_space_for_point_group_comparisons.py +++ b/cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.15 on 2024-11-01 11:59 +# Generated by Django 4.2.15 on 2024-11-01 15:40 from django.db import migrations, models @@ -12,7 +12,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name="qualitysettings", - name="use_image_space_for_point_group_comparisons", - field=models.BooleanField(default=False), + name="use_bbox_size_for_points", + field=models.BooleanField(default=True), + preserve_default=False, ), ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index 76c441c66bd5..5280cd03cd3a 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -205,7 +205,7 @@ class QualitySettings(models.Model): low_overlap_threshold = models.FloatField() - use_image_space_for_point_group_comparisons = models.BooleanField(default=False) + use_bbox_size_for_points = models.BooleanField() compare_line_orientation = models.BooleanField() line_orientation_threshold = models.FloatField() diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 606c0133bf82..03308e624558 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -187,8 +187,8 @@ class ComparisonParameters(_Serializable): oks_sigma: float = 0.09 "Like IoU threshold, but for points, % of the bbox area to match a pair of points" - use_image_space_for_point_group_comparisons: bool = False - "Compare points in the image space, instead of using the point group bbox" + use_bbox_size_for_points: bool = True + "Compare point groups using the group bbox size instead of the image size" line_thickness: float = 0.01 "Thickness of polylines, relatively to the (image area) ^ 0.5" @@ -958,7 +958,7 @@ def __init__( # https://cocodataset.org/#keypoints-eval # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L523 oks_sigma: float = 0.09, - use_image_space_for_point_group_comparisons: bool = False, + use_bbox_size_for_points: bool = True, compare_line_orientation: bool = False, line_torso_radius: float = 0.01, panoptic_comparison: bool = False, @@ -972,10 +972,8 @@ def __init__( self.oks_sigma = oks_sigma "% of the shape area" - self.use_image_space_for_point_group_comparisons = ( - use_image_space_for_point_group_comparisons - ) - "Do not infer bbox for point group comparison, use the image space" + self.use_bbox_size_for_points = use_bbox_size_for_points + "Compare point groups using the group bbox size instead of the image size" self.compare_line_orientation = compare_line_orientation "Whether lines are oriented or not" @@ -1303,7 +1301,7 @@ def _distance(a: dm.Points, b: dm.Points) -> float: # Complex case: multiple points, grouped points, points with a bbox # Try to align points and then return the metric - if self.use_image_space_for_point_group_comparisons: + if not self.use_bbox_size_for_points: scale = img_h * img_w else: # match points in their bbox space @@ -1539,9 +1537,7 @@ def __init__(self, categories: dm.CategoriesInfo, *, settings: ComparisonParamet panoptic_comparison=settings.panoptic_comparison, iou_threshold=settings.iou_threshold, oks_sigma=settings.oks_sigma, - use_image_space_for_point_group_comparisons=( - settings.use_image_space_for_point_group_comparisons - ), + use_bbox_size_for_points=settings.use_bbox_size_for_points, line_torso_radius=settings.line_thickness, compare_line_orientation=False, # should not be taken from outside, handled differently ) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index e12b7c7056e3..b4425f777993 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -81,7 +81,7 @@ class Meta: "max_validations_per_job", "iou_threshold", "oks_sigma", - "use_image_space_for_point_group_comparisons", + "use_bbox_size_for_points", "line_thickness", "low_overlap_threshold", "compare_line_orientation", @@ -99,9 +99,6 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} - extra_kwargs.setdefault("use_image_space_for_point_group_comparisons", {}).setdefault( - "default", False - ) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -123,9 +120,11 @@ class Meta: where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval """, - "use_image_space_for_point_group_comparisons": """ - Compare point groups in the image space, instead of using the point group bbox. - Useful if point groups may not represent a single object or grouped boxes + "use_bbox_size_for_points": """ + When comparing point groups, OKS sigma defines the matching area for a GT point. + If enabled, the area size is based on the point group bbox size. + If disabled, the image size is used. + Useful if point groups do not represent a single object or boxes attached to points do not represent object boundaries. """, "line_thickness": """ diff --git a/cvat/schema.yml b/cvat/schema.yml index 45f42362ca55..2ee22d2d7c54 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9661,12 +9661,13 @@ components: The percent of the bbox area, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval - use_image_space_for_point_group_comparisons: + use_bbox_size_for_points: type: boolean - default: false description: | - Compare point groups in the image space, instead of using the point group bbox. - Useful if point groups may not represent a single object or grouped boxes + When comparing point groups, OKS sigma defines the matching area for a GT point. + If enabled, the area size is based on the point group bbox size. + If disabled, the image size is used. + Useful if point groups do not represent a single object or boxes attached to points do not represent object boundaries. line_thickness: type: number @@ -10148,12 +10149,13 @@ components: The percent of the bbox area, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval - use_image_space_for_point_group_comparisons: + use_bbox_size_for_points: type: boolean - default: false description: | - Compare point groups in the image space, instead of using the point group bbox. - Useful if point groups may not represent a single object or grouped boxes + When comparing point groups, OKS sigma defines the matching area for a GT point. + If enabled, the area size is based on the point group bbox size. + If disabled, the image size is used. + Useful if point groups do not represent a single object or boxes attached to points do not represent object boundaries. line_thickness: type: number From bc141e7ee48a64e87496155eb12c5d7a140be8df Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 1 Nov 2024 17:52:07 +0200 Subject: [PATCH 06/18] Update test assets --- tests/python/shared/assets/cvat_db/data.json | 24 +++++++ .../shared/assets/quality_settings.json | 72 ++++++++++++------- 2 files changed, 72 insertions(+), 24 deletions(-) diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 43b972c79e4e..dce23272347e 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18164,6 +18164,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18186,6 +18187,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18208,6 +18210,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18230,6 +18233,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18252,6 +18256,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18274,6 +18279,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18296,6 +18302,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18318,6 +18325,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18340,6 +18348,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18362,6 +18371,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18384,6 +18394,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18406,6 +18417,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18428,6 +18440,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18450,6 +18463,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18472,6 +18486,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18494,6 +18509,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18516,6 +18532,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18538,6 +18555,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18560,6 +18578,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18582,6 +18601,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18604,6 +18624,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18626,6 +18647,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18648,6 +18670,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18670,6 +18693,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "use_bbox_size_for_points": true, "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 54e0c18c63a3..6c005a035fac 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -20,7 +20,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 2 + "task_id": 2, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -39,7 +40,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 5 + "task_id": 5, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -58,7 +60,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 6 + "task_id": 6, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -77,7 +80,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 7 + "task_id": 7, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -96,7 +100,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 8 + "task_id": 8, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -115,7 +120,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 9 + "task_id": 9, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -134,7 +140,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 11 + "task_id": 11, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -153,7 +160,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 12 + "task_id": 12, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -172,7 +180,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 13 + "task_id": 13, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -191,7 +200,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 14 + "task_id": 14, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -210,7 +220,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 15 + "task_id": 15, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -229,7 +240,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 17 + "task_id": 17, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -248,7 +260,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 18 + "task_id": 18, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -267,7 +280,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 19 + "task_id": 19, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -286,7 +300,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 20 + "task_id": 20, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -305,7 +320,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 21 + "task_id": 21, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -324,7 +340,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 22 + "task_id": 22, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -343,7 +360,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 23 + "task_id": 23, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -362,7 +380,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 24 + "task_id": 24, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -381,7 +400,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 25 + "task_id": 25, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -400,7 +420,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 26 + "task_id": 26, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -419,7 +440,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 27 + "task_id": 27, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -438,7 +460,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 28 + "task_id": 28, + "use_bbox_size_for_points": true }, { "check_covered_annotations": true, @@ -457,7 +480,8 @@ "panoptic_comparison": true, "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 29 + "task_id": 29, + "use_bbox_size_for_points": true } ] } \ No newline at end of file From 51f757f42cd21076a235c3095bd6c8e712f3a414 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Mon, 4 Nov 2024 11:22:39 +0200 Subject: [PATCH 07/18] Extend quality test --- tests/python/rest_api/test_quality_control.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index 1886a0a62ac1..c74d7f678ea1 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1211,6 +1211,7 @@ def test_modified_task_produces_different_metrics( "oks_sigma", "compare_line_orientation", "panoptic_comparison", + "use_bbox_size_for_points", ], ) def test_settings_affect_metrics( From dc4667de878118ef86a6b31630cbcc056b4a9e51 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 5 Nov 2024 16:19:28 +0200 Subject: [PATCH 08/18] Add server support for empty frame matching --- .../0005_qualitysettings_match_empty.py | 18 ++++++++ cvat/apps/quality_control/models.py | 2 + cvat/apps/quality_control/quality_reports.py | 44 ++++++++++++++----- cvat/apps/quality_control/serializers.py | 7 +++ cvat/schema.yml | 14 ++++++ 5 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py diff --git a/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py new file mode 100644 index 000000000000..8f2b3ef013a9 --- /dev/null +++ b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-11-05 14:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0004_qualitysettings_use_bbox_size_for_points"), + ] + + operations = [ + migrations.AddField( + model_name="qualitysettings", + name="match_empty", + field=models.BooleanField(default=False), + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index 5280cd03cd3a..0d165a2c8520 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -220,6 +220,8 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() + match_empty = models.BooleanField(default=False) + target_metric = models.CharField( max_length=32, choices=QualityTargetMetricType.choices(), diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 03308e624558..ce331f53c0ff 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -217,6 +217,12 @@ class ComparisonParameters(_Serializable): panoptic_comparison: bool = True "Use only the visible part of the masks and polygons in comparisons" + match_empty: bool = False + """ + Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, + will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. + """ + def _value_serializer(self, v): if isinstance(v, dm.AnnotationType): return str(v.name) @@ -232,11 +238,11 @@ def from_dict(cls, d: dict): @define(kw_only=True) class ConfusionMatrix(_Serializable): labels: List[str] - rows: np.array - precision: np.array - recall: np.array - accuracy: np.array - jaccard_index: Optional[np.array] + rows: np.ndarray + precision: np.ndarray + recall: np.ndarray + accuracy: np.ndarray + jaccard_index: Optional[np.ndarray] @property def axes(self): @@ -1970,6 +1976,11 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX confusion_matrix[ds_label_idx, gt_label_idx] += 1 + if self.settings.match_empty and not gt_item.annotations and not ds_item.annotations: + # Add virtual annotations. Here we expect logic like r = v / (x or 1) + valid_labels_count = 1 + valid_shapes_count = 1 + self._frame_results[frame_id] = ComparisonReportFrameSummary( annotations=self._generate_annotations_summary( confusion_matrix, confusion_matrix_labels @@ -2013,9 +2024,8 @@ def _make_zero_confusion_matrix(self) -> Tuple[List[str], np.ndarray, Dict[int, return label_names, confusion_matrix, label_id_idx_map - @classmethod def _generate_annotations_summary( - cls, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] + self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] ) -> ComparisonReportAnnotationsSummary: matched_ann_counts = np.diag(confusion_matrix) ds_ann_counts = np.sum(confusion_matrix, axis=1) @@ -2035,10 +2045,22 @@ def _generate_annotations_summary( ) / (total_annotations_count or 1) valid_annotations_count = np.sum(matched_ann_counts) - missing_annotations_count = np.sum(confusion_matrix[cls._UNMATCHED_IDX, :]) - extra_annotations_count = np.sum(confusion_matrix[:, cls._UNMATCHED_IDX]) - ds_annotations_count = np.sum(ds_ann_counts[: cls._UNMATCHED_IDX]) - gt_annotations_count = np.sum(gt_ann_counts[: cls._UNMATCHED_IDX]) + missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :]) + extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX]) + ds_annotations_count = np.sum(ds_ann_counts[: self._UNMATCHED_IDX]) + gt_annotations_count = np.sum(gt_ann_counts[: self._UNMATCHED_IDX]) + + if self.settings.match_empty: + empty_labels = (ds_ann_counts == 0) & (gt_ann_counts == 0) + if np.any(empty_labels): + label_jaccard_indices[empty_labels] = 1 + label_precisions[empty_labels] = 1 + label_recalls[empty_labels] = 1 + label_accuracies[empty_labels] = 1 + + if total_annotations_count == 0: + # Add virtual annotations. Here we expect logic like r = v / (x or 1) + valid_annotations_count = 1 return ComparisonReportAnnotationsSummary( valid_count=valid_annotations_count, diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index b4425f777993..7de4e38fe8f8 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -92,6 +92,7 @@ class Meta: "object_visibility_threshold", "panoptic_comparison", "compare_attributes", + "match_empty", ) read_only_fields = ( "id", @@ -99,6 +100,7 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} + extra_kwargs.setdefault("match_empty", {}).setdefault("default", False) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -155,6 +157,11 @@ class Meta: Use only the visible part of the masks and polygons in comparisons """, "compare_attributes": "Enables or disables annotation attribute comparison", + "match_empty": """ + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + """, }.items(): extra_kwargs.setdefault(field_name, {}).setdefault( "help_text", textwrap.dedent(help_text.lstrip("\n")) diff --git a/cvat/schema.yml b/cvat/schema.yml index 2ee22d2d7c54..a863fac2f98b 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9718,6 +9718,13 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison + match_empty: + type: boolean + default: false + description: | + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. PatchedTaskValidationLayoutWriteRequest: type: object properties: @@ -10206,6 +10213,13 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison + match_empty: + type: boolean + default: false + description: | + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. RegisterSerializerEx: type: object properties: From c8e5dc7239a419133b8e5a7c5e25c49cae6721b0 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Tue, 5 Nov 2024 17:40:54 +0200 Subject: [PATCH 09/18] Add ui --- cvat-core/src/quality-settings.ts | 11 ++++++++ cvat-core/src/server-response-types.ts | 1 + .../quality-control/quality-control-page.tsx | 1 + .../task-quality/quality-settings-form.tsx | 27 +++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 76b06c63f26c..6251ec82e72e 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -33,6 +33,7 @@ export default class QualitySettings { #objectVisibilityThreshold: number; #panopticComparison: boolean; #compareAttributes: boolean; + #matchEmpty: boolean; #descriptions: Record; constructor(initialData: SerializedQualitySettingsData) { @@ -54,6 +55,7 @@ export default class QualitySettings { this.#objectVisibilityThreshold = initialData.object_visibility_threshold; this.#panopticComparison = initialData.panoptic_comparison; this.#compareAttributes = initialData.compare_attributes; + this.#matchEmpty = initialData.match_empty; this.#descriptions = initialData.descriptions; } @@ -193,6 +195,14 @@ export default class QualitySettings { this.#maxValidationsPerJob = newVal; } + get matchEmpty(): boolean { + return this.#matchEmpty; + } + + set matchEmpty(newVal: boolean) { + this.#matchEmpty = newVal; + } + get descriptions(): Record { const descriptions: Record = Object.keys(this.#descriptions).reduce((acc, key) => { const camelCaseKey = _.camelCase(key); @@ -221,6 +231,7 @@ export default class QualitySettings { target_metric: this.#targetMetric, target_metric_threshold: this.#targetMetricThreshold, max_validations_per_job: this.#maxValidationsPerJob, + match_empty: this.#matchEmpty, }; return result; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 8b334819705b..ffa03f491768 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -258,6 +258,7 @@ export interface SerializedQualitySettingsData { object_visibility_threshold?: number; panoptic_comparison?: boolean; compare_attributes?: boolean; + match_empty?: boolean; descriptions?: Record; } diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 5d27f58ac0f0..776904a51685 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -223,6 +223,7 @@ function QualityControlPage(): JSX.Element { settings.lowOverlapThreshold = values.lowOverlapThreshold / 100; settings.iouThreshold = values.iouThreshold / 100; settings.compareAttributes = values.compareAttributes; + settings.matchEmpty = values.matchEmpty; settings.oksSigma = values.oksSigma / 100; settings.useBboxSizeForPoints = values.useBboxSizeForPoints; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 151f70afca49..7922aeab6e08 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -33,6 +33,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element lowOverlapThreshold: settings.lowOverlapThreshold * 100, iouThreshold: settings.iouThreshold * 100, compareAttributes: settings.compareAttributes, + matchEmpty: settings.matchEmpty, oksSigma: settings.oksSigma * 100, useBboxSizeForPoints: settings.useBboxSizeForPoints, @@ -72,6 +73,8 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element <> {makeTooltipFragment('Target metric', targetMetricDescription)} {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)} + {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)} + {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmpty)} , ); @@ -174,6 +177,30 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element + + + + + Compare attributes + + + + + + + Match empty frames + + + + From fe4d8c43d0de4d10b5c8c1846fc3553125ae141e Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 6 Nov 2024 15:39:06 +0200 Subject: [PATCH 10/18] Rename --- cvat-core/src/quality-settings.ts | 14 +++++------ cvat-core/src/server-response-types.ts | 2 +- .../quality-control/quality-control-page.tsx | 2 +- .../task-quality/quality-settings-form.tsx | 6 ++--- .../0005_qualitysettings_match_empty.py | 2 +- cvat/apps/quality_control/models.py | 2 +- cvat/apps/quality_control/quality_reports.py | 2 +- cvat/apps/quality_control/serializers.py | 6 ++--- cvat/schema.yml | 4 ++-- tests/python/rest_api/test_quality_control.py | 6 ++++- tests/python/shared/assets/cvat_db/data.json | 24 +++++++++++++++++++ .../shared/assets/quality_settings.json | 24 +++++++++++++++++++ 12 files changed, 73 insertions(+), 21 deletions(-) diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 6251ec82e72e..4b6d3db96a07 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -33,7 +33,7 @@ export default class QualitySettings { #objectVisibilityThreshold: number; #panopticComparison: boolean; #compareAttributes: boolean; - #matchEmpty: boolean; + #matchEmptyFrames: boolean; #descriptions: Record; constructor(initialData: SerializedQualitySettingsData) { @@ -55,7 +55,7 @@ export default class QualitySettings { this.#objectVisibilityThreshold = initialData.object_visibility_threshold; this.#panopticComparison = initialData.panoptic_comparison; this.#compareAttributes = initialData.compare_attributes; - this.#matchEmpty = initialData.match_empty; + this.#matchEmptyFrames = initialData.match_empty_frames; this.#descriptions = initialData.descriptions; } @@ -195,12 +195,12 @@ export default class QualitySettings { this.#maxValidationsPerJob = newVal; } - get matchEmpty(): boolean { - return this.#matchEmpty; + get matchEmptyFrames(): boolean { + return this.#matchEmptyFrames; } - set matchEmpty(newVal: boolean) { - this.#matchEmpty = newVal; + set matchEmptyFrames(newVal: boolean) { + this.#matchEmptyFrames = newVal; } get descriptions(): Record { @@ -231,7 +231,7 @@ export default class QualitySettings { target_metric: this.#targetMetric, target_metric_threshold: this.#targetMetricThreshold, max_validations_per_job: this.#maxValidationsPerJob, - match_empty: this.#matchEmpty, + match_empty_frames: this.#matchEmptyFrames, }; return result; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index ffa03f491768..ea260826c2e7 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData { object_visibility_threshold?: number; panoptic_comparison?: boolean; compare_attributes?: boolean; - match_empty?: boolean; + match_empty_frames?: boolean; descriptions?: Record; } diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 776904a51685..23bae2d929b5 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element { settings.lowOverlapThreshold = values.lowOverlapThreshold / 100; settings.iouThreshold = values.iouThreshold / 100; settings.compareAttributes = values.compareAttributes; - settings.matchEmpty = values.matchEmpty; + settings.matchEmptyFrames = values.matchEmptyFrames; settings.oksSigma = values.oksSigma / 100; settings.useBboxSizeForPoints = values.useBboxSizeForPoints; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 7922aeab6e08..47c496e0ac81 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -33,7 +33,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element lowOverlapThreshold: settings.lowOverlapThreshold * 100, iouThreshold: settings.iouThreshold * 100, compareAttributes: settings.compareAttributes, - matchEmpty: settings.matchEmpty, + matchEmptyFrames: settings.matchEmptyFrames, oksSigma: settings.oksSigma * 100, useBboxSizeForPoints: settings.useBboxSizeForPoints, @@ -74,7 +74,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element {makeTooltipFragment('Target metric', targetMetricDescription)} {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)} {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)} - {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmpty)} + {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)} , ); @@ -191,7 +191,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element diff --git a/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py index 8f2b3ef013a9..00d3b2dd5552 100644 --- a/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py +++ b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name="qualitysettings", - name="match_empty", + name="match_empty_frames", field=models.BooleanField(default=False), ), ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index 0d165a2c8520..f55db347b37d 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -220,7 +220,7 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() - match_empty = models.BooleanField(default=False) + match_empty_frames = models.BooleanField(default=False) target_metric = models.CharField( max_length=32, diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index ce331f53c0ff..471e992752c8 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -217,7 +217,7 @@ class ComparisonParameters(_Serializable): panoptic_comparison: bool = True "Use only the visible part of the masks and polygons in comparisons" - match_empty: bool = False + match_empty_frames: bool = False """ Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 7de4e38fe8f8..6367a29aa6ba 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -92,7 +92,7 @@ class Meta: "object_visibility_threshold", "panoptic_comparison", "compare_attributes", - "match_empty", + "match_empty_frames", ) read_only_fields = ( "id", @@ -100,7 +100,7 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} - extra_kwargs.setdefault("match_empty", {}).setdefault("default", False) + extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -157,7 +157,7 @@ class Meta: Use only the visible part of the masks and polygons in comparisons """, "compare_attributes": "Enables or disables annotation attribute comparison", - "match_empty": """ + "match_empty_frames": """ Count empty frames as matching. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. diff --git a/cvat/schema.yml b/cvat/schema.yml index a863fac2f98b..dad3510aaa19 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9718,7 +9718,7 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty: + match_empty_frames: type: boolean default: false description: | @@ -10213,7 +10213,7 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty: + match_empty_frames: type: boolean default: false description: | diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index c74d7f678ea1..0e940effdedc 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1212,6 +1212,7 @@ def test_modified_task_produces_different_metrics( "compare_line_orientation", "panoptic_comparison", "use_bbox_size_for_points", + "match_empty_frames", ], ) def test_settings_affect_metrics( @@ -1238,7 +1239,10 @@ def test_settings_affect_metrics( ) new_report = self.create_quality_report(admin_user, task_id) - assert new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] + if parameter == "match_empty_frames": + assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"] + else: + assert new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] def test_old_report_can_be_loaded(self, admin_user, quality_reports): report = min((r for r in quality_reports if r["task_id"]), key=lambda r: r["id"]) diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index dce23272347e..4ddd1218a1be 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18173,6 +18173,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18196,6 +18197,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18219,6 +18221,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18242,6 +18245,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18265,6 +18269,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18288,6 +18293,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18311,6 +18317,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18334,6 +18341,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18357,6 +18365,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18380,6 +18389,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18403,6 +18413,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18426,6 +18437,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18449,6 +18461,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18472,6 +18485,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18495,6 +18509,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18518,6 +18533,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18541,6 +18557,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18564,6 +18581,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18587,6 +18605,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18610,6 +18629,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18633,6 +18653,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18656,6 +18677,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18679,6 +18701,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18702,6 +18725,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 6c005a035fac..3be3f36fafa0 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -14,6 +14,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -34,6 +35,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -54,6 +56,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -74,6 +77,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -94,6 +98,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -114,6 +119,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -134,6 +140,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -154,6 +161,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -174,6 +182,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -194,6 +203,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -214,6 +224,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -234,6 +245,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -254,6 +266,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -274,6 +287,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -294,6 +308,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -314,6 +329,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -334,6 +350,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -354,6 +371,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -374,6 +392,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -394,6 +413,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -414,6 +434,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -434,6 +455,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -454,6 +476,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -474,6 +497,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, From 41f467a9ed1afbde2837fe76d88079288c383529 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 6 Nov 2024 15:54:38 +0200 Subject: [PATCH 11/18] t --- cvat/apps/quality_control/quality_reports.py | 121 +++++++++++++------ 1 file changed, 81 insertions(+), 40 deletions(-) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 471e992752c8..7bc883c754d0 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -1976,13 +1976,16 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX confusion_matrix[ds_label_idx, gt_label_idx] += 1 - if self.settings.match_empty and not gt_item.annotations and not ds_item.annotations: - # Add virtual annotations. Here we expect logic like r = v / (x or 1) + if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations: + # Add virtual annotations for empty frames valid_labels_count = 1 + total_labels_count = 1 + valid_shapes_count = 1 + total_shapes_count = 1 self._frame_results[frame_id] = ComparisonReportFrameSummary( - annotations=self._generate_annotations_summary( + annotations=self._generate_frame_annotations_summary( confusion_matrix, confusion_matrix_labels ), annotation_components=ComparisonReportAnnotationComponentsSummary( @@ -2024,7 +2027,7 @@ def _make_zero_confusion_matrix(self) -> Tuple[List[str], np.ndarray, Dict[int, return label_names, confusion_matrix, label_id_idx_map - def _generate_annotations_summary( + def _compute_annotation_summary( self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] ) -> ComparisonReportAnnotationsSummary: matched_ann_counts = np.diag(confusion_matrix) @@ -2044,13 +2047,7 @@ def _generate_annotations_summary( # ... = TP + TN ) / (total_annotations_count or 1) - valid_annotations_count = np.sum(matched_ann_counts) - missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :]) - extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX]) - ds_annotations_count = np.sum(ds_ann_counts[: self._UNMATCHED_IDX]) - gt_annotations_count = np.sum(gt_ann_counts[: self._UNMATCHED_IDX]) - - if self.settings.match_empty: + if self.settings.match_empty_frames: empty_labels = (ds_ann_counts == 0) & (gt_ann_counts == 0) if np.any(empty_labels): label_jaccard_indices[empty_labels] = 1 @@ -2058,9 +2055,11 @@ def _generate_annotations_summary( label_recalls[empty_labels] = 1 label_accuracies[empty_labels] = 1 - if total_annotations_count == 0: - # Add virtual annotations. Here we expect logic like r = v / (x or 1) - valid_annotations_count = 1 + valid_annotations_count = np.sum(matched_ann_counts) + missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :]) + extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX]) + ds_annotations_count = np.sum(ds_ann_counts[: self._UNMATCHED_IDX]) + gt_annotations_count = np.sum(gt_ann_counts[: self._UNMATCHED_IDX]) return ComparisonReportAnnotationsSummary( valid_count=valid_annotations_count, @@ -2079,12 +2078,24 @@ def _generate_annotations_summary( ), ) - def generate_report(self) -> ComparisonReport: - self._find_gt_conflicts() + def _generate_frame_annotations_summary( + self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] + ) -> ComparisonReportAnnotationsSummary: + summary = self._compute_annotation_summary(confusion_matrix, confusion_matrix_labels) + + if self.settings.match_empty_frames and summary.total_count == 0: + # Add virtual annotations for empty frames + summary.valid_count = 1 + summary.total_count = 1 + summary.ds_count = 1 + summary.gt_count = 1 + return summary + + def _generate_dataset_annotations_summary( + self, frame_summaries: Dict[int, ComparisonReportFrameSummary] + ) -> Tuple[ComparisonReportAnnotationsSummary, ComparisonReportAnnotationComponentsSummary]: # accumulate stats - intersection_frames = [] - conflicts = [] annotation_components = ComparisonReportAnnotationComponentsSummary( shape=ComparisonReportAnnotationShapeSummary( valid_count=0, @@ -2102,19 +2113,66 @@ def generate_report(self) -> ComparisonReport: ), ) mean_ious = [] + empty_frames = [] confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix() - for frame_id, frame_result in self._frame_results.items(): - intersection_frames.append(frame_id) - conflicts += frame_result.conflicts + for frame_id, frame_result in frame_summaries.items(): confusion_matrix += frame_result.annotations.confusion_matrix.rows if annotation_components is None: annotation_components = deepcopy(frame_result.annotation_components) else: annotation_components.accumulate(frame_result.annotation_components) + mean_ious.append(frame_result.annotation_components.shape.mean_iou) + if frame_result.annotations.ds_count == 0 and frame_result.annotations.gt_count == 0: + empty_frames.append(frame_id) + + annotation_summary = self._compute_annotation_summary( + confusion_matrix, confusion_matrix_labels + ) + + annotation_components = ComparisonReportAnnotationComponentsSummary( + shape=ComparisonReportAnnotationShapeSummary( + valid_count=annotation_components.shape.valid_count, + missing_count=annotation_components.shape.missing_count, + extra_count=annotation_components.shape.extra_count, + total_count=annotation_components.shape.total_count, + ds_count=annotation_components.shape.ds_count, + gt_count=annotation_components.shape.gt_count, + mean_iou=np.mean(mean_ious), + ), + label=ComparisonReportAnnotationLabelSummary( + valid_count=annotation_components.label.valid_count, + invalid_count=annotation_components.label.invalid_count, + total_count=annotation_components.label.total_count, + ), + ) + + if self.settings.match_empty_frames: + # Add virtual annotations for empty frames + empty_frame_count = len(empty_frames) + annotation_summary.valid_count += empty_frame_count + annotation_summary.total_count += empty_frame_count + annotation_summary.ds_count += empty_frame_count + annotation_summary.gt_count += empty_frame_count + + return annotation_summary, annotation_components + + def generate_report(self) -> ComparisonReport: + self._find_gt_conflicts() + + intersection_frames = [] + conflicts = [] + for frame_id, frame_result in self._frame_results.items(): + intersection_frames.append(frame_id) + conflicts += frame_result.conflicts + + annotation_summary, annotations_component_summary = ( + self._generate_dataset_annotations_summary(self._frame_results) + ) + return ComparisonReport( parameters=self.settings, comparison_summary=ComparisonReportComparisonSummary( @@ -2130,25 +2188,8 @@ def generate_report(self) -> ComparisonReport: [c for c in conflicts if c.severity == AnnotationConflictSeverity.ERROR] ), conflicts_by_type=Counter(c.type for c in conflicts), - annotations=self._generate_annotations_summary( - confusion_matrix, confusion_matrix_labels - ), - annotation_components=ComparisonReportAnnotationComponentsSummary( - shape=ComparisonReportAnnotationShapeSummary( - valid_count=annotation_components.shape.valid_count, - missing_count=annotation_components.shape.missing_count, - extra_count=annotation_components.shape.extra_count, - total_count=annotation_components.shape.total_count, - ds_count=annotation_components.shape.ds_count, - gt_count=annotation_components.shape.gt_count, - mean_iou=np.mean(mean_ious), - ), - label=ComparisonReportAnnotationLabelSummary( - valid_count=annotation_components.label.valid_count, - invalid_count=annotation_components.label.invalid_count, - total_count=annotation_components.label.total_count, - ), - ), + annotations=annotation_summary, + annotation_components=annotations_component_summary, ), frame_results=self._frame_results, ) From 418b33364f98a24400932b046485d5d38d0c1bf7 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 6 Nov 2024 17:02:32 +0200 Subject: [PATCH 12/18] Update quality report numbers, extend setting description --- cvat/apps/quality_control/quality_reports.py | 42 ++++--------------- cvat/apps/quality_control/serializers.py | 1 + tests/python/rest_api/test_quality_control.py | 4 +- 3 files changed, 13 insertions(+), 34 deletions(-) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 7bc883c754d0..403a7d455df9 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -221,6 +221,7 @@ class ComparisonParameters(_Serializable): """ Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. """ def _value_serializer(self, v): @@ -2047,14 +2048,6 @@ def _compute_annotation_summary( # ... = TP + TN ) / (total_annotations_count or 1) - if self.settings.match_empty_frames: - empty_labels = (ds_ann_counts == 0) & (gt_ann_counts == 0) - if np.any(empty_labels): - label_jaccard_indices[empty_labels] = 1 - label_precisions[empty_labels] = 1 - label_recalls[empty_labels] = 1 - label_accuracies[empty_labels] = 1 - valid_annotations_count = np.sum(matched_ann_counts) missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :]) extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX]) @@ -2113,12 +2106,15 @@ def _generate_dataset_annotations_summary( ), ) mean_ious = [] - empty_frames = [] + empty_frame_count = 0 confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix() - for frame_id, frame_result in frame_summaries.items(): + for frame_result in frame_summaries.values(): confusion_matrix += frame_result.annotations.confusion_matrix.rows + if not np.any(frame_result.annotations.confusion_matrix.rows): + empty_frame_count += 1 + if annotation_components is None: annotation_components = deepcopy(frame_result.annotation_components) else: @@ -2126,33 +2122,13 @@ def _generate_dataset_annotations_summary( mean_ious.append(frame_result.annotation_components.shape.mean_iou) - if frame_result.annotations.ds_count == 0 and frame_result.annotations.gt_count == 0: - empty_frames.append(frame_id) - annotation_summary = self._compute_annotation_summary( confusion_matrix, confusion_matrix_labels ) - annotation_components = ComparisonReportAnnotationComponentsSummary( - shape=ComparisonReportAnnotationShapeSummary( - valid_count=annotation_components.shape.valid_count, - missing_count=annotation_components.shape.missing_count, - extra_count=annotation_components.shape.extra_count, - total_count=annotation_components.shape.total_count, - ds_count=annotation_components.shape.ds_count, - gt_count=annotation_components.shape.gt_count, - mean_iou=np.mean(mean_ious), - ), - label=ComparisonReportAnnotationLabelSummary( - valid_count=annotation_components.label.valid_count, - invalid_count=annotation_components.label.invalid_count, - total_count=annotation_components.label.total_count, - ), - ) - - if self.settings.match_empty_frames: - # Add virtual annotations for empty frames - empty_frame_count = len(empty_frames) + if self.settings.match_empty_frames and empty_frame_count: + # Add virtual annotations for empty frames, + # they are not included in the confusion matrix annotation_summary.valid_count += empty_frame_count annotation_summary.total_count += empty_frame_count annotation_summary.ds_count += empty_frame_count diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 6367a29aa6ba..31e4cfcd3082 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -161,6 +161,7 @@ class Meta: Count empty frames as matching. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. """, }.items(): extra_kwargs.setdefault(field_name, {}).setdefault( diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index 0e940effdedc..277096c3bd50 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1242,7 +1242,9 @@ def test_settings_affect_metrics( if parameter == "match_empty_frames": assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"] else: - assert new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] + assert ( + new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] + ) def test_old_report_can_be_loaded(self, admin_user, quality_reports): report = min((r for r in quality_reports if r["task_id"]), key=lambda r: r["id"]) From 01d5a0705ba8d57e978b47f3569387eb3bebc775 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 6 Nov 2024 17:07:28 +0200 Subject: [PATCH 13/18] Update changelog --- changelog.d/20241106_170626_mzhiltso_match_empty_frames.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/20241106_170626_mzhiltso_match_empty_frames.md diff --git a/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md b/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md new file mode 100644 index 000000000000..e615de6354c5 --- /dev/null +++ b/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md @@ -0,0 +1,4 @@ +### Added + +- A quality check option to consider empty frames matching + () From f943343c4ccc404ef9b6107fa46b5ad2ff5b6e6a Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 6 Nov 2024 18:41:20 +0200 Subject: [PATCH 14/18] Update parameter --- cvat-core/src/quality-settings.ts | 19 ++-- cvat-core/src/server-response-types.ts | 2 +- .../quality-control/quality-control-page.tsx | 2 +- .../task-quality/quality-settings-form.tsx | 31 ++++-- .../0004_qualitysettings_point_size_base.py | 24 +++++ ...ualitysettings_use_bbox_size_for_points.py | 19 ---- cvat/apps/quality_control/models.py | 16 +++- cvat/apps/quality_control/quality_reports.py | 18 ++-- cvat/apps/quality_control/serializers.py | 26 +++-- cvat/schema.yml | 64 +++++++++---- tests/python/rest_api/test_quality_control.py | 8 +- tests/python/shared/assets/cvat_db/data.json | 48 +++++----- .../shared/assets/quality_settings.json | 96 +++++++++---------- 13 files changed, 228 insertions(+), 145 deletions(-) create mode 100644 cvat/apps/quality_control/migrations/0004_qualitysettings_point_size_base.py delete mode 100644 cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 76b06c63f26c..8a3b5d28888c 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -14,6 +14,11 @@ export enum TargetMetric { RECALL = 'recall', } +export enum PointSizeBase { + IMAGE_SIZE = 'image_size', + GROUP_BBOX_SIZE = 'group_bbox_size', +} + export default class QualitySettings { #id: number; #targetMetric: TargetMetric; @@ -22,7 +27,7 @@ export default class QualitySettings { #task: number; #iouThreshold: number; #oksSigma: number; - #useBboxSizeForPoints: boolean; + #pointSizeBase: PointSizeBase; #lineThickness: number; #lowOverlapThreshold: number; #orientedLines: boolean; @@ -43,7 +48,7 @@ export default class QualitySettings { this.#maxValidationsPerJob = initialData.max_validations_per_job; this.#iouThreshold = initialData.iou_threshold; this.#oksSigma = initialData.oks_sigma; - this.#useBboxSizeForPoints = initialData.use_bbox_size_for_points; + this.#pointSizeBase = initialData.point_size_base as PointSizeBase; this.#lineThickness = initialData.line_thickness; this.#lowOverlapThreshold = initialData.low_overlap_threshold; this.#orientedLines = initialData.compare_line_orientation; @@ -81,12 +86,12 @@ export default class QualitySettings { this.#oksSigma = newVal; } - get useBboxSizeForPoints(): boolean { - return this.#useBboxSizeForPoints; + get pointSizeBase(): PointSizeBase { + return this.#pointSizeBase; } - set useBboxSizeForPoints(newVal: boolean) { - this.#useBboxSizeForPoints = newVal; + set pointSizeBase(newVal: PointSizeBase) { + this.#pointSizeBase = newVal; } get lineThickness(): number { @@ -207,7 +212,7 @@ export default class QualitySettings { const result: SerializedQualitySettingsData = { iou_threshold: this.#iouThreshold, oks_sigma: this.#oksSigma, - use_bbox_size_for_points: this.#useBboxSizeForPoints, + point_size_base: this.#pointSizeBase, line_thickness: this.#lineThickness, low_overlap_threshold: this.#lowOverlapThreshold, compare_line_orientation: this.#orientedLines, diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 8b334819705b..649b2c509a23 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -247,7 +247,7 @@ export interface SerializedQualitySettingsData { max_validations_per_job?: number; iou_threshold?: number; oks_sigma?: number; - use_bbox_size_for_points?: boolean; + point_size_base?: string; line_thickness?: number; low_overlap_threshold?: number; compare_line_orientation?: boolean; diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 5d27f58ac0f0..0a3914712f87 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -225,7 +225,7 @@ function QualityControlPage(): JSX.Element { settings.compareAttributes = values.compareAttributes; settings.oksSigma = values.oksSigma / 100; - settings.useBboxSizeForPoints = values.useBboxSizeForPoints; + settings.pointSizeBase = values.pointSizeBase; settings.lineThickness = values.lineThickness / 100; settings.lineOrientationThreshold = values.lineOrientationThreshold / 100; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 151f70afca49..2633fae6bfbe 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -14,6 +14,7 @@ import Button from 'antd/lib/button'; import Select from 'antd/lib/select'; import CVATTooltip from 'components/common/cvat-tooltip'; import { QualitySettings, TargetMetric } from 'cvat-core-wrapper'; +import { PointSizeBase } from 'cvat-core/src/quality-settings'; interface Props { form: FormInstance; @@ -35,7 +36,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element compareAttributes: settings.compareAttributes, oksSigma: settings.oksSigma * 100, - useBboxSizeForPoints: settings.useBboxSizeForPoints, + pointSizeBase: settings.pointSizeBase, lineThickness: settings.lineThickness * 100, lineOrientationThreshold: settings.lineOrientationThreshold * 100, @@ -51,7 +52,13 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element const targetMetricDescription = `${settings.descriptions.targetMetric .replaceAll(/\* [a-z` -]+[A-Z]+/g, '') - .replaceAll(/\n/g, '')}.`; + .replaceAll(/\n/g, '') + }`; + + const pointSizeBaseDescription = `${settings.descriptions.pointSizeBase + .substring(0, settings.descriptions.pointSizeBase.indexOf('\n\n\n')) + .replaceAll(/\n/g, ' ') + }`; const makeTooltipFragment = (metric: string, description: string): JSX.Element => (
@@ -91,7 +98,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element ); const pointTooltip = makeTooltip( - makeTooltipFragment('Use image space', settings.descriptions.useBboxSizeForPoints), + makeTooltipFragment('Point size base', pointSizeBaseDescription), ); const linesTooltip = makeTooltip( @@ -267,13 +274,21 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element - - Use bbox size - + diff --git a/cvat/apps/quality_control/migrations/0004_qualitysettings_point_size_base.py b/cvat/apps/quality_control/migrations/0004_qualitysettings_point_size_base.py new file mode 100644 index 000000000000..024d263356ac --- /dev/null +++ b/cvat/apps/quality_control/migrations/0004_qualitysettings_point_size_base.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.15 on 2024-11-06 15:39 + +from django.db import migrations, models + +import cvat.apps.quality_control.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0003_qualityreport_assignee_last_updated_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="qualitysettings", + name="point_size_base", + field=models.CharField( + choices=[("image_size", "IMAGE_SIZE"), ("group_bbox_size", "GROUP_BBOX_SIZE")], + default=cvat.apps.quality_control.models.PointSizeBase["GROUP_BBOX_SIZE"], + max_length=32, + ), + ), + ] diff --git a/cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py b/cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py deleted file mode 100644 index 9325a21a3f2a..000000000000 --- a/cvat/apps/quality_control/migrations/0004_qualitysettings_use_bbox_size_for_points.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.15 on 2024-11-01 15:40 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("quality_control", "0003_qualityreport_assignee_last_updated_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="qualitysettings", - name="use_bbox_size_for_points", - field=models.BooleanField(default=True), - preserve_default=False, - ), - ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index 5280cd03cd3a..cfa2522541df 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -196,6 +196,18 @@ def clean(self) -> None: raise ValidationError(f"Unexpected type value '{self.type}'") +class PointSizeBase(str, Enum): + IMAGE_SIZE = "image_size" + GROUP_BBOX_SIZE = "group_bbox_size" + + def __str__(self) -> str: + return self.value + + @classmethod + def choices(cls): + return tuple((x.value, x.name) for x in cls) + + class QualitySettings(models.Model): task = models.OneToOneField(Task, on_delete=models.CASCADE, related_name="quality_settings") @@ -205,7 +217,9 @@ class QualitySettings(models.Model): low_overlap_threshold = models.FloatField() - use_bbox_size_for_points = models.BooleanField() + point_size_base = models.CharField( + max_length=32, choices=PointSizeBase.choices(), default=PointSizeBase.GROUP_BBOX_SIZE + ) compare_line_orientation = models.BooleanField() line_orientation_threshold = models.FloatField() diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 03308e624558..72a54bf10a77 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -187,8 +187,8 @@ class ComparisonParameters(_Serializable): oks_sigma: float = 0.09 "Like IoU threshold, but for points, % of the bbox area to match a pair of points" - use_bbox_size_for_points: bool = True - "Compare point groups using the group bbox size instead of the image size" + point_size_base: models.PointSizeBase = models.PointSizeBase.GROUP_BBOX_SIZE + "Determines how to obtain the object size for point comparisons" line_thickness: float = 0.01 "Thickness of polylines, relatively to the (image area) ^ 0.5" @@ -958,7 +958,7 @@ def __init__( # https://cocodataset.org/#keypoints-eval # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py#L523 oks_sigma: float = 0.09, - use_bbox_size_for_points: bool = True, + point_size_base: models.PointSizeBase = models.PointSizeBase.GROUP_BBOX_SIZE, compare_line_orientation: bool = False, line_torso_radius: float = 0.01, panoptic_comparison: bool = False, @@ -972,8 +972,8 @@ def __init__( self.oks_sigma = oks_sigma "% of the shape area" - self.use_bbox_size_for_points = use_bbox_size_for_points - "Compare point groups using the group bbox size instead of the image size" + self.point_size_base = point_size_base + "Compare point groups using the group bbox size or the image size" self.compare_line_orientation = compare_line_orientation "Whether lines are oriented or not" @@ -1301,9 +1301,9 @@ def _distance(a: dm.Points, b: dm.Points) -> float: # Complex case: multiple points, grouped points, points with a bbox # Try to align points and then return the metric - if not self.use_bbox_size_for_points: + if self.point_size_base == models.PointSizeBase.IMAGE_SIZE: scale = img_h * img_w - else: + elif self.point_size_base == models.PointSizeBase.GROUP_BBOX_SIZE: # match points in their bbox space if dm.ops.bbox_iou(a_bbox, b_bbox) <= 0: @@ -1312,6 +1312,8 @@ def _distance(a: dm.Points, b: dm.Points) -> float: bbox = dm.ops.mean_bbox([a_bbox, b_bbox]) scale = bbox[2] * bbox[3] + else: + assert False, f"Unknown point size base {self.point_size_base}" a_points = np.reshape(a.points, (-1, 2)) b_points = np.reshape(b.points, (-1, 2)) @@ -1537,7 +1539,7 @@ def __init__(self, categories: dm.CategoriesInfo, *, settings: ComparisonParamet panoptic_comparison=settings.panoptic_comparison, iou_threshold=settings.iou_threshold, oks_sigma=settings.oks_sigma, - use_bbox_size_for_points=settings.use_bbox_size_for_points, + point_size_base=settings.point_size_base, line_torso_radius=settings.line_thickness, compare_line_orientation=False, # should not be taken from outside, handled differently ) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index b4425f777993..5b23de399cf8 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -81,7 +81,7 @@ class Meta: "max_validations_per_job", "iou_threshold", "oks_sigma", - "use_bbox_size_for_points", + "point_size_base", "line_thickness", "low_overlap_threshold", "compare_line_orientation", @@ -116,17 +116,25 @@ class Meta: """, "oks_sigma": """ Like IoU threshold, but for points. - The percent of the bbox area, used as the radius of the circle around the GT point, + The percent of the bbox side, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval """, - "use_bbox_size_for_points": """ - When comparing point groups, OKS sigma defines the matching area for a GT point. - If enabled, the area size is based on the point group bbox size. - If disabled, the image size is used. - Useful if point groups do not represent a single object or boxes attached to points - do not represent object boundaries. - """, + "point_size_base": """ + When comparing point annotations (including both separate points and point groups), + the OKS sigma parameter defines matching area for each GT point based to the + object size. The point size base parameter allows to configure how to determine + the object size. + If {image_size}, the image size is used. Useful if each point + annotation represents a separate object or boxes grouped with points do not + represent object boundaries. + If {group_bbox_size}, the object size is based on + the point group bbox size. Useful if each point group represents an object + or there is a bbox grouped with points, representing the object size. + """.format( + image_size=models.PointSizeBase.IMAGE_SIZE, + group_bbox_size=models.PointSizeBase.GROUP_BBOX_SIZE, + ), "line_thickness": """ Thickness of polylines, relatively to the (image area) ^ 0.5. The distance to the boundary around the GT line, diff --git a/cvat/schema.yml b/cvat/schema.yml index 2ee22d2d7c54..4602e36b1c34 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9658,17 +9658,27 @@ components: format: double description: | Like IoU threshold, but for points. - The percent of the bbox area, used as the radius of the circle around the GT point, + The percent of the bbox side, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval - use_bbox_size_for_points: - type: boolean - description: | - When comparing point groups, OKS sigma defines the matching area for a GT point. - If enabled, the area size is based on the point group bbox size. - If disabled, the image size is used. - Useful if point groups do not represent a single object or boxes attached to points - do not represent object boundaries. + point_size_base: + allOf: + - $ref: '#/components/schemas/PointSizeBaseEnum' + description: |- + When comparing point annotations (including both separate points and point groups), + the OKS sigma parameter defines matching area for each GT point based to the + object size. The point size base parameter allows to configure how to determine + the object size. + If image_size, the image size is used. Useful if each point + annotation represents a separate object or boxes grouped with points do not + represent object boundaries. + If group_bbox_size, the object size is based on + the point group bbox size. Useful if each point group represents an object + or there is a bbox grouped with points, representing the object size. + + + * `image_size` - IMAGE_SIZE + * `group_bbox_size` - GROUP_BBOX_SIZE line_thickness: type: number format: double @@ -9862,6 +9872,14 @@ components: - GIT_INTEGRATION - MODELS - PREDICT + PointSizeBaseEnum: + enum: + - image_size + - group_bbox_size + type: string + description: |- + * `image_size` - IMAGE_SIZE + * `group_bbox_size` - GROUP_BBOX_SIZE ProjectFileRequest: type: object properties: @@ -10146,17 +10164,27 @@ components: format: double description: | Like IoU threshold, but for points. - The percent of the bbox area, used as the radius of the circle around the GT point, + The percent of the bbox side, used as the radius of the circle around the GT point, where the checked point is expected to be. Read more: https://cocodataset.org/#keypoints-eval - use_bbox_size_for_points: - type: boolean - description: | - When comparing point groups, OKS sigma defines the matching area for a GT point. - If enabled, the area size is based on the point group bbox size. - If disabled, the image size is used. - Useful if point groups do not represent a single object or boxes attached to points - do not represent object boundaries. + point_size_base: + allOf: + - $ref: '#/components/schemas/PointSizeBaseEnum' + description: |- + When comparing point annotations (including both separate points and point groups), + the OKS sigma parameter defines matching area for each GT point based to the + object size. The point size base parameter allows to configure how to determine + the object size. + If image_size, the image size is used. Useful if each point + annotation represents a separate object or boxes grouped with points do not + represent object boundaries. + If group_bbox_size, the object size is based on + the point group bbox size. Useful if each point group represents an object + or there is a bbox grouped with points, representing the object size. + + + * `image_size` - IMAGE_SIZE + * `group_bbox_size` - GROUP_BBOX_SIZE line_thickness: type: number format: double diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index c74d7f678ea1..d3b2799b4cd9 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1211,7 +1211,7 @@ def test_modified_task_produces_different_metrics( "oks_sigma", "compare_line_orientation", "panoptic_comparison", - "use_bbox_size_for_points", + "point_size_base", ], ) def test_settings_affect_metrics( @@ -1229,6 +1229,12 @@ def test_settings_affect_metrics( settings[parameter] = 1 - settings[parameter] if parameter == "group_match_threshold": settings[parameter] = 0.9 + elif parameter == "point_size_base": + settings[parameter] = next( + v + for v in models.PointSizeBaseEnum.allowed_values[("value",)].values() + if v != settings[parameter] + ) else: assert False diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index dce23272347e..e4e912b95863 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18164,7 +18164,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18187,7 +18187,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18210,7 +18210,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18233,7 +18233,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18256,7 +18256,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18279,7 +18279,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18302,7 +18302,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18325,7 +18325,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18348,7 +18348,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18371,7 +18371,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18394,7 +18394,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18417,7 +18417,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18440,7 +18440,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18463,7 +18463,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18486,7 +18486,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18509,7 +18509,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18532,7 +18532,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18555,7 +18555,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18578,7 +18578,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18601,7 +18601,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18624,7 +18624,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18647,7 +18647,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18670,7 +18670,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, @@ -18693,7 +18693,7 @@ "oks_sigma": 0.09, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "use_bbox_size_for_points": true, + "point_size_base": "group_bbox_size", "compare_line_orientation": true, "line_orientation_threshold": 0.1, "compare_groups": true, diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 6c005a035fac..e6e4cba929b1 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -18,10 +18,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 2, - "use_bbox_size_for_points": true + "task_id": 2 }, { "check_covered_annotations": true, @@ -38,10 +38,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 5, - "use_bbox_size_for_points": true + "task_id": 5 }, { "check_covered_annotations": true, @@ -58,10 +58,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 6, - "use_bbox_size_for_points": true + "task_id": 6 }, { "check_covered_annotations": true, @@ -78,10 +78,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 7, - "use_bbox_size_for_points": true + "task_id": 7 }, { "check_covered_annotations": true, @@ -98,10 +98,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 8, - "use_bbox_size_for_points": true + "task_id": 8 }, { "check_covered_annotations": true, @@ -118,10 +118,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 9, - "use_bbox_size_for_points": true + "task_id": 9 }, { "check_covered_annotations": true, @@ -138,10 +138,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 11, - "use_bbox_size_for_points": true + "task_id": 11 }, { "check_covered_annotations": true, @@ -158,10 +158,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 12, - "use_bbox_size_for_points": true + "task_id": 12 }, { "check_covered_annotations": true, @@ -178,10 +178,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 13, - "use_bbox_size_for_points": true + "task_id": 13 }, { "check_covered_annotations": true, @@ -198,10 +198,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 14, - "use_bbox_size_for_points": true + "task_id": 14 }, { "check_covered_annotations": true, @@ -218,10 +218,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 15, - "use_bbox_size_for_points": true + "task_id": 15 }, { "check_covered_annotations": true, @@ -238,10 +238,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 17, - "use_bbox_size_for_points": true + "task_id": 17 }, { "check_covered_annotations": true, @@ -258,10 +258,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 18, - "use_bbox_size_for_points": true + "task_id": 18 }, { "check_covered_annotations": true, @@ -278,10 +278,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 19, - "use_bbox_size_for_points": true + "task_id": 19 }, { "check_covered_annotations": true, @@ -298,10 +298,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 20, - "use_bbox_size_for_points": true + "task_id": 20 }, { "check_covered_annotations": true, @@ -318,10 +318,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 21, - "use_bbox_size_for_points": true + "task_id": 21 }, { "check_covered_annotations": true, @@ -338,10 +338,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 22, - "use_bbox_size_for_points": true + "task_id": 22 }, { "check_covered_annotations": true, @@ -358,10 +358,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 23, - "use_bbox_size_for_points": true + "task_id": 23 }, { "check_covered_annotations": true, @@ -378,10 +378,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 24, - "use_bbox_size_for_points": true + "task_id": 24 }, { "check_covered_annotations": true, @@ -398,10 +398,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 25, - "use_bbox_size_for_points": true + "task_id": 25 }, { "check_covered_annotations": true, @@ -418,10 +418,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 26, - "use_bbox_size_for_points": true + "task_id": 26 }, { "check_covered_annotations": true, @@ -438,10 +438,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 27, - "use_bbox_size_for_points": true + "task_id": 27 }, { "check_covered_annotations": true, @@ -458,10 +458,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 28, - "use_bbox_size_for_points": true + "task_id": 28 }, { "check_covered_annotations": true, @@ -478,10 +478,10 @@ "object_visibility_threshold": 0.05, "oks_sigma": 0.09, "panoptic_comparison": true, + "point_size_base": "group_bbox_size", "target_metric": "accuracy", "target_metric_threshold": 0.7, - "task_id": 29, - "use_bbox_size_for_points": true + "task_id": 29 } ] } \ No newline at end of file From d83a38fdf4e63d595674c308b9562cb7ed0375b3 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 7 Nov 2024 11:12:12 +0200 Subject: [PATCH 15/18] Improve oks sigma side description --- cvat/apps/quality_control/serializers.py | 3 ++- cvat/schema.yml | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 5b23de399cf8..fb38677636fd 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -117,7 +117,8 @@ class Meta: "oks_sigma": """ Like IoU threshold, but for points. The percent of the bbox side, used as the radius of the circle around the GT point, - where the checked point is expected to be. + where the checked point is expected to be. For boxes with different width and + height, the "side" is computed as a geometric mean of the width and height. Read more: https://cocodataset.org/#keypoints-eval """, "point_size_base": """ diff --git a/cvat/schema.yml b/cvat/schema.yml index 4602e36b1c34..e48b9e4fb08b 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9659,7 +9659,8 @@ components: description: | Like IoU threshold, but for points. The percent of the bbox side, used as the radius of the circle around the GT point, - where the checked point is expected to be. + where the checked point is expected to be. For boxes with different width and + height, the "side" is computed as a geometric mean of the width and height. Read more: https://cocodataset.org/#keypoints-eval point_size_base: allOf: @@ -10165,7 +10166,8 @@ components: description: | Like IoU threshold, but for points. The percent of the bbox side, used as the radius of the circle around the GT point, - where the checked point is expected to be. + where the checked point is expected to be. For boxes with different width and + height, the "side" is computed as a geometric mean of the width and height. Read more: https://cocodataset.org/#keypoints-eval point_size_base: allOf: From 7e84f3a91e0b9287c115f3bf3fbfa3a6e04b7332 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Thu, 7 Nov 2024 11:24:59 +0200 Subject: [PATCH 16/18] Update schema --- cvat/schema.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cvat/schema.yml b/cvat/schema.yml index d020defcb7c7..1938cabc5071 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9736,6 +9736,7 @@ components: Count empty frames as matching. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. PatchedTaskValidationLayoutWriteRequest: type: object properties: @@ -10250,6 +10251,7 @@ components: Count empty frames as matching. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. RegisterSerializerEx: type: object properties: From f8a82495aa709d520080712c1e843cd7660512c5 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 8 Nov 2024 12:33:13 +0200 Subject: [PATCH 17/18] Fix missing mean_iou value --- cvat/apps/quality_control/quality_reports.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index dda218dbe414..a28cfec509df 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -2136,6 +2136,9 @@ def _generate_dataset_annotations_summary( annotation_summary.ds_count += empty_frame_count annotation_summary.gt_count += empty_frame_count + # Cannot be computed in accumulate() + annotation_components.shape.mean_iou = np.mean(mean_ious) + return annotation_summary, annotation_components def generate_report(self) -> ComparisonReport: From b42daa3573ae99e6ce1ba83e6ca8a0c4e63e67ad Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 8 Nov 2024 18:29:05 +0200 Subject: [PATCH 18/18] Add missing shapes counts, rename function --- cvat/apps/quality_control/quality_reports.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index a28cfec509df..f5e527468aa3 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -1986,6 +1986,8 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): valid_shapes_count = 1 total_shapes_count = 1 + ds_shapes_count = 1 + gt_shapes_count = 1 self._frame_results[frame_id] = ComparisonReportFrameSummary( annotations=self._generate_frame_annotations_summary( @@ -2030,7 +2032,7 @@ def _make_zero_confusion_matrix(self) -> Tuple[List[str], np.ndarray, Dict[int, return label_names, confusion_matrix, label_id_idx_map - def _compute_annotation_summary( + def _compute_annotations_summary( self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] ) -> ComparisonReportAnnotationsSummary: matched_ann_counts = np.diag(confusion_matrix) @@ -2076,7 +2078,7 @@ def _compute_annotation_summary( def _generate_frame_annotations_summary( self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] ) -> ComparisonReportAnnotationsSummary: - summary = self._compute_annotation_summary(confusion_matrix, confusion_matrix_labels) + summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels) if self.settings.match_empty_frames and summary.total_count == 0: # Add virtual annotations for empty frames @@ -2124,7 +2126,7 @@ def _generate_dataset_annotations_summary( mean_ious.append(frame_result.annotation_components.shape.mean_iou) - annotation_summary = self._compute_annotation_summary( + annotation_summary = self._compute_annotations_summary( confusion_matrix, confusion_matrix_labels )