diff --git a/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md b/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md new file mode 100644 index 000000000000..e615de6354c5 --- /dev/null +++ b/changelog.d/20241106_170626_mzhiltso_match_empty_frames.md @@ -0,0 +1,4 @@ +### Added + +- A quality check option to consider empty frames matching + () diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 8a3b5d28888c..7c591e371cc4 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -38,6 +38,7 @@ export default class QualitySettings { #objectVisibilityThreshold: number; #panopticComparison: boolean; #compareAttributes: boolean; + #matchEmptyFrames: boolean; #descriptions: Record; constructor(initialData: SerializedQualitySettingsData) { @@ -59,6 +60,7 @@ export default class QualitySettings { this.#objectVisibilityThreshold = initialData.object_visibility_threshold; this.#panopticComparison = initialData.panoptic_comparison; this.#compareAttributes = initialData.compare_attributes; + this.#matchEmptyFrames = initialData.match_empty_frames; this.#descriptions = initialData.descriptions; } @@ -198,6 +200,14 @@ export default class QualitySettings { this.#maxValidationsPerJob = newVal; } + get matchEmptyFrames(): boolean { + return this.#matchEmptyFrames; + } + + set matchEmptyFrames(newVal: boolean) { + this.#matchEmptyFrames = newVal; + } + get descriptions(): Record { const descriptions: Record = Object.keys(this.#descriptions).reduce((acc, key) => { const camelCaseKey = _.camelCase(key); @@ -226,6 +236,7 @@ export default class QualitySettings { target_metric: this.#targetMetric, target_metric_threshold: this.#targetMetricThreshold, max_validations_per_job: this.#maxValidationsPerJob, + match_empty_frames: this.#matchEmptyFrames, }; return result; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index 649b2c509a23..ea97c0730aaa 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -258,6 +258,7 @@ export interface SerializedQualitySettingsData { object_visibility_threshold?: number; panoptic_comparison?: boolean; compare_attributes?: boolean; + match_empty_frames?: boolean; descriptions?: Record; } diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index 0a3914712f87..cbaa26a8dd09 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -223,6 +223,7 @@ function QualityControlPage(): JSX.Element { settings.lowOverlapThreshold = values.lowOverlapThreshold / 100; settings.iouThreshold = values.iouThreshold / 100; settings.compareAttributes = values.compareAttributes; + settings.matchEmptyFrames = values.matchEmptyFrames; settings.oksSigma = values.oksSigma / 100; settings.pointSizeBase = values.pointSizeBase; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 2633fae6bfbe..87a727f9772b 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -34,6 +34,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element lowOverlapThreshold: settings.lowOverlapThreshold * 100, iouThreshold: settings.iouThreshold * 100, compareAttributes: settings.compareAttributes, + matchEmptyFrames: settings.matchEmptyFrames, oksSigma: settings.oksSigma * 100, pointSizeBase: settings.pointSizeBase, @@ -79,6 +80,8 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element <> {makeTooltipFragment('Target metric', targetMetricDescription)} {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)} + {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)} + {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)} , ); @@ -181,6 +184,30 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element + + + + + Compare attributes + + + + + + + Match empty frames + + + + diff --git a/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py new file mode 100644 index 000000000000..dba6a0c9bc43 --- /dev/null +++ b/cvat/apps/quality_control/migrations/0005_qualitysettings_match_empty.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-11-05 14:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0004_qualitysettings_point_size_base"), + ] + + operations = [ + migrations.AddField( + model_name="qualitysettings", + name="match_empty_frames", + field=models.BooleanField(default=False), + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index cfa2522541df..b8cf76873597 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -234,6 +234,8 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() + match_empty_frames = models.BooleanField(default=False) + target_metric = models.CharField( max_length=32, choices=QualityTargetMetricType.choices(), diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 72a54bf10a77..f5e527468aa3 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -217,6 +217,13 @@ class ComparisonParameters(_Serializable): panoptic_comparison: bool = True "Use only the visible part of the masks and polygons in comparisons" + match_empty_frames: bool = False + """ + Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, + will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. + """ + def _value_serializer(self, v): if isinstance(v, dm.AnnotationType): return str(v.name) @@ -232,11 +239,11 @@ def from_dict(cls, d: dict): @define(kw_only=True) class ConfusionMatrix(_Serializable): labels: List[str] - rows: np.array - precision: np.array - recall: np.array - accuracy: np.array - jaccard_index: Optional[np.array] + rows: np.ndarray + precision: np.ndarray + recall: np.ndarray + accuracy: np.ndarray + jaccard_index: Optional[np.ndarray] @property def axes(self): @@ -1972,8 +1979,18 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX confusion_matrix[ds_label_idx, gt_label_idx] += 1 + if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations: + # Add virtual annotations for empty frames + valid_labels_count = 1 + total_labels_count = 1 + + valid_shapes_count = 1 + total_shapes_count = 1 + ds_shapes_count = 1 + gt_shapes_count = 1 + self._frame_results[frame_id] = ComparisonReportFrameSummary( - annotations=self._generate_annotations_summary( + annotations=self._generate_frame_annotations_summary( confusion_matrix, confusion_matrix_labels ), annotation_components=ComparisonReportAnnotationComponentsSummary( @@ -2015,9 +2032,8 @@ def _make_zero_confusion_matrix(self) -> Tuple[List[str], np.ndarray, Dict[int, return label_names, confusion_matrix, label_id_idx_map - @classmethod - def _generate_annotations_summary( - cls, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] + def _compute_annotations_summary( + self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] ) -> ComparisonReportAnnotationsSummary: matched_ann_counts = np.diag(confusion_matrix) ds_ann_counts = np.sum(confusion_matrix, axis=1) @@ -2037,10 +2053,10 @@ def _generate_annotations_summary( ) / (total_annotations_count or 1) valid_annotations_count = np.sum(matched_ann_counts) - missing_annotations_count = np.sum(confusion_matrix[cls._UNMATCHED_IDX, :]) - extra_annotations_count = np.sum(confusion_matrix[:, cls._UNMATCHED_IDX]) - ds_annotations_count = np.sum(ds_ann_counts[: cls._UNMATCHED_IDX]) - gt_annotations_count = np.sum(gt_ann_counts[: cls._UNMATCHED_IDX]) + missing_annotations_count = np.sum(confusion_matrix[self._UNMATCHED_IDX, :]) + extra_annotations_count = np.sum(confusion_matrix[:, self._UNMATCHED_IDX]) + ds_annotations_count = np.sum(ds_ann_counts[: self._UNMATCHED_IDX]) + gt_annotations_count = np.sum(gt_ann_counts[: self._UNMATCHED_IDX]) return ComparisonReportAnnotationsSummary( valid_count=valid_annotations_count, @@ -2059,12 +2075,24 @@ def _generate_annotations_summary( ), ) - def generate_report(self) -> ComparisonReport: - self._find_gt_conflicts() + def _generate_frame_annotations_summary( + self, confusion_matrix: np.ndarray, confusion_matrix_labels: List[str] + ) -> ComparisonReportAnnotationsSummary: + summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels) + + if self.settings.match_empty_frames and summary.total_count == 0: + # Add virtual annotations for empty frames + summary.valid_count = 1 + summary.total_count = 1 + summary.ds_count = 1 + summary.gt_count = 1 + return summary + + def _generate_dataset_annotations_summary( + self, frame_summaries: Dict[int, ComparisonReportFrameSummary] + ) -> Tuple[ComparisonReportAnnotationsSummary, ComparisonReportAnnotationComponentsSummary]: # accumulate stats - intersection_frames = [] - conflicts = [] annotation_components = ComparisonReportAnnotationComponentsSummary( shape=ComparisonReportAnnotationShapeSummary( valid_count=0, @@ -2082,19 +2110,52 @@ def generate_report(self) -> ComparisonReport: ), ) mean_ious = [] + empty_frame_count = 0 confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix() - for frame_id, frame_result in self._frame_results.items(): - intersection_frames.append(frame_id) - conflicts += frame_result.conflicts + for frame_result in frame_summaries.values(): confusion_matrix += frame_result.annotations.confusion_matrix.rows + if not np.any(frame_result.annotations.confusion_matrix.rows): + empty_frame_count += 1 + if annotation_components is None: annotation_components = deepcopy(frame_result.annotation_components) else: annotation_components.accumulate(frame_result.annotation_components) + mean_ious.append(frame_result.annotation_components.shape.mean_iou) + annotation_summary = self._compute_annotations_summary( + confusion_matrix, confusion_matrix_labels + ) + + if self.settings.match_empty_frames and empty_frame_count: + # Add virtual annotations for empty frames, + # they are not included in the confusion matrix + annotation_summary.valid_count += empty_frame_count + annotation_summary.total_count += empty_frame_count + annotation_summary.ds_count += empty_frame_count + annotation_summary.gt_count += empty_frame_count + + # Cannot be computed in accumulate() + annotation_components.shape.mean_iou = np.mean(mean_ious) + + return annotation_summary, annotation_components + + def generate_report(self) -> ComparisonReport: + self._find_gt_conflicts() + + intersection_frames = [] + conflicts = [] + for frame_id, frame_result in self._frame_results.items(): + intersection_frames.append(frame_id) + conflicts += frame_result.conflicts + + annotation_summary, annotations_component_summary = ( + self._generate_dataset_annotations_summary(self._frame_results) + ) + return ComparisonReport( parameters=self.settings, comparison_summary=ComparisonReportComparisonSummary( @@ -2110,25 +2171,8 @@ def generate_report(self) -> ComparisonReport: [c for c in conflicts if c.severity == AnnotationConflictSeverity.ERROR] ), conflicts_by_type=Counter(c.type for c in conflicts), - annotations=self._generate_annotations_summary( - confusion_matrix, confusion_matrix_labels - ), - annotation_components=ComparisonReportAnnotationComponentsSummary( - shape=ComparisonReportAnnotationShapeSummary( - valid_count=annotation_components.shape.valid_count, - missing_count=annotation_components.shape.missing_count, - extra_count=annotation_components.shape.extra_count, - total_count=annotation_components.shape.total_count, - ds_count=annotation_components.shape.ds_count, - gt_count=annotation_components.shape.gt_count, - mean_iou=np.mean(mean_ious), - ), - label=ComparisonReportAnnotationLabelSummary( - valid_count=annotation_components.label.valid_count, - invalid_count=annotation_components.label.invalid_count, - total_count=annotation_components.label.total_count, - ), - ), + annotations=annotation_summary, + annotation_components=annotations_component_summary, ), frame_results=self._frame_results, ) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index fb38677636fd..6164abc12200 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -92,6 +92,7 @@ class Meta: "object_visibility_threshold", "panoptic_comparison", "compare_attributes", + "match_empty_frames", ) read_only_fields = ( "id", @@ -99,6 +100,7 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} + extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -164,6 +166,12 @@ class Meta: Use only the visible part of the masks and polygons in comparisons """, "compare_attributes": "Enables or disables annotation attribute comparison", + "match_empty_frames": """ + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. + """, }.items(): extra_kwargs.setdefault(field_name, {}).setdefault( "help_text", textwrap.dedent(help_text.lstrip("\n")) diff --git a/cvat/schema.yml b/cvat/schema.yml index e48b9e4fb08b..1938cabc5071 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9729,6 +9729,14 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison + match_empty_frames: + type: boolean + default: false + description: | + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. PatchedTaskValidationLayoutWriteRequest: type: object properties: @@ -10236,6 +10244,14 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison + match_empty_frames: + type: boolean + default: false + description: | + Count empty frames as matching. This affects target metrics like accuracy in cases + there are no annotations. If disabled, frames without annotations + are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. + This will also add virtual annotations to empty frames in the comparison results. RegisterSerializerEx: type: object properties: diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index 1865d59f94cd..d03675c9156e 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1213,6 +1213,7 @@ def test_modified_task_produces_different_metrics( "compare_line_orientation", "panoptic_comparison", "point_size_base", + "match_empty_frames", ], ) def test_settings_affect_metrics( @@ -1245,7 +1246,12 @@ def test_settings_affect_metrics( ) new_report = self.create_quality_report(admin_user, task_id) - assert new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] + if parameter == "match_empty_frames": + assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"] + else: + assert ( + new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] + ) def test_old_report_can_be_loaded(self, admin_user, quality_reports): report = min((r for r in quality_reports if r["task_id"]), key=lambda r: r["id"]) diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index e4e912b95863..5b30d421cb5a 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18173,6 +18173,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18196,6 +18197,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18219,6 +18221,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18242,6 +18245,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18265,6 +18269,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18288,6 +18293,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18311,6 +18317,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18334,6 +18341,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18357,6 +18365,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18380,6 +18389,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18403,6 +18413,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18426,6 +18437,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18449,6 +18461,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18472,6 +18485,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18495,6 +18509,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18518,6 +18533,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18541,6 +18557,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18564,6 +18581,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18587,6 +18605,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18610,6 +18629,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18633,6 +18653,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18656,6 +18677,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18679,6 +18701,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18702,6 +18725,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, + "match_empty_frames": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index e6e4cba929b1..7ddc589bc7bf 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -14,6 +14,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -34,6 +35,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -54,6 +56,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -74,6 +77,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -94,6 +98,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -114,6 +119,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -134,6 +140,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -154,6 +161,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -174,6 +182,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -194,6 +203,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -214,6 +224,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -234,6 +245,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -254,6 +266,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -274,6 +287,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -294,6 +308,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -314,6 +329,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -334,6 +350,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -354,6 +371,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -374,6 +392,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -394,6 +413,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -414,6 +434,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -434,6 +455,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -454,6 +476,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -474,6 +497,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, + "match_empty_frames": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09,