From 33a2dac86f8f815bbc28612d71dbc169adfd35b4 Mon Sep 17 00:00:00 2001 From: brimoor Date: Tue, 31 Dec 2024 19:31:50 -0500 Subject: [PATCH] store matches instead --- fiftyone/utils/eval/segmentation.py | 148 ++++++++--------------- plugins/panels/model_evaluation.py | 176 ++++++++++++++++++---------- 2 files changed, 161 insertions(+), 163 deletions(-) diff --git a/fiftyone/utils/eval/segmentation.py b/fiftyone/utils/eval/segmentation.py index b63355dc2fe..e3878608a6c 100644 --- a/fiftyone/utils/eval/segmentation.py +++ b/fiftyone/utils/eval/segmentation.py @@ -8,6 +8,7 @@ from copy import deepcopy import logging import inspect +import itertools import warnings import numpy as np @@ -349,9 +350,7 @@ def evaluate_samples( nc = len(values) confusion_matrix = np.zeros((nc, nc), dtype=int) - weights_dict = {} - ytrue_ids_dict = {} - ypred_ids_dict = {} + matches = [] bandwidth = self.config.bandwidth average = self.config.average @@ -394,18 +393,16 @@ def evaluate_samples( ) sample_conf_mat += image_conf_mat - for index in zip(*np.nonzero(image_conf_mat)): - if index not in weights_dict: - weights_dict[index] = [] - weights_dict[index].append(int(image_conf_mat[index])) - - if index not in ytrue_ids_dict: - ytrue_ids_dict[index] = [] - ytrue_ids_dict[index].append(gt_seg.id) - - if index not in ypred_ids_dict: - ypred_ids_dict[index] = [] - ypred_ids_dict[index].append(pred_seg.id) + for i, j in zip(*np.nonzero(image_conf_mat)): + matches.append( + ( + classes[i], + classes[j], + int(image_conf_mat[i, j]), + gt_seg.id, + pred_seg.id, + ) + ) if processing_frames and save: facc, fpre, frec = _compute_accuracy_precision_recall( @@ -440,9 +437,7 @@ def evaluate_samples( eval_key, confusion_matrix, classes, - weights_dict=weights_dict, - ytrue_ids_dict=ytrue_ids_dict, - ypred_ids_dict=ypred_ids_dict, + matches=matches, missing=missing, backend=self, ) @@ -457,11 +452,9 @@ class SegmentationResults(BaseEvaluationResults): eval_key: the evaluation key pixel_confusion_matrix: a pixel value confusion matrix classes: a list of class labels corresponding to the confusion matrix - weights_dict (None): a dict mapping ``(i, j)`` tuples to pixel counts - ytrue_ids_dict (None): a dict mapping ``(i, j)`` tuples to lists of - ground truth IDs - ypred_ids_dict (None): a dict mapping ``(i, j)`` tuples to lists of - predicted label IDs + matches (None): a list of + ``(gt_label, pred_label, pixel_count, gt_id, pred_id)`` + matches missing (None): a missing (background) class backend (None): a :class:`SegmentationEvaluation` backend """ @@ -473,27 +466,20 @@ def __init__( eval_key, pixel_confusion_matrix, classes, - weights_dict=None, - ytrue_ids_dict=None, - ypred_ids_dict=None, + matches=None, missing=None, backend=None, ): pixel_confusion_matrix = np.asarray(pixel_confusion_matrix) - ( - ytrue, - ypred, - weights, - ytrue_ids, - ypred_ids, - ) = self._parse_confusion_matrix( - pixel_confusion_matrix, - classes, - weights_dict=weights_dict, - ytrue_ids_dict=ytrue_ids_dict, - ypred_ids_dict=ypred_ids_dict, - ) + if matches is not None: + ytrue, ypred, weights, ytrue_ids, ypred_ids = zip(*matches) + else: + ytrue, ypred, weights = self._parse_confusion_matrix( + pixel_confusion_matrix, classes + ) + ytrue_ids = None + ypred_ids = None super().__init__( samples, @@ -510,20 +496,6 @@ def __init__( ) self.pixel_confusion_matrix = pixel_confusion_matrix - self.weights_dict = weights_dict - self.ytrue_ids_dict = ytrue_ids_dict - self.ypred_ids_dict = ypred_ids_dict - - def attributes(self): - return [ - "cls", - "pixel_confusion_matrix", - "classes", - "weights_dict", - "ytrue_ids_dict", - "ypred_ids_dict", - "missing", - ] def dice_score(self): """Computes the Dice score across all samples in the evaluation. @@ -535,70 +507,50 @@ def dice_score(self): @classmethod def _from_dict(cls, d, samples, config, eval_key, **kwargs): + ytrue = d.get("ytrue", None) + ypred = d.get("ypred", None) + weights = d.get("weights", None) + ytrue_ids = d.get("ytrue_ids", None) + ypred_ids = d.get("ypred_ids", None) + + if ytrue is not None and ypred is not None and weights is not None: + if ytrue_ids is None: + ytrue_ids = itertools.repeat(None) + + if ypred_ids is None: + ypred_ids = itertools.repeat(None) + + matches = list(zip(ytrue, ypred, weights, ytrue_ids, ypred_ids)) + else: + # Legacy format segmentations + matches = None + return cls( samples, config, eval_key, d["pixel_confusion_matrix"], d["classes"], - weights_dict=_parse_index_dict(d.get("weights_dict", None)), - ytrue_ids_dict=_parse_index_dict(d.get("ytrue_ids_dict", None)), - ypred_ids_dict=_parse_index_dict(d.get("ypred_ids_dict", None)), + matches=matches, missing=d.get("missing", None), **kwargs, ) @staticmethod - def _parse_confusion_matrix( - confusion_matrix, - classes, - weights_dict=None, - ytrue_ids_dict=None, - ypred_ids_dict=None, - ): - have_ids = ytrue_ids_dict is not None and ypred_ids_dict is not None - + def _parse_confusion_matrix(confusion_matrix, classes): ytrue = [] ypred = [] weights = [] - if have_ids: - ytrue_ids = [] - ypred_ids = [] - else: - ytrue_ids = None - ypred_ids = None - nrows, ncols = confusion_matrix.shape for i in range(nrows): for j in range(ncols): cij = confusion_matrix[i, j] if cij > 0: - if have_ids: - index = (i, j) - classi = classes[i] - classj = classes[j] - for weight, ytrue_id, ypred_id in zip( - weights_dict[index], - ytrue_ids_dict[index], - ypred_ids_dict[index], - ): - ytrue.append(classi) - ypred.append(classj) - weights.append(weight) - ytrue_ids.append(ytrue_id) - ypred_ids.append(ypred_id) - else: - ytrue.append(classes[i]) - ypred.append(classes[j]) - weights.append(cij) - - return ytrue, ypred, weights, ytrue_ids, ypred_ids - - -def _parse_index_dict(d): - import ast - - return {ast.literal_eval(k): v for k, v in d.items()} + ytrue.append(classes[i]) + ypred.append(classes[j]) + weights.append(cij) + + return ytrue, ypred, weights def _parse_config(pred_field, gt_field, method, **kwargs): diff --git a/plugins/panels/model_evaluation.py b/plugins/panels/model_evaluation.py index 95b5f150909..f5f70e3ee1e 100644 --- a/plugins/panels/model_evaluation.py +++ b/plugins/panels/model_evaluation.py @@ -327,26 +327,6 @@ def get_mask_targets(self, dataset, gt_field): return None - def get_classes_map(self, dataset, results, gt_field): - classes = results.classes - - # - # `results.classes` could contain any of the following: - # 1. stringified pixel values - # 2. RGB hex strings - # 3. label strings - # - # If mask targets are available, then App callbacks will use label - # strings, so we convert to label strings here - # - mask_targets = self.get_mask_targets(dataset, gt_field) - if mask_targets is not None: - # `str()` handles cases 1 and 2, and `.get(c, c)` handles case 3 - mask_targets = {str(k): v for k, v in mask_targets.items()} - classes = [mask_targets.get(c, c) for c in classes] - - return {c: i for i, c in enumerate(classes)} - def load_evaluation(self, ctx): view_state = ctx.panel.get_state("view") or {} eval_key = view_state.get("key") @@ -367,13 +347,15 @@ def load_evaluation(self, ctx): {"error": "unsupported", "info": serialized_info}, ) return - gt_field = info.config.gt_field - mask_targets = ( - self.get_mask_targets(ctx.dataset, gt_field) - if evaluation_type == "segmentation" - else None - ) + results = ctx.dataset.load_evaluation_results(computed_eval_key) + gt_field = info.config.gt_field + mask_targets = None + + if evaluation_type == "segmentation": + mask_targets = self.get_mask_targets(ctx.dataset, gt_field) + _init_segmentation_results(results, mask_targets) + metrics = results.metrics() per_class_metrics = self.get_per_class_metrics(info, results) metrics["average_confidence"] = self.get_avg_confidence( @@ -612,12 +594,8 @@ def load_view(self, ctx): ) elif info.config.type == "segmentation": results = ctx.dataset.load_evaluation_results(eval_key) - classes_map = self.get_classes_map(ctx.dataset, results, gt_field) - if ( - results.ytrue_ids_dict is None - or results.ypred_ids_dict is None - ): - # legacy segmentation evaluation + if results.ytrue_ids is None or results.ypred_ids is None: + # Legacy format segmentations return if eval_key2: @@ -625,14 +603,8 @@ def load_view(self, ctx): gt_field2 = gt_field results2 = ctx.dataset.load_evaluation_results(eval_key2) - classes_map2 = self.get_classes_map( - ctx.dataset, results2, gt_field2 - ) - if ( - results2.ytrue_ids_dict is None - or results2.ypred_ids_dict is None - ): - # legacy segmentation evaluation + if results2.ytrue_ids is None or results2.ypred_ids is None: + # Legacy format segmentations return else: results2 = None @@ -648,26 +620,21 @@ def load_view(self, ctx): if view_type == "class": # All GT/predictions that contain class `x` - k = classes_map[x] - ytrue_ids, ypred_ids = _get_ids_slice(results, k) + ytrue_ids, ypred_ids = _get_segmentation_class_ids(results, x) expr = F(gt_id).is_in(ytrue_ids) expr |= F(pred_id).is_in(ypred_ids) if results2 is not None: - k2 = classes_map2[x] - ytrue_ids2, ypred_ids2 = _get_ids_slice(results2, k2) + ytrue_ids2, ypred_ids2 = _get_segmentation_class_ids( + results2, x + ) expr |= F(gt_id2).is_in(ytrue_ids2) expr |= F(pred_id2).is_in(ypred_ids2) view = eval_view.match(expr) elif view_type == "matrix": # Specific confusion matrix cell - i = classes_map[x] - j = classes_map[y] - ytrue_ids = _to_object_ids( - results.ytrue_ids_dict.get((i, j), []) - ) - ypred_ids = _to_object_ids( - results.ypred_ids_dict.get((i, j), []) + ytrue_ids, ypred_ids = _get_segmentation_conf_mat_ids( + results, x, y ) expr = F(gt_id).is_in(ytrue_ids) expr &= F(pred_id).is_in(ypred_ids) @@ -675,22 +642,24 @@ def load_view(self, ctx): elif view_type == "field": if field == "tp": # All true positives - inds = results.ytrue == results.ypred - ytrue_ids = _to_object_ids(results.ytrue_ids[inds]) - ypred_ids = _to_object_ids(results.ypred_ids[inds]) + ytrue_ids, ypred_ids = _get_segmentation_tp_fp_fn_ids( + results, field + ) expr = F(gt_id).is_in(ytrue_ids) expr &= F(pred_id).is_in(ypred_ids) view = eval_view.match(expr) elif field == "fn": # All false negatives - inds = results.ypred == missing - ytrue_ids = _to_object_ids(results.ytrue_ids[inds]) + ytrue_ids, _ = _get_segmentation_tp_fp_fn_ids( + results, field + ) expr = F(gt_id).is_in(ytrue_ids) view = eval_view.match(expr) else: # All false positives - inds = results.ytrue == missing - ypred_ids = _to_object_ids(results.ypred_ids[inds]) + _, ypred_ids = _get_segmentation_tp_fp_fn_ids( + results, field + ) expr = F(pred_id).is_in(ypred_ids) view = eval_view.match(expr) @@ -715,23 +684,100 @@ def render(self, ctx): ) -def _to_object_ids(ids): - return [ObjectId(_id) for _id in ids] - - -def _get_ids_slice(results, k): +def _init_segmentation_results(results, mask_targets): + if results.ytrue_ids is None or results.ypred_ids is None: + # Legacy format segmentations + return + + # + # `results.classes` and App callbacks could contain any of the + # following: + # 1. stringified pixel values + # 2. RGB hex strings + # 3. label strings + # + # so we must construct `classes_map` that can map any of these possible + # values to integer indexes + # + classes_map = {c: i for i, c in enumerate(results.classes)} + + if mask_targets is not None: + # `str()` handles cases 1 and 2, and `.get(c, c)` handles case 3 + mask_targets = {str(k): v for k, v in mask_targets.items()} + classes = [mask_targets.get(c, c) for c in results.classes] + classes_map.update({c: i for i, c in enumerate(classes)}) + + # + # Generate mapping from `(i, j)` to ID lists for use in App callbacks + # + + ytrue_ids_dict = {} + ypred_ids_dict = {} + for ytrue, ypred, ytrue_id, ypred_id in zip( + results.ytrue, results.ypred, results.ytrue_ids, results.ypred_ids + ): + i = classes_map[ytrue] + j = classes_map[ypred] + index = (i, j) + + if index not in ytrue_ids_dict: + ytrue_ids_dict[index] = [] + ytrue_ids_dict[index].append(ytrue_id) + + if index not in ypred_ids_dict: + ypred_ids_dict[index] = [] + ypred_ids_dict[index].append(ypred_id) + + results._classes_map = classes_map + results._ytrue_ids_dict = ytrue_ids_dict + results._ypred_ids_dict = ypred_ids_dict + + +def _get_segmentation_class_ids(results, x): + k = results._classes_map[x] nrows, ncols = results.pixel_confusion_matrix.shape ytrue_ids = [] for j in range(ncols): - _ytrue_ids = results.ytrue_ids_dict.get((k, j), None) + _ytrue_ids = results._ytrue_ids_dict.get((k, j), None) if _ytrue_ids is not None: ytrue_ids.extend(_ytrue_ids) ypred_ids = [] for i in range(nrows): - _ypred_ids = results.ypred_ids_dict.get((i, k), None) + _ypred_ids = results._ypred_ids_dict.get((i, k), None) if _ypred_ids is not None: ypred_ids.extend(_ypred_ids) return _to_object_ids(ytrue_ids), _to_object_ids(ypred_ids) + + +def _get_segmentation_conf_mat_ids(results, x, y): + i = results._classes_map[x] + j = results._classes_map[y] + ytrue_ids = _to_object_ids(results._ytrue_ids_dict.get((i, j), [])) + ypred_ids = _to_object_ids(results._ypred_ids_dict.get((i, j), [])) + return ytrue_ids, ypred_ids + + +def _get_segmentation_tp_fp_fn_ids(results, field): + if field == "tp": + # True positives + inds = results.ytrue == results.ypred + ytrue_ids = _to_object_ids(results.ytrue_ids[inds]) + ypred_ids = _to_object_ids(results.ypred_ids[inds]) + return ytrue_ids, ypred_ids + elif field == "fn": + # False negatives + inds = results.ypred == results.missing + ytrue_ids = _to_object_ids(results.ytrue_ids[inds]) + return ytrue_ids, None + else: + # False positives + inds = results.ytrue == results.missing + ypred_ids = _to_object_ids(results.ypred_ids[inds]) + return None, ypred_ids + + +def _to_object_ids(ids): + return [ObjectId(_id) for _id in ids]