From 8d56e67c8f22ca08e586bd42add624f72f8b7658 Mon Sep 17 00:00:00 2001 From: imanjra Date: Tue, 5 Nov 2024 12:01:26 -0500 Subject: [PATCH] add pending evaluation cards --- .../NativeModelEvaluationView/Overview.tsx | 86 +++++++++++++++---- .../NativeModelEvaluationView/index.tsx | 3 + .../panels/model_evaluation/__init__.py | 53 +++++++++++- 3 files changed, 122 insertions(+), 20 deletions(-) diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Overview.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Overview.tsx index 942376f275e..85532c38a1a 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Overview.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Overview.tsx @@ -1,4 +1,5 @@ -import { Card, CardActionArea, Stack, Typography } from "@mui/material"; +import { LoadingDots } from "@fiftyone/components"; +import { Card, CardActionArea, Chip, Stack, Typography } from "@mui/material"; import React from "react"; import Evaluate from "./Evaluate"; import EvaluationNotes from "./EvaluationNotes"; @@ -12,6 +13,7 @@ export default function Overview(props: OverviewProps) { statuses = {}, notes = {}, permissions = {}, + pending_evaluations, } = props; const count = evaluations.length; @@ -24,36 +26,67 @@ export default function Overview(props: OverviewProps) { {count} Model Evaluations - + {evaluations.map((evaluation) => { const { key, id } = evaluation; const status = statuses[id] || "needs_review"; const note = notes[id]; - return ( - - { - onSelect(key, id); - }} - > - - - {key} - - - - - - + + ); + })} + {pending_evaluations.map((evaluation) => { + const { eval_key } = evaluation; + return ( + ); })} ); } +function EvaluationCard(props: EvaluationCardProps) { + const { pending, onSelect, eval_key, note, status, id } = props; + return ( + + { + onSelect(eval_key, id); + }} + > + + + {eval_key} + + {pending && ( + } /> + )} + {status && } + + {note && } + + + ); +} + type OverviewProps = { evaluations: EvaluationType[]; onSelect: (key: string, id: string) => void; @@ -61,6 +94,7 @@ type OverviewProps = { statuses?: Record; notes?: Record; permissions?: Record; + pending_evaluations: PendingEvaluationType[]; }; type EvaluationType = { @@ -69,3 +103,17 @@ type EvaluationType = { description: string; status: string; }; + +type PendingEvaluationType = { + eval_key: string; + doc_id?: string; +}; + +type EvaluationCardProps = { + eval_key: string; + id?: string; + note?: string; + onSelect: OverviewProps["onSelect"]; + pending?: boolean; + status?: string; +}; diff --git a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx index 37e52e04a7a..191189e567f 100644 --- a/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx +++ b/app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx @@ -23,6 +23,7 @@ export default function NativeModelEvaluationView(props) { statuses = {}, notes = {}, permissions = {}, + pending_evaluations = [], } = data; const computedEvaluations = useMemo(() => { return evaluations.map(({ key, id }) => ({ @@ -77,6 +78,7 @@ export default function NativeModelEvaluationView(props) { onEvaluate={() => { triggerEvent(on_evaluate_model); }} + permissions={permissions} /> ) : ( ))} diff --git a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py index 7949c57ac29..dd5633f8904 100644 --- a/fiftyone/operators/builtins/panels/model_evaluation/__init__.py +++ b/fiftyone/operators/builtins/panels/model_evaluation/__init__.py @@ -30,6 +30,9 @@ def config(self): is_new=is_new("2024-11-07"), ) + def get_dataset_id(self, ctx): + return str(ctx.dataset._doc.id) + def get_store(self, ctx): return ctx.create_store(STORE_NAME) @@ -76,6 +79,7 @@ def on_load(self, ctx): ctx.panel.set_data("statuses", statuses) ctx.panel.set_data("notes", notes) ctx.panel.set_data("permissions", permissions) + self.load_pending_evaluations(ctx) # keys = ctx.dataset.list_evaluations() # ctx.panel.set_state("keys", keys) @@ -266,6 +270,50 @@ def on_change_view(self, ctx): # Used only for triggering re-renders when the view changes pass + def load_pending_evaluations(self, ctx, skip_update=False): + store = self.get_store(ctx) + dataset_id = self.get_dataset_id(ctx) + pending_evaluations = store.get("pending_evaluations") or {} + pending = pending_evaluations.get(dataset_id, []) + if not skip_update: + eval_keys = ctx.dataset.list_evaluations() + updated_pending = [] + update = False + for item in pending: + pending_eval_key = item.get("eval_key") + if pending_eval_key in eval_keys: + update = True + else: + updated_pending.append(item) + if update: + pending_evaluations[dataset_id] = updated_pending + store.set("pending_evaluations", pending_evaluations) + pending = updated_pending + ctx.panel.set_data("pending_evaluations", pending) + + def on_evaluate_model_success(self, ctx): + dataset_id = self.get_dataset_id(ctx) + store = self.get_store(ctx) + result = ctx.params.get("result", {}) + doc_id = result.get("id") + delegated_eval_key = ( + result.get("context", {}).get("params", {}).get("eval_key") + ) + eval_key = result.get("eval_key", delegated_eval_key) + pending = {} + if doc_id is None: + pending["eval_key"] = eval_key + else: + pending["doc_id"] = str(doc_id) + pending["eval_key"] = eval_key + + pending_evaluations = store.get("pending_evaluations") or {} + if dataset_id not in pending_evaluations: + pending_evaluations[dataset_id] = [] + pending_evaluations[dataset_id].append(pending) + store.set("pending_evaluations", pending_evaluations) + self.load_pending_evaluations(ctx, True) + def on_evaluate_model(self, ctx): if not self.can_evaluate(ctx): ctx.ops.notify( @@ -274,7 +322,10 @@ def on_evaluate_model(self, ctx): ) return # Called when you click the "Evaluate Model" button - ctx.prompt("@voxel51/evaluation/evaluate_model") + ctx.prompt( + "@voxel51/operators/evaluate_model_async", + on_success=self.on_evaluate_model_success, + ) # ctx.panel.state.view = "eval" def load_view(self, ctx):