From 40b908d59d3951fb2a5f36f74ea14806148518f0 Mon Sep 17 00:00:00 2001 From: Ben Epstein Date: Thu, 18 May 2023 12:19:23 -0400 Subject: [PATCH 1/2] inference + alert clearing for reprocess_run (#610) 1. Remove alerts before reprocessing 2. Handle inference and non-inference case # Tested with a run with inference and without ![image](https://github.com/rungalileo/dataquality/assets/22605641/1c171502-5a5e-46e4-9319-5cc1c405cd4e) --- dataquality/clients/api.py | 37 +++++++++++++++++++++++++++++++++--- dataquality/internal.py | 18 +++++++++++++----- dataquality/metrics.py | 8 ++++---- dataquality/schemas/route.py | 2 +- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/dataquality/clients/api.py b/dataquality/clients/api.py index d7f1b7704..dc39c5520 100644 --- a/dataquality/clients/api.py +++ b/dataquality/clients/api.py @@ -619,20 +619,51 @@ def get_column_distribution( body = {"task": task, "filter_params": filter_params or {}} return self.make_request(RequestType.POST, url, body=body, params=params) - def get_xray_cards( + def get_alerts( self, project_name: str, run_name: str, split: str, inference_name: Optional[str] = None, ) -> List[Dict[str, str]]: - """Queries API for xray cards for a run/split""" + """Queries API for alerts for a run/split""" project, run = self._get_project_run_id(project_name, run_name) path = Route.content_path(project, run, split) - url = f"{config.api_url}/{path}/{Route.xray}" + url = f"{config.api_url}/{path}/{Route.alerts}" params = {"inference_name": inference_name} if inference_name else None return self.make_request(RequestType.GET, url, params=params) + def delete_alerts_for_split( + self, project_id: UUID4, run_id: UUID4, split: str + ) -> None: + path = Route.content_path(project_id, run_id, split) + url = f"{config.api_url}/{path}/{Route.alerts}" + alerts = [] + if split == "inference": + inference_names = self.get_inference_names(project_id, run_id) + for inf_name in inference_names["inference_names"]: + params = {"inference_name": inf_name} + res = self.make_request(RequestType.GET, url, params=params) + alerts.extend([alert["id"] for alert in res]) + else: + res = self.make_request(RequestType.GET, url) + alerts.extend([alert["id"] for alert in res]) + path = Route.content_path(project_id, run_id, split) + for alert_id in alerts: + url = f"{config.api_url}/{path}/{Route.alerts}/{alert_id}" + self.make_request(RequestType.DELETE, url) + + def delete_alerts( + self, + project_name: str, + run_name: str, + ) -> None: + """Delete all alerts for a run""" + project_id, run_id = self._get_project_run_id(project_name, run_name) + for split in self.get_splits(project_id, run_id)["splits"]: + self.delete_alerts_for_split(project_id, run_id, split) + print(f"All alerts removed for run {project_name}/{run_name}") + def get_edits( self, project_name: str, diff --git a/dataquality/internal.py b/dataquality/internal.py index af1fd3b1c..f376b14c5 100644 --- a/dataquality/internal.py +++ b/dataquality/internal.py @@ -10,7 +10,7 @@ def reprocess_run( - project_name: str, run_name: str, alerts: bool = False, wait: bool = True + project_name: str, run_name: str, alerts: bool = True, wait: bool = True ) -> None: """Reprocesses a run that has already been processed by Galileo @@ -19,9 +19,8 @@ def reprocess_run( :param project_name: The name of the project :param run_name: The name of the run - :param alerts: Whether to create the alerts. Currently, this will not delete the - existing alerts, so they will be duplicated if they already exist. This - feature will come soon + :param alerts: Whether to create the alerts. If True, all alerts for the run will + be removed, and recreated during processing. Default True :param wait: Whether to wait for the run to complete processing on the server. If True, this will block execution, printing out the status updates of the run. Useful if you want to know exactly when your run completes. Otherwise, this will @@ -52,8 +51,10 @@ def reprocess_run( tasks = api_client.get_tasks_for_run(project_name, run_name) if task_type == TaskType.text_ner: # In NER, dq.metrics.get_labels_for_run will return the _full_ label set in NER - # form (ie B-PER, I-PER, O-PER, B-LOC, etc) which is needed for processeing + # form (ie B-PER, I-PER, O-PER, B-LOC, etc) which is needed for processing ner_labels = dataquality.metrics.get_labels_for_run(project_name, run_name) + if alerts: + api_client.delete_alerts(project_name, run_name) body = dict( project_id=str(project), run_id=str(run), @@ -62,6 +63,13 @@ def reprocess_run( task_type=task_type, ner_labels=ner_labels, xray=alerts, + # We set the job name to inference and non_inference_logged to True because + # This will force the server to first reprocess the non-inference splits, + # and then reprocess all of the inference splits. If there are no inference + # splits, this will still work as expected, inference will just be skipped + job_name="inference", + process_existing_inference_runs=True, + non_inference_logged=True, ) res = api_client.make_request( RequestType.POST, url=f"{config.api_url}/{Route.jobs}", body=body diff --git a/dataquality/metrics.py b/dataquality/metrics.py index b27feb522..716f69ecc 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -643,15 +643,15 @@ def get_raw_data( ) -def get_xray_cards( +def get_alerts( project_name: str, run_name: str, split: Split, inference_name: Optional[str] = None ) -> List[Dict[str, str]]: - """Get xray cards for a project/run/split + """Get alerts for a project/run/split - Xray cards are automatic insights calculated and provided by Galileo on your data + Alerts are automatic insights calculated and provided by Galileo on your data """ split = conform_split(split) - return api_client.get_xray_cards(project_name, run_name, split, inference_name) + return api_client.get_alerts(project_name, run_name, split, inference_name) def get_labels_for_run( diff --git a/dataquality/schemas/route.py b/dataquality/schemas/route.py index 107a17ca2..89232b794 100644 --- a/dataquality/schemas/route.py +++ b/dataquality/schemas/route.py @@ -33,7 +33,7 @@ class Route(str, Enum): groupby = "insights/groupby" metrics = "metrics" distribution = "insights/distribution" - xray = "insights/xray" + alerts = "insights/alerts" export = "export" edits = "edits" export_edits = "edits/export" From 0945629c745adae221941b811acdf57f880a7e8e Mon Sep 17 00:00:00 2001 From: franz101 Date: Thu, 18 May 2023 11:56:35 -0700 Subject: [PATCH 2/2] SetFit: When logging probs, log them not as logits (#612) This is a proposed fixed that might resolve quirks around the DEP reported for setfit --- dataquality/__init__.py | 2 +- dataquality/integrations/setfit.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dataquality/__init__.py b/dataquality/__init__.py index 40eebaf92..44afa265b 100644 --- a/dataquality/__init__.py +++ b/dataquality/__init__.py @@ -30,7 +30,7 @@ dataquality.get_insights() """ -__version__ = "v0.8.42" +__version__ = "v0.8.43" import sys from typing import Any, List, Optional diff --git a/dataquality/integrations/setfit.py b/dataquality/integrations/setfit.py index 248b64a33..450789276 100644 --- a/dataquality/integrations/setfit.py +++ b/dataquality/integrations/setfit.py @@ -246,7 +246,7 @@ def __call__(self, *args: Any, **kwds: Any) -> Any: # 🔭🌕 Galileo logging dq.log_model_outputs( ids=batch["id"], - logits=dq_store["output"], + probs=dq_store["output"], embs=dq_store["input_args"][0], split=split, epoch=0, @@ -362,7 +362,7 @@ def dq_evaluate( # 🔭🌕 Galileo logging dq.log_model_outputs( ids=batch[id_col], - logits=dq_store["output"], + probs=dq_store["output"], embs=dq_store["input_args"][0], split=split, epoch=0,