From 88b9c1580da7fc0fbe0b05c16da34180a17b41d6 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Mon, 2 Dec 2024 10:45:34 -0500 Subject: [PATCH 01/20] [STAGE-21961] - [API, NLP] export parity between dq.get_dataframe() and NLP studio 1. Added `approved_only` optional argument to `ApiClient.get_edits()` and `ApiClient.export_edits()` procedures. This requires new `api` package version - which takes extra `approved_only` argument for its Get Run Edits request 2. Added `approved_only` optional argument to `metrics.get_edited_dataframe()` procedure Signed-off-by: Valeriy Pogrebitskiy --- dataquality/clients/api.py | 11 +++++++++-- dataquality/metrics.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dataquality/clients/api.py b/dataquality/clients/api.py index 9811ba253..312f2fd1a 100644 --- a/dataquality/clients/api.py +++ b/dataquality/clients/api.py @@ -21,6 +21,7 @@ class ApiClient: + def _refresh_jwt_token(self) -> str: username = os.getenv("GALILEO_USERNAME") password = os.getenv("GALILEO_PASSWORD") @@ -757,6 +758,7 @@ def get_edits( run_name: str, split: str, inference_name: Optional[str] = None, + approved_only: Optional[bool] = False, ) -> List: """Gets all edits for a run/split""" project, run = self._get_project_run_id(project_name, run_name) @@ -765,7 +767,7 @@ def get_edits( url = ( f"{config.api_url}/{Route.content_path(project, run, split)}/{Route.edits}" ) - params = {"inference_name": inference_name} if inference_name else None + params = {"inference_name": inference_name, "approved_only": approved_only} return self.make_request(RequestType.GET, url, params=params) def export_edits( @@ -779,6 +781,7 @@ def export_edits( col_mapping: Optional[Dict[str, str]] = None, hf_format: bool = False, tagging_schema: Optional[TaggingSchema] = None, + approved_only: Optional[bool] = False, ) -> None: """Export the edits of a project/run/split to disk as a file @@ -797,8 +800,12 @@ def export_edits( If hf_format is True, you must pass a tagging schema :param filter_params: Filters to apply to the dataframe before exporting. Only rows with matching filters will be included in the exported data. If a slice + :param approved_only: Whether to export only approved edits or all edits. + Default: False (all edits) """ - edits = self.get_edits(project_name, run_name, split, inference_name) + edits = self.get_edits( + project_name, run_name, split, inference_name, approved_only=approved_only + ) ext = os.path.splitext(file_name)[-1].lstrip(".") assert ext in list(FileType), f"File must be one of {list(FileType)}" diff --git a/dataquality/metrics.py b/dataquality/metrics.py index fe5b8ad22..ec5794f33 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -355,6 +355,7 @@ def get_edited_dataframe( include_token_indices: bool = False, hf_format: bool = False, tagging_schema: Optional[TaggingSchema] = None, + approved_only: Optional[bool] = False, as_pandas: bool = True, include_data_embs: bool = False, ) -> Union[pd.DataFrame, DataFrame]: @@ -383,6 +384,8 @@ def get_edited_dataframe( Whether to export the dataframe in a HuggingFace compatible format :param tagging_schema: (NER only) If hf_format is True, you must pass a tagging schema + :param approved_only: Whether to export only approved edits or all edits. + Default: False (all edits) :param as_pandas: Whether to return the dataframe as a pandas df (or vaex if False) If you are having memory issues (the data is too large), set this to False, and vaex will memory map the data. If any columns returned are multi-dimensional @@ -403,6 +406,7 @@ def get_edited_dataframe( file_name=file_name, hf_format=hf_format, tagging_schema=tagging_schema, + approved_only=approved_only, ) data_df = vaex.open(file_name) return _process_exported_dataframe( From b112fc113898995dfe1f822b1076aa7f0b4d88db Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Wed, 4 Dec 2024 15:44:38 -0500 Subject: [PATCH 02/20] ... Signed-off-by: Valeriy Pogrebitskiy --- dataquality/clients/api.py | 10 +++++----- dataquality/metrics.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dataquality/clients/api.py b/dataquality/clients/api.py index 312f2fd1a..68e2eeef8 100644 --- a/dataquality/clients/api.py +++ b/dataquality/clients/api.py @@ -758,7 +758,7 @@ def get_edits( run_name: str, split: str, inference_name: Optional[str] = None, - approved_only: Optional[bool] = False, + reviewed_only: Optional[bool] = False, ) -> List: """Gets all edits for a run/split""" project, run = self._get_project_run_id(project_name, run_name) @@ -767,7 +767,7 @@ def get_edits( url = ( f"{config.api_url}/{Route.content_path(project, run, split)}/{Route.edits}" ) - params = {"inference_name": inference_name, "approved_only": approved_only} + params = {"inference_name": inference_name, "reviewed_only": reviewed_only} return self.make_request(RequestType.GET, url, params=params) def export_edits( @@ -781,7 +781,7 @@ def export_edits( col_mapping: Optional[Dict[str, str]] = None, hf_format: bool = False, tagging_schema: Optional[TaggingSchema] = None, - approved_only: Optional[bool] = False, + reviewed_only: Optional[bool] = False, ) -> None: """Export the edits of a project/run/split to disk as a file @@ -800,11 +800,11 @@ def export_edits( If hf_format is True, you must pass a tagging schema :param filter_params: Filters to apply to the dataframe before exporting. Only rows with matching filters will be included in the exported data. If a slice - :param approved_only: Whether to export only approved edits or all edits. + :param reviewed_only: Whether to export only reviewed edits or all edits. Default: False (all edits) """ edits = self.get_edits( - project_name, run_name, split, inference_name, approved_only=approved_only + project_name, run_name, split, inference_name, reviewed_only=reviewed_only ) ext = os.path.splitext(file_name)[-1].lstrip(".") diff --git a/dataquality/metrics.py b/dataquality/metrics.py index ec5794f33..bd52888eb 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -355,7 +355,7 @@ def get_edited_dataframe( include_token_indices: bool = False, hf_format: bool = False, tagging_schema: Optional[TaggingSchema] = None, - approved_only: Optional[bool] = False, + reviewed_only: Optional[bool] = False, as_pandas: bool = True, include_data_embs: bool = False, ) -> Union[pd.DataFrame, DataFrame]: @@ -384,7 +384,7 @@ def get_edited_dataframe( Whether to export the dataframe in a HuggingFace compatible format :param tagging_schema: (NER only) If hf_format is True, you must pass a tagging schema - :param approved_only: Whether to export only approved edits or all edits. + :param reviewed_only: Whether to export only reviewed edits or all edits. Default: False (all edits) :param as_pandas: Whether to return the dataframe as a pandas df (or vaex if False) If you are having memory issues (the data is too large), set this to False, @@ -406,7 +406,7 @@ def get_edited_dataframe( file_name=file_name, hf_format=hf_format, tagging_schema=tagging_schema, - approved_only=approved_only, + reviewed_only=reviewed_only, ) data_df = vaex.open(file_name) return _process_exported_dataframe( From 02996c8afcd8c5af567f6dda53b892546cfc8ad5 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Thu, 5 Dec 2024 13:10:10 -0500 Subject: [PATCH 03/20] [SHORTCUT-21961] [API, NLP] export parity between dq.get_dataframe() and NLP studio Added optional argument `reviewed_only` to `metrics.get_edited_dataframe()` and a code to filter DataFrame to only contain reviewed edits when this argument is set to True. Signed-off-by: Valeriy Pogrebitskiy --- dataquality/metrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dataquality/metrics.py b/dataquality/metrics.py index bd52888eb..799ba3e6b 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -406,9 +406,13 @@ def get_edited_dataframe( file_name=file_name, hf_format=hf_format, tagging_schema=tagging_schema, - reviewed_only=reviewed_only, ) data_df = vaex.open(file_name) + + if reviewed_only: + # [SHORTCUT-21961] Filter DataFrame to contain only reviewed edits + data_df = data_df[data_df.reviewers.is_not_null().any(dim="list")] + return _process_exported_dataframe( data_df, project_name, From 280a9bac250fca78459842e3f01ae4c39ae9e7d3 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Thu, 5 Dec 2024 13:11:58 -0500 Subject: [PATCH 04/20] ... Signed-off-by: Valeriy Pogrebitskiy --- dataquality/clients/api.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/dataquality/clients/api.py b/dataquality/clients/api.py index 68e2eeef8..9811ba253 100644 --- a/dataquality/clients/api.py +++ b/dataquality/clients/api.py @@ -21,7 +21,6 @@ class ApiClient: - def _refresh_jwt_token(self) -> str: username = os.getenv("GALILEO_USERNAME") password = os.getenv("GALILEO_PASSWORD") @@ -758,7 +757,6 @@ def get_edits( run_name: str, split: str, inference_name: Optional[str] = None, - reviewed_only: Optional[bool] = False, ) -> List: """Gets all edits for a run/split""" project, run = self._get_project_run_id(project_name, run_name) @@ -767,7 +765,7 @@ def get_edits( url = ( f"{config.api_url}/{Route.content_path(project, run, split)}/{Route.edits}" ) - params = {"inference_name": inference_name, "reviewed_only": reviewed_only} + params = {"inference_name": inference_name} if inference_name else None return self.make_request(RequestType.GET, url, params=params) def export_edits( @@ -781,7 +779,6 @@ def export_edits( col_mapping: Optional[Dict[str, str]] = None, hf_format: bool = False, tagging_schema: Optional[TaggingSchema] = None, - reviewed_only: Optional[bool] = False, ) -> None: """Export the edits of a project/run/split to disk as a file @@ -800,12 +797,8 @@ def export_edits( If hf_format is True, you must pass a tagging schema :param filter_params: Filters to apply to the dataframe before exporting. Only rows with matching filters will be included in the exported data. If a slice - :param reviewed_only: Whether to export only reviewed edits or all edits. - Default: False (all edits) """ - edits = self.get_edits( - project_name, run_name, split, inference_name, reviewed_only=reviewed_only - ) + edits = self.get_edits(project_name, run_name, split, inference_name) ext = os.path.splitext(file_name)[-1].lstrip(".") assert ext in list(FileType), f"File must be one of {list(FileType)}" From 00f6601f24dcd9abc2450fb5775eb520c7090538 Mon Sep 17 00:00:00 2001 From: Valeriy Date: Thu, 5 Dec 2024 17:46:25 -0500 Subject: [PATCH 05/20] Update dataquality/metrics.py Co-authored-by: Setu Shah --- dataquality/metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dataquality/metrics.py b/dataquality/metrics.py index 799ba3e6b..9eb9411b4 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -410,7 +410,6 @@ def get_edited_dataframe( data_df = vaex.open(file_name) if reviewed_only: - # [SHORTCUT-21961] Filter DataFrame to contain only reviewed edits data_df = data_df[data_df.reviewers.is_not_null().any(dim="list")] return _process_exported_dataframe( From 9e8fea9a715dd7614d187f8e35dd2c02cb5b0513 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 12:10:30 -0500 Subject: [PATCH 06/20] ... Signed-off-by: Valeriy Pogrebitskiy --- dataquality/metrics.py | 2 +- tests/test_metrics.py | 154 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tests/test_metrics.py diff --git a/dataquality/metrics.py b/dataquality/metrics.py index 9eb9411b4..07239b66f 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -410,7 +410,7 @@ def get_edited_dataframe( data_df = vaex.open(file_name) if reviewed_only: - data_df = data_df[data_df.reviewers.is_not_null().any(dim="list")] + data_df = data_df[data_df.reviewers] return _process_exported_dataframe( data_df, diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 000000000..01cdb6759 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,154 @@ +from unittest.mock import Mock +from dataquality import metrics +import vaex + + +def test_get_edited_dataframe_all_edits(mocker): + reviewed_only = False + + project_id = "project_id" + project_name = "project_name" + run_id = "run_id" + run_name = "run_name" + task_type = "task_type" + file_type = Mock() + uuid = "test-uuid" + inference_name = "" + hf_format = False + tagging_schema = Mock() + as_pandas = True + include_embs = False + include_data_embs = False + include_probs = False + include_token_indices = False + + test_df = vaex.from_dict({ + "id": range(0, 10), + "confidence": [0.7] * 10, + "is_drifted": [False] * 7 + [True] * 3, + "reviewers": [[]] * 7 + [["review1"]] * 3, + }) + + api_mock = mocker.patch.object(metrics, "api_client") + split = Mock() + conform_split_mock = mocker.patch("dataquality.metrics.conform_split") + split_mock = conform_split_mock.return_value + + api_mock._get_project_run_id.return_value = [project_id, run_id] + api_mock.get_task_type.return_value = task_type + + mocker.patch("dataquality.metrics.uuid4", return_value = uuid) + mocker.patch("dataquality.vaex.open", return_value = test_df) + + _process_exported_dataframe_mock = Mock("dataquality.metrics._process_exported_dataframe") + + response = metrics.get_edited_dataframe(project_name, run_name, split, inference_name, file_type, include_embs, include_probs, include_token_indices, hf_format, tagging_schema, reviewed_only, as_pandas, include_data_embs) + + assert response == _process_exported_dataframe_mock.return_value + assert conform_split_mock.assert_called_once_with(split) + assert api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) + assert api_mock.get_task_type.assert_called_once_with(project_id, run_id) + + assert api_mock.export_edits.assert_called_once_with( + project_name, + run_name, + split_mock, + inference_name = inference_name, + file_name = f"/tmp/{uuid}-data.{file_type}", + hf_format = hf_format, + tagging_schema = tagging_schema, + ) + + assert _process_exported_dataframe_mock.assert_called_once_with( + test_df, + project_name, + run_name, + split, + task_type, + inference_name, + include_embs, + include_probs, + include_token_indices, + hf_format, + as_pandas, + include_data_embs, + ) + + +def test_get_edited_dataframe_reviewed_only_edits(mocker): + reviewed_only = True + + project_id = "project_id" + project_name = "project_name" + run_id = "run_id" + run_name = "run_name" + task_type = "task_type" + file_type = Mock() + uuid = "test-uuid" + inference_name = "" + hf_format = False + tagging_schema = Mock() + as_pandas = True + include_embs = False + include_data_embs = False + include_probs = False + include_token_indices = False + + test_df = vaex.from_dict({ + "id": range(0, 10), + "confidence": [0.7] * 10, + "is_drifted": [False] * 7 + [True] * 3, + "reviewers": [[]] * 7 + [["review1"]] * 3, + }) + + expected_df = vaex.from_dict({ + "id": range(7, 10), + "confidence": [0.7] * 3, + "is_drifted": [True] * 3, + "reviewers": [["review1"]] * 3, + }) + + api_mock = mocker.patch.object(metrics, "api_client") + split = Mock() + conform_split_mock = mocker.patch("dataquality.metrics.conform_split") + split_mock = conform_split_mock.return_value + + api_mock._get_project_run_id.return_value = [project_id, run_id] + api_mock.get_task_type.return_value = task_type + + mocker.patch("dataquality.metrics.uuid4", return_value = uuid) + mocker.patch("dataquality.vaex.open", return_value = test_df) + + _process_exported_dataframe_mock = Mock("dataquality.metrics._process_exported_dataframe") + + response = metrics.get_edited_dataframe(project_name, run_name, split, inference_name, file_type, include_embs, include_probs, include_token_indices, hf_format, tagging_schema, reviewed_only, as_pandas, include_data_embs) + + assert response == _process_exported_dataframe_mock.return_value + assert conform_split_mock.assert_called_once_with(split) + assert api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) + assert api_mock.get_task_type.assert_called_once_with(project_id, run_id) + + assert api_mock.export_edits.assert_called_once_with( + project_name, + run_name, + split_mock, + inference_name = inference_name, + file_name = f"/tmp/{uuid}-data.{file_type}", + hf_format = hf_format, + tagging_schema = tagging_schema, + ) + + assert _process_exported_dataframe_mock.assert_called_once_with( + expected_df, + project_name, + run_name, + split, + task_type, + inference_name, + include_embs, + include_probs, + include_token_indices, + hf_format, + as_pandas, + include_data_embs, + ) From abda6c02c486a34783e08914aaee094c0a287b7f Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 12:15:13 -0500 Subject: [PATCH 07/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 110 ++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 01cdb6759..fb9629c98 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,7 +1,9 @@ from unittest.mock import Mock -from dataquality import metrics + import vaex +from dataquality import metrics + def test_get_edited_dataframe_all_edits(mocker): reviewed_only = False @@ -22,12 +24,14 @@ def test_get_edited_dataframe_all_edits(mocker): include_probs = False include_token_indices = False - test_df = vaex.from_dict({ - "id": range(0, 10), - "confidence": [0.7] * 10, - "is_drifted": [False] * 7 + [True] * 3, - "reviewers": [[]] * 7 + [["review1"]] * 3, - }) + test_df = vaex.from_dict( + { + "id": range(0, 10), + "confidence": [0.7] * 10, + "is_drifted": [False] * 7 + [True] * 3, + "reviewers": [[]] * 7 + [["review1"]] * 3, + } + ) api_mock = mocker.patch.object(metrics, "api_client") split = Mock() @@ -37,12 +41,28 @@ def test_get_edited_dataframe_all_edits(mocker): api_mock._get_project_run_id.return_value = [project_id, run_id] api_mock.get_task_type.return_value = task_type - mocker.patch("dataquality.metrics.uuid4", return_value = uuid) - mocker.patch("dataquality.vaex.open", return_value = test_df) + mocker.patch("dataquality.metrics.uuid4", return_value=uuid) + mocker.patch("dataquality.vaex.open", return_value=test_df) - _process_exported_dataframe_mock = Mock("dataquality.metrics._process_exported_dataframe") + _process_exported_dataframe_mock = Mock( + "dataquality.metrics._process_exported_dataframe" + ) - response = metrics.get_edited_dataframe(project_name, run_name, split, inference_name, file_type, include_embs, include_probs, include_token_indices, hf_format, tagging_schema, reviewed_only, as_pandas, include_data_embs) + response = metrics.get_edited_dataframe( + project_name, + run_name, + split, + inference_name, + file_type, + include_embs, + include_probs, + include_token_indices, + hf_format, + tagging_schema, + reviewed_only, + as_pandas, + include_data_embs, + ) assert response == _process_exported_dataframe_mock.return_value assert conform_split_mock.assert_called_once_with(split) @@ -53,10 +73,10 @@ def test_get_edited_dataframe_all_edits(mocker): project_name, run_name, split_mock, - inference_name = inference_name, - file_name = f"/tmp/{uuid}-data.{file_type}", - hf_format = hf_format, - tagging_schema = tagging_schema, + inference_name=inference_name, + file_name=f"/tmp/{uuid}-data.{file_type}", + hf_format=hf_format, + tagging_schema=tagging_schema, ) assert _process_exported_dataframe_mock.assert_called_once_with( @@ -94,19 +114,23 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): include_probs = False include_token_indices = False - test_df = vaex.from_dict({ - "id": range(0, 10), - "confidence": [0.7] * 10, - "is_drifted": [False] * 7 + [True] * 3, - "reviewers": [[]] * 7 + [["review1"]] * 3, - }) + test_df = vaex.from_dict( + { + "id": range(0, 10), + "confidence": [0.7] * 10, + "is_drifted": [False] * 7 + [True] * 3, + "reviewers": [[]] * 7 + [["review1"]] * 3, + } + ) - expected_df = vaex.from_dict({ - "id": range(7, 10), - "confidence": [0.7] * 3, - "is_drifted": [True] * 3, - "reviewers": [["review1"]] * 3, - }) + expected_df = vaex.from_dict( + { + "id": range(7, 10), + "confidence": [0.7] * 3, + "is_drifted": [True] * 3, + "reviewers": [["review1"]] * 3, + } + ) api_mock = mocker.patch.object(metrics, "api_client") split = Mock() @@ -116,12 +140,28 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): api_mock._get_project_run_id.return_value = [project_id, run_id] api_mock.get_task_type.return_value = task_type - mocker.patch("dataquality.metrics.uuid4", return_value = uuid) - mocker.patch("dataquality.vaex.open", return_value = test_df) + mocker.patch("dataquality.metrics.uuid4", return_value=uuid) + mocker.patch("dataquality.vaex.open", return_value=test_df) - _process_exported_dataframe_mock = Mock("dataquality.metrics._process_exported_dataframe") + _process_exported_dataframe_mock = Mock( + "dataquality.metrics._process_exported_dataframe" + ) - response = metrics.get_edited_dataframe(project_name, run_name, split, inference_name, file_type, include_embs, include_probs, include_token_indices, hf_format, tagging_schema, reviewed_only, as_pandas, include_data_embs) + response = metrics.get_edited_dataframe( + project_name, + run_name, + split, + inference_name, + file_type, + include_embs, + include_probs, + include_token_indices, + hf_format, + tagging_schema, + reviewed_only, + as_pandas, + include_data_embs, + ) assert response == _process_exported_dataframe_mock.return_value assert conform_split_mock.assert_called_once_with(split) @@ -132,10 +172,10 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): project_name, run_name, split_mock, - inference_name = inference_name, - file_name = f"/tmp/{uuid}-data.{file_type}", - hf_format = hf_format, - tagging_schema = tagging_schema, + inference_name=inference_name, + file_name=f"/tmp/{uuid}-data.{file_type}", + hf_format=hf_format, + tagging_schema=tagging_schema, ) assert _process_exported_dataframe_mock.assert_called_once_with( From 333c2fdade029b9e10596b2862351b42f34a8323 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 12:37:16 -0500 Subject: [PATCH 08/20] ... Signed-off-by: Valeriy Pogrebitskiy --- poetry.lock | 23 ++++++++++++++++++++--- pyproject.toml | 1 + 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 86cc529fe..2c0733056 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "absl-py" @@ -3428,9 +3428,9 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, ] [[package]] @@ -4388,6 +4388,23 @@ tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} [package.extras] test = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "pytest-mock (>=3.12)"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "pytest-xdist" version = "3.5.0" @@ -7400,4 +7417,4 @@ setfit = ["setfit"] [metadata] lock-version = "2.0" python-versions = "^3.9,<3.11" -content-hash = "7d03febb39290bca86f7c019e8ac5ad852e09bcb13326d570cd7f4ffe3ac659c" +content-hash = "3eb4814e38ec5f561d112aba3659a9fe36081b4ebdd5c78a7a7d428d0376327c" diff --git a/pyproject.toml b/pyproject.toml index 9582b51b7..aff2332bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ accelerate = ">=0.19.0" typing-inspect = "==0.8.0" typing-extensions = ">=4.9.0" lightning = "^2.3.1" # Assuming you want the latest version as no version was specified +pytest-mock = "^3.14.0" From 0bdbb0c5e0de180a1bcfbca1bf6ad2a2eb1e4926 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 12:47:09 -0500 Subject: [PATCH 09/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index fb9629c98..b263cd3be 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -42,7 +42,7 @@ def test_get_edited_dataframe_all_edits(mocker): api_mock.get_task_type.return_value = task_type mocker.patch("dataquality.metrics.uuid4", return_value=uuid) - mocker.patch("dataquality.vaex.open", return_value=test_df) + mocker.patch("dataquality.metrics.vaex.open", return_value=test_df) _process_exported_dataframe_mock = Mock( "dataquality.metrics._process_exported_dataframe" From 3af05bc688eee81cbe588c0df09c1f0dfb042b76 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 13:55:09 -0500 Subject: [PATCH 10/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index b263cd3be..a50c07551 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -44,7 +44,7 @@ def test_get_edited_dataframe_all_edits(mocker): mocker.patch("dataquality.metrics.uuid4", return_value=uuid) mocker.patch("dataquality.metrics.vaex.open", return_value=test_df) - _process_exported_dataframe_mock = Mock( + _process_exported_dataframe_mock = mocker.patch( "dataquality.metrics._process_exported_dataframe" ) @@ -141,9 +141,9 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): api_mock.get_task_type.return_value = task_type mocker.patch("dataquality.metrics.uuid4", return_value=uuid) - mocker.patch("dataquality.vaex.open", return_value=test_df) + mocker.patch("dataquality.metrics.vaex.open", return_value=test_df) - _process_exported_dataframe_mock = Mock( + _process_exported_dataframe_mock = mocker.patch( "dataquality.metrics._process_exported_dataframe" ) From 8406952d316b1c0f895241675943d8bce65eb7aa Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 14:25:02 -0500 Subject: [PATCH 11/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index a50c07551..e00029fb3 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -65,11 +65,11 @@ def test_get_edited_dataframe_all_edits(mocker): ) assert response == _process_exported_dataframe_mock.return_value - assert conform_split_mock.assert_called_once_with(split) - assert api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) - assert api_mock.get_task_type.assert_called_once_with(project_id, run_id) + conform_split_mock.assert_called_once_with(split) + api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) + api_mock.get_task_type.assert_called_once_with(project_id, run_id) - assert api_mock.export_edits.assert_called_once_with( + api_mock.export_edits.assert_called_once_with( project_name, run_name, split_mock, @@ -79,7 +79,7 @@ def test_get_edited_dataframe_all_edits(mocker): tagging_schema=tagging_schema, ) - assert _process_exported_dataframe_mock.assert_called_once_with( + _process_exported_dataframe_mock.assert_called_once_with( test_df, project_name, run_name, @@ -164,11 +164,11 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): ) assert response == _process_exported_dataframe_mock.return_value - assert conform_split_mock.assert_called_once_with(split) - assert api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) - assert api_mock.get_task_type.assert_called_once_with(project_id, run_id) + conform_split_mock.assert_called_once_with(split) + api_mock._get_project_run_id.assert_called_once_with(project_name, run_name) + api_mock.get_task_type.assert_called_once_with(project_id, run_id) - assert api_mock.export_edits.assert_called_once_with( + api_mock.export_edits.assert_called_once_with( project_name, run_name, split_mock, @@ -178,7 +178,7 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): tagging_schema=tagging_schema, ) - assert _process_exported_dataframe_mock.assert_called_once_with( + _process_exported_dataframe_mock.assert_called_once_with( expected_df, project_name, run_name, From 2b4f93117573a7dedcd4af6ceeb617f399248627 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 14:40:08 -0500 Subject: [PATCH 12/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index e00029fb3..825d6aa8b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -83,7 +83,7 @@ def test_get_edited_dataframe_all_edits(mocker): test_df, project_name, run_name, - split, + conform_split_mock, task_type, inference_name, include_embs, @@ -182,7 +182,7 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): expected_df, project_name, run_name, - split, + conform_split_mock, task_type, inference_name, include_embs, From 7ca385a33f1f95e7fa3d754d1e2587a884070aee Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 14:55:56 -0500 Subject: [PATCH 13/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 825d6aa8b..b84fc8df7 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -83,7 +83,7 @@ def test_get_edited_dataframe_all_edits(mocker): test_df, project_name, run_name, - conform_split_mock, + split_mock, task_type, inference_name, include_embs, @@ -182,7 +182,7 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): expected_df, project_name, run_name, - conform_split_mock, + split_mock, task_type, inference_name, include_embs, From 09f034d7d61cd5202866bb9bb8c237bbca6e3eb8 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 15:12:59 -0500 Subject: [PATCH 14/20] ... Signed-off-by: Valeriy Pogrebitskiy --- dataquality/metrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dataquality/metrics.py b/dataquality/metrics.py index 07239b66f..825d75793 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -410,7 +410,9 @@ def get_edited_dataframe( data_df = vaex.open(file_name) if reviewed_only: + print(f"DataFrame before reviewed_only filtering: {data_df}") data_df = data_df[data_df.reviewers] + print(f"DataFrame after reviewed_only filtering: {data_df}") return _process_exported_dataframe( data_df, From 44da0ed97f05f52dfca384869fca7ffb6c2767c5 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 17:43:12 -0500 Subject: [PATCH 15/20] ... Signed-off-by: Valeriy Pogrebitskiy --- dataquality/metrics.py | 2 -- tests/test_metrics.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dataquality/metrics.py b/dataquality/metrics.py index 825d75793..07239b66f 100644 --- a/dataquality/metrics.py +++ b/dataquality/metrics.py @@ -410,9 +410,7 @@ def get_edited_dataframe( data_df = vaex.open(file_name) if reviewed_only: - print(f"DataFrame before reviewed_only filtering: {data_df}") data_df = data_df[data_df.reviewers] - print(f"DataFrame after reviewed_only filtering: {data_df}") return _process_exported_dataframe( data_df, diff --git a/tests/test_metrics.py b/tests/test_metrics.py index b84fc8df7..1044b72c6 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -128,9 +128,11 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): "id": range(7, 10), "confidence": [0.7] * 3, "is_drifted": [True] * 3, - "reviewers": [["review1"]] * 3, } ) + pandas_df = expected_df.to_pandas_df() + pandas_df["reviewers"] = [["review1"]] * 3 + expected_df = vaex.from_pandas(pandas_df) api_mock = mocker.patch.object(metrics, "api_client") split = Mock() From febd50c8acef147d22464a17dd5ca9ad56c15772 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 18:04:27 -0500 Subject: [PATCH 16/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 1044b72c6..54fe3ebbb 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock +from unittest.mock import ANY, Mock import vaex @@ -181,7 +181,7 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): ) _process_exported_dataframe_mock.assert_called_once_with( - expected_df, + ANY, project_name, run_name, split_mock, @@ -194,3 +194,14 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): as_pandas, include_data_embs, ) + + print(f"CALL ARGS LIST: {_process_exported_dataframe_mock.call_args_list}") + print(f"CALL ARGS LIST [0]: {_process_exported_dataframe_mock.call_args_list[0]}") + print( + f"CALL ARGS LIST [0][0]: {_process_exported_dataframe_mock.call_args_list[0][0]}" + ) + + assert ( + _process_exported_dataframe_mock.call_args_list[0][0].reviewers + == [["reviewer1"]] * 3 + ) From f68a34ffa3c43685d8051f26b5e4b21c8124a49d Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 18:22:59 -0500 Subject: [PATCH 17/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 54fe3ebbb..2ca4e7c3f 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -195,13 +195,7 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): include_data_embs, ) - print(f"CALL ARGS LIST: {_process_exported_dataframe_mock.call_args_list}") - print(f"CALL ARGS LIST [0]: {_process_exported_dataframe_mock.call_args_list[0]}") - print( - f"CALL ARGS LIST [0][0]: {_process_exported_dataframe_mock.call_args_list[0][0]}" - ) - - assert ( - _process_exported_dataframe_mock.call_args_list[0][0].reviewers - == [["reviewer1"]] * 3 - ) + call_df = _process_exported_dataframe_mock.call_args_list[0][0][0] + print(f"CALL_DF: {_process_exported_dataframe_mock.call_args_list[0][0][0]}") + assert call_df.reviewers == [["reviewer1"]] * 3 + assert call_df.equals(expected_df) From 4a4e858685db95a63a97e03b3295ebd6f355e70e Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 18:31:13 -0500 Subject: [PATCH 18/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 2ca4e7c3f..32434b79f 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -197,5 +197,5 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): call_df = _process_exported_dataframe_mock.call_args_list[0][0][0] print(f"CALL_DF: {_process_exported_dataframe_mock.call_args_list[0][0][0]}") - assert call_df.reviewers == [["reviewer1"]] * 3 + assert call_df.reviewers == [["review1"]] * 3 assert call_df.equals(expected_df) From 88d30839db290e3554b6c8933987cf7857579f42 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 18:43:46 -0500 Subject: [PATCH 19/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 32434b79f..5dc6909e8 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -196,6 +196,5 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): ) call_df = _process_exported_dataframe_mock.call_args_list[0][0][0] - print(f"CALL_DF: {_process_exported_dataframe_mock.call_args_list[0][0][0]}") - assert call_df.reviewers == [["review1"]] * 3 + assert call_df.reviewers.tolist() == [["review1"]] * 3 assert call_df.equals(expected_df) From d83b1646243058a64ed0a7d5626e75743414ae38 Mon Sep 17 00:00:00 2001 From: Valeriy Pogrebitskiy Date: Fri, 6 Dec 2024 18:51:48 -0500 Subject: [PATCH 20/20] ... Signed-off-by: Valeriy Pogrebitskiy --- tests/test_metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 5dc6909e8..2d248942e 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -197,4 +197,3 @@ def test_get_edited_dataframe_reviewed_only_edits(mocker): call_df = _process_exported_dataframe_mock.call_args_list[0][0][0] assert call_df.reviewers.tolist() == [["review1"]] * 3 - assert call_df.equals(expected_df)