Feat/cv save local paths

dataquality/__init__.py

@@ -31,7 +31,7 @@
 """
 
 
-__version__ = "0.9.8"
+__version__ = "0.10.0"
 
 import sys
 from typing import Any, List, Optional

dataquality/core/log.py

@@ -123,30 +123,28 @@ def log_data_sample(*, text: str, id: int, **kwargs: Any) -> None:
 def log_image_dataset(
     dataset: DataSet,
     *,
-    imgs_colname: Optional[str] = None,
-    imgs_location_colname: Optional[str] = None,
-    imgs_remote_location: Optional[str] = None,
+    imgs_local_colname: Optional[str] = None,
+    imgs_remote: Optional[str] = None,
     batch_size: int = ITER_CHUNK_SIZE,
     id: str = "id",
-    label: Union[str, int] = "label",
+    label: str = "label",
     split: Optional[Split] = None,
     inference_name: Optional[str] = None,
-    meta: Union[List[str], List[int], None] = None,
+    meta: Optional[List[str]] = None,
     parallel: bool = False,
     **kwargs: Any,
 ) -> None:
     """
     Log an image dataset of input samples for image classification
 
-    :param dataset: The dataset to log. This can be a Pandas/Vaex dataframe or an
+    :param dataset: The dataset to log. This can be a Pandas/HF dataframe or an
         ImageFolder (from Torchvision).
-    :param imgs_colname: If the images are passed as bytes in the dataframe, this
-        indicates the name of the column containing the images
-    :param imgs_location_colname: If the images are passed via their path in the
-        dataframe, this indicates the name of the column containing the paths.
-        These paths could be remote (skip upload) or local (upload)
-    :param imgs_remote_location: If the dataset is of type ImageFolder and the
-        images are stored remotely, pass the folder name here to avoid upload
+    :param imgs_local_colname: The name of the column containing the local images
+        (typically paths but could also be bytes for HF dataframes). Ignored for
+        ImageFolder where local paths are directly retrieved from the dataset.
+    :param imgs_remote: The name of the column containing paths to the remote images (in
+        the case of a df) or remote directory containing the images (in the case of
+        ImageFolder). Specifying this argument is required to skip uploading the images.
     :param batch_size: Number of samples to log in a batch. Default 10,000
     :param id: The name of the column containing the ids (in the dataframe)
     :param label: The name of the column containing the labels (in the dataframe)
@@ -165,11 +163,13 @@ def log_image_dataset(
         "This method is only supported for image tasks. "
         "Please use dq.log_samples for text tasks."
     )
+
+    # TODO: raise warning if imgs_local is None (and we provide no smart features)
+
     data_logger.log_image_dataset(
         dataset=dataset,
-        imgs_colname=imgs_colname,
-        imgs_location_colname=imgs_location_colname,
-        imgs_remote_location=imgs_remote_location,
+        imgs_local_colname=imgs_local_colname,
+        imgs_remote=imgs_remote,
         batch_size=batch_size,
         id=id,
         label=label,

dataquality/integrations/fastai.py

@@ -15,6 +15,7 @@
 from dataquality.analytics import Analytics
 from dataquality.clients.api import ApiClient
 from dataquality.exceptions import GalileoException
+from dataquality.loggers.data_logger.image_classification import GAL_LOCAL_IMAGES_PATHS
 from dataquality.loggers.logger_config.base_logger_config import BaseLoggerConfig
 from dataquality.schemas.split import Split
 from dataquality.utils.helpers import galileo_disabled
@@ -396,7 +397,7 @@ def convert_img_dl_to_df(dl: DataLoader, x_col: str = "image") -> pd.DataFrame:
     a.log_function("fastai/convert_img_dl_to_df")
     additional_data = {}
     if x_col == "image":
-        additional_data["text"] = dl.items
+        additional_data[GAL_LOCAL_IMAGES_PATHS] = dl.items
     x, y = [], []
     for x_item, y_item in dl.dataset:
         x.append(x_item)

dataquality/loggers/data_logger/base_data_logger.py

@@ -10,6 +10,7 @@
 import numpy as np
 import pandas as pd
 import vaex
+from datasets.arrow_dataset import Dataset as HFDataset
 from huggingface_hub.utils import HfHubHTTPError
 from vaex.dataframe import DataFrame
 
@@ -40,7 +41,7 @@
 )
 
 DATA_FOLDERS = ["emb", "prob", "data"]
-DataSet = TypeVar("DataSet", bound=Union[Iterable, pd.DataFrame, DataFrame])
+DataSet = TypeVar("DataSet", bound=Union[Iterable, pd.DataFrame, HFDataset, DataFrame])
 MetasType = TypeVar("MetasType", bound=Dict[str, List[Union[str, float, int]]])
 MetaType = TypeVar("MetaType", bound=Dict[str, Union[str, float, int]])
 ITER_CHUNK_SIZE = 100_000
@@ -219,6 +220,26 @@ def export_df(self, df: vaex.DataFrame) -> None:
     def support_data_embs(self) -> bool:
         return True
 
+    def apply_column_map(self, dataset: DataSet, column_map: Dict[str, str]) -> DataSet:
+        """Rename columns in the dataset according to the column_map
+
+        This function works for both pandas and HF datasets
+        """
+        # Remove any columns that are mapped to themselves
+        column_map = {k: v for k, v in column_map.items() if k != v}
+
+        if isinstance(dataset, pd.DataFrame):
+            dataset = dataset.rename(columns=column_map)
+        elif self.is_hf_dataset(dataset):
+            import datasets
+
+            assert isinstance(dataset, datasets.Dataset)
+            for old_col, new_col in column_map.items():
+                if old_col in dataset.column_names:  # HF breaks if col doesn't exist
+                    dataset = dataset.rename_column(old_col, new_col)
+
+        return dataset
+
     def upload(
         self, last_epoch: Optional[int] = None, create_data_embs: bool = False
     ) -> None: