fix for subset handling -> display ID updates (#137)

* clear hash from SUBSET_HASH_TO_PARENT_DATA during update formatting

* info->debug logging & general logging updates

* fix for filter attribute error; logging updates

* fix filter types

src/dx/filtering.py

@@ -31,17 +31,19 @@ def store_sample_to_history(df: pd.DataFrame, display_id: str, filters: list) ->
     datalink_metadata = metadata["datalink"]
 
     sample_time = pd.Timestamp("now").strftime(settings.DATETIME_STRING_FORMAT)
+    # convert from FilterTypes to dicts
+    dex_filters = [dex_filter.dict() for dex_filter in filters]
     sample = {
         "sampling_time": sample_time,
-        "filters": [dex_filter.dict() for dex_filter in filters],
+        "filters": dex_filters,
         "dataframe_info": get_df_dimensions(df, prefix="truncated"),
     }
     datalink_metadata["sample_history"].append(sample)
     # only storing the last n-number of samples, not everything
     datalink_metadata["sample_history"] = datalink_metadata["sample_history"][
         -settings.NUM_PAST_SAMPLES_TRACKED :
     ]
-    datalink_metadata["applied_filters"] = filters
+    datalink_metadata["applied_filters"] = dex_filters
     datalink_metadata["sampling_time"] = sample_time
 
     metadata["datalink"] = datalink_metadata

src/dx/formatters/main.py

@@ -1,3 +1,4 @@
+import os
 import uuid
 from typing import Optional
 
@@ -98,7 +99,7 @@ def handle_format(
             extra_metadata=extra_metadata,
         )
     except Exception as e:
-        logger.debug(f"Error in datalink_processing: {e}")
+        logger.exception(f"Error in datalink_processing: {e}")
         # fall back to default processing
         payload, metadata = format_output(
             obj,
@@ -236,9 +237,10 @@ def determine_parent_display_id(dxdf: DXDataFrame) -> Optional[str]:
     else:
         logger.debug(f"df is subset of existing {parent_display_id=}")
 
+    last_executed_cell_id = os.environ.get("LAST_EXECUTED_CELL_ID")
     parent_cell_id = parent_dataset_info.get("cell_id")
     different_cell_output = parent_cell_id != dxdf.cell_id
-    logger.debug(f"{dxdf.cell_id=} & {parent_cell_id=}")
+    logger.debug(f"{dxdf.cell_id=} | {parent_cell_id=} | {last_executed_cell_id=}")
     if different_cell_output and parent_display_id is not None:
         logger.debug(
             f"disregarding {parent_display_id=} and using {dxdf.display_id=} since this is a new cell_id",
@@ -248,6 +250,15 @@ def determine_parent_display_id(dxdf: DXDataFrame) -> Optional[str]:
         # doesn't matter if this dataset was associated with another,
         # we shouldn't be re-rendering the display ID from another cell ID
         parent_display_id = None
+
+    if parent_display_id is not None:
+        logger.debug(
+            f"updating existing display handler {parent_display_id=}",
+            parent_cell_id=parent_cell_id,
+            cell_id=dxdf.cell_id,
+        )
+        # if we don't remove this, we'll keep updating the same display handler
+        SUBSET_HASH_TO_PARENT_DATA.pop(dxdf.hash)
     return parent_display_id
 
 

src/dx/plotting/dashboards.py

@@ -67,7 +67,7 @@ def make_dashboard(
                 by_alias=True,
                 exclude_none=True,
             )
-            logger.info(f"{dex_dashboard_view_dict=}")
+            logger.debug(f"{dex_dashboard_view_dict=}")
             dex_metadata.views.append(dex_dashboard_view_dict)
             # define the view positioning
             multiview = {

src/dx/plotting/dex/basic.py

@@ -187,7 +187,7 @@ def line(
         "timeseries_sort": x,
         "zero_baseline": zero_baseline,
     }
-    logger.info(f"{chart_settings=}")
+    logger.debug(f"{chart_settings=}")
     return handle_view(
         df,
         chart_mode="line",

src/dx/plotting/main.py

@@ -58,7 +58,7 @@ def plot(df: dict, kind: str, **kwargs) -> None:
         exclude_none=True,
         by_alias=True,
     )
-    logger.info(f"{view_metadata=}")
+    logger.debug(f"{view_metadata=}")
 
     with settings_context(generate_dex_metadata=True):
         # if someone is calling one of these functions with the dx plotting backend,
@@ -79,7 +79,7 @@ def handle_view(
     and either passes it to be handled by the display formatter,
     or returns the view.
     """
-    logger.info(f"{chart=}")
+    logger.debug(f"{chart=}")
 
     view_params = {
         "chart_mode": chart_mode,
@@ -100,7 +100,7 @@ def handle_view(
         exclude_none=True,
         by_alias=True,
     )
-    logger.info(f"{view_metadata=}")
+    logger.debug(f"{view_metadata=}")
     handle_format(
         df,
         extra_metadata=view_metadata,

src/dx/types/dex_metadata.py

@@ -275,5 +275,5 @@ def add_view(self, **kwargs):
             is_default=is_default,
             **kwargs,
         )
-        logger.info(f"adding {new_view=}")
+        logger.debug(f"adding {new_view=}")
         self.views.append(new_view)

src/dx/utils/formatting.py

@@ -257,7 +257,7 @@ def generate_metadata(
         existing_metadata = parent_dxdf.metadata
         parent_dataframe_info = existing_metadata.get("datalink", {}).get("dataframe_info", {})
         dex_metadata = DEXMetadata.parse_obj(existing_metadata.get("dx", {}))
-        logger.info(f"existing {dex_metadata=}")
+        logger.debug(f"existing {dex_metadata=}")
         if parent_dataframe_info:
             # if this comes after a resampling operation, we need to make sure the
             # original dimensions aren't overwritten by this new dataframe_info,
@@ -271,7 +271,7 @@ def generate_metadata(
             dataframe_info = parent_dataframe_info
         # these are set whenever store_sample_to_history() is called after a filter action from the frontend
         sample_history = existing_metadata.get("datalink", {}).get("sample_history", [])
-        filters = [dex_filter.dict() for dex_filter in parent_dxdf.filters]
+        filters = parent_dxdf.filters
 
     metadata = {
         "datalink": {
@@ -335,7 +335,7 @@ def add_dex_metadata(
     variable_name: str,
 ) -> dict:
     if not dex_metadata.views:
-        logger.info("no views found, adding default view")
+        logger.debug("no views found, adding default view")
         dex_metadata.add_view(
             variable_name=variable_name,
             display_id=display_id,
@@ -385,7 +385,7 @@ def handle_extra_metadata(
             logger.warning(f"not sure what to do with {extra_metadata=}")
     except Exception as e:
         logger.error(f'error updating metadata: "{e}"')
-    logger.info(f"done handling extra metadata, {metadata=}")
+    logger.debug(f"done handling extra metadata, {metadata=}")
     return metadata
 
 
@@ -416,7 +416,7 @@ def update_dex_view_metadata(
             metadata.views[i] = extra_metadata
             updated_existing_view = True
         elif not updated_existing_view and view.variable_name == variable_name:
-            logger.info(f"updating {view.display_id=} with {extra_metadata=}")
+            logger.debug(f"updating {view.display_id=} with {extra_metadata=}")
             view = view.copy(update=extra_metadata)
             updated_existing_view = True
         else:
@@ -426,7 +426,7 @@ def update_dex_view_metadata(
         updated_views.append(view)
 
     if not updated_existing_view:
-        logger.info(f"didn't match to existing view; adding new view with {extra_metadata=}")
+        logger.debug(f"didn't match to existing view; adding new view with {extra_metadata=}")
         metadata.add_view(**extra_metadata)
     elif updated_views:
         metadata.views = updated_views
@@ -441,7 +441,7 @@ def update_dex_metadata(metadata: DEXMetadata, extra_metadata: dict) -> DEXMetad
     """
     Convenience method to update top-level DEX metadata; similar to update_dex_view_metadata().
     """
-    logger.info(f"updating metadata with {extra_metadata=}")
+    logger.debug(f"updating metadata with {extra_metadata=}")
     return metadata.copy(update=extra_metadata)
 
 

src/dx/utils/tracking.py

@@ -92,14 +92,14 @@ def get_cell_id(self) -> str:
         cell_id = SUBSET_HASH_TO_PARENT_DATA.get(self.hash, {}).get(
             "cell_id", last_executed_cell_id
         )
-        logger.debug(f"{last_executed_cell_id=} / last associated {cell_id=}")
+        logger.debug(f"DXDF {last_executed_cell_id=} / last associated {cell_id=}")
         return cell_id
 
     def get_display_id(self) -> str:
         display_id = SUBSET_HASH_TO_PARENT_DATA.get(self.hash, {}).get(
             "display_id", str(uuid.uuid4())
         )
-        logger.debug(f"{display_id=}")
+        logger.debug(f"DXDF {display_id=}")
         return display_id