diff --git a/src/dx/filtering.py b/src/dx/filtering.py index b6d56a0e..6fd37218 100644 --- a/src/dx/filtering.py +++ b/src/dx/filtering.py @@ -31,9 +31,11 @@ def store_sample_to_history(df: pd.DataFrame, display_id: str, filters: list) -> datalink_metadata = metadata["datalink"] sample_time = pd.Timestamp("now").strftime(settings.DATETIME_STRING_FORMAT) + # convert from FilterTypes to dicts + dex_filters = [dex_filter.dict() for dex_filter in filters] sample = { "sampling_time": sample_time, - "filters": [dex_filter.dict() for dex_filter in filters], + "filters": dex_filters, "dataframe_info": get_df_dimensions(df, prefix="truncated"), } datalink_metadata["sample_history"].append(sample) @@ -41,7 +43,7 @@ def store_sample_to_history(df: pd.DataFrame, display_id: str, filters: list) -> datalink_metadata["sample_history"] = datalink_metadata["sample_history"][ -settings.NUM_PAST_SAMPLES_TRACKED : ] - datalink_metadata["applied_filters"] = filters + datalink_metadata["applied_filters"] = dex_filters datalink_metadata["sampling_time"] = sample_time metadata["datalink"] = datalink_metadata diff --git a/src/dx/formatters/main.py b/src/dx/formatters/main.py index 1110610e..f0fecacd 100644 --- a/src/dx/formatters/main.py +++ b/src/dx/formatters/main.py @@ -1,3 +1,4 @@ +import os import uuid from typing import Optional @@ -98,7 +99,7 @@ def handle_format( extra_metadata=extra_metadata, ) except Exception as e: - logger.debug(f"Error in datalink_processing: {e}") + logger.exception(f"Error in datalink_processing: {e}") # fall back to default processing payload, metadata = format_output( obj, @@ -236,9 +237,10 @@ def determine_parent_display_id(dxdf: DXDataFrame) -> Optional[str]: else: logger.debug(f"df is subset of existing {parent_display_id=}") + last_executed_cell_id = os.environ.get("LAST_EXECUTED_CELL_ID") parent_cell_id = parent_dataset_info.get("cell_id") different_cell_output = parent_cell_id != dxdf.cell_id - logger.debug(f"{dxdf.cell_id=} & {parent_cell_id=}") + logger.debug(f"{dxdf.cell_id=} | {parent_cell_id=} | {last_executed_cell_id=}") if different_cell_output and parent_display_id is not None: logger.debug( f"disregarding {parent_display_id=} and using {dxdf.display_id=} since this is a new cell_id", @@ -248,6 +250,15 @@ def determine_parent_display_id(dxdf: DXDataFrame) -> Optional[str]: # doesn't matter if this dataset was associated with another, # we shouldn't be re-rendering the display ID from another cell ID parent_display_id = None + + if parent_display_id is not None: + logger.debug( + f"updating existing display handler {parent_display_id=}", + parent_cell_id=parent_cell_id, + cell_id=dxdf.cell_id, + ) + # if we don't remove this, we'll keep updating the same display handler + SUBSET_HASH_TO_PARENT_DATA.pop(dxdf.hash) return parent_display_id diff --git a/src/dx/plotting/dashboards.py b/src/dx/plotting/dashboards.py index c71900ef..e74d8cc3 100644 --- a/src/dx/plotting/dashboards.py +++ b/src/dx/plotting/dashboards.py @@ -67,7 +67,7 @@ def make_dashboard( by_alias=True, exclude_none=True, ) - logger.info(f"{dex_dashboard_view_dict=}") + logger.debug(f"{dex_dashboard_view_dict=}") dex_metadata.views.append(dex_dashboard_view_dict) # define the view positioning multiview = { diff --git a/src/dx/plotting/dex/basic.py b/src/dx/plotting/dex/basic.py index 2ae680ce..831e9191 100644 --- a/src/dx/plotting/dex/basic.py +++ b/src/dx/plotting/dex/basic.py @@ -187,7 +187,7 @@ def line( "timeseries_sort": x, "zero_baseline": zero_baseline, } - logger.info(f"{chart_settings=}") + logger.debug(f"{chart_settings=}") return handle_view( df, chart_mode="line", diff --git a/src/dx/plotting/main.py b/src/dx/plotting/main.py index 13887a3a..238485b5 100644 --- a/src/dx/plotting/main.py +++ b/src/dx/plotting/main.py @@ -58,7 +58,7 @@ def plot(df: dict, kind: str, **kwargs) -> None: exclude_none=True, by_alias=True, ) - logger.info(f"{view_metadata=}") + logger.debug(f"{view_metadata=}") with settings_context(generate_dex_metadata=True): # if someone is calling one of these functions with the dx plotting backend, @@ -79,7 +79,7 @@ def handle_view( and either passes it to be handled by the display formatter, or returns the view. """ - logger.info(f"{chart=}") + logger.debug(f"{chart=}") view_params = { "chart_mode": chart_mode, @@ -100,7 +100,7 @@ def handle_view( exclude_none=True, by_alias=True, ) - logger.info(f"{view_metadata=}") + logger.debug(f"{view_metadata=}") handle_format( df, extra_metadata=view_metadata, diff --git a/src/dx/types/dex_metadata.py b/src/dx/types/dex_metadata.py index 12847923..7a2c32e7 100644 --- a/src/dx/types/dex_metadata.py +++ b/src/dx/types/dex_metadata.py @@ -275,5 +275,5 @@ def add_view(self, **kwargs): is_default=is_default, **kwargs, ) - logger.info(f"adding {new_view=}") + logger.debug(f"adding {new_view=}") self.views.append(new_view) diff --git a/src/dx/utils/formatting.py b/src/dx/utils/formatting.py index cfd5d481..df96b09f 100644 --- a/src/dx/utils/formatting.py +++ b/src/dx/utils/formatting.py @@ -257,7 +257,7 @@ def generate_metadata( existing_metadata = parent_dxdf.metadata parent_dataframe_info = existing_metadata.get("datalink", {}).get("dataframe_info", {}) dex_metadata = DEXMetadata.parse_obj(existing_metadata.get("dx", {})) - logger.info(f"existing {dex_metadata=}") + logger.debug(f"existing {dex_metadata=}") if parent_dataframe_info: # if this comes after a resampling operation, we need to make sure the # original dimensions aren't overwritten by this new dataframe_info, @@ -271,7 +271,7 @@ def generate_metadata( dataframe_info = parent_dataframe_info # these are set whenever store_sample_to_history() is called after a filter action from the frontend sample_history = existing_metadata.get("datalink", {}).get("sample_history", []) - filters = [dex_filter.dict() for dex_filter in parent_dxdf.filters] + filters = parent_dxdf.filters metadata = { "datalink": { @@ -335,7 +335,7 @@ def add_dex_metadata( variable_name: str, ) -> dict: if not dex_metadata.views: - logger.info("no views found, adding default view") + logger.debug("no views found, adding default view") dex_metadata.add_view( variable_name=variable_name, display_id=display_id, @@ -385,7 +385,7 @@ def handle_extra_metadata( logger.warning(f"not sure what to do with {extra_metadata=}") except Exception as e: logger.error(f'error updating metadata: "{e}"') - logger.info(f"done handling extra metadata, {metadata=}") + logger.debug(f"done handling extra metadata, {metadata=}") return metadata @@ -416,7 +416,7 @@ def update_dex_view_metadata( metadata.views[i] = extra_metadata updated_existing_view = True elif not updated_existing_view and view.variable_name == variable_name: - logger.info(f"updating {view.display_id=} with {extra_metadata=}") + logger.debug(f"updating {view.display_id=} with {extra_metadata=}") view = view.copy(update=extra_metadata) updated_existing_view = True else: @@ -426,7 +426,7 @@ def update_dex_view_metadata( updated_views.append(view) if not updated_existing_view: - logger.info(f"didn't match to existing view; adding new view with {extra_metadata=}") + logger.debug(f"didn't match to existing view; adding new view with {extra_metadata=}") metadata.add_view(**extra_metadata) elif updated_views: metadata.views = updated_views @@ -441,7 +441,7 @@ def update_dex_metadata(metadata: DEXMetadata, extra_metadata: dict) -> DEXMetad """ Convenience method to update top-level DEX metadata; similar to update_dex_view_metadata(). """ - logger.info(f"updating metadata with {extra_metadata=}") + logger.debug(f"updating metadata with {extra_metadata=}") return metadata.copy(update=extra_metadata) diff --git a/src/dx/utils/tracking.py b/src/dx/utils/tracking.py index d9759d3e..2a2e9524 100644 --- a/src/dx/utils/tracking.py +++ b/src/dx/utils/tracking.py @@ -92,14 +92,14 @@ def get_cell_id(self) -> str: cell_id = SUBSET_HASH_TO_PARENT_DATA.get(self.hash, {}).get( "cell_id", last_executed_cell_id ) - logger.debug(f"{last_executed_cell_id=} / last associated {cell_id=}") + logger.debug(f"DXDF {last_executed_cell_id=} / last associated {cell_id=}") return cell_id def get_display_id(self) -> str: display_id = SUBSET_HASH_TO_PARENT_DATA.get(self.hash, {}).get( "display_id", str(uuid.uuid4()) ) - logger.debug(f"{display_id=}") + logger.debug(f"DXDF {display_id=}") return display_id