diff --git a/src/hub/dataload/sources/civic/civic_parser.py b/src/hub/dataload/sources/civic/civic_parser.py index 4d850c2c..09956bb3 100644 --- a/src/hub/dataload/sources/civic/civic_parser.py +++ b/src/hub/dataload/sources/civic/civic_parser.py @@ -25,22 +25,27 @@ def merge_dicts(d1, d2): def remove_nodes_and_edges(data): if isinstance(data, dict): + # If the current data is a dictionary, iterate through its keys new_data = {} for key, value in data.items(): if key in ['node', 'nodes', 'edge', 'edges']: - # If the value is a list (like 'edges'), process each item and keep them in a list + # If the key is 'nodes' or 'edges', recursively process the value if isinstance(value, list): - new_data = remove_nodes_and_edges(value) # Keep all items as a list + # If 'edges' is a list, take each element (each 'nodes') and process + for item in value: + new_data.update(remove_nodes_and_edges(item.get(key, item))) else: + # If 'nodes' is a dictionary, just update with its content new_data.update(remove_nodes_and_edges(value)) else: + # Process the value recursively for other keys new_data[key] = remove_nodes_and_edges(value) return new_data elif isinstance(data, list): - # If it's a list, return a list of processed items + # If it's a list, apply the function recursively to each element return [remove_nodes_and_edges(item) for item in data] else: - # If it's neither a dict nor a list, return the value as is + # If it's neither a dict nor a list, return the value return data