From a268e3e7bc56cfc5a70d69f9b5794240f8fe874c Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Mon, 13 Jan 2025 12:39:05 -0500 Subject: [PATCH 01/30] merged changes from master --- .../ingestion/source/common/subtypes.py | 1 + .../ingestion/source/snowflake/constants.py | 1 + .../source/snowflake/snowflake_config.py | 10 + .../source/snowflake/snowflake_lineage_v2.py | 33 +++ .../source/snowflake/snowflake_queries.py | 37 +++- .../source/snowflake/snowflake_query.py | 27 ++- .../source/snowflake/snowflake_report.py | 7 + .../source/snowflake/snowflake_schema.py | 125 +++++++++++ .../source/snowflake/snowflake_schema_gen.py | 202 +++++++++++++++++- .../source/snowflake/snowflake_utils.py | 11 +- .../source/snowflake/snowflake_v2.py | 50 ++++- .../ingestion/source/sql/sql_config.py | 4 + 12 files changed, 483 insertions(+), 25 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index a5eecf198a9b49..8eb53259df8062 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -24,6 +24,7 @@ class DatasetSubTypes(StrEnum): SAC_LIVE_DATA_MODEL = "Live Data Model" NEO4J_NODE = "Neo4j Node" NEO4J_RELATIONSHIP = "Neo4j Relationship" + SNOWFLAKE_STREAM = "Snowflake Stream" # TODO: Create separate entity... NOTEBOOK = "Notebook" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py index fc464cc5ea9bb5..84ccd2c67c95a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py @@ -53,6 +53,7 @@ class SnowflakeObjectDomain(StrEnum): SCHEMA = "schema" COLUMN = "column" ICEBERG_TABLE = "iceberg table" + STREAM = "stream" GENERIC_PERMISSION_ERROR_KEY = "permission-error" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 2d61ce59857778..9d46e68cf52b22 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -263,6 +263,16 @@ class SnowflakeV2Config( description="List of regex patterns for tags to include in ingestion. Only used if `extract_tags` is enabled.", ) + include_streams: bool = Field( + default=True, + description="If enabled, streams will be ingested as separate entities from tables/views.", + ) + + stream_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for streams to filter in ingestion.", + ) + # This is required since access_history table does not capture whether the table was temporary table. temporary_tables_pattern: List[str] = Field( default=DEFAULT_TEMP_TABLES_PATTERNS, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index e93ecf30171f65..826787a2314ca7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -20,6 +20,7 @@ ) from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report +from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeStream from datahub.ingestion.source.snowflake.snowflake_utils import ( SnowflakeCommonMixin, SnowflakeFilter, @@ -540,3 +541,35 @@ def report_status(self, step: str, status: bool) -> None: def close(self) -> None: pass + + def populate_stream_upstreams(self, streams: List[SnowflakeStream]) -> None: + try: + for stream in streams: + stream_fully_qualified = ".".join( + [ + stream.database_name.lower(), + stream.schema_name.lower(), + stream.name.lower(), + ] + ) + + logger.info( + f"Stream Info: upstream_urn: {stream.table_name.lower()}, downstream_urn: {stream_fully_qualified}" + ) + known_lineage = KnownLineageMapping( + upstream_urn=self.identifiers.gen_dataset_urn( + stream.table_name.lower() + ), + downstream_urn=self.identifiers.gen_dataset_urn( + stream_fully_qualified + ), + ) + self.report.num_streams_with_known_upstreams += 1 + self.sql_aggregator.add(known_lineage) + except Exception as e: + logger.debug(e, exc_info=e) + self.warn_if_stateful_else_error( + "stream-lineage", + f"Failed to extract stream lineage due to error {e}", + ) + self.report_status("STREAM_LINEAGE", False) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index b82734cbbe84ea..3dc9d239c8178b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -49,6 +49,7 @@ from datahub.sql_parsing.schema_resolver import SchemaResolver from datahub.sql_parsing.sql_parsing_aggregator import ( KnownLineageMapping, + ObservedQuery, PreparsedQuery, SqlAggregatorReport, SqlParsingAggregator, @@ -398,6 +399,36 @@ def _parse_audit_log_row( pass else: return None + + user = CorpUserUrn( + self.identifiers.get_user_identifier( + res["user_name"], users.get(res["user_name"]) + ) + ) + + # Check if any of the accessed objects are streams + has_stream_objects = any( + obj.get("objectDomain") == "Stream" for obj in direct_objects_accessed + ) + + # If a stream is used, default to query parsing. + if has_stream_objects: + logger.debug("Found matching stream object") + self.aggregator.add_observed_query( + observed=ObservedQuery( + query=res["query_text"], + session_id=res["session_id"], + timestamp=res["query_start_time"].astimezone(timezone.utc), + user=user, + default_db=res["default_db"], + default_schema=res["default_schema"], + query_hash=get_query_fingerprint( + res["query_text"], self.identifiers.platform, fast=True + ), + ), + ) + return None + upstreams = [] column_usage = {} @@ -460,12 +491,6 @@ def _parse_audit_log_row( ) ) - user = CorpUserUrn( - self.identifiers.get_user_identifier( - res["user_name"], users.get(res["user_name"]) - ) - ) - timestamp: datetime = res["query_start_time"] timestamp = timestamp.astimezone(timezone.utc) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 40bcfb514efd23..cbb2ace8f85b1c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -9,6 +9,7 @@ from datahub.utilities.prefix_batch_builder import PrefixGroup SHOW_VIEWS_MAX_PAGE_SIZE = 10000 +SHOW_STREAM_MAX_PAGE_SIZE = 10000 def create_deny_regex_sql_filter( @@ -36,6 +37,7 @@ class SnowflakeQuery: SnowflakeObjectDomain.VIEW.capitalize(), SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize(), SnowflakeObjectDomain.ICEBERG_TABLE.capitalize(), + SnowflakeObjectDomain.STREAM.capitalize(), } ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = "({})".format( @@ -44,7 +46,8 @@ class SnowflakeQuery: ACCESS_HISTORY_TABLE_DOMAINS_FILTER = ( "(" f"'{SnowflakeObjectDomain.TABLE.capitalize()}'," - f"'{SnowflakeObjectDomain.VIEW.capitalize()}'" + f"'{SnowflakeObjectDomain.VIEW.capitalize()}'," + f"'{SnowflakeObjectDomain.STREAM.capitalize()}'," ")" ) @@ -952,3 +955,25 @@ def dmf_assertion_results(start_time_millis: int, end_time_millis: int) -> str: @staticmethod def get_all_users() -> str: return """SELECT name as "NAME", email as "EMAIL" FROM SNOWFLAKE.ACCOUNT_USAGE.USERS""" + + @staticmethod + def streams_for_database( + db_name: str, + limit: int = SHOW_STREAM_MAX_PAGE_SIZE, + stream_pagination_marker: Optional[str] = None, + ) -> str: + # SHOW VIEWS can return a maximum of 10000 rows. + # https://docs.snowflake.com/en/sql-reference/sql/show-streams#usage-notes + assert limit <= SHOW_STREAM_MAX_PAGE_SIZE + + # To work around this, we paginate through the results using the FROM clause. + from_clause = ( + f"""FROM '{stream_pagination_marker}'""" + if SHOW_STREAM_MAX_PAGE_SIZE + else "" + ) + return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" + + @staticmethod + def streams_for_schema(schema_name: str, db_name: str) -> str: + return f"""SHOW STREAMS IN SCHEMA {db_name}.{schema_name}""" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index b24471f8666afa..fa389640e1a781 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -64,6 +64,7 @@ class SnowflakeReport(SQLSourceReport, BaseTimeWindowReport): num_table_to_view_edges_scanned: int = 0 num_view_to_table_edges_scanned: int = 0 num_external_table_edges_scanned: int = 0 + num_stream_edges_scanned: int = 0 ignore_start_time_lineage: Optional[bool] = None upstream_lineage_in_report: Optional[bool] = None upstream_lineage: Dict[str, List[str]] = field(default_factory=dict) @@ -103,6 +104,7 @@ class SnowflakeV2Report( schemas_scanned: int = 0 databases_scanned: int = 0 tags_scanned: int = 0 + streams_scanned: int = 0 include_usage_stats: bool = False include_operational_stats: bool = False @@ -112,6 +114,7 @@ class SnowflakeV2Report( table_lineage_query_secs: float = -1 external_lineage_queries_secs: float = -1 num_tables_with_known_upstreams: int = 0 + num_streams_with_known_upstreams: int = 0 num_upstream_lineage_edge_parsing_failed: int = 0 num_secure_views_missing_definition: int = 0 @@ -129,6 +132,8 @@ class SnowflakeV2Report( num_get_tags_for_object_queries: int = 0 num_get_tags_on_columns_for_table_queries: int = 0 + num_get_streams_for_schema_queries: int = 0 + rows_zero_objects_modified: int = 0 _processed_tags: MutableSet[str] = field(default_factory=set) @@ -155,6 +160,8 @@ def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: return self._scanned_tags.add(name) self.tags_scanned += 1 + elif ent_type == "stream": + self.streams_scanned += 1 else: raise KeyError(f"Unknown entity {ent_type}.") diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 780effc82b0163..d81cce5f59e124 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -115,6 +115,7 @@ class SnowflakeSchema: comment: Optional[str] tables: List[str] = field(default_factory=list) views: List[str] = field(default_factory=list) + streams: List[str] = field(default_factory=list) tags: Optional[List[SnowflakeTag]] = None @@ -128,6 +129,29 @@ class SnowflakeDatabase: tags: Optional[List[SnowflakeTag]] = None +@dataclass +class SnowflakeStream: + name: str + created: datetime + owner: str + comment: str + source_type: str + type: str + stale: str + mode: str + invalid_reason: str + owner_role_type: str + database_name: str + schema_name: str + table_name: str + columns: List[SnowflakeColumn] = field(default_factory=list) + stale_after: Optional[datetime] = None + base_tables: Optional[str] = None + tags: Optional[List[SnowflakeTag]] = None + column_tags: Dict[str, List[SnowflakeTag]] = field(default_factory=dict) + last_altered: Optional[datetime] = None + + class _SnowflakeTagCache: def __init__(self) -> None: # self._database_tags[] = list of tags applied to database @@ -205,6 +229,7 @@ def as_obj(self) -> Dict[str, Dict[str, int]]: self.get_tables_for_database, self.get_views_for_database, self.get_columns_for_schema, + self.get_streams_for_database, self.get_pk_constraints_for_schema, self.get_fk_constraints_for_schema, ] @@ -591,3 +616,103 @@ def get_tags_on_columns_for_table( tags[column_name].append(snowflake_tag) return tags + + @serialized_lru_cache(maxsize=1) + def get_streams_for_database( + self, db_name: str + ) -> Dict[str, List[SnowflakeStream]]: + page_limit = SHOW_VIEWS_MAX_PAGE_SIZE + + streams: Dict[str, List[SnowflakeStream]] = {} + + first_iteration = True + stream_pagination_marker: Optional[str] = None + while first_iteration or stream_pagination_marker is not None: + cur = self.connection.query( + SnowflakeQuery.streams_for_database( + db_name, + limit=page_limit, + stream_pagination_marker=stream_pagination_marker, + ) + ) + + first_iteration = False + stream_pagination_marker = None + + result_set_size = 0 + for stream in cur: + result_set_size += 1 + + stream_name = stream["name"] + schema_name = stream["schema_name"] + if schema_name not in streams: + streams[schema_name] = [] + streams[stream["schema_name"]].append( + SnowflakeStream( + name=stream["name"], + created=stream["created_on"], + owner=stream["owner"], + comment=stream["comment"], + source_type=stream["source_type"], + type=stream["type"], + stale=stream["stale"], + mode=stream["mode"], + database_name=stream["database_name"], + schema_name=stream["schema_name"], + invalid_reason=stream["invalid_reason"], + owner_role_type=stream["owner_role_type"], + stale_after=stream["stale_after"], + table_name=stream["table_name"], + base_tables=stream["base_tables"], + last_altered=stream["created_on"], + ) + ) + + if result_set_size >= page_limit: + # If we hit the limit, we need to send another request to get the next page. + logger.info( + f"Fetching next page of views for {db_name} - after {stream_name}" + ) + stream_pagination_marker = stream_name + + return streams + + @serialized_lru_cache(maxsize=1) + def get_streams_for_schema( + self, schema_name: str, db_name: str + ) -> Optional[List[SnowflakeStream]]: + try: + streams: List[SnowflakeStream] = [] + cur = self.connection.query( + SnowflakeQuery.streams_for_schema(schema_name, db_name), + ) + + for stream in cur: + streams.append( + SnowflakeStream( + name=stream["name"], + created=stream["created_on"], + owner=stream["owner"], + comment=stream["comment"], + source_type=stream["source_type"], + type=stream["type"], + stale=stream["stale"], + mode=stream["mode"], + database_name=stream["database_name"], + schema_name=stream["schema_name"], + invalid_reason=stream["invalid_reason"], + owner_role_type=stream["owner_role_type"], + stale_after=stream["stale_after"], + table_name=stream["table_name"], + base_tables=stream["base_tables"], + last_altered=stream["created_on"], + ) + ) + return streams + + except Exception as e: + logger.debug( + f"Failed to get all streams for schema - {db_name}.{schema_name}", + exc_info=e, + ) + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 6f09c26b08da2d..6eb83c71c50a50 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -46,6 +46,7 @@ SnowflakeFK, SnowflakePK, SnowflakeSchema, + SnowflakeStream, SnowflakeTable, SnowflakeTag, SnowflakeView, @@ -419,6 +420,11 @@ def _process_schema( if self.config.include_views: views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) + if self.config.include_streams: + streams = self.fetch_streams_for_schema( + snowflake_schema, db_name, schema_name + ) + if self.config.include_tables: db_tables[schema_name] = tables @@ -464,10 +470,22 @@ def _process_schema( for tag in snowflake_schema.tags: yield from self._process_tag(tag) - if not snowflake_schema.views and not snowflake_schema.tables: + if self.config.include_streams: + for stream in streams: + yield from self._process_stream(stream, snowflake_schema, db_name) + + if self.config.include_technical_schema and snowflake_schema.tags: + for tag in snowflake_schema.tags: + yield from self._process_tag(tag) + + if ( + not snowflake_schema.views + and not snowflake_schema.tables + and not snowflake_schema.streams + ): self.structured_reporter.info( - title="No tables/views found in schema", - message="If tables exist, please grant REFERENCES or SELECT permissions on them.", + title="No tables/views/streams found in schema", + message="If objects exist, please grant REFERENCES or SELECT permissions on them.", context=f"{db_name}.{schema_name}", ) @@ -684,7 +702,7 @@ def _process_tag(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: def gen_dataset_workunits( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> Iterable[MetadataWorkUnit]: @@ -739,7 +757,9 @@ def gen_dataset_workunits( subTypes = SubTypes( typeNames=( - [DatasetSubTypes.VIEW] + [DatasetSubTypes.SNOWFLAKE_STREAM] + if isinstance(table, SnowflakeStream) + else [DatasetSubTypes.VIEW] if isinstance(table, SnowflakeView) else [DatasetSubTypes.TABLE] ) @@ -786,7 +806,7 @@ def gen_dataset_workunits( def get_dataset_properties( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> DatasetProperties: @@ -808,6 +828,34 @@ def get_dataset_properties( if isinstance(table, SnowflakeView) and table.is_secure: custom_properties["IS_SECURE"] = "true" + elif isinstance(table, SnowflakeStream): + if table.source_type: + custom_properties["SOURCE_TYPE"] = table.source_type + + if table.type: + custom_properties["TYPE"] = table.type + + if table.stale: + custom_properties["STALE"] = table.stale + + if table.mode: + custom_properties["MODE"] = table.mode + + if table.invalid_reason: + custom_properties["INVALID_REASON"] = table.invalid_reason + + if table.owner_role_type: + custom_properties["OWNER_ROLE_TYPE"] = table.owner_role_type + + if table.table_name: + custom_properties["TABLE_NAME"] = table.table_name + + if table.base_tables: + custom_properties["BASE_TABLES"] = table.base_tables + + if table.stale_after: + custom_properties["STALE_AFTER"] = table.stale_after.isoformat() + return DatasetProperties( name=table.name, created=( @@ -853,7 +901,7 @@ def gen_tag_workunits(self, tag: SnowflakeTag) -> Iterable[MetadataWorkUnit]: def gen_schema_metadata( self, - table: Union[SnowflakeTable, SnowflakeView], + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], schema_name: str, db_name: str, ) -> SchemaMetadata: @@ -1130,3 +1178,143 @@ def _external_tables_ddl_lineage( "External table ddl lineage extraction failed", exc=e, ) + + def fetch_streams_for_schema( + self, snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str + ) -> List[SnowflakeStream]: + try: + streams: List[SnowflakeStream] = [] + for stream in self.get_streams_for_schema(schema_name, db_name): + stream_identifier = self.identifiers.get_dataset_identifier( + stream.name.lower(), schema_name, db_name + ) + + self.report.report_entity_scanned(stream_identifier, "stream") + + if not self.filters.filter_config.table_pattern.allowed( + stream_identifier + ): + self.report.report_dropped(stream_identifier) + else: + streams.append(stream) + snowflake_schema.streams = [stream.name.lower() for stream in streams] + return streams + except Exception as e: + if isinstance(e, SnowflakePermissionError): + error_msg = f"Failed to get streams for schema {db_name}.{schema_name}. Please check permissions." + raise SnowflakePermissionError(error_msg) from e.__cause__ + else: + self.structured_reporter.warning( + "Failed to get streams for schema", + f"{db_name}.{schema_name}", + exc=e, + ) + return [] + + def get_streams_for_schema( + self, schema_name: str, db_name: str + ) -> List[SnowflakeStream]: + streams = self.data_dictionary.get_streams_for_database(db_name) + + return streams.get(schema_name, []) + + def _process_stream( + self, + stream: SnowflakeStream, + snowflake_schema: SnowflakeSchema, + db_name: str, + ) -> Iterable[MetadataWorkUnit]: + schema_name = snowflake_schema.name + stream_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + + try: + stream.columns = self.get_columns_for_stream( + stream.name, snowflake_schema, db_name, stream.table_name + ) + if self.config.extract_tags != TagOption.skip: + stream.column_tags = self.tag_extractor.get_column_tags_for_table( + stream.name, schema_name, db_name + ) + except Exception as e: + self.structured_reporter.warning( + "Failed to get columns for stream", stream_identifier, exc=e + ) + + yield from self.gen_dataset_workunits(stream, schema_name, db_name) + + def get_columns_for_stream( + self, + stream_name: str, + snowflake_schema: SnowflakeSchema, + db_name: str, + source_object: str, # Qualified name of source table/view + ) -> List[SnowflakeColumn]: + """ + Get column information for a stream by getting source object columns and adding metadata columns. + Stream includes all columns from source object plus metadata columns like: + - METADATA$ACTION + - METADATA$ISUPDATE + - METADATA$ROW_ID + """ + columns: List[SnowflakeColumn] = [] + + # First get the source object columns by reusing existing table/view column logic + # Extract database, schema, table name from source_object + source_parts = source_object.split(".") + if len(source_parts) != 3: + self.report.warning(f"Invalid source object name format: {source_object}") + return columns + + source_db, source_schema, source_name = source_parts + + # Get columns from source object + source_columns = self.get_columns_for_table( + source_name, + SnowflakeSchema( + name=source_schema, created=None, last_altered=None, comment=None + ), + source_db, + ) + + # Add all source columns + columns.extend(source_columns) + + # Add standard stream metadata columns + metadata_columns = [ + SnowflakeColumn( + name="METADATA$ACTION", + ordinal_position=len(columns) + 1, + is_nullable=False, + data_type="VARCHAR", + comment="Type of DML operation (INSERT/DELETE)", + character_maximum_length=10, + numeric_precision=None, + numeric_scale=None, + ), + SnowflakeColumn( + name="METADATA$ISUPDATE", + ordinal_position=len(columns) + 2, + is_nullable=False, + data_type="BOOLEAN", + comment="Whether row is from UPDATE operation", + character_maximum_length=None, + numeric_precision=None, + numeric_scale=None, + ), + SnowflakeColumn( + name="METADATA$ROW_ID", + ordinal_position=len(columns) + 3, + is_nullable=False, + data_type="NUMBER", + comment="Unique row identifier", + character_maximum_length=None, + numeric_precision=38, + numeric_scale=0, + ), + ] + + columns.extend(metadata_columns) + + return columns diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 885bee1ccdb908..512c0270d8e3cf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -125,6 +125,7 @@ def is_dataset_pattern_allowed( SnowflakeObjectDomain.VIEW, SnowflakeObjectDomain.MATERIALIZED_VIEW, SnowflakeObjectDomain.ICEBERG_TABLE, + SnowflakeObjectDomain.STREAM, ): return False if _is_sys_table(dataset_name): @@ -137,7 +138,7 @@ def is_dataset_pattern_allowed( message=f"Found a {dataset_type} with an unexpected number of parts. Database and schema filtering will not work as expected, but table filtering will still work.", context=dataset_name, ) - # We fall-through here so table/view filtering still works. + # We fall-through here so table/view/stream filtering still works. if ( len(dataset_params) >= 1 @@ -170,6 +171,14 @@ def is_dataset_pattern_allowed( ): return False + if ( + dataset_type.lower() == SnowflakeObjectDomain.STREAM + and not self.filter_config.stream_pattern.allowed( + _cleanup_qualified_name(dataset_name, self.structured_reporter) + ) + ): + return False + return True diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index c0385a8d5af30a..9aa5838df5f175 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -53,7 +53,10 @@ ) from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeDataDictionary +from datahub.ingestion.source.snowflake.snowflake_schema import ( + SnowflakeDataDictionary, + SnowflakeStream, +) from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( SnowflakeSchemaGenerator, ) @@ -502,15 +505,25 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: for schema in db.schemas for table_name in schema.views ] + discovered_streams: List[str] = [ + self.identifiers.get_dataset_identifier(stream_name, schema.name, db.name) + for db in databases + for schema in db.schemas + for stream_name in schema.streams + ] - if len(discovered_tables) == 0 and len(discovered_views) == 0: + if ( + len(discovered_tables) == 0 + and len(discovered_views) == 0 + and len(discovered_streams) == 0 + ): self.structured_reporter.failure( GENERIC_PERMISSION_ERROR_KEY, - "No tables/views found. Please check permissions.", + "No tables/views/streams found. Please check permissions.", ) return - discovered_datasets = discovered_tables + discovered_views + discovered_datasets = discovered_tables + discovered_views + discovered_streams if self.config.use_queries_v2: with self.report.new_stage(f"*: {VIEW_PARSING}"): @@ -538,13 +551,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: discovered_tables=discovered_datasets, graph=self.ctx.graph, ) + with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): + if self.lineage_extractor and self.config.include_streams: + self.lineage_extractor.populate_stream_upstreams( + self.get_streams(databases) + ) + yield from auto_workunit(self.aggregator.gen_metadata()) - # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs - # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors, - # it should be pretty straightforward to refactor this and only initialize the aggregator once. - self.report.queries_extractor = queries_extractor.report - yield from queries_extractor.get_workunits_internal() - queries_extractor.close() + # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs + # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors, + # it should be pretty straightforward to refactor this and only initialize the aggregator once. + self.report.queries_extractor = queries_extractor.report + yield from queries_extractor.get_workunits_internal() + queries_extractor.close() else: if self.lineage_extractor: @@ -738,6 +757,17 @@ def _snowflake_clear_ocsp_cache(self) -> None: except Exception: logger.debug(f'Failed to remove OCSP cache file at "{file_path}"') + def get_streams(self, databases: List) -> List[SnowflakeStream]: + streams = [] + for db in databases: + for schema in db.schemas: + schema_streams = self.data_dictionary.get_streams_for_schema( + schema.name, db.name + ) + if schema_streams: + streams.extend(schema_streams) + return streams + def close(self) -> None: super().close() StatefulIngestionSourceBase.close(self) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py index 7d82d99412ffe8..d194b059b8d923 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py @@ -51,6 +51,10 @@ class SQLFilterConfig(ConfigModel): default=AllowDenyPattern.allow_all(), description="Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", ) + stream_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for streams to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", + ) @pydantic.root_validator(pre=True) def view_pattern_is_table_pattern_unless_specified( From b7dfa7cba093ce3adfc5be91bfcfe2a033282b14 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:06:37 -0500 Subject: [PATCH 02/30] - moved stream_pattern from sql_config.py to snowflake_config.py - moved stream lineage from snowflake_v2.py & snowflake_lineage_v2.py to snowflake_schema_gen.py - updated snowflake_schema_gen.py to use snowflake_utils.py --- .../source/snowflake/snowflake_config.py | 5 +++ .../source/snowflake/snowflake_lineage_v2.py | 33 -------------- .../source/snowflake/snowflake_query.py | 2 +- .../source/snowflake/snowflake_schema_gen.py | 45 +++++++++++++++---- .../source/snowflake/snowflake_v2.py | 27 +++++------ .../ingestion/source/sql/sql_config.py | 4 -- 6 files changed, 52 insertions(+), 64 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 9d46e68cf52b22..108f0214f80f6e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -98,6 +98,11 @@ class SnowflakeFilterConfig(SQLFilterConfig): ) # table_pattern and view_pattern are inherited from SQLFilterConfig + stream_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for streams to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", + ) + match_fully_qualified_names: bool = Field( default=False, description="Whether `schema_pattern` is matched against fully qualified schema name `.`.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index 826787a2314ca7..e93ecf30171f65 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -20,7 +20,6 @@ ) from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeStream from datahub.ingestion.source.snowflake.snowflake_utils import ( SnowflakeCommonMixin, SnowflakeFilter, @@ -541,35 +540,3 @@ def report_status(self, step: str, status: bool) -> None: def close(self) -> None: pass - - def populate_stream_upstreams(self, streams: List[SnowflakeStream]) -> None: - try: - for stream in streams: - stream_fully_qualified = ".".join( - [ - stream.database_name.lower(), - stream.schema_name.lower(), - stream.name.lower(), - ] - ) - - logger.info( - f"Stream Info: upstream_urn: {stream.table_name.lower()}, downstream_urn: {stream_fully_qualified}" - ) - known_lineage = KnownLineageMapping( - upstream_urn=self.identifiers.gen_dataset_urn( - stream.table_name.lower() - ), - downstream_urn=self.identifiers.gen_dataset_urn( - stream_fully_qualified - ), - ) - self.report.num_streams_with_known_upstreams += 1 - self.sql_aggregator.add(known_lineage) - except Exception as e: - logger.debug(e, exc_info=e) - self.warn_if_stateful_else_error( - "stream-lineage", - f"Failed to extract stream lineage due to error {e}", - ) - self.report_status("STREAM_LINEAGE", False) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index cbb2ace8f85b1c..09b05b92f07ced 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -962,7 +962,7 @@ def streams_for_database( limit: int = SHOW_STREAM_MAX_PAGE_SIZE, stream_pagination_marker: Optional[str] = None, ) -> str: - # SHOW VIEWS can return a maximum of 10000 rows. + # SHOW STREAMS can return a maximum of 10000 rows. # https://docs.snowflake.com/en/sql-reference/sql/show-streams#usage-notes assert limit <= SHOW_STREAM_MAX_PAGE_SIZE diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 6eb83c71c50a50..424d169aa50a86 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -57,6 +57,7 @@ SnowflakeIdentifierBuilder, SnowflakeStructuredReportMixin, SnowsightUrlBuilder, + _split_qualified_name, ) from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, @@ -69,6 +70,7 @@ ) from datahub.ingestion.source_report.ingestion_stage import ( EXTERNAL_TABLE_DDL_LINEAGE, + LINEAGE_EXTRACTION, METADATA_EXTRACTION, PROFILING, ) @@ -234,6 +236,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if self.aggregator: for entry in self._external_tables_ddl_lineage(discovered_tables): self.aggregator.add(entry) + with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): + if self.config.include_streams and self.aggregator: + for entry in self.populate_stream_upstreams(): + self.aggregator.add(entry) except SnowflakePermissionError as e: self.structured_reporter.failure( @@ -421,7 +427,7 @@ def _process_schema( views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) if self.config.include_streams: - streams = self.fetch_streams_for_schema( + self.streams = self.fetch_streams_for_schema( snowflake_schema, db_name, schema_name ) @@ -471,7 +477,7 @@ def _process_schema( yield from self._process_tag(tag) if self.config.include_streams: - for stream in streams: + for stream in self.streams: yield from self._process_stream(stream, snowflake_schema, db_name) if self.config.include_technical_schema and snowflake_schema.tags: @@ -1186,7 +1192,7 @@ def fetch_streams_for_schema( streams: List[SnowflakeStream] = [] for stream in self.get_streams_for_schema(schema_name, db_name): stream_identifier = self.identifiers.get_dataset_identifier( - stream.name.lower(), schema_name, db_name + stream.name, schema_name, db_name ) self.report.report_entity_scanned(stream_identifier, "stream") @@ -1260,12 +1266,7 @@ def get_columns_for_stream( """ columns: List[SnowflakeColumn] = [] - # First get the source object columns by reusing existing table/view column logic - # Extract database, schema, table name from source_object - source_parts = source_object.split(".") - if len(source_parts) != 3: - self.report.warning(f"Invalid source object name format: {source_object}") - return columns + source_parts = _split_qualified_name(source_object) source_db, source_schema, source_name = source_parts @@ -1318,3 +1319,29 @@ def get_columns_for_stream( columns.extend(metadata_columns) return columns + + def populate_stream_upstreams(self) -> Iterable[KnownLineageMapping]: + try: + for stream in self.streams: + stream_fully_qualified = ".".join( + [ + stream.database_name.lower(), + stream.schema_name.lower(), + stream.name.lower(), + ] + ) + + yield KnownLineageMapping( + upstream_urn=self.identifiers.gen_dataset_urn( + stream.table_name.lower() + ), + downstream_urn=self.identifiers.gen_dataset_urn( + stream_fully_qualified + ), + ) + self.report.num_streams_with_known_upstreams += 1 + except Exception as e: + self.structured_reporter.warning( + "Stream lineage extraction failed", + exc=e, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 9aa5838df5f175..1e7d9d5710b9e7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -55,7 +55,6 @@ from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report from datahub.ingestion.source.snowflake.snowflake_schema import ( SnowflakeDataDictionary, - SnowflakeStream, ) from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( SnowflakeSchemaGenerator, @@ -551,12 +550,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: discovered_tables=discovered_datasets, graph=self.ctx.graph, ) - with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): - if self.lineage_extractor and self.config.include_streams: - self.lineage_extractor.populate_stream_upstreams( - self.get_streams(databases) - ) - yield from auto_workunit(self.aggregator.gen_metadata()) # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors, @@ -757,16 +750,16 @@ def _snowflake_clear_ocsp_cache(self) -> None: except Exception: logger.debug(f'Failed to remove OCSP cache file at "{file_path}"') - def get_streams(self, databases: List) -> List[SnowflakeStream]: - streams = [] - for db in databases: - for schema in db.schemas: - schema_streams = self.data_dictionary.get_streams_for_schema( - schema.name, db.name - ) - if schema_streams: - streams.extend(schema_streams) - return streams + # def get_streams(self, databases: List) -> List[SnowflakeStream]: + # streams = [] + # for db in databases: + # for schema in db.schemas: + # schema_streams = self.data_dictionary.get_streams_for_schema( + # schema.name, db.name + # ) + # if schema_streams: + # streams.extend(schema_streams) + # return streams def close(self) -> None: super().close() diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py index d194b059b8d923..7d82d99412ffe8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_config.py @@ -51,10 +51,6 @@ class SQLFilterConfig(ConfigModel): default=AllowDenyPattern.allow_all(), description="Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", ) - stream_pattern: AllowDenyPattern = Field( - default=AllowDenyPattern.allow_all(), - description="Regex patterns for streams to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", - ) @pydantic.root_validator(pre=True) def view_pattern_is_table_pattern_unless_specified( From 30f2e530b29a5fcb01d9ad8582fa369ec8bf766d Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Tue, 14 Jan 2025 13:30:44 -0500 Subject: [PATCH 03/30] added streams to docs --- metadata-ingestion/docs/sources/snowflake/snowflake_pre.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md index 4cfbc470e8c239..37d56655a366d3 100644 --- a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md +++ b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md @@ -50,7 +50,7 @@ The details of each granted privilege can be viewed in [snowflake docs](https:// If the warehouse is already running during ingestion or has auto-resume enabled, this permission is not required. - `usage` is required for us to run queries using the warehouse -- `usage` on `database` and `schema` are required because without it tables and views inside them are not accessible. If an admin does the required grants on `table` but misses the grants on `schema` or the `database` in which the table/view exists then we will not be able to get metadata for the table/view. +- `usage` on `database` and `schema` are required because without it tables, views, and streams inside them are not accessible. If an admin does the required grants on `table` but misses the grants on `schema` or the `database` in which the table/view/stream exists then we will not be able to get metadata for the table/view/stream. - If metadata is required only on some schemas then you can grant the usage privilieges only on a particular schema like ```sql From e0b3c3b2e23f8c517698cc9338894a3b9812adfa Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Tue, 14 Jan 2025 23:31:14 -0500 Subject: [PATCH 04/30] - removed unused method - changed from clone table logic to manually mapping columns since metadata columns were attempting to be mapped from the stream source --- .../source/snowflake/snowflake_schema.py | 40 -------- .../source/snowflake/snowflake_schema_gen.py | 93 ++++++++++++------- .../source/snowflake/snowflake_v2.py | 15 +-- 3 files changed, 62 insertions(+), 86 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index d81cce5f59e124..8ac9a11929196a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -676,43 +676,3 @@ def get_streams_for_database( stream_pagination_marker = stream_name return streams - - @serialized_lru_cache(maxsize=1) - def get_streams_for_schema( - self, schema_name: str, db_name: str - ) -> Optional[List[SnowflakeStream]]: - try: - streams: List[SnowflakeStream] = [] - cur = self.connection.query( - SnowflakeQuery.streams_for_schema(schema_name, db_name), - ) - - for stream in cur: - streams.append( - SnowflakeStream( - name=stream["name"], - created=stream["created_on"], - owner=stream["owner"], - comment=stream["comment"], - source_type=stream["source_type"], - type=stream["type"], - stale=stream["stale"], - mode=stream["mode"], - database_name=stream["database_name"], - schema_name=stream["schema_name"], - invalid_reason=stream["invalid_reason"], - owner_role_type=stream["owner_role_type"], - stale_after=stream["stale_after"], - table_name=stream["table_name"], - base_tables=stream["base_tables"], - last_altered=stream["created_on"], - ) - ) - return streams - - except Exception as e: - logger.debug( - f"Failed to get all streams for schema - {db_name}.{schema_name}", - exc_info=e, - ) - return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 424d169aa50a86..f800c8180d9391 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -104,8 +104,14 @@ from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties from datahub.sql_parsing.sql_parsing_aggregator import ( KnownLineageMapping, + KnownQueryLineageInfo, SqlParsingAggregator, ) +from datahub.sql_parsing.sqlglot_lineage import ( + ColumnLineageInfo, + ColumnRef, + DownstreamColumnRef, +) from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor @@ -236,10 +242,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if self.aggregator: for entry in self._external_tables_ddl_lineage(discovered_tables): self.aggregator.add(entry) - with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): - if self.config.include_streams and self.aggregator: - for entry in self.populate_stream_upstreams(): - self.aggregator.add(entry) except SnowflakePermissionError as e: self.structured_reporter.failure( @@ -1203,7 +1205,7 @@ def fetch_streams_for_schema( self.report.report_dropped(stream_identifier) else: streams.append(stream) - snowflake_schema.streams = [stream.name.lower() for stream in streams] + snowflake_schema.streams = [stream.name for stream in streams] return streams except Exception as e: if isinstance(e, SnowflakePermissionError): @@ -1231,9 +1233,6 @@ def _process_stream( db_name: str, ) -> Iterable[MetadataWorkUnit]: schema_name = snowflake_schema.name - stream_identifier = self.identifiers.get_dataset_identifier( - stream.name, schema_name, db_name - ) try: stream.columns = self.get_columns_for_stream( @@ -1243,9 +1242,13 @@ def _process_stream( stream.column_tags = self.tag_extractor.get_column_tags_for_table( stream.name, schema_name, db_name ) + if self.config.include_column_lineage: + with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): + self.populate_stream_upstreams(stream, db_name, schema_name) + except Exception as e: self.structured_reporter.warning( - "Failed to get columns for stream", stream_identifier, exc=e + "Failed to get columns for stream:", stream.name, exc=e ) yield from self.gen_dataset_workunits(stream, schema_name, db_name) @@ -1320,28 +1323,54 @@ def get_columns_for_stream( return columns - def populate_stream_upstreams(self) -> Iterable[KnownLineageMapping]: - try: - for stream in self.streams: - stream_fully_qualified = ".".join( - [ - stream.database_name.lower(), - stream.schema_name.lower(), - stream.name.lower(), - ] - ) + def populate_stream_upstreams( + self, stream: SnowflakeStream, db_name: str, schema_name: str + ) -> None: + """ + Populate Streams upstream tables excluding the metadata columns + """ + if self.aggregator: + source_parts = _split_qualified_name(stream.table_name) + source_db, source_schema, source_name = source_parts - yield KnownLineageMapping( - upstream_urn=self.identifiers.gen_dataset_urn( - stream.table_name.lower() - ), - downstream_urn=self.identifiers.gen_dataset_urn( - stream_fully_qualified - ), - ) - self.report.num_streams_with_known_upstreams += 1 - except Exception as e: - self.structured_reporter.warning( - "Stream lineage extraction failed", - exc=e, + downstream_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) + + upstream_identifier = self.identifiers.get_dataset_identifier( + source_name, source_schema, source_db ) + upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) + + column_lineage = [] + for col in stream.columns: + if not col.name.startswith("METADATA$"): + column_lineage.append( + ColumnLineageInfo( + downstream=DownstreamColumnRef( + dataset=downstream_urn, + column=self.identifiers.snowflake_identifier(col.name), + ), + upstreams=[ + ColumnRef( + table=upstream_urn, + column=self.identifiers.snowflake_identifier( + col.name + ), + ) + ], + ) + ) + + if column_lineage: + self.aggregator.add_known_query_lineage( + known_query_lineage=KnownQueryLineageInfo( + query_id=f"stream_lineage_{stream.name}", + query_text="", + upstreams=[upstream_urn], + downstream=downstream_urn, + column_lineage=column_lineage, + timestamp=stream.created if stream.created else None, + ) + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 1e7d9d5710b9e7..61344d93c2ddde 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -53,9 +53,7 @@ ) from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_schema import ( - SnowflakeDataDictionary, -) +from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeDataDictionary from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( SnowflakeSchemaGenerator, ) @@ -750,17 +748,6 @@ def _snowflake_clear_ocsp_cache(self) -> None: except Exception: logger.debug(f'Failed to remove OCSP cache file at "{file_path}"') - # def get_streams(self, databases: List) -> List[SnowflakeStream]: - # streams = [] - # for db in databases: - # for schema in db.schemas: - # schema_streams = self.data_dictionary.get_streams_for_schema( - # schema.name, db.name - # ) - # if schema_streams: - # streams.extend(schema_streams) - # return streams - def close(self) -> None: super().close() StatefulIngestionSourceBase.close(self) From 2ff19ba8ec4dc6981c7c9dafb3510e7bcec84fae Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:17:02 -0500 Subject: [PATCH 05/30] merge changes --- .../src/datahub/ingestion/source/snowflake/snowflake_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index d453ac3c9dddbe..99a1fd606e3c5a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -273,7 +273,6 @@ class SnowflakeV2Config( description="List of regex patterns for tags to include in ingestion. Only used if `extract_tags` is enabled.", ) -snowflake-streams-v2 include_streams: bool = Field( default=True, description="If enabled, streams will be ingested as separate entities from tables/views.", From fd87ca9da573e64da7c6301d153085e8a29ea025 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Wed, 15 Jan 2025 20:58:41 -0500 Subject: [PATCH 06/30] lintfix --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 4c0bc893ceac64..bcc9f91543d721 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -973,7 +973,7 @@ def gen_tag_as_structured_property_workunits( ).as_workunit() def gen_column_tags_as_structured_properties( - self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView] + self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream] ) -> Iterable[MetadataWorkUnit]: for column_name in table.column_tags: schema_field_urn = SchemaFieldUrn(dataset_urn, column_name).urn() From 11c6b8c9b1464867cafb4adc5573298336e0c080 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 10:35:35 -0500 Subject: [PATCH 07/30] Refactored _process_schema into smaller functions --- .../source/snowflake/snowflake_schema_gen.py | 139 +++++++++++------- 1 file changed, 82 insertions(+), 57 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index bcc9f91543d721..abb75c49c992b7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -427,78 +427,36 @@ def _process_schema( schema_name = snowflake_schema.name if self.config.extract_tags != TagOption.skip: - snowflake_schema.tags = self.tag_extractor.get_tags_on_object( - schema_name=schema_name, db_name=db_name, domain="schema" - ) + self._process_tags(snowflake_schema, schema_name, db_name, domain="schema") if self.config.include_technical_schema: yield from self.gen_schema_containers(snowflake_schema, db_name) - # We need to do this first so that we can use it when fetching columns. + tables, views, streams = [], [], [] + if self.config.include_tables: tables = self.fetch_tables_for_schema( snowflake_schema, db_name, schema_name ) + db_tables[schema_name] = tables + yield from self._process_tables( + tables, snowflake_schema, db_name, schema_name + ) + if self.config.include_views: views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) + yield from self._process_views( + views, snowflake_schema, db_name, schema_name + ) if self.config.include_streams: - self.streams = self.fetch_streams_for_schema( + streams = self.fetch_streams_for_schema( snowflake_schema, db_name, schema_name ) - - if self.config.include_tables: - db_tables[schema_name] = tables - - if self.config.include_technical_schema: - data_reader = self.make_data_reader() - for table in tables: - table_wu_generator = self._process_table( - table, snowflake_schema, db_name - ) - - yield from classification_workunit_processor( - table_wu_generator, - self.classification_handler, - data_reader, - [db_name, schema_name, table.name], - ) - - if self.config.include_views: - if self.aggregator: - for view in views: - view_identifier = self.identifiers.get_dataset_identifier( - view.name, schema_name, db_name - ) - if view.is_secure and not view.view_definition: - view.view_definition = self.fetch_secure_view_definition( - view.name, schema_name, db_name - ) - if view.view_definition: - self.aggregator.add_view_definition( - view_urn=self.identifiers.gen_dataset_urn(view_identifier), - view_definition=view.view_definition, - default_db=db_name, - default_schema=schema_name, - ) - elif view.is_secure: - self.report.num_secure_views_missing_definition += 1 - - if self.config.include_technical_schema: - for view in views: - yield from self._process_view(view, snowflake_schema, db_name) - - if self.config.include_technical_schema and snowflake_schema.tags: - for tag in snowflake_schema.tags: - yield from self._process_tag(tag) - - if self.config.include_streams: - for stream in self.streams: - yield from self._process_stream(stream, snowflake_schema, db_name) + yield from self._process_streams(streams, snowflake_schema, db_name) if self.config.include_technical_schema and snowflake_schema.tags: - for tag in snowflake_schema.tags: - yield from self._process_tag(tag) + yield from self._process_tags_in_schema(snowflake_schema) if ( not snowflake_schema.views @@ -511,6 +469,71 @@ def _process_schema( context=f"{db_name}.{schema_name}", ) + def _process_tags(self, snowflake_schema, schema_name, db_name, domain): + snowflake_schema.tags = self.tag_extractor.get_tags_on_object( + schema_name=schema_name, db_name=db_name, domain=domain + ) + + def _process_tables( + self, + tables: list, + snowflake_schema: SnowflakeSchema, + db_name: str, + schema_name: str, + ) -> Iterable[MetadataWorkUnit]: + if self.config.include_technical_schema: + data_reader = self.make_data_reader() + for table in tables: + table_wu_generator = self._process_table( + table, snowflake_schema, db_name + ) + yield from classification_workunit_processor( + table_wu_generator, + self.classification_handler, + data_reader, + [db_name, schema_name, table.name], + ) + + def _process_views( + self, + views: list, + snowflake_schema: SnowflakeSchema, + db_name: str, + schema_name: str, + ) -> Iterable[MetadataWorkUnit]: + if self.aggregator: + for view in views: + view_identifier = self.identifiers.get_dataset_identifier( + view.name, schema_name, db_name + ) + if view.is_secure and not view.view_definition: + view.view_definition = self.fetch_secure_view_definition( + view.name, schema_name, db_name + ) + if view.view_definition: + self.aggregator.add_view_definition( + view_urn=self.identifiers.gen_dataset_urn(view_identifier), + view_definition=view.view_definition, + default_db=db_name, + default_schema=schema_name, + ) + elif view.is_secure: + self.report.num_secure_views_missing_definition += 1 + + if self.config.include_technical_schema: + for view in views: + yield from self._process_view(view, snowflake_schema, db_name) + + def _process_streams( + self, streams: list, snowflake_schema: SnowflakeSchema, db_name: str + ) -> Iterable[MetadataWorkUnit]: + for stream in streams: + yield from self._process_stream(stream, snowflake_schema, db_name) + + def _process_tags_in_schema(self, snowflake_schema): + for tag in snowflake_schema.tags: + yield from self._process_tag(tag) + def fetch_secure_view_definition( self, table_name: str, schema_name: str, db_name: str ) -> Optional[str]: @@ -973,7 +996,9 @@ def gen_tag_as_structured_property_workunits( ).as_workunit() def gen_column_tags_as_structured_properties( - self, dataset_urn: str, table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream] + self, + dataset_urn: str, + table: Union[SnowflakeTable, SnowflakeView, SnowflakeStream], ) -> Iterable[MetadataWorkUnit]: for column_name in table.column_tags: schema_field_urn = SchemaFieldUrn(dataset_urn, column_name).urn() From b06d0bd4fe9244130853db69ab1417961dfde735 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:42:39 -0500 Subject: [PATCH 08/30] fixed streams_for_database --- .../src/datahub/ingestion/source/snowflake/snowflake_query.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 09b05b92f07ced..b7f1ff3b323b5e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -968,9 +968,7 @@ def streams_for_database( # To work around this, we paginate through the results using the FROM clause. from_clause = ( - f"""FROM '{stream_pagination_marker}'""" - if SHOW_STREAM_MAX_PAGE_SIZE - else "" + f"""FROM '{stream_pagination_marker}'""" if stream_pagination_marker else "" ) return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" From 5f25f3cc1a38b71fe8cd1b42c362c83139b3a450 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:44:01 -0500 Subject: [PATCH 09/30] updated pytests and golden files --- .../source/snowflake/snowflake_query.py | 6 +- .../tests/integration/snowflake/common.py | 24 + .../snowflake/snowflake_golden.json | 612 +++++++++++++++++- .../snowflake_privatelink_golden.json | 383 +++++++++++ ...nowflake_structured_properties_golden.json | 314 +++++++++ 5 files changed, 1332 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index b7f1ff3b323b5e..ba5f67464f2717 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -970,8 +970,4 @@ def streams_for_database( from_clause = ( f"""FROM '{stream_pagination_marker}'""" if stream_pagination_marker else "" ) - return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" - - @staticmethod - def streams_for_schema(schema_name: str, db_name: str) -> str: - return f"""SHOW STREAMS IN SCHEMA {db_name}.{schema_name}""" + return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 7b4f5abe1cd462..5330821bae54c8 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -10,6 +10,7 @@ NUM_TABLES = 10 NUM_VIEWS = 2 +NUM_STREAMS = 1 NUM_COLS = 10 NUM_OPS = 10 NUM_USAGE = 0 @@ -175,6 +176,7 @@ def default_query_results( # noqa: C901 query, num_tables=NUM_TABLES, num_views=NUM_VIEWS, + num_streams=NUM_STREAMS, num_cols=NUM_COLS, num_ops=NUM_OPS, num_usages=NUM_USAGE, @@ -293,6 +295,28 @@ def default_query_results( # noqa: C901 ) for col_idx in range(1, num_cols + 1) ] + elif query == SnowflakeQuery.streams_for_database("TEST_DB"): + # TODO: Add tests for stream pagination. + return [ + { + "created_on": datetime(2021, 6, 8, 0, 0, 0, 0), + "name": f"STREAM_{stream_idx}", + "database_name": "TEST_DB", + "schema_name": "TEST_SCHEMA", + "owner": "ACCOUNTADMIN", + "comment": f"Comment for Stream {stream_idx}", + "table_name": f"TEST_DB.TEST_SCHEMA.TABLE_{stream_idx}", + "source_type": "Table", + "base_tables": f"TEST_DB.TEST_SCHEMA.TABLE_{stream_idx}", + "type": "DELTA", + "stale": "false", + "mode": "DEFAULT", + "stale_after": datetime(2021, 6, 22, 0, 0, 0, 0), + "invalid_reason": None, + "owner_role_type": "ROLE", + } + for stream_idx in range(1, num_streams + 1) + ] elif query in ( SnowflakeQuery.use_database("TEST_DB"), SnowflakeQuery.show_primary_keys_for_schema("TEST_SCHEMA", "TEST_DB"), diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 48ec46af069cef..1b707dd035947d 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -3951,6 +3951,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", @@ -3974,6 +3990,279 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623124800000 + }, + "lastModified": { + "time": 1623124800000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", @@ -4038,8 +4327,33 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", "lastRunId": "no-run-id-provided" } }, @@ -4361,6 +4675,158 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1623124800000, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "type": "TRANSFORMED", + "query": "urn:li:query:stream_lineage_STREAM_1" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:stream_lineage_STREAM_1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", @@ -4922,6 +5388,34 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_STREAM_1", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1623124800000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1623124800000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -4938,6 +5432,75 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_STREAM_1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_STREAM_1", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -7659,6 +8222,35 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", @@ -8162,5 +8754,21 @@ "runId": "snowflake-2024_04_15-12_58_22", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_STREAM_1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index f22cbd122361dc..2f6fe620304362 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -4101,6 +4101,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)", @@ -4117,6 +4133,344 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "PATCH", + "aspectName": "datasetProperties", + "aspect": { + "json": [ + { + "op": "add", + "path": "/name", + "value": "STREAM_1" + }, + { + "op": "add", + "path": "/description", + "value": "Comment for Stream 1" + }, + { + "op": "add", + "path": "/created", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "time": 1623110400000 + } + }, + { + "op": "add", + "path": "/qualifiedName", + "value": "TEST_DB.TEST_SCHEMA.STREAM_1" + }, + { + "op": "add", + "path": "/customProperties/SOURCE_TYPE", + "value": "Table" + }, + { + "op": "add", + "path": "/customProperties/TYPE", + "value": "DELTA" + }, + { + "op": "add", + "path": "/customProperties/STALE", + "value": "false" + }, + { + "op": "add", + "path": "/customProperties/MODE", + "value": "DEFAULT" + }, + { + "op": "add", + "path": "/customProperties/OWNER_ROLE_TYPE", + "value": "ROLE" + }, + { + "op": "add", + "path": "/customProperties/TABLE_NAME", + "value": "TEST_DB.TEST_SCHEMA.TABLE_1" + }, + { + "op": "add", + "path": "/customProperties/BASE_TABLES", + "value": "TEST_DB.TEST_SCHEMA.TABLE_1" + }, + { + "op": "add", + "path": "/customProperties/STALE_AFTER", + "value": "2021-06-22T00:00:00" + } + ] + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", @@ -4396,6 +4750,35 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,instance1)" + }, + { + "id": "urn:li:container:900b1327253068cb1537b1b3c807ddab", + "urn": "urn:li:container:900b1327253068cb1537b1b3c807ddab" + }, + { + "id": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f", + "urn": "urn:li:container:eac598ee71ef1b5e24448d650c08aa5f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00-foacth", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json index 3adedd59070396..a6c82874e64dc2 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json @@ -3729,6 +3729,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", @@ -3881,6 +3897,279 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623124800000 + }, + "lastModified": { + "time": 1623124800000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", @@ -4008,6 +4297,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", From 18770a932999e9aee28d6744a9f25376881109bc Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 20:11:10 -0500 Subject: [PATCH 10/30] lintfix --- .../src/datahub/ingestion/source/snowflake/snowflake_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index ba5f67464f2717..b68b01dbb0243c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -970,4 +970,4 @@ def streams_for_database( from_clause = ( f"""FROM '{stream_pagination_marker}'""" if stream_pagination_marker else "" ) - return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" \ No newline at end of file + return f"""SHOW STREAMS IN DATABASE {db_name} LIMIT {limit} {from_clause};""" From f726f384d9ece3822519117e79d14a2699ff15c2 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 20:37:12 -0500 Subject: [PATCH 11/30] code review updates. --- .../source/snowflake/snowflake_config.py | 5 - .../source/snowflake/snowflake_schema_gen.py | 147 ++++++++---------- .../source/snowflake/snowflake_utils.py | 14 +- 3 files changed, 76 insertions(+), 90 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 99a1fd606e3c5a..f81e7ec4e31ed1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -278,11 +278,6 @@ class SnowflakeV2Config( description="If enabled, streams will be ingested as separate entities from tables/views.", ) - stream_pattern: AllowDenyPattern = Field( - default=AllowDenyPattern.allow_all(), - description="Regex patterns for streams to filter in ingestion.", - ) - structured_property_pattern: AllowDenyPattern = Field( default=AllowDenyPattern.allow_all(), description=( diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index abb75c49c992b7..3d6d04010bbc1b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -59,7 +59,7 @@ SnowflakeIdentifierBuilder, SnowflakeStructuredReportMixin, SnowsightUrlBuilder, - _split_qualified_name, + split_qualified_name, ) from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, @@ -886,48 +886,42 @@ def get_dataset_properties( custom_properties = {} if isinstance(table, SnowflakeTable): - if table.clustering_key: - custom_properties["CLUSTERING_KEY"] = table.clustering_key - - if table.is_hybrid: - custom_properties["IS_HYBRID"] = "true" - - if table.is_dynamic: - custom_properties["IS_DYNAMIC"] = "true" - - if table.is_iceberg: - custom_properties["IS_ICEBERG"] = "true" + custom_properties.update( + { + k: v + for k, v in { + "CLUSTERING_KEY": table.clustering_key, + "IS_HYBRID": "true" if table.is_hybrid else None, + "IS_DYNAMIC": "true" if table.is_dynamic else None, + "IS_ICEBERG": "true" if table.is_iceberg else None, + }.items() + if v + } + ) if isinstance(table, SnowflakeView) and table.is_secure: custom_properties["IS_SECURE"] = "true" elif isinstance(table, SnowflakeStream): - if table.source_type: - custom_properties["SOURCE_TYPE"] = table.source_type - - if table.type: - custom_properties["TYPE"] = table.type - - if table.stale: - custom_properties["STALE"] = table.stale - - if table.mode: - custom_properties["MODE"] = table.mode - - if table.invalid_reason: - custom_properties["INVALID_REASON"] = table.invalid_reason - - if table.owner_role_type: - custom_properties["OWNER_ROLE_TYPE"] = table.owner_role_type - - if table.table_name: - custom_properties["TABLE_NAME"] = table.table_name - - if table.base_tables: - custom_properties["BASE_TABLES"] = table.base_tables - - if table.stale_after: - custom_properties["STALE_AFTER"] = table.stale_after.isoformat() + custom_properties.update( + { + k: v + for k, v in { + "SOURCE_TYPE": table.source_type, + "TYPE": table.type, + "STALE": table.stale, + "MODE": table.mode, + "INVALID_REASON": table.invalid_reason, + "OWNER_ROLE_TYPE": table.owner_role_type, + "TABLE_NAME": table.table_name, + "BASE_TABLES": table.base_tables, + "STALE_AFTER": table.stale_after.isoformat() + if table.stale_after + else None, + }.items() + if v + } + ) return DatasetProperties( name=table.name, @@ -1386,7 +1380,7 @@ def get_columns_for_stream( """ columns: List[SnowflakeColumn] = [] - source_parts = _split_qualified_name(source_object) + source_parts = split_qualified_name(source_object) source_db, source_schema, source_name = source_parts @@ -1446,48 +1440,45 @@ def populate_stream_upstreams( """ Populate Streams upstream tables excluding the metadata columns """ - if self.aggregator: - source_parts = _split_qualified_name(stream.table_name) - source_db, source_schema, source_name = source_parts + source_parts = split_qualified_name(stream.table_name) + source_db, source_schema, source_name = source_parts - downstream_identifier = self.identifiers.get_dataset_identifier( - stream.name, schema_name, db_name - ) - downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) + downstream_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) - upstream_identifier = self.identifiers.get_dataset_identifier( - source_name, source_schema, source_db - ) - upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) - - column_lineage = [] - for col in stream.columns: - if not col.name.startswith("METADATA$"): - column_lineage.append( - ColumnLineageInfo( - downstream=DownstreamColumnRef( - dataset=downstream_urn, + upstream_identifier = self.identifiers.get_dataset_identifier( + source_name, source_schema, source_db + ) + upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) + + column_lineage = [] + for col in stream.columns: + if not col.name.startswith("METADATA$"): + column_lineage.append( + ColumnLineageInfo( + downstream=DownstreamColumnRef( + dataset=downstream_urn, + column=self.identifiers.snowflake_identifier(col.name), + ), + upstreams=[ + ColumnRef( + table=upstream_urn, column=self.identifiers.snowflake_identifier(col.name), - ), - upstreams=[ - ColumnRef( - table=upstream_urn, - column=self.identifiers.snowflake_identifier( - col.name - ), - ) - ], - ) + ) + ], ) + ) - if column_lineage: - self.aggregator.add_known_query_lineage( - known_query_lineage=KnownQueryLineageInfo( - query_id=f"stream_lineage_{stream.name}", - query_text="", - upstreams=[upstream_urn], - downstream=downstream_urn, - column_lineage=column_lineage, - timestamp=stream.created if stream.created else None, - ) + if column_lineage: + self.aggregator.add_known_query_lineage( + known_query_lineage=KnownQueryLineageInfo( + query_id=f"stream_lineage_{stream.name}", + query_text="", + upstreams=[upstream_urn], + downstream=downstream_urn, + column_lineage=column_lineage, + timestamp=stream.created if stream.created else None, ) + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py index 512c0270d8e3cf..9e451c1c887fa9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_utils.py @@ -131,7 +131,7 @@ def is_dataset_pattern_allowed( if _is_sys_table(dataset_name): return False - dataset_params = _split_qualified_name(dataset_name) + dataset_params = split_qualified_name(dataset_name) if len(dataset_params) != 3: self.structured_reporter.info( title="Unexpected dataset pattern", @@ -193,17 +193,17 @@ def _is_sys_table(table_name: str) -> bool: return table_name.lower().startswith("sys$") -def _split_qualified_name(qualified_name: str) -> List[str]: +def split_qualified_name(qualified_name: str) -> List[str]: """ Split a qualified name into its constituent parts. - >>> _split_qualified_name("db.my_schema.my_table") + >>> split_qualified_name("db.my_schema.my_table") ['db', 'my_schema', 'my_table'] - >>> _split_qualified_name('"db"."my_schema"."my_table"') + >>> split_qualified_name('"db"."my_schema"."my_table"') ['db', 'my_schema', 'my_table'] - >>> _split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"') + >>> split_qualified_name('TEST_DB.TEST_SCHEMA."TABLE.WITH.DOTS"') ['TEST_DB', 'TEST_SCHEMA', 'TABLE.WITH.DOTS'] - >>> _split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE') + >>> split_qualified_name('TEST_DB."SCHEMA.WITH.DOTS".MY_TABLE') ['TEST_DB', 'SCHEMA.WITH.DOTS', 'MY_TABLE'] """ @@ -241,7 +241,7 @@ def _split_qualified_name(qualified_name: str) -> List[str]: def _cleanup_qualified_name( qualified_name: str, structured_reporter: SourceReport ) -> str: - name_parts = _split_qualified_name(qualified_name) + name_parts = split_qualified_name(qualified_name) if len(name_parts) != 3: if not _is_sys_table(qualified_name): structured_reporter.info( From 0d95fcde876b4e5dcad7b0cbb3803b58f17458b0 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 21:55:35 -0500 Subject: [PATCH 12/30] lint --- .../source/snowflake/snowflake_schema_gen.py | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 3d6d04010bbc1b..9a2735088a8f4e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -2,6 +2,8 @@ import logging from typing import Dict, Iterable, List, Optional, Union +from duckdb.duckdb import aggregate + from datahub.configuration.pattern_utils import is_schema_allowed from datahub.emitter.mce_builder import ( get_sys_time, @@ -1440,45 +1442,46 @@ def populate_stream_upstreams( """ Populate Streams upstream tables excluding the metadata columns """ - source_parts = split_qualified_name(stream.table_name) - source_db, source_schema, source_name = source_parts + if self.aggregator: + source_parts = split_qualified_name(stream.table_name) + source_db, source_schema, source_name = source_parts - downstream_identifier = self.identifiers.get_dataset_identifier( - stream.name, schema_name, db_name - ) - downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) + downstream_identifier = self.identifiers.get_dataset_identifier( + stream.name, schema_name, db_name + ) + downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) - upstream_identifier = self.identifiers.get_dataset_identifier( - source_name, source_schema, source_db - ) - upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) - - column_lineage = [] - for col in stream.columns: - if not col.name.startswith("METADATA$"): - column_lineage.append( - ColumnLineageInfo( - downstream=DownstreamColumnRef( - dataset=downstream_urn, - column=self.identifiers.snowflake_identifier(col.name), - ), - upstreams=[ - ColumnRef( - table=upstream_urn, + upstream_identifier = self.identifiers.get_dataset_identifier( + source_name, source_schema, source_db + ) + upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) + + column_lineage = [] + for col in stream.columns: + if not col.name.startswith("METADATA$"): + column_lineage.append( + ColumnLineageInfo( + downstream=DownstreamColumnRef( + dataset=downstream_urn, column=self.identifiers.snowflake_identifier(col.name), - ) - ], + ), + upstreams=[ + ColumnRef( + table=upstream_urn, + column=self.identifiers.snowflake_identifier(col.name), + ) + ], + ) ) - ) - if column_lineage: - self.aggregator.add_known_query_lineage( - known_query_lineage=KnownQueryLineageInfo( - query_id=f"stream_lineage_{stream.name}", - query_text="", - upstreams=[upstream_urn], - downstream=downstream_urn, - column_lineage=column_lineage, - timestamp=stream.created if stream.created else None, + if column_lineage: + self.aggregator.add_known_query_lineage( + known_query_lineage=KnownQueryLineageInfo( + query_id=f"stream_lineage_{stream.name}", + query_text="", + upstreams=[upstream_urn], + downstream=downstream_urn, + column_lineage=column_lineage, + timestamp=stream.created if stream.created else None, + ) ) - ) From 6ad3f70c32b2df3af27a09483faa1c6a3f2ec92a Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 16 Jan 2025 22:21:52 -0500 Subject: [PATCH 13/30] lint --- .../ingestion/source/snowflake/snowflake_schema_gen.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 9a2735088a8f4e..b557030dfc157e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -2,8 +2,6 @@ import logging from typing import Dict, Iterable, List, Optional, Union -from duckdb.duckdb import aggregate - from datahub.configuration.pattern_utils import is_schema_allowed from datahub.emitter.mce_builder import ( get_sys_time, @@ -1468,7 +1466,9 @@ def populate_stream_upstreams( upstreams=[ ColumnRef( table=upstream_urn, - column=self.identifiers.snowflake_identifier(col.name), + column=self.identifiers.snowflake_identifier( + col.name + ), ) ], ) From c20aa2d3582db066d76fe04fc3e079051eff5986 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:11:36 -0500 Subject: [PATCH 14/30] updated tests --- .../tests/integration/snowflake/test_snowflake_failures.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index de6e996a52642b..43d1da97049716 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -148,10 +148,15 @@ def test_snowflake_no_tables_causes_pipeline_failure( [SnowflakeQuery.show_views_for_database("TEST_DB")], [], ) + sf_cursor.execute.side_effect = query_permission_response_override( + no_tables_fn, + [SnowflakeQuery.streams_for_database("TEST_DB")], + [], + ) pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert "permission-error" in [ + assert [] == [ failure.message for failure in pipeline.source.get_report().failures ] From af9d421bbf0d25df3b6c4e3a3997a2feca309d57 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 17 Jan 2025 13:31:09 -0500 Subject: [PATCH 15/30] updated tests --- .../tests/integration/snowflake/common.py | 4 ++-- .../tests/integration/snowflake/snowflake_golden.json | 10 +++++----- .../snowflake/snowflake_privatelink_golden.json | 4 ++-- .../snowflake_structured_properties_golden.json | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 5330821bae54c8..4756f5c8c5ffcc 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -299,7 +299,7 @@ def default_query_results( # noqa: C901 # TODO: Add tests for stream pagination. return [ { - "created_on": datetime(2021, 6, 8, 0, 0, 0, 0), + "created_on": datetime(2021, 6, 8, 0, 0, 0, 0, tzinfo=timezone.utc), "name": f"STREAM_{stream_idx}", "database_name": "TEST_DB", "schema_name": "TEST_SCHEMA", @@ -311,7 +311,7 @@ def default_query_results( # noqa: C901 "type": "DELTA", "stale": "false", "mode": "DEFAULT", - "stale_after": datetime(2021, 6, 22, 0, 0, 0, 0), + "stale_after": datetime(2021, 6, 22, 0, 0, 0, 0, tzinfo=timezone.utc), "invalid_reason": None, "owner_role_type": "ROLE", } diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 1b707dd035947d..7a4f67ac9654d4 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -4242,24 +4242,24 @@ "OWNER_ROLE_TYPE": "ROLE", "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", - "STALE_AFTER": "2021-06-22T00:00:00" + "STALE_AFTER": "2021-06-22T00:00:00+00:00" }, "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", "name": "STREAM_1", "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", "description": "Comment for Stream 1", "created": { - "time": 1623124800000 + "time": 1623110400000 }, "lastModified": { - "time": 1623124800000 + "time": 1623110400000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_17-13_30_23-sqvsy0", "lastRunId": "no-run-id-provided" } }, @@ -4689,7 +4689,7 @@ "actor": "urn:li:corpuser:_ingestion" }, "created": { - "time": 1623124800000, + "time": 1623110400000, "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 2f6fe620304362..87eca14c7655eb 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -4443,13 +4443,13 @@ { "op": "add", "path": "/customProperties/STALE_AFTER", - "value": "2021-06-22T00:00:00" + "value": "2021-06-22T00:00:00+00:00" } ] }, "systemMetadata": { "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00-foacth", + "runId": "snowflake-2022_06_07-17_00_00-ftc9zc", "lastRunId": "no-run-id-provided" } }, diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json index a6c82874e64dc2..559856e1f654b3 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_structured_properties_golden.json @@ -4149,24 +4149,24 @@ "OWNER_ROLE_TYPE": "ROLE", "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", - "STALE_AFTER": "2021-06-22T00:00:00" + "STALE_AFTER": "2021-06-22T00:00:00+00:00" }, "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", "name": "STREAM_1", "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", "description": "Comment for Stream 1", "created": { - "time": 1623124800000 + "time": 1623110400000 }, "lastModified": { - "time": 1623124800000 + "time": 1623110400000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_38-5znjnn", + "runId": "snowflake-2025_01_17-13_30_28-vz7bmd", "lastRunId": "no-run-id-provided" } }, From c1f0be84c46be30e1aa24572f926cad7ea582870 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Sat, 18 Jan 2025 09:41:38 -0500 Subject: [PATCH 16/30] updated reporting --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index b557030dfc157e..85098d1b1103a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -450,6 +450,7 @@ def _process_schema( ) if self.config.include_streams: + self.report.num_get_streams_for_schema_queries += 1 streams = self.fetch_streams_for_schema( snowflake_schema, db_name, schema_name ) From ade6503a5b2b974233b2a151df718686d5be0478 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:24:31 -0500 Subject: [PATCH 17/30] - Updated docs with required permissions - added comments - removed unused num_stream reports - made SnowflakeStream comments optional - defined tables and view argument datatypes - updated allowed pattern - fixed indentation --- .../docs/sources/snowflake/snowflake_pre.md | 7 ++++++- .../ingestion/source/snowflake/snowflake_queries.py | 3 ++- .../ingestion/source/snowflake/snowflake_report.py | 1 - .../ingestion/source/snowflake/snowflake_schema.py | 8 +++++--- .../source/snowflake/snowflake_schema_gen.py | 9 +++++---- .../ingestion/source/snowflake/snowflake_v2.py | 12 ++++++------ 6 files changed, 24 insertions(+), 16 deletions(-) diff --git a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md index 37d56655a366d3..420715e5312cb6 100644 --- a/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md +++ b/metadata-ingestion/docs/sources/snowflake/snowflake_pre.md @@ -15,6 +15,8 @@ grant operate, usage on warehouse "" to role datahub_role; grant usage on DATABASE "" to role datahub_role; grant usage on all schemas in database "" to role datahub_role; grant usage on future schemas in database "" to role datahub_role; +grant select on all streams in database "> to role datahub_role; +grant select on future streams in database "> to role datahub_role; // If you are NOT using Snowflake Profiling or Classification feature: Grant references privileges to your tables and views grant references on all tables in database "" to role datahub_role; @@ -52,7 +54,10 @@ The details of each granted privilege can be viewed in [snowflake docs](https:// - `usage` is required for us to run queries using the warehouse - `usage` on `database` and `schema` are required because without it tables, views, and streams inside them are not accessible. If an admin does the required grants on `table` but misses the grants on `schema` or the `database` in which the table/view/stream exists then we will not be able to get metadata for the table/view/stream. - If metadata is required only on some schemas then you can grant the usage privilieges only on a particular schema like - +```sql +grant usage on schema ""."" to role datahub_role; +``` +- `select` on `streams` is required in order for stream definitions to be available. This does not allow selecting of the data (not required) unless the underlying dataset has select access as well. ```sql grant usage on schema ""."" to role datahub_role; ``` diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index e62129680fee05..f5a192aaeafffd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -406,7 +406,8 @@ def _parse_audit_log_row( ) ) - # Check if any of the accessed objects are streams + # Use direct_objects_accessed instead objects_modified + # objects_modified returns $SYS_VIEW_X with no mapping has_stream_objects = any( obj.get("objectDomain") == "Stream" for obj in direct_objects_accessed ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index fa389640e1a781..f658642ab9e57f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -64,7 +64,6 @@ class SnowflakeReport(SQLSourceReport, BaseTimeWindowReport): num_table_to_view_edges_scanned: int = 0 num_view_to_table_edges_scanned: int = 0 num_external_table_edges_scanned: int = 0 - num_stream_edges_scanned: int = 0 ignore_start_time_lineage: Optional[bool] = None upstream_lineage_in_report: Optional[bool] = None upstream_lineage: Dict[str, List[str]] = field(default_factory=dict) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 4e35493bdd26e5..c2edacea5f23fc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -137,7 +137,6 @@ class SnowflakeStream: name: str created: datetime owner: str - comment: str source_type: str type: str stale: str @@ -147,6 +146,7 @@ class SnowflakeStream: database_name: str schema_name: str table_name: str + comment: Optional[str] columns: List[SnowflakeColumn] = field(default_factory=list) stale_after: Optional[datetime] = None base_tables: Optional[str] = None @@ -448,7 +448,9 @@ def get_columns_for_schema( f"Still fetching columns for {db_name}.{schema_name} - batch {batch_index + 1} of {len(object_batches)}" ) query = SnowflakeQuery.columns_for_schema( - schema_name, db_name, object_batch + schema_name, + db_name, + object_batch, ) cur = self.connection.query(query) @@ -674,7 +676,7 @@ def get_streams_for_database( if result_set_size >= page_limit: # If we hit the limit, we need to send another request to get the next page. logger.info( - f"Fetching next page of views for {db_name} - after {stream_name}" + f"Fetching next page of streams for {db_name} - after {stream_name}" ) stream_pagination_marker = stream_name diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index dd7af7425b784d..99a19ce0dc4c40 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -477,7 +477,7 @@ def _process_tags(self, snowflake_schema, schema_name, db_name, domain): def _process_tables( self, - tables: list, + tables: List[SnowflakeTable], snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str, @@ -497,7 +497,7 @@ def _process_tables( def _process_views( self, - views: list, + views: List[SnowflakeView], snowflake_schema: SnowflakeSchema, db_name: str, schema_name: str, @@ -1311,8 +1311,8 @@ def fetch_streams_for_schema( self.report.report_entity_scanned(stream_identifier, "stream") - if not self.filters.filter_config.table_pattern.allowed( - stream_identifier + if not self.filters.is_dataset_pattern_allowed( + stream_identifier, SnowflakeObjectDomain.STREAM ): self.report.report_dropped(stream_identifier) else: @@ -1441,6 +1441,7 @@ def populate_stream_upstreams( """ Populate Streams upstream tables excluding the metadata columns """ + self.report.num_streams_with_known_upstreams += 1 if self.aggregator: source_parts = split_qualified_name(stream.table_name) source_db, source_schema, source_name = source_parts diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index d9a265fe83ef2a..5a00978c5e0d8c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -552,12 +552,12 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: graph=self.ctx.graph, ) - # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs - # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors, - # it should be pretty straightforward to refactor this and only initialize the aggregator once. - self.report.queries_extractor = queries_extractor.report - yield from queries_extractor.get_workunits_internal() - queries_extractor.close() + # TODO: This is slightly suboptimal because we create two SqlParsingAggregator instances with different configs + # but a shared schema resolver. That's fine for now though - once we remove the old lineage/usage extractors, + # it should be pretty straightforward to refactor this and only initialize the aggregator once. + self.report.queries_extractor = queries_extractor.report + yield from queries_extractor.get_workunits_internal() + queries_extractor.close() else: if self.lineage_extractor: From 3a25dd975db3ab18dc299bd0b15806ddeb5330ef Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 23 Jan 2025 20:02:29 -0500 Subject: [PATCH 18/30] - added logic to filter information schema columns for only - updated tests to accomidate new filter pattern --- .../source/snowflake/snowflake_queries.py | 23 ++++++++----------- .../source/snowflake/snowflake_schema.py | 19 +++++++++++---- .../source/snowflake/snowflake_schema_gen.py | 18 ++++----------- .../tests/integration/snowflake/common.py | 22 ++++++++++++++++++ 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index f5a192aaeafffd..d3d1111bb7ff06 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -363,7 +363,7 @@ def fetch_query_log( def _parse_audit_log_row( self, row: Dict[str, Any], users: UsersMapping - ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery]]: + ) -> Optional[Union[TableRename, TableSwap, PreparsedQuery, ObservedQuery]]: json_fields = { "DIRECT_OBJECTS_ACCESSED", "OBJECTS_MODIFIED", @@ -415,20 +415,17 @@ def _parse_audit_log_row( # If a stream is used, default to query parsing. if has_stream_objects: logger.debug("Found matching stream object") - self.aggregator.add_observed_query( - observed=ObservedQuery( - query=res["query_text"], - session_id=res["session_id"], - timestamp=res["query_start_time"].astimezone(timezone.utc), - user=user, - default_db=res["default_db"], - default_schema=res["default_schema"], - query_hash=get_query_fingerprint( - res["query_text"], self.identifiers.platform, fast=True - ), + return ObservedQuery( + query=res["query_text"], + session_id=res["session_id"], + timestamp=res["query_start_time"].astimezone(timezone.utc), + user=user, + default_db=res["default_db"], + default_schema=res["default_schema"], + query_hash=get_query_fingerprint( + res["query_text"], self.identifiers.platform, fast=True ), ) - return None upstreams = [] column_usage = {} diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index c2edacea5f23fc..9e891027cfb9d4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -14,7 +14,7 @@ ) from datahub.ingestion.source.sql.sql_generic import BaseColumn, BaseTable, BaseView from datahub.utilities.file_backed_collections import FileBackedDict -from datahub.utilities.prefix_batch_builder import build_prefix_batches +from datahub.utilities.prefix_batch_builder import PrefixGroup, build_prefix_batches from datahub.utilities.serialized_lru_cache import serialized_lru_cache logger: logging.Logger = logging.getLogger(__name__) @@ -439,9 +439,18 @@ def get_columns_for_schema( # For massive schemas, use a FileBackedDict to avoid memory issues. columns = FileBackedDict() - object_batches = build_prefix_batches( - all_objects, max_batch_size=10000, max_groups_in_batch=5 - ) + # Single prefix table case (for streams) + if len(all_objects) == 1: + object_batches = [ + PrefixGroup(prefix=all_objects[0], names=[], exact_match=True) + ] + else: + # Build batches for full schema scan + object_batches = build_prefix_batches( + all_objects, max_batch_size=10000, max_groups_in_batch=5 + ) + + # Process batches for batch_index, object_batch in enumerate(object_batches): if batch_index > 0: logger.info( @@ -450,7 +459,7 @@ def get_columns_for_schema( query = SnowflakeQuery.columns_for_schema( schema_name, db_name, - object_batch, + object_batch if isinstance(object_batch, list) else [object_batch], ) cur = self.connection.query(query) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 99a19ce0dc4c40..efa5675296df5b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -1347,9 +1347,8 @@ def _process_stream( schema_name = snowflake_schema.name try: - stream.columns = self.get_columns_for_stream( - stream.name, snowflake_schema, db_name, stream.table_name - ) + stream.columns = self.get_columns_for_stream(stream.table_name) + if self.config.extract_tags != TagOption.skip: stream.column_tags = self.tag_extractor.get_column_tags_for_table( stream.name, schema_name, db_name @@ -1367,9 +1366,6 @@ def _process_stream( def get_columns_for_stream( self, - stream_name: str, - snowflake_schema: SnowflakeSchema, - db_name: str, source_object: str, # Qualified name of source table/view ) -> List[SnowflakeColumn]: """ @@ -1386,13 +1382,9 @@ def get_columns_for_stream( source_db, source_schema, source_name = source_parts # Get columns from source object - source_columns = self.get_columns_for_table( - source_name, - SnowflakeSchema( - name=source_schema, created=None, last_altered=None, comment=None - ), - source_db, - ) + source_columns = self.data_dictionary.get_columns_for_schema( + source_schema, source_db, itertools.chain([source_name]) + ).get(source_name, []) # Add all source columns columns.extend(source_columns) diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 4756f5c8c5ffcc..d059864467ead2 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -7,6 +7,7 @@ from datahub.configuration.time_window_config import BucketDuration from datahub.ingestion.source.snowflake import snowflake_query from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery +from datahub.utilities.prefix_batch_builder import PrefixGroup NUM_TABLES = 10 NUM_VIEWS = 2 @@ -274,6 +275,27 @@ def default_query_results( # noqa: C901 for view_idx in range(1, num_views + 1) if is_secure(view_idx) ] + elif query == SnowflakeQuery.columns_for_schema( + "TEST_SCHEMA", + "TEST_DB", + [PrefixGroup(prefix="TABLE_1", names=[], exact_match=True)], + ): + return [ + { + "TABLE_CATALOG": "TEST_DB", + "TABLE_SCHEMA": "TEST_SCHEMA", + "TABLE_NAME": "TABLE_1", + "COLUMN_NAME": f"COL_{col_idx}", + "ORDINAL_POSITION": col_idx, + "IS_NULLABLE": "NO", + "DATA_TYPE": "TEXT" if col_idx > 1 else "NUMBER", + "COMMENT": "Comment for column", + "CHARACTER_MAXIMUM_LENGTH": 255 if col_idx > 1 else None, + "NUMERIC_PRECISION": None if col_idx > 1 else 38, + "NUMERIC_SCALE": None if col_idx > 1 else 0, + } + for col_idx in range(1, num_cols + 1) + ] elif query == SnowflakeQuery.columns_for_schema("TEST_SCHEMA", "TEST_DB"): return [ { From 13bb2eca70d525e5d34f612109dddf88e53b8d22 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 23 Jan 2025 23:23:21 -0500 Subject: [PATCH 19/30] - added logic to filter information schema columns for only - updated tests to accomidate new filter pattern --- .../source/snowflake/snowflake_queries.py | 18 +++++++++++++++--- .../source/snowflake/snowflake_schema.py | 6 ++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py index d3d1111bb7ff06..eb015f9d13281f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_queries.py @@ -242,7 +242,13 @@ def get_workunits_internal( use_cached_audit_log = audit_log_file.exists() queries: FileBackedList[ - Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] + Union[ + KnownLineageMapping, + PreparsedQuery, + TableRename, + TableSwap, + ObservedQuery, + ] ] if use_cached_audit_log: logger.info("Using cached audit log") @@ -253,7 +259,13 @@ def get_workunits_internal( shared_connection = ConnectionWrapper(audit_log_file) queries = FileBackedList(shared_connection) - entry: Union[KnownLineageMapping, PreparsedQuery, TableRename, TableSwap] + entry: Union[ + KnownLineageMapping, + PreparsedQuery, + TableRename, + TableSwap, + ObservedQuery, + ] with self.report.copy_history_fetch_timer: for entry in self.fetch_copy_history(): @@ -330,7 +342,7 @@ def fetch_copy_history(self) -> Iterable[KnownLineageMapping]: def fetch_query_log( self, users: UsersMapping - ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap]]: + ) -> Iterable[Union[PreparsedQuery, TableRename, TableSwap, ObservedQuery]]: query_log_query = _build_enriched_query_log_query( start_time=self.config.window.start_time, end_time=self.config.window.end_time, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py index 9e891027cfb9d4..9fd497bebb7bb9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema.py @@ -442,7 +442,7 @@ def get_columns_for_schema( # Single prefix table case (for streams) if len(all_objects) == 1: object_batches = [ - PrefixGroup(prefix=all_objects[0], names=[], exact_match=True) + [PrefixGroup(prefix=all_objects[0], names=[], exact_match=True)] ] else: # Build batches for full schema scan @@ -457,9 +457,7 @@ def get_columns_for_schema( f"Still fetching columns for {db_name}.{schema_name} - batch {batch_index + 1} of {len(object_batches)}" ) query = SnowflakeQuery.columns_for_schema( - schema_name, - db_name, - object_batch if isinstance(object_batch, list) else [object_batch], + schema_name, db_name, object_batch ) cur = self.connection.query(query) From f46a4188b16e648bdb61ff1b8b01547227294d56 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 10:46:46 -0500 Subject: [PATCH 20/30] Update metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index efa5675296df5b..bd2893a536bdb9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -531,7 +531,7 @@ def _process_streams( for stream in streams: yield from self._process_stream(stream, snowflake_schema, db_name) - def _process_tags_in_schema(self, snowflake_schema): + def _process_tags_in_schema(self, snowflake_schema:SnowflakeSchema) -> Iterable[MetadataWorkUnit]: for tag in snowflake_schema.tags: yield from self._process_tag(tag) From 51993663df5eed83dace3f633625e7bbc450d3d9 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 10:47:07 -0500 Subject: [PATCH 21/30] Update metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index bd2893a536bdb9..9148feb4589c7a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -526,7 +526,7 @@ def _process_views( yield from self._process_view(view, snowflake_schema, db_name) def _process_streams( - self, streams: list, snowflake_schema: SnowflakeSchema, db_name: str + self, streams: List[SnowflakeStream], snowflake_schema: SnowflakeSchema, db_name: str ) -> Iterable[MetadataWorkUnit]: for stream in streams: yield from self._process_stream(stream, snowflake_schema, db_name) From b719ffb32ca814cd53c73dfa931808e227f35c38 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:50:25 -0500 Subject: [PATCH 22/30] - removed tags from streams, streams don't have tags on columns --- .../source/snowflake/snowflake_schema_gen.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 26e13ea6736d90..b6ad8d3d81f5df 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -526,12 +526,17 @@ def _process_views( yield from self._process_view(view, snowflake_schema, db_name) def _process_streams( - self, streams: List[SnowflakeStream], snowflake_schema: SnowflakeSchema, db_name: str + self, + streams: List[SnowflakeStream], + snowflake_schema: SnowflakeSchema, + db_name: str, ) -> Iterable[MetadataWorkUnit]: for stream in streams: yield from self._process_stream(stream, snowflake_schema, db_name) - def _process_tags_in_schema(self, snowflake_schema:SnowflakeSchema) -> Iterable[MetadataWorkUnit]: + def _process_tags_in_schema( + self, snowflake_schema: SnowflakeSchema + ) -> Iterable[MetadataWorkUnit]: for tag in snowflake_schema.tags: yield from self._process_tag(tag) @@ -1359,10 +1364,6 @@ def _process_stream( try: stream.columns = self.get_columns_for_stream(stream.table_name) - if self.config.extract_tags != TagOption.skip: - stream.column_tags = self.tag_extractor.get_column_tags_for_table( - stream.name, schema_name, db_name - ) if self.config.include_column_lineage: with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): self.populate_stream_upstreams(stream, db_name, schema_name) @@ -1481,7 +1482,7 @@ def populate_stream_upstreams( if column_lineage: self.aggregator.add_known_query_lineage( known_query_lineage=KnownQueryLineageInfo( - query_id=f"stream_lineage_{stream.name}", + query_id=f"stream_lineage_{DatasetUrn.url_encode(downstream_urn)}", query_text="", upstreams=[upstream_urn], downstream=downstream_urn, From 7b68003447e3d4292aba91717ae9db58977a0daf Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:15:15 -0500 Subject: [PATCH 23/30] - removed tags from streams, streams don't have columns so they don't have tags on columns --- .../tests/integration/snowflake/test_snowflake.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index d2e20e784282ee..f94627b1c22f9c 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -180,7 +180,8 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): cache_info = report.data_dictionary_cache.as_obj() assert cache_info["get_tables_for_database"]["misses"] == 1 assert cache_info["get_views_for_database"]["misses"] == 1 - assert cache_info["get_columns_for_schema"]["misses"] == 1 + # When streams query specific tables, the query will not be cached resulting in 2 cache misses + assert cache_info["get_columns_for_schema"]["misses"] == 2 assert cache_info["get_pk_constraints_for_schema"]["misses"] == 1 assert cache_info["get_fk_constraints_for_schema"]["misses"] == 1 From a41aac482ac13b60612e67265fac321561cb38b3 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:19:32 -0500 Subject: [PATCH 24/30] - updated golden files --- .../snowflake/snowflake_golden.json | 250 +++++++++--------- 1 file changed, 125 insertions(+), 125 deletions(-) diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 7a4f67ac9654d4..45bcc1cbf0e80d 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -4647,6 +4647,103 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1623110400000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1623110400000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", @@ -4694,7 +4791,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" } ], "fineGrainedLineages": [ @@ -4708,7 +4805,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4720,7 +4817,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4732,7 +4829,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4744,7 +4841,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4756,7 +4853,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4768,7 +4865,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4780,7 +4877,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4792,7 +4889,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4804,7 +4901,7 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4816,14 +4913,14 @@ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_STREAM_1" + "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_24-12_18_35-7jt669", "lastRunId": "no-run-id-provided" } }, @@ -5388,34 +5485,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_STREAM_1", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "json": { - "statement": { - "value": "", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1623124800000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1623124800000, - "actor": "urn:li:corpuser:_ingestion" - } - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -5432,75 +5501,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_STREAM_1", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_STREAM_1", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -8643,6 +8643,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", @@ -8754,21 +8770,5 @@ "runId": "snowflake-2024_04_15-12_58_22", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_STREAM_1", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file From 8b411015a7126b95607b5fb9147adf50a6bec962 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:08:00 -0500 Subject: [PATCH 25/30] - test fix --- .../ingestion/source/snowflake/snowflake_schema_gen.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index b6ad8d3d81f5df..23ae71807a29da 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -537,8 +537,9 @@ def _process_streams( def _process_tags_in_schema( self, snowflake_schema: SnowflakeSchema ) -> Iterable[MetadataWorkUnit]: - for tag in snowflake_schema.tags: - yield from self._process_tag(tag) + if snowflake_schema.tags: + for tag in snowflake_schema.tags: + yield from self._process_tag(tag) def fetch_secure_view_definition( self, table_name: str, schema_name: str, db_name: str From 619efc35fb955b3186decb60c27c5177e9502c1b Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:32:36 -0500 Subject: [PATCH 26/30] - test fix --- .../tests/integration/snowflake/test_snowflake_failures.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index f529428e0bf69e..808734c13801ec 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -143,20 +143,19 @@ def test_snowflake_no_tables_causes_pipeline_failure( [SnowflakeQuery.tables_for_schema("TEST_SCHEMA", "TEST_DB")], [], ) - sf_cursor.execute.side_effect = query_permission_response_override( + no_views_fn = query_permission_response_override( no_tables_fn, [SnowflakeQuery.show_views_for_database("TEST_DB")], [], ) sf_cursor.execute.side_effect = query_permission_response_override( - no_tables_fn, + no_views_fn, [SnowflakeQuery.streams_for_database("TEST_DB")], [], ) - pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() - assert [] == [ + assert "permission-error" in [ failure.message for failure in pipeline.source.get_report().failures ] From 0527c233504f50c66d9bfad5a1f4ec63cf661ee4 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:07:19 -0500 Subject: [PATCH 27/30] reverted to add_known_lineage_mapping for streams --- .../source/snowflake/snowflake_schema_gen.py | 53 +- .../snowflake/snowflake_golden.json | 4064 +++++++++-------- 2 files changed, 2142 insertions(+), 1975 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 23ae71807a29da..8f5df5228880eb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -85,6 +85,7 @@ TimeStamp, ) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( + DatasetLineageTypeClass, DatasetProperties, ViewProperties, ) @@ -118,14 +119,8 @@ ) from datahub.sql_parsing.sql_parsing_aggregator import ( KnownLineageMapping, - KnownQueryLineageInfo, SqlParsingAggregator, ) -from datahub.sql_parsing.sqlglot_lineage import ( - ColumnLineageInfo, - ColumnRef, - DownstreamColumnRef, -) from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor @@ -1363,7 +1358,9 @@ def _process_stream( schema_name = snowflake_schema.name try: + # Retrieve and register the schema without metadata to prevent columns from mapping upstream stream.columns = self.get_columns_for_stream(stream.table_name) + yield from self.gen_dataset_workunits(stream, schema_name, db_name) if self.config.include_column_lineage: with self.report.new_stage(f"*: {LINEAGE_EXTRACTION}"): @@ -1443,51 +1440,29 @@ def populate_stream_upstreams( self, stream: SnowflakeStream, db_name: str, schema_name: str ) -> None: """ - Populate Streams upstream tables excluding the metadata columns + Populate Streams upstream tables """ self.report.num_streams_with_known_upstreams += 1 if self.aggregator: source_parts = split_qualified_name(stream.table_name) source_db, source_schema, source_name = source_parts - downstream_identifier = self.identifiers.get_dataset_identifier( + dataset_identifier = self.identifiers.get_dataset_identifier( stream.name, schema_name, db_name ) - downstream_urn = self.identifiers.gen_dataset_urn(downstream_identifier) + dataset_urn = self.identifiers.gen_dataset_urn(dataset_identifier) upstream_identifier = self.identifiers.get_dataset_identifier( source_name, source_schema, source_db ) upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) - column_lineage = [] - for col in stream.columns: - if not col.name.startswith("METADATA$"): - column_lineage.append( - ColumnLineageInfo( - downstream=DownstreamColumnRef( - dataset=downstream_urn, - column=self.identifiers.snowflake_identifier(col.name), - ), - upstreams=[ - ColumnRef( - table=upstream_urn, - column=self.identifiers.snowflake_identifier( - col.name - ), - ) - ], - ) - ) + logger.info( + f"""upstream_urn: {upstream_urn}, downstream_urn: {dataset_urn}""" + ) - if column_lineage: - self.aggregator.add_known_query_lineage( - known_query_lineage=KnownQueryLineageInfo( - query_id=f"stream_lineage_{DatasetUrn.url_encode(downstream_urn)}", - query_text="", - upstreams=[upstream_urn], - downstream=downstream_urn, - column_lineage=column_lineage, - timestamp=stream.created if stream.created else None, - ) - ) + self.aggregator.add_known_lineage_mapping( + upstream_urn=upstream_urn, + downstream_urn=dataset_urn, + lineage_type=DatasetLineageTypeClass.COPY, + ) diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index 45bcc1cbf0e80d..4b3bd16deceeae 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -16,16 +16,16 @@ "description": "Comment for TEST_DB", "env": "PROD", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_08_20-16_18_24", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -40,8 +40,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -56,8 +56,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -74,8 +74,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -94,8 +94,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -110,8 +110,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -127,8 +127,24 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -150,16 +166,16 @@ "description": "comment for TEST_DB.TEST_SCHEMA", "env": "PROD", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_08_20-16_18_24", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -174,8 +190,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -190,8 +206,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -208,8 +224,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -228,24 +244,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -265,8 +265,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -281,8 +281,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -329,7 +329,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -353,7 +353,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -377,7 +377,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -478,8 +478,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -498,17 +498,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -523,8 +523,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -541,8 +541,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -566,8 +566,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -582,8 +582,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -630,7 +630,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -654,7 +654,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -678,7 +678,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -779,8 +779,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -799,17 +799,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -824,8 +824,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -842,8 +842,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -867,8 +867,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -883,8 +883,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -931,7 +931,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -955,7 +955,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -979,7 +979,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1080,8 +1080,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1100,17 +1100,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1125,8 +1125,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1143,8 +1143,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1168,8 +1168,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1184,8 +1184,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1232,7 +1232,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1256,7 +1256,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1280,7 +1280,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1381,8 +1381,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1401,17 +1401,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1426,8 +1426,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1444,8 +1444,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1469,8 +1469,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1485,8 +1485,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1533,7 +1533,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1557,7 +1557,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1581,7 +1581,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1682,8 +1682,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1702,17 +1702,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1727,8 +1727,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1745,8 +1745,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1770,8 +1770,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1786,8 +1786,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -1834,7 +1834,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1858,7 +1858,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1882,7 +1882,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -1983,8 +1983,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2003,17 +2003,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2028,8 +2028,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2046,8 +2046,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2071,8 +2071,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2087,8 +2087,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2135,7 +2135,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2159,7 +2159,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2183,7 +2183,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2284,8 +2284,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2304,17 +2304,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2329,8 +2329,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2347,8 +2347,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2372,8 +2372,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2388,8 +2388,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2436,7 +2436,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2460,7 +2460,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2484,7 +2484,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2585,8 +2585,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2605,17 +2605,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2630,8 +2630,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2648,8 +2648,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2673,8 +2673,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2689,8 +2689,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2737,7 +2737,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2761,7 +2761,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2785,7 +2785,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -2886,8 +2886,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2906,17 +2906,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2931,8 +2931,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2949,8 +2949,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2974,8 +2974,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -2990,8 +2990,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3038,7 +3038,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3062,7 +3062,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3086,7 +3086,7 @@ } ], "auditStamp": { - "time": 1654621200000, + "time": 1615443388097, "actor": "urn:li:corpuser:datahub" } }, @@ -3187,8 +3187,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3207,17 +3207,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10", "description": "Comment for Table", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3232,8 +3232,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3250,8 +3250,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3275,8 +3275,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3292,8 +3292,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3308,8 +3308,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3479,8 +3479,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3499,17 +3499,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_1", "description": "Comment for View", "created": { - "time": 1623090600000 + "time": 1623124800000 }, "lastModified": { - "time": 1623090600000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_12_16-15_30_20-649nax", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3524,8 +3524,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3542,8 +3542,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3561,7 +3561,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3585,8 +3585,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3602,8 +3602,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3619,8 +3619,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3636,8 +3636,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3652,8 +3652,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3816,8 +3816,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3834,17 +3834,17 @@ "qualifiedName": "TEST_DB.TEST_SCHEMA.VIEW_2", "description": "Comment for View", "created": { - "time": 1623103200000 + "time": 1623124800000 }, "lastModified": { - "time": 1623103200000 + "time": 1623124800000 }, "tags": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_12_18-10_16_09", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3859,8 +3859,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3877,8 +3877,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3903,33 +3903,8 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -3947,46 +3922,48 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "status", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "removed": false } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4189,41 +4166,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Snowflake Stream" - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4259,76 +4202,41 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_17-13_30_23-sqvsy0", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "container", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "subTypes", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "typeNames": [ + "Snowflake Stream" + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4336,58 +4244,324 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", - "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" - }, - { - "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", - "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" - } - ] + "removed": false } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "schemaMetadata", "aspect": { "json": { - "timestampMillis": 1654621200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 10000, - "columnCount": 10, - "sizeInBytes": 1024 + "schemaName": "test_db.test_schema.stream_1", + "platform": "urn:li:dataPlatform:snowflake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "col_1", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_2", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_3", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_4", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_5", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_6", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_7", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_8", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_9", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "col_10", + "nullable": false, + "description": "Comment for column", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$action", + "nullable": false, + "description": "Type of DML operation (INSERT/DELETE)", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(10)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$isupdate", + "nullable": false, + "description": "Whether row is from UPDATE operation", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata$row_id", + "nullable": false, + "description": "Unique row identifier", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "NUMBER(38,0)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "SOURCE_TYPE": "Table", + "TYPE": "DELTA", + "STALE": "false", + "MODE": "DEFAULT", + "OWNER_ROLE_TYPE": "ROLE", + "TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_1", + "BASE_TABLES": "TEST_DB.TEST_SCHEMA.TABLE_1", + "STALE_AFTER": "2021-06-22T00:00:00+00:00" + }, + "externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/view/STREAM_1/", + "name": "STREAM_1", + "qualifiedName": "TEST_DB.TEST_SCHEMA.STREAM_1", + "description": "Comment for Stream 1", + "created": { + "time": 1623110400000 + }, + "lastModified": { + "time": 1623110400000 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snowflake Stream" + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5e359958be02ce647cd9ac196dbd4585", + "urn": "urn:li:container:5e359958be02ce647cd9ac196dbd4585" + }, + { + "id": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c", + "urn": "urn:li:container:94c696a054bab40b73e640a7f82e3b1c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516740, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4398,19 +4572,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516750, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4421,19 +4595,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516750, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4444,19 +4618,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516751, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4467,19 +4641,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1738040516751, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -4490,285 +4664,123 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "datasetProfile", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 1615443388097, - "actor": "urn:li:corpuser:_ingestion" - }, - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "type": "VIEW", - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" - } - ] + "timestampMillis": 1738040516752, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetProfile", "aspect": { "json": { - "statement": { - "value": "", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1623110400000, - "actor": "urn:li:corpuser:_ingestion" + "timestampMillis": 1738040516752, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1623110400000, - "actor": "urn:li:corpuser:_ingestion" - } + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetProfile", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" - } - ] + "timestampMillis": 1738040516753, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "datasetProfile", "aspect": { "json": { - "platform": "urn:li:dataPlatform:snowflake" + "timestampMillis": 1738040516754, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetProfile", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_1\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "timestampMillis": 1738040516754, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "rowCount": 10000, + "columnCount": 10, + "sizeInBytes": 1024 } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -4786,12 +4798,11 @@ "actor": "urn:li:corpuser:_ingestion" }, "created": { - "time": 1623110400000, + "time": 0, "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "type": "TRANSFORMED", - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "type": "COPY" } ], "fineGrainedLineages": [ @@ -4804,8 +4815,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_1)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4816,8 +4826,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_2)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4828,8 +4837,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_3)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4840,8 +4848,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_4)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4852,8 +4859,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_5)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4864,8 +4870,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_6)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4876,8 +4881,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_7)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4888,8 +4892,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_8)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4900,8 +4903,7 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_9)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", @@ -4912,245 +4914,47 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),col_10)" ], - "confidenceScore": 1.0, - "query": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_24-12_18_35-7jt669", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 1615443388097, - "actor": "urn:li:corpuser:_ingestion" - }, - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "VIEW", - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$action)" ], "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" - ], - "confidenceScore": 0.9, - "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$action)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$isupdate)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$isupdate)" + ], + "confidenceScore": 1.0 }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),metadata$row_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD),metadata$row_id)" + ], + "confidenceScore": 1.0 } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5329,13 +5133,116 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_1\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5352,7 +5259,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ], "fineGrainedLineages": [ @@ -5363,10 +5270,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5375,10 +5282,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5387,10 +5294,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5399,10 +5306,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5411,10 +5318,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5423,10 +5330,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5435,10 +5342,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5447,10 +5354,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5459,10 +5366,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5471,33 +5378,17 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5525,41 +5416,82 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "querySubjects", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_2\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5576,7 +5508,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ], "fineGrainedLineages": [ @@ -5587,10 +5519,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5599,10 +5531,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5611,10 +5543,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5623,10 +5555,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5635,10 +5567,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5647,10 +5579,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5659,10 +5591,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5671,10 +5603,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5683,10 +5615,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -5695,17 +5627,45 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_2\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5717,9 +5677,6 @@ "aspect": { "json": { "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, { "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, @@ -5758,7 +5715,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -5774,66 +5731,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", - "changeType": "UPSERT", - "aspectName": "querySubjects", - "aspect": { - "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -5850,7 +5754,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" } ], "fineGrainedLineages": [ @@ -5861,10 +5765,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5873,10 +5777,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5885,10 +5789,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5897,10 +5801,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5909,10 +5813,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5921,10 +5825,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5933,10 +5837,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5945,10 +5849,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5957,10 +5861,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -5969,23 +5873,120 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_3\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6002,7 +6003,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ], "fineGrainedLineages": [ @@ -6013,10 +6014,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6025,10 +6026,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6037,10 +6038,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6049,10 +6050,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6061,10 +6062,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6073,10 +6074,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6085,10 +6086,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6097,10 +6098,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6109,10 +6110,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -6121,17 +6122,17 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -6159,13 +6160,66 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -6175,13 +6229,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6198,7 +6252,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ], "fineGrainedLineages": [ @@ -6209,10 +6263,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6221,10 +6275,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6233,10 +6287,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6245,10 +6299,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6257,10 +6311,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6269,10 +6323,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6281,10 +6335,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6293,10 +6347,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6305,10 +6359,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -6317,23 +6371,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_5\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -6343,44 +6425,44 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -6388,33 +6470,21 @@ "entityType": "query", "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_5\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "platform": "urn:li:dataPlatform:snowflake" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6431,7 +6501,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ], "fineGrainedLineages": [ @@ -6442,10 +6512,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6454,10 +6524,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6466,10 +6536,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6478,10 +6548,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6490,10 +6560,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6502,10 +6572,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6514,10 +6584,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6526,10 +6596,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6538,10 +6608,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -6550,23 +6620,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_6\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -6576,66 +6674,50 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -6645,13 +6727,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6668,7 +6750,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ], "fineGrainedLineages": [ @@ -6679,10 +6761,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6691,10 +6773,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6703,10 +6785,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6715,10 +6797,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6727,10 +6809,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6739,10 +6821,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6751,10 +6833,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6763,10 +6845,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6775,10 +6857,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -6787,29 +6869,29 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", "changeType": "UPSERT", "aspectName": "queryProperties", "aspect": { "json": { "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_3\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_7\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", "language": "SQL" }, "source": "SYSTEM", @@ -6825,13 +6907,66 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -6841,13 +6976,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6864,7 +6999,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ], "fineGrainedLineages": [ @@ -6875,10 +7010,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6887,10 +7022,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6899,10 +7034,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6911,10 +7046,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6923,10 +7058,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6935,10 +7070,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6947,10 +7082,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6959,10 +7094,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6971,10 +7106,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -6983,23 +7118,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_8\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -7009,50 +7172,66 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -7069,7 +7248,7 @@ }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "type": "TRANSFORMED", - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" } ], "fineGrainedLineages": [ @@ -7080,10 +7259,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7092,10 +7271,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7104,10 +7283,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7116,10 +7295,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7128,10 +7307,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7140,10 +7319,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7152,10 +7331,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7164,10 +7343,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7176,10 +7355,10 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -7188,257 +7367,51 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" ], "confidenceScore": 1.0, - "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_6\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", + "aspectName": "queryProperties", "aspect": { "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_9\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { @@ -7448,44 +7421,44 @@ "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7501,93 +7474,159 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", - "aspect": { - "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] - } - }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", - "changeType": "UPSERT", - "aspectName": "queryProperties", - "aspect": { - "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_8\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" - }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", + "aspectName": "upstreamLineage", "aspect": { "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 - }, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "uniqueUserCount": 0, - "totalSqlQueries": 0, - "topSqlQueries": [], - "userCounts": [], - "fieldCounts": [] + "upstreams": [ + { + "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + } + ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7608,67 +7647,97 @@ "actor": "urn:li:corpuser:_ingestion" }, "lastModified": { - "time": 1710193011317, + "time": 1738040516852, "actor": "urn:li:corpuser:_ingestion" } } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7676,82 +7745,278 @@ "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "CREATE VIEW view_2 AS\nSELECT\n *\nFROM table_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1738040516859, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", "aspectName": "querySubjects", "aspect": { "json": { "subjects": [ { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" }, { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" }, { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -7767,41 +8032,42 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_9\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -7822,84 +8088,44 @@ "fieldCounts": [] } }, - "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:snowflake" - } - }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_22-10_26_30", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -7921,42 +8147,43 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_7\nSELECT\n *\nFROM TEST_DB.TEST_SCHEMA.TABLE_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1654473600000, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -7978,178 +8205,101 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", "changeType": "UPSERT", - "aspectName": "queryProperties", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "statement": { - "value": "CREATE VIEW view_2 AS\nSELECT\n *\nFROM table_2", - "language": "SQL" + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 }, - "source": "SYSTEM", - "created": { - "time": 0, - "actor": "urn:li:corpuser:_ingestion" + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, - "lastModified": { - "time": 1710193011336, - "actor": "urn:li:corpuser:_ingestion" - } + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_11-14_36_03", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", - "aspectName": "querySubjects", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "subjects": [ - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" - }, - { - "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" - }, - { - "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" - } - ] + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_07_09-14_45_18", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -8171,31 +8321,37 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -8203,28 +8359,34 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", - "aspectName": "operation", + "aspectName": "datasetUsageStatistics", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "actor": "urn:li:corpuser:abc@xyz.com", - "operationType": "CREATE", - "lastUpdatedTimestamp": 1654144861367 + "uniqueUserCount": 0, + "totalSqlQueries": 0, + "topSqlQueries": [], + "userCounts": [], + "fieldCounts": [] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.stream_1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", "aspectName": "datasetUsageStatistics", "aspect": { @@ -8247,18 +8409,18 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_16-14_38_33-6dhd1g", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8269,19 +8431,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8292,19 +8454,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8315,19 +8477,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8338,19 +8500,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8361,19 +8523,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8384,19 +8546,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8407,19 +8569,19 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", "changeType": "UPSERT", "aspectName": "operation", "aspect": { "json": { - "timestampMillis": 1654621200000, + "timestampMillis": 1615443388097, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -8430,30 +8592,60 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "operation", "aspect": { "json": { - "removed": false + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "actor": "urn:li:corpuser:abc@xyz.com", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1654144861367 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8462,14 +8654,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8478,14 +8670,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8494,14 +8686,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8510,14 +8702,14 @@ } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8527,7 +8719,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -8543,13 +8735,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8559,7 +8751,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -8575,13 +8767,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8591,13 +8783,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8607,13 +8799,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8623,13 +8815,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "query", - "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8639,13 +8831,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:stream_lineage_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.stream_1%2CPROD%29", + "entityType": "tag", + "entityUrn": "urn:li:tag:other_db.other_schema.my_other_tag:other", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8655,13 +8847,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2025_01_24-12_18_35-7jt669", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_0:my_value_0", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8671,13 +8863,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_1:my_value_1", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8687,13 +8879,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.my_tag_2:my_value_2", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8703,13 +8895,13 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { - "entityType": "query", - "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "entityType": "tag", + "entityUrn": "urn:li:tag:test_db.test_schema.security:pii", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -8719,7 +8911,7 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_03_01-16_29_41", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, @@ -8735,39 +8927,39 @@ }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "glossaryTerm", - "entityUrn": "urn:li:glossaryTerm:Email_Address", + "entityUrn": "urn:li:glossaryTerm:CloudRegion", "changeType": "UPSERT", "aspectName": "glossaryTermKey", "aspect": { "json": { - "name": "Email_Address" + "name": "CloudRegion" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } }, { "entityType": "glossaryTerm", - "entityUrn": "urn:li:glossaryTerm:CloudRegion", + "entityUrn": "urn:li:glossaryTerm:Email_Address", "changeType": "UPSERT", "aspectName": "glossaryTermKey", "aspect": { "json": { - "name": "CloudRegion" + "name": "Email_Address" } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2024_04_15-12_58_22", + "runId": "snowflake-2025_01_28-00_01_52-5vkne0", "lastRunId": "no-run-id-provided" } } From 16bdca27b9ae7a48a5b0cf364982e1a556ecc41c Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Thu, 30 Jan 2025 14:07:18 -0500 Subject: [PATCH 28/30] Update metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 8f5df5228880eb..5ad6c98313963d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -1457,7 +1457,7 @@ def populate_stream_upstreams( ) upstream_urn = self.identifiers.gen_dataset_urn(upstream_identifier) - logger.info( + logger.debug( f"""upstream_urn: {upstream_urn}, downstream_urn: {dataset_urn}""" ) From 457dab79bc7411ef50393305661218d618c380a2 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Fri, 31 Jan 2025 09:16:12 -0500 Subject: [PATCH 29/30] removed extra yield --- .../datahub/ingestion/source/snowflake/snowflake_schema_gen.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index 5ad6c98313963d..e945ab53b67657 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -1371,8 +1371,6 @@ def _process_stream( "Failed to get columns for stream:", stream.name, exc=e ) - yield from self.gen_dataset_workunits(stream, schema_name, db_name) - def get_columns_for_stream( self, source_object: str, # Qualified name of source table/view From f82685e4e50c6b0a623dfcc630dcca05cff61350 Mon Sep 17 00:00:00 2001 From: Brock Griffey <52086127+brock-acryl@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:03:12 -0500 Subject: [PATCH 30/30] lintfix --- .../src/datahub/ingestion/source/snowflake/snowflake_v2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 583b9029ae8201..daeb839e5f54da 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -557,7 +557,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) return - self.discovered_datasets = discovered_tables + discovered_views + discovered_streams + self.discovered_datasets = ( + discovered_tables + discovered_views + discovered_streams + ) if self.config.use_queries_v2: with self.report.new_stage(f"*: {VIEW_PARSING}"):