Merge branch 'master' into cus-2206+powerbi-lineage-issue

datahub-project · Aug 6, 2024 · 6b7b778 · 6b7b778
2 parents 3bd58a7 + 41fbae5
commit 6b7b778
Show file tree

Hide file tree

Showing 39 changed files with 216 additions and 81 deletions.
diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java
@@ -66,7 +66,8 @@ public SsoCallbackController(
 
   public CompletionStage<Result> handleCallback(String protocol, Http.Request request) {
     if (shouldHandleCallback(protocol)) {
-      log.debug(String.format("Handling SSO callback. Protocol: %s", protocol));
+      log.debug("Handling SSO callback. Protocol: {}",
+        _ssoManager.getSsoProvider().protocol().getCommonName());
       return callback(request)
           .handle(
               (res, e) -> {

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -1032,6 +1032,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher("mlModel", getResolver(mlModelType))
                 .dataFetcher("mlModelGroup", getResolver(mlModelGroupType))
                 .dataFetcher("assertion", getResolver(assertionType))
+                .dataFetcher("form", getResolver(formType))
+                .dataFetcher("view", getResolver(dataHubViewType))
                 .dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient))
                 .dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver())
                 .dataFetcher("listUsers", new ListUsersResolver(this.entityClient))

diff --git a/.../src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java b/.../src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java
@@ -41,7 +41,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
           final QueryContext context = environment.getContext();
           final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class);
 
-          log.info("User {} revoking access token {}", context.getActorUrn(), tokenId);
+          log.info("User {} revoking access token", context.getActorUrn());
 
           if (isAuthorizedToRevokeToken(context, tokenId)) {
             try {

diff --git a/.../main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java b/.../main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java
@@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) {
   public CompletableFuture<String> get(final DataFetchingEnvironment environment) throws Exception {
     final QueryContext context = environment.getContext();
     if (IngestionAuthUtils.canManageSecrets(context)) {
-      final String secretUrn = environment.getArgument("urn");
-      final Urn urn = Urn.createFromString(secretUrn);
+      final String inputUrn = environment.getArgument("urn");
+      final Urn urn = Urn.createFromString(inputUrn);
       return GraphQLConcurrencyUtils.supplyAsync(
           () -> {
             try {
               _entityClient.deleteEntity(context.getOperationContext(), urn);
-              return secretUrn;
+              return inputUrn;
             } catch (Exception e) {
               throw new RuntimeException(
-                  String.format("Failed to perform delete against secret with urn %s", secretUrn),
+                  String.format("Failed to perform delete against secret with urn %s", inputUrn),
                   e);
             }
           },

diff --git a/...-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java b/...-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java
@@ -25,7 +25,7 @@ static String encrypt(String value, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.ENCRYPT_MODE, secretKey);
       return Base64.getEncoder()
           .encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8)));
@@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) {
       } catch (NoSuchAlgorithmException e) {
         e.printStackTrace();
       }
-      Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
+      Cipher cipher = Cipher.getInstance("AES");
       cipher.init(Cipher.DECRYPT_MODE, secretKey);
       return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue)));
     } catch (Exception e) {

diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -68,6 +68,17 @@ type Query {
     Fetch a Tag by primary key (urn)
     """
     tag(urn: String!): Tag
+
+    """
+    Fetch a View by primary key (urn)
+    """
+    view(urn: String!): DataHubView
+
+    """
+    Fetch a Form by primary key (urn)
+    """
+    form(urn: String!): Form
+
     """
     Fetch a Role by primary key (urn)
     """

diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx
@@ -1,6 +1,5 @@
 import React from 'react';
 import Cookies from 'js-cookie';
-import { message } from 'antd';
 import { BrowserRouter as Router } from 'react-router-dom';
 import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client';
 import { onError } from '@apollo/client/link/error';
@@ -21,7 +20,7 @@ import { useCustomTheme } from './customThemeContext';
 const httpLink = createHttpLink({ uri: '/api/v2/graphql' });
 
 const errorLink = onError((error) => {
-    const { networkError, graphQLErrors } = error;
+    const { networkError } = error;
     if (networkError) {
         const serverError = networkError as ServerError;
         if (serverError.statusCode === ErrorCodes.Unauthorized) {
@@ -31,13 +30,14 @@ const errorLink = onError((error) => {
             window.location.replace(`${PageRoutes.AUTHENTICATE}?redirect_uri=${encodeURIComponent(currentPath)}`);
         }
     }
-    if (graphQLErrors && graphQLErrors.length) {
-        const firstError = graphQLErrors[0];
-        const { extensions } = firstError;
-        const errorCode = extensions && (extensions.code as number);
-        // Fallback in case the calling component does not handle.
-        message.error(`${firstError.message} (code ${errorCode})`, 3);
-    }
+    // Disabled behavior for now -> Components are expected to handle their errors.
+    // if (graphQLErrors && graphQLErrors.length) {
+    //     const firstError = graphQLErrors[0];
+    //     const { extensions } = firstError;
+    //     const errorCode = extensions && (extensions.code as number);
+    //     // Fallback in case the calling component does not handle.
+    //     message.error(`${firstError.message} (code ${errorCode})`, 3); // TODO: Decide if we want this back.
+    // }
 });
 
 const client = new ApolloClient({

diff --git a/datahub-web-react/src/graphql/view.graphql b/datahub-web-react/src/graphql/view.graphql
@@ -39,6 +39,12 @@ query listGlobalViews($start: Int!, $count: Int!, $query: String) {
     }
 }
 
+query getView($urn: String!) {
+    view(urn: $urn) {
+        ...view
+    }
+}
+
 mutation createView($input: CreateViewInput!) {
     createView(input: $input) {
         ...view

diff --git a/docs-website/src/pages/cloud/CompanyLogos/customersData.json b/docs-website/src/pages/cloud/CompanyLogos/customersData.json
@@ -2,38 +2,50 @@
   "customers": [
     {
       "link": {
-        "href": "https://robinhood.com",
+        "href": "https://www.depop.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/acertus.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/345d124a249d43bf0f20b608f8bfa2f7683311fa-360x180.png"
         },
-        "alt": "Robinhood"
+        "alt": "depop"
       }
     },
     {
       "link": {
-        "href": "https://www.dpgmediagroup.com/",
+        "href": "https://riskified.com",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/autoscout24.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c982e0459bed565273a9b696d9d40aed76f84b1e-360x180.png"
         },
-        "alt": "DPG Media"
+        "alt": "Riskified"
       }
     },
     {
       "link": {
-        "href": "https://www.twilio.com",
+        "href": "https://get.betterup.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/betterup.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5988a55b3c090a12ddc3f3cae07b290ac3134771-360x180.png"
         },
-        "alt": "Twilio"
+        "alt": "Betterup"
+      }
+    },
+    {
+      "link": {
+        "href": "https://www.ovoenergy.com/",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/5e7bd32dfbc769849dca136947ebd2fc2f5e91f3-540x270.png"
+        },
+        "alt": "OVO Energy"
       }
     },
     {
@@ -43,45 +55,69 @@
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/depop.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/3d7c10e1bd7c7a062250e092d6d9d0553fb57790-360x180.png"
         },
         "alt": "Myob"
       }
     },
     {
       "link": {
-        "href": "https://regeneron.com",
+        "href": "https://www.dpgmediagroup.com/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/dpg_media.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/b446f595b4b13a72ee82a285924715f950e012ca-540x270.png"
         },
-        "alt": "Regeneron"
+        "alt": "DPG Megia"
       }
     },
     {
       "link": {
-        "href": "https://riskified.com",
+        "href": "https://www.notion.so/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/myob.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/c2e84f93572cd1baf30ea7ab8da234ff44182eb6-540x270.png"
         },
-        "alt": "Riskified"
+        "alt": "Notion"
+      }
+    },
+    {
+      "link": {
+        "href": "https://www.sae.org/",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/a9e8586635cb4039cbfc5836a6a5cacdeba9e6b3-540x270.png"
+        },
+        "alt": "SAE International"
+      }
+    },
+    {
+      "link": {
+        "href": "https://viator.com",
+        "blank": true
+      },
+      "logo": {
+        "asset": {
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/0c17141cad5baa053da18dffb17d9c182d242e69-1200x475.png"
+        },
+        "alt": "Viator"
       }
     },
     {
       "link": {
-        "href": "https://xero.com",
+        "href": "https://www.tripadvisor.co.uk/",
         "blank": true
       },
       "logo": {
         "asset": {
-          "_ref": "/img/logos/scrollingCompanies/notion.webp"
+          "_ref": "https://cdn.sanity.io/images/cqo9wkgf/production/28255a28d261a074a83d1ee8632f0338bf5cf57e-1112x256.png"
         },
-        "alt": "Xero"
+        "alt": "Trip Advisor"
       }
     }
   ]

diff --git a/docs-website/src/pages/cloud/CompanyLogos/index.js b/docs-website/src/pages/cloud/CompanyLogos/index.js
@@ -32,7 +32,7 @@ const ScrollingCustomers = ({ noOverlay = true, spacing, ...rest }) => {
             to={customer.link.href}
             target={customer.link.blank ? '_blank' : '_self'}
             rel={customer.link.blank ? 'noopener noreferrer' : ''}
-            style={{ minWidth: 'max-content', padding: '0 3.25rem' }}
+            style={{ minWidth: 'max-content', padding: '0 1.8rem' }}
           >
             <img
               src={customer.logo.asset._ref}

diff --git a/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss b/docs-website/src/pages/cloud/CompanyLogos/logos.module.scss
@@ -33,7 +33,7 @@
 }
 
 .scrollingCustomers__inner img {
-  max-width: 100px!important;
+  max-width: 160px !important;
   min-width: unset;
   filter: invert(1) brightness(0) contrast(100); // make image black
 }

diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js
@@ -34,11 +34,11 @@ function Home() {
               <div className={clsx("hero__subtitle", styles.hero__subtitle)}>
                 Introducing DataHub as a Managed Service
                 <div style={{ fontWeight: "500"}}>with Data Observability and Data Governance built-in.</div>
-                <div className={styles.learnMore}>
+                {/* <div className={styles.learnMore}>
                   <a className={styles.link} href="https://acryldata.io" target="_blank">
                     Learn More  →
                   </a>
-                </div>
+                </div> */}
               </div>
               <Link className="button button--primary button--lg" to="https://www.acryldata.io/datahub-sign-up?utm_source=datahub&utm_medium=referral&utm_campaign=acryl_signup">
                 Book Demo

diff --git a/docs/how/search.md b/docs/how/search.md
@@ -148,24 +148,42 @@ The same GraphQL API that powers the Search UI can be used
 for integrations and programmatic use-cases. 
 
 ```
-# Example query
-{
-  searchAcrossEntities(
-    input: {types: [], query: "*", start: 0, count: 10, filters: [{field: "fieldTags", value: "urn:li:tag:Dimension"}]}
+# Example query - search for datasets matching the example_query_text who have the Dimension tag applied to a schema field and are from the data platform looker
+query searchEntities {
+  search(
+    input: {
+      type: DATASET,
+      query: "example_query_text",
+      orFilters: [
+        {
+          and: [
+            {
+              field: "fieldTags",
+              values: ["urn:li:tag:Dimension"]
+            },
+            {
+              field: "platform",
+              values: ["urn:li:dataPlatform:looker"]
+            }
+          ]
+        }
+      ],
+      start: 0,
+      count: 10
+    }
   ) {
     start
     count
     total
     searchResults {
       entity {
+        urn
         type
         ... on Dataset {
-          urn
-          type
+          name
           platform {
             name
           }
-          name
         }
       }
     }

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java
@@ -63,7 +63,7 @@ public static <T extends FieldSpec> Map<T, List<Object>> extractFields(
         } else {
           List<Object> valueList = (List<Object>) value.get();
           // If the field is a nested list of values, flatten it
-          for (int i = 0; i < numArrayWildcards - 1; i++) {
+          for (long i = 0; i < numArrayWildcards - 1; i++) {
             valueList =
                 valueList.stream()
                     .flatMap(v -> ((List<Object>) v).stream())

diff --git a/metadata-ingestion/src/datahub/cli/cli_utils.py b/metadata-ingestion/src/datahub/cli/cli_utils.py
@@ -1,5 +1,6 @@
 import json
 import logging
+import time
 import typing
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
@@ -403,6 +404,8 @@ def ensure_has_system_metadata(
     if event.systemMetadata is None:
         event.systemMetadata = SystemMetadataClass()
     metadata = event.systemMetadata
+    if metadata.lastObserved == 0:
+        metadata.lastObserved = int(time.time() * 1000)
     if metadata.properties is None:
         metadata.properties = {}
     props = metadata.properties

diff --git a/metadata-ingestion/src/datahub/ingestion/source/abs/source.py b/metadata-ingestion/src/datahub/ingestion/source/abs/source.py
@@ -198,11 +198,6 @@ class TableData:
 @support_status(SupportStatus.INCUBATING)
 @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration")
 @capability(SourceCapability.TAGS, "Can extract ABS object/container tags if enabled")
-@capability(
-    SourceCapability.DELETION_DETECTION,
-    "Optionally enabled via `stateful_ingestion.remove_stale_metadata`",
-    supported=True,
-)
 class ABSSource(StatefulIngestionSourceBase):
     source_config: DataLakeSourceConfig
     report: DataLakeSourceReport