Skip to content

Commit

Permalink
Merge branch 'master' into Fivetran-add-bigquery-destination-support
Browse files Browse the repository at this point in the history
  • Loading branch information
hsheth2 authored Jan 10, 2024
2 parents 0bf2ba0 + 8efe30f commit dadf654
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 73 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.linkedin.datahub.graphql.types.rolemetadata.mappers;

import com.linkedin.common.RoleAssociationArray;
import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.Role;
Expand All @@ -19,8 +20,10 @@ public com.linkedin.datahub.graphql.generated.Access apply(
@Nonnull final com.linkedin.common.Access access, @Nonnull final Urn entityUrn) {
com.linkedin.datahub.graphql.generated.Access result =
new com.linkedin.datahub.graphql.generated.Access();
RoleAssociationArray roles =
access.getRoles() != null ? access.getRoles() : new RoleAssociationArray();
result.setRoles(
access.getRoles().stream()
roles.stream()
.map(association -> this.mapRoleAssociation(association, entityUrn))
.collect(Collectors.toList()));
return result;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class SqlParsingBuilder:
def __post_init__(self) -> None:
if self.usage_config:
self._usage_aggregator = UsageAggregator(self.usage_config)
else:
elif self.generate_usage_statistics:
logger.info("No usage config provided, not generating usage statistics")
self.generate_usage_statistics = False

Expand Down
4 changes: 4 additions & 0 deletions metadata-ingestion/src/datahub/ingestion/run/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ class CliReport(Report):

disk_info: Optional[dict] = None
peak_disk_usage: Optional[str] = None
_initial_disk_usage: int = -1
_peak_disk_usage: int = 0

thread_count: Optional[int] = None
Expand All @@ -156,12 +157,15 @@ def compute_stats(self) -> None:

try:
disk_usage = shutil.disk_usage("/")
if self._initial_disk_usage < 0:
self._initial_disk_usage = disk_usage.used
if self._peak_disk_usage < disk_usage.used:
self._peak_disk_usage = disk_usage.used
self.peak_disk_usage = humanfriendly.format_size(self._peak_disk_usage)
self.disk_info = {
"total": humanfriendly.format_size(disk_usage.total),
"used": humanfriendly.format_size(disk_usage.used),
"used_initally": humanfriendly.format_size(self._initial_disk_usage),
"free": humanfriendly.format_size(disk_usage.free),
}
except Exception as e:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1283,9 +1283,13 @@ def create_bigquery_temp_table(
# temporary table dance. However, that would require either a) upgrading to
# use GE's batch v3 API or b) bypassing GE altogether.

query_job: Optional[
"google.cloud.bigquery.job.query.QueryJob"
] = cursor._query_job
query_job: Optional["google.cloud.bigquery.job.query.QueryJob"] = (
# In google-cloud-bigquery 3.15.0, the _query_job attribute was
# made public and renamed to query_job.
cursor.query_job
if hasattr(cursor, "query_job")
else cursor._query_job # type: ignore[attr-defined]
)
assert query_job
temp_destination_table = query_job.destination
bigquery_temp_table = f"{temp_destination_table.project}.{temp_destination_table.dataset_id}.{temp_destination_table.table_id}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -353,13 +353,13 @@ def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None:

self._populate_external_lineage_from_copy_history(discovered_tables)
logger.info(
"Done populating external lineage from copy history."
"Done populating external lineage from copy history. "
f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far."
)

self._populate_external_lineage_from_show_query(discovered_tables)
logger.info(
"Done populating external lineage from show external tables."
"Done populating external lineage from show external tables. "
f"Found {self.report.num_external_table_edges_scanned} external lineage edges so far."
)

Expand Down
128 changes: 62 additions & 66 deletions smoke-test/tests/cypress/cypress/e2e/query/query_tab.js
Original file line number Diff line number Diff line change
@@ -1,75 +1,71 @@
const DATASET_URN = 'urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleCypressHdfsDataset,PROD)';
const runId = Date.now()

describe("manage queries", () => {
beforeEach(() => {
cy.login();
cy.goToDataset(
DATASET_URN,
"SampleCypressHdfsDataset"
);
cy.hideOnboardingTour();
cy.openEntityTab("Queries")
})

it("go to queries tab on dataset page then, create, edit, make default, delete a view", () => {
const runId = Date.now()

// Headers
cy.waitTextVisible("Highlighted Queries");
cy.ensureTextNotPresent("Recent Queries");

// Add new Query
cy.get('[data-testid="add-query-button"]').click();
cy.get('[class="query-builder-editor-input"]').click();
cy.get('[class="query-builder-editor-input"]').type(` + Test Query-${runId}`);
cy.get('[data-testid="query-builder-title-input"]').click();
cy.get('[data-testid="query-builder-title-input"]').type(`Test Table-${runId}`);
cy.get('.ProseMirror').click();
cy.get('.ProseMirror').type(`Test Description-${runId}`);
cy.get('[data-testid="query-builder-save-button"]').click();
cy.waitTextVisible("Created Query!");

// Verify the card
cy.waitTextVisible(`+ Test Query-${runId}`);
cy.waitTextVisible(`Test Table-${runId}`);
cy.waitTextVisible(`Test Description-${runId}`);
cy.waitTextVisible("Created on");

// View the Query
cy.get('[data-testid="query-content-0"]').click();
cy.get('.ant-modal-content').waitTextVisible(`+ Test Query-${runId}`);
cy.get('.ant-modal-content').waitTextVisible(`Test Table-${runId}`);
cy.get('.ant-modal-content').waitTextVisible(`Test Description-${runId}`);
cy.get('[data-testid="query-modal-close-button"]').click();
const addNewQuery = () => {
cy.get('[data-testid="add-query-button"]').click();
cy.get('[data-mode-id="sql"]').click()
.type(` + Test Query-${runId}`);
cy.get('[data-testid="query-builder-title-input"]').click()
.type(`Test Table-${runId}`);
cy.get('.ProseMirror').click()
.type(`Test Description-${runId}`);
cy.get('[data-testid="query-builder-save-button"]').click();
cy.waitTextVisible("Created Query!");
}

// Edit the Query
cy.get('[data-testid="query-edit-button-0"]').click()
cy.get('[class="query-builder-editor-input"]').click();
cy.get('[class="query-builder-editor-input"]').type(` + Edited Query-${runId}`);
cy.get('[data-testid="query-builder-title-input"]').click();
cy.get('[data-testid="query-builder-title-input"]').clear();
cy.get('[data-testid="query-builder-title-input"]').type(`Edited Table-${runId}`);
cy.get('.ProseMirror').click();
cy.get('.ProseMirror').clear();
cy.get('.ProseMirror').type(`Edited Description-${runId}`);
cy.get('[data-testid="query-builder-save-button"]').click();
cy.waitTextVisible("Edited Query!");
const editQuery = () => {
cy.get('[data-testid="query-edit-button-0"]').click()
cy.get('[data-mode-id="sql"]').click()
.type(` + Edited Query-${runId}`);
cy.get('[data-testid="query-builder-title-input"]').click().clear()
.type(`Edited Table-${runId}`);
cy.get('.ProseMirror').click().clear()
.type(`Edited Description-${runId}`);
cy.get('[data-testid="query-builder-save-button"]').click();
cy.waitTextVisible("Edited Query!");
}

// Verify edited Query card
cy.get('[data-testid="query-content-0"]').scrollIntoView().should('be.visible');
cy.waitTextVisible(`+ Test Query-${runId} + Edited Query-${runId}`);
cy.waitTextVisible(`Edited Table-${runId}`);
cy.waitTextVisible(`Edited Description-${runId}`);

// Delete the Query
const deleteQuery = () => {
cy.get('[data-testid="query-more-button-0"]').click();
cy.get('[data-testid="query-delete-button-0"]').click();
cy.contains('Yes').click();
cy.clickOptionWithText("Delete");
cy.clickOptionWithText('Yes')
cy.waitTextVisible("Deleted Query!");
}

const verifyViewCardDetails = (query,title,description) => {
cy.get('[data-testid="query-content-0"]').scrollIntoView().should('be.visible').click()
cy.get('.ant-modal-content').waitTextVisible(query);
cy.get('.ant-modal-content').waitTextVisible(title);
cy.get('.ant-modal-content').waitTextVisible(description);
};

// Query should be gone
cy.ensureTextNotPresent(`+ Test Query-${runId} + Edited Query-${runId}`);
cy.ensureTextNotPresent(`Edited Table-${runId}`);
cy.ensureTextNotPresent(`Edited Description-${runId}`);
describe("manage queries", () => {
beforeEach(() => {
cy.loginWithCredentials();
cy.goToDataset(DATASET_URN,"SampleCypressHdfsDataset");
cy.openEntityTab("Queries")
})

it("go to queries tab on dataset page then create query and verify & view the card", () => {
cy.waitTextVisible("Highlighted Queries");
cy.ensureTextNotPresent("Recent Queries");
addNewQuery();
cy.waitTextVisible(`+ Test Query-${runId}`);
cy.waitTextVisible(`Test Table-${runId}`);
cy.waitTextVisible(`Test Description-${runId}`);
cy.waitTextVisible("Created on");
verifyViewCardDetails(`+ Test Query-${runId}`,`Test Table-${runId}`,`Test Description-${runId}`)
});

it("go to queries tab on dataset page then edit the query and verify edited Query card", () => {
editQuery();
verifyViewCardDetails(`+ Test Query-${runId} + Edited Query-${runId}`,`Edited Table-${runId}`,`Edited Description-${runId}`)
});

it("go to queries tab on dataset page then delete the query and verify that query should be gone", () => {
deleteQuery();
cy.ensureTextNotPresent(`+ Test Query-${runId} + Edited Query-${runId}`);
cy.ensureTextNotPresent(`Edited Table-${runId}`);
cy.ensureTextNotPresent(`Edited Description-${runId}`);
});
});

0 comments on commit dadf654

Please sign in to comment.