Skip to content

Commit

Permalink
Avoid resource conflicts in integration tests by using a random dir n…
Browse files Browse the repository at this point in the history
…ame (#2865)

## Changes
Replace hard-coded dir name with random one

### Linked issues
Resolves #2797 

### Functionality
None

### Tests
- [x] ran integration tests

---------

Co-authored-by: Eric Vergnaud <eric.vergnaud@databricks.com>
  • Loading branch information
ericvergnaud and ericvergnaud authored Oct 9, 2024
1 parent e06fd44 commit d4e9eeb
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 10 deletions.
6 changes: 3 additions & 3 deletions tests/integration/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -804,12 +804,12 @@ def config(self) -> WorkspaceConfig:
renamed_group_prefix=f'tmp-{self.inventory_database}-',
)

def save_locations(self) -> None:
def save_locations(self, make_random) -> None:
locations: list[ExternalLocation] = []
if self.workspace_client.config.is_azure:
locations = [ExternalLocation("abfss://things@labsazurethings.dfs.core.windows.net/a", 1)]
locations = [ExternalLocation(f"abfss://things@labsazurethings.dfs.core.windows.net/a", 1)]
if self.workspace_client.config.is_aws:
locations = [ExternalLocation("s3://labs-things/a", 1)]
locations = [ExternalLocation(f"s3://labs-things/a", 1)]
self.sql_backend.save_table(
f"{self.inventory_database}.external_locations",
locations,
Expand Down
6 changes: 3 additions & 3 deletions tests/integration/hive_metastore/test_migrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,6 @@ def test_migrate_table_in_mount(
env_or_skip,
make_random,
runtime_ctx,
make_acc_group,
):
if not ws.config.is_azure:
pytest.skip("temporary: only works in azure test env")
Expand All @@ -710,17 +709,18 @@ def test_migrate_table_in_mount(
connect=ws.config,
)
runtime_ctx = runtime_ctx.replace(config=config)
b_dir = make_random(4).lower()
table_path = make_random(4).lower()
src_schema = make_schema(
catalog_name="hive_metastore",
name=f"mounted_{env_or_skip('TEST_MOUNT_NAME')}_{table_path}",
)
src_external_table = runtime_ctx.make_table(
schema_name=src_schema.name,
external_delta=f"dbfs:/mnt/{env_or_skip('TEST_MOUNT_NAME')}/a/b/{table_path}",
external_delta=f"dbfs:/mnt/{env_or_skip('TEST_MOUNT_NAME')}/a/{b_dir}/{table_path}",
columns=[("1-0`.0-ugly-column", "STRING")], # Test with column that needs escaping
)
table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{table_path}"
table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{table_path}"
# TODO: Remove this hack below
# This is done because we have to create the external table in a mount point, but TablesInMounts() has to use the adls/ path
# Otherwise, if we keep the dbfs:/ path, the entire logic of TablesInMounts won't work
Expand Down
9 changes: 5 additions & 4 deletions tests/integration/hive_metastore/test_table_move.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,20 +201,21 @@ def test_alias_tables(


def test_move_tables_table_properties_mismatch_preserves_original(
ws, sql_backend, make_catalog, make_schema, make_table, make_acc_group, make_random, env_or_skip
):
ws, sql_backend, make_catalog, make_schema, make_table, make_random
) -> None:
table_move = TableMove(ws, sql_backend)
from_catalog = make_catalog()
from_schema = make_schema(catalog_name=from_catalog.name)
b_dir = make_random(4).lower()
tbl_path = make_random(4).lower()
tbl_properties = {"delta.enableDeletionVectors": "true"}
from_table_1 = make_table(
catalog_name=from_catalog.name,
schema_name=from_schema.name,
external_delta=f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{tbl_path}",
external_delta=f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{tbl_path}",
tbl_properties=tbl_properties,
)
table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{tbl_path}"
table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{tbl_path}"
from_table_1.storage_location = table_in_mount_location

to_catalog = make_catalog()
Expand Down

0 comments on commit d4e9eeb

Please sign in to comment.