From d4e9eeb118dd2896f2bd90664750640f67205a02 Mon Sep 17 00:00:00 2001 From: Eric Vergnaud Date: Wed, 9 Oct 2024 12:20:38 +0200 Subject: [PATCH] Avoid resource conflicts in integration tests by using a random dir name (#2865) ## Changes Replace hard-coded dir name with random one ### Linked issues Resolves #2797 ### Functionality None ### Tests - [x] ran integration tests --------- Co-authored-by: Eric Vergnaud --- tests/integration/conftest.py | 6 +++--- tests/integration/hive_metastore/test_migrate.py | 6 +++--- tests/integration/hive_metastore/test_table_move.py | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index de6fd1ae21..b7c131529a 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -804,12 +804,12 @@ def config(self) -> WorkspaceConfig: renamed_group_prefix=f'tmp-{self.inventory_database}-', ) - def save_locations(self) -> None: + def save_locations(self, make_random) -> None: locations: list[ExternalLocation] = [] if self.workspace_client.config.is_azure: - locations = [ExternalLocation("abfss://things@labsazurethings.dfs.core.windows.net/a", 1)] + locations = [ExternalLocation(f"abfss://things@labsazurethings.dfs.core.windows.net/a", 1)] if self.workspace_client.config.is_aws: - locations = [ExternalLocation("s3://labs-things/a", 1)] + locations = [ExternalLocation(f"s3://labs-things/a", 1)] self.sql_backend.save_table( f"{self.inventory_database}.external_locations", locations, diff --git a/tests/integration/hive_metastore/test_migrate.py b/tests/integration/hive_metastore/test_migrate.py index 3f9d60e092..041e398562 100644 --- a/tests/integration/hive_metastore/test_migrate.py +++ b/tests/integration/hive_metastore/test_migrate.py @@ -700,7 +700,6 @@ def test_migrate_table_in_mount( env_or_skip, make_random, runtime_ctx, - make_acc_group, ): if not ws.config.is_azure: pytest.skip("temporary: only works in azure test env") @@ -710,6 +709,7 @@ def test_migrate_table_in_mount( connect=ws.config, ) runtime_ctx = runtime_ctx.replace(config=config) + b_dir = make_random(4).lower() table_path = make_random(4).lower() src_schema = make_schema( catalog_name="hive_metastore", @@ -717,10 +717,10 @@ def test_migrate_table_in_mount( ) src_external_table = runtime_ctx.make_table( schema_name=src_schema.name, - external_delta=f"dbfs:/mnt/{env_or_skip('TEST_MOUNT_NAME')}/a/b/{table_path}", + external_delta=f"dbfs:/mnt/{env_or_skip('TEST_MOUNT_NAME')}/a/{b_dir}/{table_path}", columns=[("1-0`.0-ugly-column", "STRING")], # Test with column that needs escaping ) - table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{table_path}" + table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{table_path}" # TODO: Remove this hack below # This is done because we have to create the external table in a mount point, but TablesInMounts() has to use the adls/ path # Otherwise, if we keep the dbfs:/ path, the entire logic of TablesInMounts won't work diff --git a/tests/integration/hive_metastore/test_table_move.py b/tests/integration/hive_metastore/test_table_move.py index aa06c14b36..c1a6d7e4b8 100644 --- a/tests/integration/hive_metastore/test_table_move.py +++ b/tests/integration/hive_metastore/test_table_move.py @@ -201,20 +201,21 @@ def test_alias_tables( def test_move_tables_table_properties_mismatch_preserves_original( - ws, sql_backend, make_catalog, make_schema, make_table, make_acc_group, make_random, env_or_skip -): + ws, sql_backend, make_catalog, make_schema, make_table, make_random +) -> None: table_move = TableMove(ws, sql_backend) from_catalog = make_catalog() from_schema = make_schema(catalog_name=from_catalog.name) + b_dir = make_random(4).lower() tbl_path = make_random(4).lower() tbl_properties = {"delta.enableDeletionVectors": "true"} from_table_1 = make_table( catalog_name=from_catalog.name, schema_name=from_schema.name, - external_delta=f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{tbl_path}", + external_delta=f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{tbl_path}", tbl_properties=tbl_properties, ) - table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/b/{tbl_path}" + table_in_mount_location = f"abfss://things@labsazurethings.dfs.core.windows.net/a/{b_dir}/{tbl_path}" from_table_1.storage_location = table_in_mount_location to_catalog = make_catalog()