From 7ae52d1683eec79baa5ab87a5d7977d1c1a76b0c Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 3 Jan 2025 15:39:04 +0000 Subject: [PATCH 01/20] Support roles in grants --- .../dremio/macros/adapters/apply_grants.sql | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index 1345f986..637e427b 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -16,6 +16,23 @@ limitations under the License.*/ {{ return(False) }} {%- endmacro -%} +{%- macro dremio__split_grantee(grantee) -%} + {%- set splitted = grantee.split(':') %} + {%- if splitted | length == 2 %} + {%- set type = splitted[0] %} + {%- if type in ['user', 'role'] %} + {%- set name = splitted[1] %} + {%- else %} + {% do exceptions.CompilationError("Invalid prefix. Use either user or role") %} + {%- endif %} + {%- else %} + {%- set type = 'user' %} + {%- set name = grantee %} + {%- endif %} + + {{ return((type, name)) }} +{%- endmacro -%} + {% macro dremio__get_show_grant_sql(relation) %} {%- if relation.type == 'table' -%} {%- set relation_without_double_quotes = target.datalake ~ '.' ~ target.root_path ~ '.' ~ relation.identifier-%} @@ -36,7 +53,10 @@ limitations under the License.*/ {% endmacro %} {%- macro dremio__get_grant_sql(relation, privilege, grantees) -%} - grant {{ privilege }} on {{relation.type}} {{ relation }} to user {{adapter.quote(grantees[0])}} + {%- set type, name = dremio__split_grantee(grantees[0]) %} + + grant {{ privilege }} on {{ relation.type }} {{ relation }} + to {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} {%- macro default__get_revoke_sql(relation, privilege, grantees) -%} From 9a5b7b911680911b3469c8da0cdce60e44f05fe1 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 3 Jan 2025 17:01:21 +0000 Subject: [PATCH 02/20] revoke --- dbt/include/dremio/macros/adapters/apply_grants.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index 637e427b..e97fbde7 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -59,8 +59,11 @@ limitations under the License.*/ to {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} -{%- macro default__get_revoke_sql(relation, privilege, grantees) -%} - revoke {{ privilege }} on {{ relation.type }} {{ relation }} from user {{adapter.quote(grantees[0])}} +{%- macro dremio__get_revoke_sql(relation, privilege, grantees) -%} + {%- set type, name = dremio__split_grantee(grantees[0]) %} + + revoke {{ privilege }} on {{ relation.type }} {{ relation }} + from {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} {% macro dremio__call_dcl_statements(dcl_statement_list) %} From e8d3a6c2163ab2e2b34643b17436709bb3aad6f0 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Wed, 8 Jan 2025 14:00:41 +0000 Subject: [PATCH 03/20] make revokes actually work --- dbt/adapters/dremio/impl.py | 6 ++++-- dbt/include/dremio/macros/adapters/apply_grants.sql | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/dremio/impl.py b/dbt/adapters/dremio/impl.py index c5175ab6..14bf7820 100644 --- a/dbt/adapters/dremio/impl.py +++ b/dbt/adapters/dremio/impl.py @@ -148,10 +148,12 @@ def standardize_grants_dict(self, grants_table: agate.Table) -> dict: # Just needed to change these two values to match Dremio cols grantee = row["grantee_id"] privilege = row["privilege"] + grantee_type = row["grantee_type"] + if privilege in grants_dict.keys(): - grants_dict[privilege].append(grantee) + grants_dict[privilege].append(f"{grantee_type}:{grantee}") else: - grants_dict.update({privilege: [grantee]}) + grants_dict.update({privilege: [f"{grantee_type}:{grantee}"]}) return grants_dict # This is for use in the test suite diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index e97fbde7..e849f5b0 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -47,7 +47,7 @@ limitations under the License.*/ {%- else -%} {% do exceptions.CompilationError("Invalid profile configuration: please only specify one of cloud_host or software_host in profiles.yml") %} {%- endif %} - SELECT privilege, grantee_id + SELECT privilege, grantee_type, grantee_id FROM {{privileges_table}} WHERE object_id='{{ relation_without_double_quotes }}' {% endmacro %} From 8c77441763a1d30c5d4cf83761ba19885cf15635 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 9 Jan 2025 13:04:47 +0000 Subject: [PATCH 04/20] tests --- .github/scripts/create_env_file.sh | 2 + .../adapter/grants/test_model_grants.py | 26 ++-- .../adapter/grants/test_roles_model_grants.py | 115 ++++++++++++++++++ 3 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 tests/functional/adapter/grants/test_roles_model_grants.py diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index 75e09707..a279f2c1 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,6 +16,8 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 +DBT_TEST_ROLE_1=PUBLIC +DBT_TEST_ROLE_2=ADMIN EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c9b99b33..347126cc 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -31,7 +31,6 @@ class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): - # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip @@ -46,9 +45,10 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] expected = {select_privilege_name: [test_users[0]]} + expected_grants = {select_privilege_name: ["user: " + test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" - self.assert_expected_grants_match_actual(project, "my_model", expected) + self.assert_expected_grants_match_actual(project, "my_model", expected_grants) # View materialization, change select grant user updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml) @@ -74,6 +74,7 @@ def get_grants_on_relation(self, project, relation_name): # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): + # we want the test to fail, not silently skip test_users = get_test_users select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] @@ -87,7 +88,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -98,34 +99,27 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees - updated_yaml = self.interpolate_name_overrides( - multiple_users_table_model_schema_yml - ) + updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0], test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges - updated_yaml = self.interpolate_name_overrides( - multiple_privileges_table_model_schema_yml - ) + updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = { - select_privilege_name: [test_users[0]], - insert_privilege_name: [test_users[1]], - } - self.assert_expected_grants_match_actual(project, "my_model", expected) + expected = {select_privilege_name: ["user:" + test_users[0]], insert_privilege_name: ["user:" + test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_roles_model_grants.py b/tests/functional/adapter/grants/test_roles_model_grants.py new file mode 100644 index 00000000..35e1fd61 --- /dev/null +++ b/tests/functional/adapter/grants/test_roles_model_grants.py @@ -0,0 +1,115 @@ +import pytest, os +from dbt.tests.adapter.grants.test_model_grants import ( + BaseModelGrants, + user2_model_schema_yml, + user2_table_model_schema_yml, + table_model_schema_yml, + multiple_users_table_model_schema_yml, + multiple_privileges_table_model_schema_yml, +) +from dbt.tests.util import ( + run_dbt, + get_manifest, + write_file, + get_connection, + run_dbt_and_capture +) +from tests.functional.adapter.grants.base_grants import BaseGrantsDremio +from tests.utils.util import relation_from_name + +TEST_ROLE_ENV_VARS = ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"] + +my_model_sql = """ + select 1 as fun +""" + +role1_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +multiple_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +users_and_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}", "role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +class TestViewRolesGrantsDremio(BaseGrantsDremio, BaseModelGrants): + @pytest.fixture(scope="class", autouse=True) + def get_test_roles(self, project): + test_roles = [] + for env_var in TEST_ROLE_ENV_VARS: + role = os.getenv(env_var) + if role: + test_roles.append(role) + return test_roles + + def test_view_role1_grants(self, project, get_test_roles): + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_roles) == 2 + + # View materialization, single select grant + updated_yaml = self.interpolate_name_overrides(role1_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + def test_view_multiple_roles_grants(self, project, get_test_roles): + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_roles) == 2 + + # View materialization, single select grant + updated_yaml = self.interpolate_name_overrides(multiple_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + def test_view_multiple_users_and_roles(self, project, get_test_users, get_test_roles): + test_users = get_test_users + assert len(test_users) == 3 + test_roles = get_test_roles + assert len(test_roles) == 2 + + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # View materialization, multiple select grants + updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1], "role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file From e38fde712deb56ae3a0e3e14271b6b5974148729 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 23 Jan 2025 17:02:08 +0000 Subject: [PATCH 05/20] Revert "tests" This reverts commit 8c77441763a1d30c5d4cf83761ba19885cf15635. --- .github/scripts/create_env_file.sh | 2 - .../adapter/grants/test_model_grants.py | 26 ++-- .../adapter/grants/test_roles_model_grants.py | 115 ------------------ 3 files changed, 16 insertions(+), 127 deletions(-) delete mode 100644 tests/functional/adapter/grants/test_roles_model_grants.py diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index a279f2c1..75e09707 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,8 +16,6 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 -DBT_TEST_ROLE_1=PUBLIC -DBT_TEST_ROLE_2=ADMIN EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index 347126cc..c9b99b33 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -31,6 +31,7 @@ class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): + # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip @@ -45,10 +46,9 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] expected = {select_privilege_name: [test_users[0]]} - expected_grants = {select_privilege_name: ["user: " + test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" - self.assert_expected_grants_match_actual(project, "my_model", expected_grants) + self.assert_expected_grants_match_actual(project, "my_model", expected) # View materialization, change select grant user updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml) @@ -74,7 +74,6 @@ def get_grants_on_relation(self, project, relation_name): # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): - # we want the test to fail, not silently skip test_users = get_test_users select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] @@ -88,7 +87,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0]]} + expected = {select_privilege_name: [test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -99,27 +98,34 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[1]]} + expected = {select_privilege_name: [test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees - updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml) + updated_yaml = self.interpolate_name_overrides( + multiple_users_table_model_schema_yml + ) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} + expected = {select_privilege_name: [test_users[0], test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges - updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml) + updated_yaml = self.interpolate_name_overrides( + multiple_privileges_table_model_schema_yml + ) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0]], insert_privilege_name: ["user:" + test_users[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file + expected = { + select_privilege_name: [test_users[0]], + insert_privilege_name: [test_users[1]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) diff --git a/tests/functional/adapter/grants/test_roles_model_grants.py b/tests/functional/adapter/grants/test_roles_model_grants.py deleted file mode 100644 index 35e1fd61..00000000 --- a/tests/functional/adapter/grants/test_roles_model_grants.py +++ /dev/null @@ -1,115 +0,0 @@ -import pytest, os -from dbt.tests.adapter.grants.test_model_grants import ( - BaseModelGrants, - user2_model_schema_yml, - user2_table_model_schema_yml, - table_model_schema_yml, - multiple_users_table_model_schema_yml, - multiple_privileges_table_model_schema_yml, -) -from dbt.tests.util import ( - run_dbt, - get_manifest, - write_file, - get_connection, - run_dbt_and_capture -) -from tests.functional.adapter.grants.base_grants import BaseGrantsDremio -from tests.utils.util import relation_from_name - -TEST_ROLE_ENV_VARS = ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"] - -my_model_sql = """ - select 1 as fun -""" - -role1_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] -""" - -multiple_roles_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] -""" - -users_and_roles_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}", "role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] -""" - -class TestViewRolesGrantsDremio(BaseGrantsDremio, BaseModelGrants): - @pytest.fixture(scope="class", autouse=True) - def get_test_roles(self, project): - test_roles = [] - for env_var in TEST_ROLE_ENV_VARS: - role = os.getenv(env_var) - if role: - test_roles.append(role) - return test_roles - - def test_view_role1_grants(self, project, get_test_roles): - test_roles = get_test_roles - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - assert len(test_roles) == 2 - - # View materialization, single select grant - updated_yaml = self.interpolate_name_overrides(role1_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["role:" + test_roles[0]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) - - def test_view_multiple_roles_grants(self, project, get_test_roles): - test_roles = get_test_roles - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - assert len(test_roles) == 2 - - # View materialization, single select grant - updated_yaml = self.interpolate_name_overrides(multiple_roles_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) - - def test_view_multiple_users_and_roles(self, project, get_test_users, get_test_roles): - test_users = get_test_users - assert len(test_users) == 3 - test_roles = get_test_roles - assert len(test_roles) == 2 - - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - - # View materialization, multiple select grants - updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1], "role:" + test_roles[0], "role:" + test_roles[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file From a54544738ad7e134a2a5fc164b1170faef1a4924 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 23 Jan 2025 17:10:25 +0000 Subject: [PATCH 06/20] skip grant tests only supported by EE/DC current pipeline is OSS only --- tests/functional/adapter/grants/base_grants.py | 1 + .../adapter/grants/test_incremental_grants.py | 7 +++++-- .../adapter/grants/test_invalid_grants.py | 2 ++ .../functional/adapter/grants/test_model_grants.py | 14 +++++++++----- .../functional/adapter/grants/test_seed_grants.py | 3 +++ .../adapter/grants/test_snapshot_grants.py | 1 + 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/functional/adapter/grants/base_grants.py b/tests/functional/adapter/grants/base_grants.py index bbf7b05b..18e468a6 100644 --- a/tests/functional/adapter/grants/base_grants.py +++ b/tests/functional/adapter/grants/base_grants.py @@ -18,6 +18,7 @@ from tests.utils.util import BUCKET +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class BaseGrantsDremio(BaseGrants): # This is overridden to change insert privilege to alter def privilege_grantee_name_overrides(self): diff --git a/tests/functional/adapter/grants/test_incremental_grants.py b/tests/functional/adapter/grants/test_incremental_grants.py index ff5624d4..ca5ba627 100644 --- a/tests/functional/adapter/grants/test_incremental_grants.py +++ b/tests/functional/adapter/grants/test_incremental_grants.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.util import ( run_dbt_and_capture, get_manifest, @@ -27,6 +29,7 @@ ) +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestIncrementalGrantsDremio(BaseGrantsDremio, BaseIncrementalGrants): # Define this here to use our modified version of relation_from_name def get_grants_on_relation(self, project, relation_name): @@ -53,7 +56,7 @@ def test_incremental_grants(self, project, get_test_users): model_id = "model.test.my_incremental_model" model = manifest.nodes[model_id] assert model.config.materialized == "incremental" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual( project, "my_incremental_model", expected ) @@ -80,7 +83,7 @@ def test_incremental_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "incremental" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual( project, "my_incremental_model", expected ) diff --git a/tests/functional/adapter/grants/test_invalid_grants.py b/tests/functional/adapter/grants/test_invalid_grants.py index 56689199..2e319e37 100644 --- a/tests/functional/adapter/grants/test_invalid_grants.py +++ b/tests/functional/adapter/grants/test_invalid_grants.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio @@ -19,6 +20,7 @@ from dbt.tests.util import get_connection +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestInvalidGrantsDremio(BaseGrantsDremio, BaseInvalidGrants): def grantee_does_not_exist_error(self): return "Grant on catalog entity failed. User invalid_user does not exist." diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c9b99b33..c1487af6 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.adapter.grants.test_model_grants import ( BaseModelGrants, user2_model_schema_yml, @@ -30,6 +32,7 @@ from tests.utils.util import relation_from_name +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): # Overridden to only include view materialization @@ -59,6 +62,7 @@ def test_view_table_grants(self, project, get_test_users): self.assert_expected_grants_match_actual(project, "my_model", expected) +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestTableGrantsDremio(BaseGrantsDremio, BaseModelGrants): # Need to override this to make sure it uses our modified version of relation_from_name # This isn't needed for views, as dbt-core's version defaults to database/schema path @@ -87,7 +91,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -98,7 +102,7 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees @@ -111,7 +115,7 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0], test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges @@ -125,7 +129,7 @@ def test_view_table_grants(self, project, get_test_users): model = manifest.nodes[model_id] assert model.config.materialized == "table" expected = { - select_privilege_name: [test_users[0]], - insert_privilege_name: [test_users[1]], + select_privilege_name: ["user:" + test_users[0]], + insert_privilege_name: ["user:" + test_users[1]], } self.assert_expected_grants_match_actual(project, "my_model", expected) diff --git a/tests/functional/adapter/grants/test_seed_grants.py b/tests/functional/adapter/grants/test_seed_grants.py index cb41028a..df85d58b 100644 --- a/tests/functional/adapter/grants/test_seed_grants.py +++ b/tests/functional/adapter/grants/test_seed_grants.py @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name from dbt.tests.util import get_connection +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestSeedGrantsDremio(BaseGrantsDremio, BaseSeedGrants): # Grants are reapplied if dbt run is ran twice without changes diff --git a/tests/functional/adapter/grants/test_snapshot_grants.py b/tests/functional/adapter/grants/test_snapshot_grants.py index ce60d919..00ea86a2 100644 --- a/tests/functional/adapter/grants/test_snapshot_grants.py +++ b/tests/functional/adapter/grants/test_snapshot_grants.py @@ -37,6 +37,7 @@ """.strip() +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestSnapshotGrantsDremio(BaseGrantsDremio, BaseSnapshotGrants): # Override this to use our modified snapshot model @pytest.fixture(scope="class") From b9fd66e62a1c6379e24cfd1e27103c2b62ae357d Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 14:45:04 +0000 Subject: [PATCH 07/20] tests for roles granting --- .github/scripts/create_env_file.sh | 2 + .../functional/adapter/grants/base_grants.py | 1 - .../adapter/grants/test_model_grants.py | 170 +++++++++++++++++- 3 files changed, 170 insertions(+), 3 deletions(-) diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index 75e09707..44472f34 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,6 +16,8 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 +DBT_TEST_ROLE_1=dbt_test_role_1 +DBT_TEST_ROLE_2=dbt_test_role_2 EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/base_grants.py b/tests/functional/adapter/grants/base_grants.py index 18e468a6..bbf7b05b 100644 --- a/tests/functional/adapter/grants/base_grants.py +++ b/tests/functional/adapter/grants/base_grants.py @@ -18,7 +18,6 @@ from tests.utils.util import BUCKET -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class BaseGrantsDremio(BaseGrants): # This is overridden to change insert privilege to alter def privilege_grantee_name_overrides(self): diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c1487af6..804e45bd 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.adapter.grants.test_model_grants import ( BaseModelGrants, @@ -31,14 +31,101 @@ from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name +role_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +role2_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +table_role_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +role2_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +multiple_roles_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +multiple_privileges_role_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +users_and_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}", "user:{{ env_var('DBT_TEST_USER_3') }}"] +""" + +users_and_roles_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}", "user:{{ env_var('DBT_TEST_USER_3') }}"] +""" @pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): + def get_test_roles(self): + test_roles = [] + for env_var in ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"]: + role_name = os.getenv(env_var) + if role_name: + test_roles.append(role_name) + return test_roles # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip test_users = get_test_users + test_roles = self.get_test_roles() select_privilege_name = self.privilege_grantee_name_overrides()["select"] assert len(test_users) == 3 @@ -51,6 +138,7 @@ def test_view_table_grants(self, project, get_test_users): expected = {select_privilege_name: [test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # View materialization, change select grant user @@ -58,7 +146,34 @@ def test_view_table_grants(self, project, get_test_users): write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 - expected = {select_privilege_name: [get_test_users[1]]} + expected = {select_privilege_name: ["user:" + get_test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + # roles + updated_yaml = self.interpolate_name_overrides(role_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(role2_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] + expected = { + select_privilege_name: ["role:" + test_roles[0], "user:" + get_test_users[0], + "user:" + get_test_users[1]], + insert_privilege_name: ["role:" + test_roles[1], "user:" + get_test_users[2]], + } self.assert_expected_grants_match_actual(project, "my_model", expected) @@ -76,9 +191,18 @@ def get_grants_on_relation(self, project, relation_name): actual_grants = adapter.standardize_grants_dict(grant_table) return actual_grants + def get_test_roles(self): + test_roles = [] + for env_var in ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"]: + role_name = os.getenv(env_var) + if role_name: + test_roles.append(role_name) + return test_roles + # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): test_users = get_test_users + test_roles = self.get_test_roles() select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] assert len(test_users) == 3 @@ -133,3 +257,45 @@ def test_view_table_grants(self, project, get_test_users): insert_privilege_name: ["user:" + test_users[1]], } self.assert_expected_grants_match_actual(project, "my_model", expected) + + # roles + updated_yaml = self.interpolate_name_overrides(table_role_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(role2_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(multiple_roles_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(multiple_privileges_role_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = { + select_privilege_name: ["role:" + test_roles[0]], + insert_privilege_name: ["role:" + test_roles[1]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(users_and_roles_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = { + select_privilege_name: ["role:" + test_roles[0], "user:" + get_test_users[0], "user:" + get_test_users[1]], + insert_privilege_name: ["role:" + test_roles[1], "user:" + get_test_users[2]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) From be6fc70421133d2ff75e46cd0b8b07213a170977 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 14:59:32 +0000 Subject: [PATCH 08/20] Update expected_failures.txt --- .github/expected_failures.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/expected_failures.txt b/.github/expected_failures.txt index c92e6daf..9627fc12 100644 --- a/.github/expected_failures.txt +++ b/.github/expected_failures.txt @@ -3,12 +3,6 @@ tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestCloneNotPossibleDremio tests/functional/adapter/dremio_specific/test_drop_temp_table.py::TestDropTempTableDremio::test_drop_temp_table tests/functional/adapter/dremio_specific/test_schema_parsing.py::TestSchemaParsingDremio::test_schema_with_dots tests/functional/adapter/dremio_specific/test_verify_ssl.py::TestVerifyCertificateDremio::test_insecure_request_warning_not_exist -tests/functional/adapter/grants/test_incremental_grants.py::TestIncrementalGrantsDremio::test_incremental_grants -tests/functional/adapter/grants/test_invalid_grants.py::TestInvalidGrantsDremio::test_invalid_grants -tests/functional/adapter/grants/test_model_grants.py::TestViewGrantsDremio::test_view_table_grants -tests/functional/adapter/grants/test_model_grants.py::TestTableGrantsDremio::test_view_table_grants -tests/functional/adapter/grants/test_seed_grants.py::TestSeedGrantsDremio::test_seed_grants -tests/functional/adapter/grants/test_snapshot_grants.py::TestSnapshotGrantsDremio::test_snapshot_grants tests/functional/adapter/relation/test_get_relation_last_modified.py::TestGetLastRelationModified::test_get_last_relation_modified tests/functional/adapter/unit_testing/test_unit_testing.py::TestDremioUnitTestingTypes::test_unit_test_data_type tests/functional/adapter/unit_testing/test_unit_testing.py::TestDremioUnitTestCaseInsensitivity::test_case_insensitivity From 5207206676ab47fe2953d405c600b43b5c0d7ed2 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 3 Jan 2025 15:39:04 +0000 Subject: [PATCH 09/20] Support roles in grants --- .../dremio/macros/adapters/apply_grants.sql | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index 1345f986..637e427b 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -16,6 +16,23 @@ limitations under the License.*/ {{ return(False) }} {%- endmacro -%} +{%- macro dremio__split_grantee(grantee) -%} + {%- set splitted = grantee.split(':') %} + {%- if splitted | length == 2 %} + {%- set type = splitted[0] %} + {%- if type in ['user', 'role'] %} + {%- set name = splitted[1] %} + {%- else %} + {% do exceptions.CompilationError("Invalid prefix. Use either user or role") %} + {%- endif %} + {%- else %} + {%- set type = 'user' %} + {%- set name = grantee %} + {%- endif %} + + {{ return((type, name)) }} +{%- endmacro -%} + {% macro dremio__get_show_grant_sql(relation) %} {%- if relation.type == 'table' -%} {%- set relation_without_double_quotes = target.datalake ~ '.' ~ target.root_path ~ '.' ~ relation.identifier-%} @@ -36,7 +53,10 @@ limitations under the License.*/ {% endmacro %} {%- macro dremio__get_grant_sql(relation, privilege, grantees) -%} - grant {{ privilege }} on {{relation.type}} {{ relation }} to user {{adapter.quote(grantees[0])}} + {%- set type, name = dremio__split_grantee(grantees[0]) %} + + grant {{ privilege }} on {{ relation.type }} {{ relation }} + to {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} {%- macro default__get_revoke_sql(relation, privilege, grantees) -%} From 7cf68f242155a7b5d85258755280376e0237f872 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 3 Jan 2025 17:01:21 +0000 Subject: [PATCH 10/20] revoke --- dbt/include/dremio/macros/adapters/apply_grants.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index 637e427b..e97fbde7 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -59,8 +59,11 @@ limitations under the License.*/ to {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} -{%- macro default__get_revoke_sql(relation, privilege, grantees) -%} - revoke {{ privilege }} on {{ relation.type }} {{ relation }} from user {{adapter.quote(grantees[0])}} +{%- macro dremio__get_revoke_sql(relation, privilege, grantees) -%} + {%- set type, name = dremio__split_grantee(grantees[0]) %} + + revoke {{ privilege }} on {{ relation.type }} {{ relation }} + from {{ type }} {{ adapter.quote(name) }} {%- endmacro -%} {% macro dremio__call_dcl_statements(dcl_statement_list) %} From a946b058794f0e8edf8dfdb5441822892a66a915 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Wed, 8 Jan 2025 14:00:41 +0000 Subject: [PATCH 11/20] make revokes actually work --- dbt/adapters/dremio/impl.py | 6 ++++-- dbt/include/dremio/macros/adapters/apply_grants.sql | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dbt/adapters/dremio/impl.py b/dbt/adapters/dremio/impl.py index 2f24acdb..3210ea18 100644 --- a/dbt/adapters/dremio/impl.py +++ b/dbt/adapters/dremio/impl.py @@ -149,10 +149,12 @@ def standardize_grants_dict(self, grants_table: agate.Table) -> dict: # Just needed to change these two values to match Dremio cols grantee = row["grantee_id"] privilege = row["privilege"] + grantee_type = row["grantee_type"] + if privilege in grants_dict.keys(): - grants_dict[privilege].append(grantee) + grants_dict[privilege].append(f"{grantee_type}:{grantee}") else: - grants_dict.update({privilege: [grantee]}) + grants_dict.update({privilege: [f"{grantee_type}:{grantee}"]}) return grants_dict # This is for use in the test suite diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index e97fbde7..e849f5b0 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -47,7 +47,7 @@ limitations under the License.*/ {%- else -%} {% do exceptions.CompilationError("Invalid profile configuration: please only specify one of cloud_host or software_host in profiles.yml") %} {%- endif %} - SELECT privilege, grantee_id + SELECT privilege, grantee_type, grantee_id FROM {{privileges_table}} WHERE object_id='{{ relation_without_double_quotes }}' {% endmacro %} From 1a73e23a96b8a293e50487a65958759acebda7a2 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 9 Jan 2025 13:04:47 +0000 Subject: [PATCH 12/20] tests --- .github/scripts/create_env_file.sh | 2 + .../adapter/grants/test_model_grants.py | 26 ++-- .../adapter/grants/test_roles_model_grants.py | 115 ++++++++++++++++++ 3 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 tests/functional/adapter/grants/test_roles_model_grants.py diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index 75e09707..a279f2c1 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,6 +16,8 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 +DBT_TEST_ROLE_1=PUBLIC +DBT_TEST_ROLE_2=ADMIN EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c9b99b33..347126cc 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -31,7 +31,6 @@ class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): - # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip @@ -46,9 +45,10 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] expected = {select_privilege_name: [test_users[0]]} + expected_grants = {select_privilege_name: ["user: " + test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" - self.assert_expected_grants_match_actual(project, "my_model", expected) + self.assert_expected_grants_match_actual(project, "my_model", expected_grants) # View materialization, change select grant user updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml) @@ -74,6 +74,7 @@ def get_grants_on_relation(self, project, relation_name): # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): + # we want the test to fail, not silently skip test_users = get_test_users select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] @@ -87,7 +88,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -98,34 +99,27 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees - updated_yaml = self.interpolate_name_overrides( - multiple_users_table_model_schema_yml - ) + updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0], test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges - updated_yaml = self.interpolate_name_overrides( - multiple_privileges_table_model_schema_yml - ) + updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = { - select_privilege_name: [test_users[0]], - insert_privilege_name: [test_users[1]], - } - self.assert_expected_grants_match_actual(project, "my_model", expected) + expected = {select_privilege_name: ["user:" + test_users[0]], insert_privilege_name: ["user:" + test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_roles_model_grants.py b/tests/functional/adapter/grants/test_roles_model_grants.py new file mode 100644 index 00000000..35e1fd61 --- /dev/null +++ b/tests/functional/adapter/grants/test_roles_model_grants.py @@ -0,0 +1,115 @@ +import pytest, os +from dbt.tests.adapter.grants.test_model_grants import ( + BaseModelGrants, + user2_model_schema_yml, + user2_table_model_schema_yml, + table_model_schema_yml, + multiple_users_table_model_schema_yml, + multiple_privileges_table_model_schema_yml, +) +from dbt.tests.util import ( + run_dbt, + get_manifest, + write_file, + get_connection, + run_dbt_and_capture +) +from tests.functional.adapter.grants.base_grants import BaseGrantsDremio +from tests.utils.util import relation_from_name + +TEST_ROLE_ENV_VARS = ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"] + +my_model_sql = """ + select 1 as fun +""" + +role1_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +multiple_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +users_and_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}", "role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +class TestViewRolesGrantsDremio(BaseGrantsDremio, BaseModelGrants): + @pytest.fixture(scope="class", autouse=True) + def get_test_roles(self, project): + test_roles = [] + for env_var in TEST_ROLE_ENV_VARS: + role = os.getenv(env_var) + if role: + test_roles.append(role) + return test_roles + + def test_view_role1_grants(self, project, get_test_roles): + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_roles) == 2 + + # View materialization, single select grant + updated_yaml = self.interpolate_name_overrides(role1_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + def test_view_multiple_roles_grants(self, project, get_test_roles): + test_roles = get_test_roles + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + assert len(test_roles) == 2 + + # View materialization, single select grant + updated_yaml = self.interpolate_name_overrides(multiple_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + def test_view_multiple_users_and_roles(self, project, get_test_users, get_test_roles): + test_users = get_test_users + assert len(test_users) == 3 + test_roles = get_test_roles + assert len(test_roles) == 2 + + select_privilege_name = self.privilege_grantee_name_overrides()["select"] + + # View materialization, multiple select grants + updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + model = manifest.nodes[model_id] + assert model.config.materialized == "view" + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1], "role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file From f4f3acc3298c3a5c9c6d151910e39206478b4341 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 23 Jan 2025 17:02:08 +0000 Subject: [PATCH 13/20] Revert "tests" This reverts commit 8c77441763a1d30c5d4cf83761ba19885cf15635. --- .github/scripts/create_env_file.sh | 2 - .../adapter/grants/test_model_grants.py | 26 ++-- .../adapter/grants/test_roles_model_grants.py | 115 ------------------ 3 files changed, 16 insertions(+), 127 deletions(-) delete mode 100644 tests/functional/adapter/grants/test_roles_model_grants.py diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index a279f2c1..75e09707 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,8 +16,6 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 -DBT_TEST_ROLE_1=PUBLIC -DBT_TEST_ROLE_2=ADMIN EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index 347126cc..c9b99b33 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -31,6 +31,7 @@ class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): + # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip @@ -45,10 +46,9 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] expected = {select_privilege_name: [test_users[0]]} - expected_grants = {select_privilege_name: ["user: " + test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" - self.assert_expected_grants_match_actual(project, "my_model", expected_grants) + self.assert_expected_grants_match_actual(project, "my_model", expected) # View materialization, change select grant user updated_yaml = self.interpolate_name_overrides(user2_model_schema_yml) @@ -74,7 +74,6 @@ def get_grants_on_relation(self, project, relation_name): # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): - # we want the test to fail, not silently skip test_users = get_test_users select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] @@ -88,7 +87,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0]]} + expected = {select_privilege_name: [test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -99,27 +98,34 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[1]]} + expected = {select_privilege_name: [test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees - updated_yaml = self.interpolate_name_overrides(multiple_users_table_model_schema_yml) + updated_yaml = self.interpolate_name_overrides( + multiple_users_table_model_schema_yml + ) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} + expected = {select_privilege_name: [test_users[0], test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges - updated_yaml = self.interpolate_name_overrides(multiple_privileges_table_model_schema_yml) + updated_yaml = self.interpolate_name_overrides( + multiple_privileges_table_model_schema_yml + ) write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: ["user:" + test_users[0]], insert_privilege_name: ["user:" + test_users[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file + expected = { + select_privilege_name: [test_users[0]], + insert_privilege_name: [test_users[1]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) diff --git a/tests/functional/adapter/grants/test_roles_model_grants.py b/tests/functional/adapter/grants/test_roles_model_grants.py deleted file mode 100644 index 35e1fd61..00000000 --- a/tests/functional/adapter/grants/test_roles_model_grants.py +++ /dev/null @@ -1,115 +0,0 @@ -import pytest, os -from dbt.tests.adapter.grants.test_model_grants import ( - BaseModelGrants, - user2_model_schema_yml, - user2_table_model_schema_yml, - table_model_schema_yml, - multiple_users_table_model_schema_yml, - multiple_privileges_table_model_schema_yml, -) -from dbt.tests.util import ( - run_dbt, - get_manifest, - write_file, - get_connection, - run_dbt_and_capture -) -from tests.functional.adapter.grants.base_grants import BaseGrantsDremio -from tests.utils.util import relation_from_name - -TEST_ROLE_ENV_VARS = ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"] - -my_model_sql = """ - select 1 as fun -""" - -role1_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] -""" - -multiple_roles_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] -""" - -users_and_roles_model_schema_yml = """ -version: 2 -models: - - name: my_model - config: - grants: - select: ["user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}", "role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] -""" - -class TestViewRolesGrantsDremio(BaseGrantsDremio, BaseModelGrants): - @pytest.fixture(scope="class", autouse=True) - def get_test_roles(self, project): - test_roles = [] - for env_var in TEST_ROLE_ENV_VARS: - role = os.getenv(env_var) - if role: - test_roles.append(role) - return test_roles - - def test_view_role1_grants(self, project, get_test_roles): - test_roles = get_test_roles - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - assert len(test_roles) == 2 - - # View materialization, single select grant - updated_yaml = self.interpolate_name_overrides(role1_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["role:" + test_roles[0]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) - - def test_view_multiple_roles_grants(self, project, get_test_roles): - test_roles = get_test_roles - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - assert len(test_roles) == 2 - - # View materialization, single select grant - updated_yaml = self.interpolate_name_overrides(multiple_roles_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) - - def test_view_multiple_users_and_roles(self, project, get_test_users, get_test_roles): - test_users = get_test_users - assert len(test_users) == 3 - test_roles = get_test_roles - assert len(test_roles) == 2 - - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - - # View materialization, multiple select grants - updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) - write_file(updated_yaml, project.project_root, "models", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "run"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - model_id = "model.test.my_model" - model = manifest.nodes[model_id] - assert model.config.materialized == "view" - expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1], "role:" + test_roles[0], "role:" + test_roles[1]]} - self.assert_expected_grants_match_actual(project, "my_model", expected) \ No newline at end of file From 3c596b4e261fc8b2228483a7cf6fa1de060e7a62 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Thu, 23 Jan 2025 17:10:25 +0000 Subject: [PATCH 14/20] skip grant tests only supported by EE/DC current pipeline is OSS only --- tests/functional/adapter/grants/base_grants.py | 1 + .../adapter/grants/test_incremental_grants.py | 7 +++++-- .../adapter/grants/test_invalid_grants.py | 2 ++ .../functional/adapter/grants/test_model_grants.py | 14 +++++++++----- .../functional/adapter/grants/test_seed_grants.py | 3 +++ .../adapter/grants/test_snapshot_grants.py | 1 + 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/functional/adapter/grants/base_grants.py b/tests/functional/adapter/grants/base_grants.py index bbf7b05b..18e468a6 100644 --- a/tests/functional/adapter/grants/base_grants.py +++ b/tests/functional/adapter/grants/base_grants.py @@ -18,6 +18,7 @@ from tests.utils.util import BUCKET +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class BaseGrantsDremio(BaseGrants): # This is overridden to change insert privilege to alter def privilege_grantee_name_overrides(self): diff --git a/tests/functional/adapter/grants/test_incremental_grants.py b/tests/functional/adapter/grants/test_incremental_grants.py index ff5624d4..ca5ba627 100644 --- a/tests/functional/adapter/grants/test_incremental_grants.py +++ b/tests/functional/adapter/grants/test_incremental_grants.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.util import ( run_dbt_and_capture, get_manifest, @@ -27,6 +29,7 @@ ) +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestIncrementalGrantsDremio(BaseGrantsDremio, BaseIncrementalGrants): # Define this here to use our modified version of relation_from_name def get_grants_on_relation(self, project, relation_name): @@ -53,7 +56,7 @@ def test_incremental_grants(self, project, get_test_users): model_id = "model.test.my_incremental_model" model = manifest.nodes[model_id] assert model.config.materialized == "incremental" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual( project, "my_incremental_model", expected ) @@ -80,7 +83,7 @@ def test_incremental_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "incremental" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual( project, "my_incremental_model", expected ) diff --git a/tests/functional/adapter/grants/test_invalid_grants.py b/tests/functional/adapter/grants/test_invalid_grants.py index 56689199..2e319e37 100644 --- a/tests/functional/adapter/grants/test_invalid_grants.py +++ b/tests/functional/adapter/grants/test_invalid_grants.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio @@ -19,6 +20,7 @@ from dbt.tests.util import get_connection +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestInvalidGrantsDremio(BaseGrantsDremio, BaseInvalidGrants): def grantee_does_not_exist_error(self): return "Grant on catalog entity failed. User invalid_user does not exist." diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c9b99b33..c1487af6 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.adapter.grants.test_model_grants import ( BaseModelGrants, user2_model_schema_yml, @@ -30,6 +32,7 @@ from tests.utils.util import relation_from_name +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): # Overridden to only include view materialization @@ -59,6 +62,7 @@ def test_view_table_grants(self, project, get_test_users): self.assert_expected_grants_match_actual(project, "my_model", expected) +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestTableGrantsDremio(BaseGrantsDremio, BaseModelGrants): # Need to override this to make sure it uses our modified version of relation_from_name # This isn't needed for views, as dbt-core's version defaults to database/schema path @@ -87,7 +91,7 @@ def test_view_table_grants(self, project, get_test_users): model_id = "model.test.my_model" model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0]]} + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, change select grant user @@ -98,7 +102,7 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple grantees @@ -111,7 +115,7 @@ def test_view_table_grants(self, project, get_test_users): manifest = get_manifest(project.project_root) model = manifest.nodes[model_id] assert model.config.materialized == "table" - expected = {select_privilege_name: [test_users[0], test_users[1]]} + expected = {select_privilege_name: ["user:" + test_users[0], "user:" + test_users[1]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # Table materialization, multiple privileges @@ -125,7 +129,7 @@ def test_view_table_grants(self, project, get_test_users): model = manifest.nodes[model_id] assert model.config.materialized == "table" expected = { - select_privilege_name: [test_users[0]], - insert_privilege_name: [test_users[1]], + select_privilege_name: ["user:" + test_users[0]], + insert_privilege_name: ["user:" + test_users[1]], } self.assert_expected_grants_match_actual(project, "my_model", expected) diff --git a/tests/functional/adapter/grants/test_seed_grants.py b/tests/functional/adapter/grants/test_seed_grants.py index cb41028a..df85d58b 100644 --- a/tests/functional/adapter/grants/test_seed_grants.py +++ b/tests/functional/adapter/grants/test_seed_grants.py @@ -12,12 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name from dbt.tests.util import get_connection +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestSeedGrantsDremio(BaseGrantsDremio, BaseSeedGrants): # Grants are reapplied if dbt run is ran twice without changes diff --git a/tests/functional/adapter/grants/test_snapshot_grants.py b/tests/functional/adapter/grants/test_snapshot_grants.py index ce60d919..00ea86a2 100644 --- a/tests/functional/adapter/grants/test_snapshot_grants.py +++ b/tests/functional/adapter/grants/test_snapshot_grants.py @@ -37,6 +37,7 @@ """.strip() +@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestSnapshotGrantsDremio(BaseGrantsDremio, BaseSnapshotGrants): # Override this to use our modified snapshot model @pytest.fixture(scope="class") From 41dbfb62ffa3c399c46353e2d66de902cfba59ea Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 14:45:04 +0000 Subject: [PATCH 15/20] tests for roles granting --- .github/scripts/create_env_file.sh | 2 + .../functional/adapter/grants/base_grants.py | 1 - .../adapter/grants/test_model_grants.py | 170 +++++++++++++++++- 3 files changed, 170 insertions(+), 3 deletions(-) diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index 75e09707..44472f34 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -16,6 +16,8 @@ DREMIO_DATABASE=dbt_test DBT_TEST_USER_1=dbt_test_user_1 DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 +DBT_TEST_ROLE_1=dbt_test_role_1 +DBT_TEST_ROLE_2=dbt_test_role_2 EOF echo ".env file created successfully." \ No newline at end of file diff --git a/tests/functional/adapter/grants/base_grants.py b/tests/functional/adapter/grants/base_grants.py index 18e468a6..bbf7b05b 100644 --- a/tests/functional/adapter/grants/base_grants.py +++ b/tests/functional/adapter/grants/base_grants.py @@ -18,7 +18,6 @@ from tests.utils.util import BUCKET -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class BaseGrantsDremio(BaseGrants): # This is overridden to change insert privilege to alter def privilege_grantee_name_overrides(self): diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index c1487af6..804e45bd 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.adapter.grants.test_model_grants import ( BaseModelGrants, @@ -31,14 +31,101 @@ from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name +role_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +role2_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +table_role_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] +""" + +role2_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +multiple_roles_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +multiple_privileges_role_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}"] +""" + +users_and_roles_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}", "user:{{ env_var('DBT_TEST_USER_3') }}"] +""" + +users_and_roles_table_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + materialized: table + grants: + select: ["role:{{ env_var('DBT_TEST_ROLE_1') }}", "user:{{ env_var('DBT_TEST_USER_1') }}", "user:{{ env_var('DBT_TEST_USER_2') }}"] + insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}", "user:{{ env_var('DBT_TEST_USER_3') }}"] +""" @pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): + def get_test_roles(self): + test_roles = [] + for env_var in ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"]: + role_name = os.getenv(env_var) + if role_name: + test_roles.append(role_name) + return test_roles # Overridden to only include view materialization def test_view_table_grants(self, project, get_test_users): # we want the test to fail, not silently skip test_users = get_test_users + test_roles = self.get_test_roles() select_privilege_name = self.privilege_grantee_name_overrides()["select"] assert len(test_users) == 3 @@ -51,6 +138,7 @@ def test_view_table_grants(self, project, get_test_users): expected = {select_privilege_name: [test_users[0]]} assert model.config.grants == expected assert model.config.materialized == "view" + expected = {select_privilege_name: ["user:" + test_users[0]]} self.assert_expected_grants_match_actual(project, "my_model", expected) # View materialization, change select grant user @@ -58,7 +146,34 @@ def test_view_table_grants(self, project, get_test_users): write_file(updated_yaml, project.project_root, "models", "schema.yml") (results, log_output) = run_dbt_and_capture(["--debug", "run"]) assert len(results) == 1 - expected = {select_privilege_name: [get_test_users[1]]} + expected = {select_privilege_name: ["user:" + get_test_users[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + # roles + updated_yaml = self.interpolate_name_overrides(role_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(role2_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(users_and_roles_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] + expected = { + select_privilege_name: ["role:" + test_roles[0], "user:" + get_test_users[0], + "user:" + get_test_users[1]], + insert_privilege_name: ["role:" + test_roles[1], "user:" + get_test_users[2]], + } self.assert_expected_grants_match_actual(project, "my_model", expected) @@ -76,9 +191,18 @@ def get_grants_on_relation(self, project, relation_name): actual_grants = adapter.standardize_grants_dict(grant_table) return actual_grants + def get_test_roles(self): + test_roles = [] + for env_var in ["DBT_TEST_ROLE_1", "DBT_TEST_ROLE_2"]: + role_name = os.getenv(env_var) + if role_name: + test_roles.append(role_name) + return test_roles + # Overridden to only include table materializations def test_view_table_grants(self, project, get_test_users): test_users = get_test_users + test_roles = self.get_test_roles() select_privilege_name = self.privilege_grantee_name_overrides()["select"] insert_privilege_name = self.privilege_grantee_name_overrides()["insert"] assert len(test_users) == 3 @@ -133,3 +257,45 @@ def test_view_table_grants(self, project, get_test_users): insert_privilege_name: ["user:" + test_users[1]], } self.assert_expected_grants_match_actual(project, "my_model", expected) + + # roles + updated_yaml = self.interpolate_name_overrides(table_role_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(role2_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(multiple_roles_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:" + test_roles[0], "role:" + test_roles[1]]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(multiple_privileges_role_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = { + select_privilege_name: ["role:" + test_roles[0]], + insert_privilege_name: ["role:" + test_roles[1]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) + + updated_yaml = self.interpolate_name_overrides(users_and_roles_table_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = { + select_privilege_name: ["role:" + test_roles[0], "user:" + get_test_users[0], "user:" + get_test_users[1]], + insert_privilege_name: ["role:" + test_roles[1], "user:" + get_test_users[2]], + } + self.assert_expected_grants_match_actual(project, "my_model", expected) From 17c7b7f98f1647910791334df957a43241d6bea5 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 14:59:32 +0000 Subject: [PATCH 16/20] Update expected_failures.txt --- .github/expected_failures.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/expected_failures.txt b/.github/expected_failures.txt index c92e6daf..9627fc12 100644 --- a/.github/expected_failures.txt +++ b/.github/expected_failures.txt @@ -3,12 +3,6 @@ tests/functional/adapter/dbt_clone/test_dbt_clone.py::TestCloneNotPossibleDremio tests/functional/adapter/dremio_specific/test_drop_temp_table.py::TestDropTempTableDremio::test_drop_temp_table tests/functional/adapter/dremio_specific/test_schema_parsing.py::TestSchemaParsingDremio::test_schema_with_dots tests/functional/adapter/dremio_specific/test_verify_ssl.py::TestVerifyCertificateDremio::test_insecure_request_warning_not_exist -tests/functional/adapter/grants/test_incremental_grants.py::TestIncrementalGrantsDremio::test_incremental_grants -tests/functional/adapter/grants/test_invalid_grants.py::TestInvalidGrantsDremio::test_invalid_grants -tests/functional/adapter/grants/test_model_grants.py::TestViewGrantsDremio::test_view_table_grants -tests/functional/adapter/grants/test_model_grants.py::TestTableGrantsDremio::test_view_table_grants -tests/functional/adapter/grants/test_seed_grants.py::TestSeedGrantsDremio::test_seed_grants -tests/functional/adapter/grants/test_snapshot_grants.py::TestSnapshotGrantsDremio::test_snapshot_grants tests/functional/adapter/relation/test_get_relation_last_modified.py::TestGetLastRelationModified::test_get_last_relation_modified tests/functional/adapter/unit_testing/test_unit_testing.py::TestDremioUnitTestingTypes::test_unit_test_data_type tests/functional/adapter/unit_testing/test_unit_testing.py::TestDremioUnitTestCaseInsensitivity::test_case_insensitivity From df246201ad24ecdea3269e2b77fc834267f0fd70 Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 15:03:01 +0000 Subject: [PATCH 17/20] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b29cb2f..6f6a4efb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## Changes - When naming reflections, if a `name` config is not set, the `alias` config parameter will be used instead. If also undefined, it will refer to the model name instead of using `Unnamed Reflection` +- Grants can now be set for both users and for roles. A prefix was added to handle this, with `user:` and `role:` being the valid prefixes. For example, `user:dbt_test_user_1` and `role:dbt_test_role_1`. If no prefix is provided, defaults to user for backwards compatibility. + +## Features + +- [#259](https://github.com/dremio/dbt-dremio/pull/259) Added support for roles in grants # dbt-dremio v1.8.1 From 53b4dd370f3cd4cd32bbae82d3b1ade38163f47a Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Fri, 24 Jan 2025 16:45:13 +0000 Subject: [PATCH 18/20] fix roles with ":" --- .../dremio/macros/adapters/apply_grants.sql | 24 +++++++++---------- .../adapter/grants/test_model_grants.py | 17 +++++++++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index e849f5b0..943e87b6 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -17,20 +17,20 @@ limitations under the License.*/ {%- endmacro -%} {%- macro dremio__split_grantee(grantee) -%} - {%- set splitted = grantee.split(':') %} - {%- if splitted | length == 2 %} - {%- set type = splitted[0] %} - {%- if type in ['user', 'role'] %} - {%- set name = splitted[1] %} - {%- else %} + {%- set splitted = grantee.split(':') -%} + + {%- if splitted | length < 2 -%} + {{ return(("user", grantee)) }} + {%- else -%} + {%- set prefix = splitted[0] -%} + {%- set remainder = splitted[1:] | join(':') -%} + + {%- if prefix not in ['user', 'role'] -%} {% do exceptions.CompilationError("Invalid prefix. Use either user or role") %} - {%- endif %} - {%- else %} - {%- set type = 'user' %} - {%- set name = grantee %} - {%- endif %} + {%- endif -%} - {{ return((type, name)) }} + {{ return((prefix, remainder)) }} + {%- endif -%} {%- endmacro -%} {% macro dremio__get_show_grant_sql(relation) %} diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index 804e45bd..cc80823c 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -111,6 +111,15 @@ insert: ["role:{{ env_var('DBT_TEST_ROLE_2') }}", "user:{{ env_var('DBT_TEST_USER_3') }}"] """ +special_character_role_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + grants: + select: ["role:test:role"] +""" + @pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): def get_test_roles(self): @@ -176,6 +185,14 @@ def test_view_table_grants(self, project, get_test_users): } self.assert_expected_grants_match_actual(project, "my_model", expected) + # special character (:) in role name + updated_yaml = self.interpolate_name_overrides(special_character_role_model_schema_yml) + write_file(updated_yaml, project.project_root, "models", "schema.yml") + (results, log_output) = run_dbt_and_capture(["--debug", "run"]) + assert len(results) == 1 + expected = {select_privilege_name: ["role:test:role"]} + self.assert_expected_grants_match_actual(project, "my_model", expected) + @pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") class TestTableGrantsDremio(BaseGrantsDremio, BaseModelGrants): From e3fb17304e2d4325d2c2232ab94191656c511baf Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Mon, 27 Jan 2025 13:52:54 +0000 Subject: [PATCH 19/20] reviews --- .github/scripts/create_env_file.sh | 1 + dbt/include/dremio/macros/adapters/apply_grants.sql | 1 + tests/functional/adapter/grants/test_model_grants.py | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/scripts/create_env_file.sh b/.github/scripts/create_env_file.sh index 44472f34..ddd58f32 100644 --- a/.github/scripts/create_env_file.sh +++ b/.github/scripts/create_env_file.sh @@ -18,6 +18,7 @@ DBT_TEST_USER_2=dbt_test_user_2 DBT_TEST_USER_3=dbt_test_user_3 DBT_TEST_ROLE_1=dbt_test_role_1 DBT_TEST_ROLE_2=dbt_test_role_2 +DREMIO_EDITION=community EOF echo ".env file created successfully." \ No newline at end of file diff --git a/dbt/include/dremio/macros/adapters/apply_grants.sql b/dbt/include/dremio/macros/adapters/apply_grants.sql index 943e87b6..409cdfa1 100644 --- a/dbt/include/dremio/macros/adapters/apply_grants.sql +++ b/dbt/include/dremio/macros/adapters/apply_grants.sql @@ -20,6 +20,7 @@ limitations under the License.*/ {%- set splitted = grantee.split(':') -%} {%- if splitted | length < 2 -%} + {{ log("Deprecation warning: grants to users will soon require the user: prefix", info=True) }} {{ return(("user", grantee)) }} {%- else -%} {%- set prefix = splitted[0] -%} diff --git a/tests/functional/adapter/grants/test_model_grants.py b/tests/functional/adapter/grants/test_model_grants.py index cc80823c..47f8a32e 100644 --- a/tests/functional/adapter/grants/test_model_grants.py +++ b/tests/functional/adapter/grants/test_model_grants.py @@ -31,6 +31,8 @@ from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name +DREMIO_EDITION = os.getenv("DREMIO_EDITION") + role_model_schema_yml = """ version: 2 models: @@ -120,7 +122,7 @@ select: ["role:test:role"] """ -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestViewGrantsDremio(BaseGrantsDremio, BaseModelGrants): def get_test_roles(self): test_roles = [] @@ -194,7 +196,7 @@ def test_view_table_grants(self, project, get_test_users): self.assert_expected_grants_match_actual(project, "my_model", expected) -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestTableGrantsDremio(BaseGrantsDremio, BaseModelGrants): # Need to override this to make sure it uses our modified version of relation_from_name # This isn't needed for views, as dbt-core's version defaults to database/schema path From 180c4f8b7cc2abbe9cf5c2c0441ee13eb78a7e6c Mon Sep 17 00:00:00 2001 From: Bernardo Meireles Date: Mon, 27 Jan 2025 14:47:05 +0000 Subject: [PATCH 20/20] skipif in all relevant tests --- tests/functional/adapter/grants/test_incremental_grants.py | 5 +++-- tests/functional/adapter/grants/test_invalid_grants.py | 5 +++-- tests/functional/adapter/grants/test_seed_grants.py | 5 +++-- tests/functional/adapter/grants/test_snapshot_grants.py | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/functional/adapter/grants/test_incremental_grants.py b/tests/functional/adapter/grants/test_incremental_grants.py index ca5ba627..7767a1d2 100644 --- a/tests/functional/adapter/grants/test_incremental_grants.py +++ b/tests/functional/adapter/grants/test_incremental_grants.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.util import ( run_dbt_and_capture, @@ -28,8 +28,9 @@ user2_incremental_model_schema_yml, ) +DREMIO_EDITION = os.getenv("DREMIO_EDITION") -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestIncrementalGrantsDremio(BaseGrantsDremio, BaseIncrementalGrants): # Define this here to use our modified version of relation_from_name def get_grants_on_relation(self, project, relation_name): diff --git a/tests/functional/adapter/grants/test_invalid_grants.py b/tests/functional/adapter/grants/test_invalid_grants.py index 2e319e37..c4ac745a 100644 --- a/tests/functional/adapter/grants/test_invalid_grants.py +++ b/tests/functional/adapter/grants/test_invalid_grants.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name from dbt.tests.util import get_connection +DREMIO_EDITION = os.getenv("DREMIO_EDITION") -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestInvalidGrantsDremio(BaseGrantsDremio, BaseInvalidGrants): def grantee_does_not_exist_error(self): return "Grant on catalog entity failed. User invalid_user does not exist." diff --git a/tests/functional/adapter/grants/test_seed_grants.py b/tests/functional/adapter/grants/test_seed_grants.py index df85d58b..df4c7bf2 100644 --- a/tests/functional/adapter/grants/test_seed_grants.py +++ b/tests/functional/adapter/grants/test_seed_grants.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants from tests.functional.adapter.grants.base_grants import BaseGrantsDremio from tests.utils.util import relation_from_name from dbt.tests.util import get_connection +DREMIO_EDITION = os.getenv("DREMIO_EDITION") -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestSeedGrantsDremio(BaseGrantsDremio, BaseSeedGrants): # Grants are reapplied if dbt run is ran twice without changes diff --git a/tests/functional/adapter/grants/test_snapshot_grants.py b/tests/functional/adapter/grants/test_snapshot_grants.py index 00ea86a2..38cdfa00 100644 --- a/tests/functional/adapter/grants/test_snapshot_grants.py +++ b/tests/functional/adapter/grants/test_snapshot_grants.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import pytest, os from dbt.tests.adapter.grants.test_snapshot_grants import ( BaseSnapshotGrants, snapshot_schema_yml, @@ -36,8 +36,9 @@ {% endsnapshot %} """.strip() +DREMIO_EDITION = os.getenv("DREMIO_EDITION") -@pytest.mark.skip(reason="Dremio only supports grants in EE/DC editions.") +@pytest.mark.skipif(DREMIO_EDITION == "community", reason="Dremio only supports grants in EE/DC editions.") class TestSnapshotGrantsDremio(BaseGrantsDremio, BaseSnapshotGrants): # Override this to use our modified snapshot model @pytest.fixture(scope="class")