From 0cebcb4e0b90372cefb4f82d90072a6af9a5d5dc Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:59:54 -0300 Subject: [PATCH] fixes from reviews --- .../api/get_language_activation_builder.py | 9 +- .../lib/tasks/upload/language_def_parser.py | 14 +- test/test_lang_activation_builder.py | 306 +++++++++--------- 3 files changed, 169 insertions(+), 160 deletions(-) diff --git a/exasol_script_languages_container_tool/lib/api/get_language_activation_builder.py b/exasol_script_languages_container_tool/lib/api/get_language_activation_builder.py index 37dda5e..9c631aa 100644 --- a/exasol_script_languages_container_tool/lib/api/get_language_activation_builder.py +++ b/exasol_script_languages_container_tool/lib/api/get_language_activation_builder.py @@ -37,12 +37,17 @@ def get_language_activation_builder( template = Template(lang_def_template) language_definition = template.render( - bucketfs_name="", bucket_name="", release_name="", path_in_bucket="" + bucketfs_name="", + bucket_name="", + release_name="", + path_in_bucket="____end_marker_bucket_path____", ) languages_defs = language_definition.split(" ") language_def_components_list = list() for lang_def in languages_defs: - alias, url = parse_language_definition(lang_def) + alias, url = parse_language_definition( + lang_def, end_marker_bucket_path="____end_marker_bucket_path____" + ) if isinstance(url, LanguageDefinitionURL): url.bucket_name = bucket_name url.bucketfs_name = bucketfs_name diff --git a/exasol_script_languages_container_tool/lib/tasks/upload/language_def_parser.py b/exasol_script_languages_container_tool/lib/tasks/upload/language_def_parser.py index 7d412d1..d59a256 100644 --- a/exasol_script_languages_container_tool/lib/tasks/upload/language_def_parser.py +++ b/exasol_script_languages_container_tool/lib/tasks/upload/language_def_parser.py @@ -31,7 +31,7 @@ def _parse_parameters(query_string: str) -> Tuple[str, List[SLCParameter]]: def parse_language_definition( - lang_def: str, + lang_def: str, end_marker_bucket_path: str ) -> Tuple[str, Union[LanguageDefinitionURL, BuiltInLanguageDefinitionURL]]: alias_end = lang_def.find("=") alias = lang_def[0:alias_end] @@ -42,14 +42,18 @@ def parse_language_definition( language, slc_parameters = _parse_parameters(parsed_url.query) # fragment is supposed to be something like: - # 'buckets/exaudf/exaudfclient_py3' + # 'buckets///____end_marker_bucket_path____exaudf/exaudfclient_py3' # We remove the given bucket prefix 'buckets/' - udf_client_path_within_container = parsed_url.fragment.replace("buckets///", "") - if not udf_client_path_within_container: + end_marker_start_idx = parsed_url.fragment.find(end_marker_bucket_path) + if end_marker_start_idx < 0: raise ValueError( - f"URL {url} for alias '{alias}' is not in expected format: Path to udf client is empty." + f"URL {url} for alias '{alias}' is not in expected format: The bucket path is invalid." ) + udf_client_path_within_container = parsed_url.fragment[ + end_marker_start_idx + len(end_marker_bucket_path) : + ] + slc_language = None for slc_language_enum in SLCLanguage: if slc_language_enum.value.lower() == language.lower(): diff --git a/test/test_lang_activation_builder.py b/test/test_lang_activation_builder.py index 196572e..016725c 100644 --- a/test/test_lang_activation_builder.py +++ b/test/test_lang_activation_builder.py @@ -23,159 +23,159 @@ def setUp(self): print(f"SetUp {self.__class__.__name__}") self.maxDiff = None - # def test_without_custom_alias_without_builtin(self): - # lang_act_build = get_language_activation_builder( - # flavor_path=self.flavor_path, - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # ) - # components_list = lang_act_build.generate_definition_components() - # self.assertEqual( - # components_list, - # [ - # LanguageDefinitionComponents( - # alias="PYTHON3_TEST", - # url=LanguageDefinitionURL( - # protocol="localzmq+protobuf", - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # udf_client_path_within_container="/exaudf/exaudfclient_py3", - # parameters=list(), - # language=SLCLanguage.Python3, - # ), - # ) - # ], - # ) - # - # def test_without_custom_alias_with_builtin(self): - # lang_act_build = get_language_activation_builder( - # flavor_path=self.flavor_path, - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # add_missing_builtin=True, - # ) - # components_list = lang_act_build.generate_definition_components() - # self.assertEqual( - # components_list, - # [ - # LanguageDefinitionComponents( - # alias="PYTHON3_TEST", - # url=LanguageDefinitionURL( - # protocol="localzmq+protobuf", - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # udf_client_path_within_container="/exaudf/exaudfclient_py3", - # parameters=list(), - # language=SLCLanguage.Python3, - # ), - # ), - # LanguageDefinitionComponents( - # alias="JAVA", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Java), - # ), - # LanguageDefinitionComponents( - # alias="PYTHON3", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Python3), - # ), - # LanguageDefinitionComponents( - # alias="R", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.R), - # ), - # ], - # ) - # - # def test_with_custom_alias_with_builtin(self): - # lang_act_build = get_language_activation_builder( - # flavor_path=self.flavor_path, - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # add_missing_builtin=True, - # ) - # lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") - # lang_act_build.add_custom_alias("JAVA", "MY_JAVA") - # components_list = lang_act_build.generate_definition_components() - # self.assertEqual( - # components_list, - # [ - # LanguageDefinitionComponents( - # alias="MY_PYTHON3", - # url=LanguageDefinitionURL( - # protocol="localzmq+protobuf", - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # udf_client_path_within_container="/exaudf/exaudfclient_py3", - # parameters=list(), - # language=SLCLanguage.Python3, - # ), - # ), - # LanguageDefinitionComponents( - # alias="JAVA", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Java), - # ), - # LanguageDefinitionComponents( - # alias="PYTHON3", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Python3), - # ), - # LanguageDefinitionComponents( - # alias="R", - # url=BuiltInLanguageDefinitionURL(language=SLCLanguage.R), - # ), - # ], - # ) - # - # def test_with_custom_alias_with_builtin_alter_session(self): - # lang_act_build = get_language_activation_builder( - # flavor_path=self.flavor_path, - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # add_missing_builtin=True, - # ) - # lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") - # lang_act_build.add_custom_alias("JAVA", "MY_JAVA") - # alter_session = lang_act_build.generate_alter_session() - # self.assertEqual( - # alter_session, - # "ALTER SESSION SET SCRIPT_LANGUAGES='MY_PYTHON3=" - # "localzmq+protobuf:///bfsdefault/default/some_path/my_release" - # "?lang=python#buckets/bfsdefault/default/some_path/my_release" - # "/exaudf/exaudfclient_py3 JAVA=builtin_java " - # "PYTHON3=builtin_python3 R=builtin_r';", - # ) - # - # def test_with_custom_alias_with_builtin_alter_system(self): - # lang_act_build = get_language_activation_builder( - # flavor_path=self.flavor_path, - # bucketfs_name="bfsdefault", - # bucket_name="default", - # container_name="my_release", - # path_in_bucket="some_path", - # add_missing_builtin=True, - # ) - # lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") - # lang_act_build.add_custom_alias("JAVA", "MY_JAVA") - # alter_session = lang_act_build.generate_alter_system() - # self.assertEqual( - # alter_session, - # "ALTER SYSTEM SET SCRIPT_LANGUAGES='MY_PYTHON3=" - # "localzmq+protobuf:///bfsdefault/default/some_path/my_release" - # "?lang=python#buckets/bfsdefault/default/some_path/my_release" - # "/exaudf/exaudfclient_py3 JAVA=builtin_java " - # "PYTHON3=builtin_python3 R=builtin_r';", - # ) + def test_without_custom_alias_without_builtin(self): + lang_act_build = get_language_activation_builder( + flavor_path=self.flavor_path, + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + ) + components_list = lang_act_build.generate_definition_components() + self.assertEqual( + components_list, + [ + LanguageDefinitionComponents( + alias="PYTHON3_TEST", + url=LanguageDefinitionURL( + protocol="localzmq+protobuf", + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + udf_client_path_within_container="/exaudf/exaudfclient_py3", + parameters=list(), + language=SLCLanguage.Python3, + ), + ) + ], + ) + + def test_without_custom_alias_with_builtin(self): + lang_act_build = get_language_activation_builder( + flavor_path=self.flavor_path, + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + add_missing_builtin=True, + ) + components_list = lang_act_build.generate_definition_components() + self.assertEqual( + components_list, + [ + LanguageDefinitionComponents( + alias="PYTHON3_TEST", + url=LanguageDefinitionURL( + protocol="localzmq+protobuf", + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + udf_client_path_within_container="/exaudf/exaudfclient_py3", + parameters=list(), + language=SLCLanguage.Python3, + ), + ), + LanguageDefinitionComponents( + alias="JAVA", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Java), + ), + LanguageDefinitionComponents( + alias="PYTHON3", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Python3), + ), + LanguageDefinitionComponents( + alias="R", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.R), + ), + ], + ) + + def test_with_custom_alias_with_builtin(self): + lang_act_build = get_language_activation_builder( + flavor_path=self.flavor_path, + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + add_missing_builtin=True, + ) + lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") + lang_act_build.add_custom_alias("JAVA", "MY_JAVA") + components_list = lang_act_build.generate_definition_components() + self.assertEqual( + components_list, + [ + LanguageDefinitionComponents( + alias="MY_PYTHON3", + url=LanguageDefinitionURL( + protocol="localzmq+protobuf", + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + udf_client_path_within_container="/exaudf/exaudfclient_py3", + parameters=list(), + language=SLCLanguage.Python3, + ), + ), + LanguageDefinitionComponents( + alias="JAVA", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Java), + ), + LanguageDefinitionComponents( + alias="PYTHON3", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.Python3), + ), + LanguageDefinitionComponents( + alias="R", + url=BuiltInLanguageDefinitionURL(language=SLCLanguage.R), + ), + ], + ) + + def test_with_custom_alias_with_builtin_alter_session(self): + lang_act_build = get_language_activation_builder( + flavor_path=self.flavor_path, + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + add_missing_builtin=True, + ) + lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") + lang_act_build.add_custom_alias("JAVA", "MY_JAVA") + alter_session = lang_act_build.generate_alter_session() + self.assertEqual( + alter_session, + "ALTER SESSION SET SCRIPT_LANGUAGES='MY_PYTHON3=" + "localzmq+protobuf:///bfsdefault/default/some_path/my_release" + "?lang=python#buckets/bfsdefault/default/some_path/my_release" + "/exaudf/exaudfclient_py3 JAVA=builtin_java " + "PYTHON3=builtin_python3 R=builtin_r';", + ) + + def test_with_custom_alias_with_builtin_alter_system(self): + lang_act_build = get_language_activation_builder( + flavor_path=self.flavor_path, + bucketfs_name="bfsdefault", + bucket_name="default", + container_name="my_release", + path_in_bucket="some_path", + add_missing_builtin=True, + ) + lang_act_build.add_custom_alias("PYTHON3_TEST", "MY_PYTHON3") + lang_act_build.add_custom_alias("JAVA", "MY_JAVA") + alter_session = lang_act_build.generate_alter_system() + self.assertEqual( + alter_session, + "ALTER SYSTEM SET SCRIPT_LANGUAGES='MY_PYTHON3=" + "localzmq+protobuf:///bfsdefault/default/some_path/my_release" + "?lang=python#buckets/bfsdefault/default/some_path/my_release" + "/exaudf/exaudfclient_py3 JAVA=builtin_java " + "PYTHON3=builtin_python3 R=builtin_r';", + ) def test_with_custom_alias_with_builtin_with_parameter_alter_system(self): with tempfile.TemporaryDirectory() as d: