diff --git a/doc/changes/changes_1.0.0.md b/doc/changes/changes_1.0.0.md index fa412f0..d2031fe 100644 --- a/doc/changes/changes_1.0.0.md +++ b/doc/changes/changes_1.0.0.md @@ -14,3 +14,4 @@ t.b.d. #219: Replaced deprecated bucketfs API by new API #171: Improved api generate_language_activation +#230: Created new method `deploy` with similar parameters as in python-extension-common diff --git a/exasol_script_languages_container_tool/cli/commands/__init__.py b/exasol_script_languages_container_tool/cli/commands/__init__.py index da578de..7711a3a 100644 --- a/exasol_script_languages_container_tool/cli/commands/__init__.py +++ b/exasol_script_languages_container_tool/cli/commands/__init__.py @@ -1,6 +1,7 @@ from .build import build from .build_test_container import build_test_container from .clean import clean_all_images, clean_flavor_images +from .deploy import deploy from .export import export from .generate_language_activation import generate_language_activation from .install_starter_scripts import install_starter_scripts diff --git a/exasol_script_languages_container_tool/cli/commands/deploy.py b/exasol_script_languages_container_tool/cli/commands/deploy.py new file mode 100644 index 0000000..512cacd --- /dev/null +++ b/exasol_script_languages_container_tool/cli/commands/deploy.py @@ -0,0 +1,166 @@ +import os +from enum import Enum +from typing import Any, Optional, Tuple + +import click +from exasol_integration_test_docker_environment.cli.cli import cli # type: ignore +from exasol_integration_test_docker_environment.cli.options.build_options import ( + build_options, +) +from exasol_integration_test_docker_environment.cli.options.docker_repository_options import ( + docker_repository_options, +) +from exasol_integration_test_docker_environment.cli.options.system_options import ( + luigi_logging_options, + system_options, +) +from exasol_integration_test_docker_environment.cli.termination_handler import ( + TerminationHandler, +) +from exasol_integration_test_docker_environment.lib.api.common import add_options + +from exasol_script_languages_container_tool.cli.options.flavor_options import ( + flavor_options, +) +from exasol_script_languages_container_tool.cli.options.goal_options import ( + release_options, +) +from exasol_script_languages_container_tool.lib import api + +# This text will be displayed instead of the actual value, if found in an environment +# variable, in a prompt. +SECRET_DISPLAY = "***" + + +class SecretParams(Enum): + """ + This enum serves as a definition of confidential parameters which values should not be + displayed in the console, unless the user types them in the command line. + + The enum name is also the name of the environment variable where the correspondent + secret value can be stored. + + The enum value is also the name of the cli parameter. + """ + + DB_PASSWORD = "db-pass" + BUCKETFS_PASSWORD = "bucketfs-password" + + +def secret_callback(ctx: click.Context, param: click.Option, value: Any): + """ + Here we try to get the secret parameter value from an environment variable. + The reason for doing this in the callback instead of using a callable default is + that we don't want the default to be displayed in the prompt. There seems to + be no way of altering this behaviour. + """ + if value == SECRET_DISPLAY: + secret_param = SecretParams(param.opts[0][2:]) + return os.environ.get(secret_param.name) + return value + + +@cli.command(short_help="Deploys the script-language-container in the database.") +@add_options(flavor_options) +@click.option("--bucketfs-host", type=str, required=True) +@click.option("--bucketfs-port", type=int, required=True) +@click.option("--bucketfs-use-https", type=bool, default=False) +@click.option("--bucketfs-user", type=str) +@click.option("--bucketfs-name", type=str, required=True) +@click.option("--bucket", type=str) +@click.option( + f"--{SecretParams.BUCKETFS_PASSWORD.value}", + type=str, + prompt="BucketFS password", + prompt_required=False, + hide_input=True, + default=SECRET_DISPLAY, + callback=secret_callback, +) +@click.option("--path-in-bucket", type=str, required=False, default="") +@add_options(release_options) +@click.option("--release-name", type=str, default=None) +@add_options(build_options) +@add_options(docker_repository_options) +@add_options(system_options) +@add_options(luigi_logging_options) +@click.option("--ssl-cert-path", type=str, default="") +@click.option("--use-ssl-cert-validation/--no-use-ssl-cert-validation", default=True) +def deploy( + flavor_path: Tuple[str, ...], + bucketfs_host: str, + bucketfs_port: int, + bucketfs_use_https: str, + bucketfs_user: str, + bucketfs_name: str, + bucket: str, + bucketfs_password: str, + path_in_bucket: str, + release_goal: Tuple[str, ...], + release_name: Optional[str], + force_rebuild: bool, + force_rebuild_from: Tuple[str, ...], + force_pull: bool, + output_directory: str, + temporary_base_directory: str, + log_build_context_content: bool, + cache_directory: Optional[str], + build_name: Optional[str], + source_docker_repository_name: str, + source_docker_tag_prefix: str, + source_docker_username: Optional[str], + source_docker_password: Optional[str], + target_docker_repository_name: str, + target_docker_tag_prefix: str, + target_docker_username: Optional[str], + target_docker_password: Optional[str], + workers: int, + task_dependencies_dot_file: Optional[str], + log_level: Optional[str], + use_job_specific_log_file: bool, + ssl_cert_path: str, + use_ssl_cert_validation: bool, +): + """ + This command uploads the whole script-language-container package of the flavor to the database. + If the stages or the packaged container do not exists locally, the system will build, pull or + export them before the upload. + """ + with TerminationHandler(): + result = api.deploy( + flavor_path=flavor_path, + bucketfs_host=bucketfs_host, + bucketfs_port=bucketfs_port, + bucketfs_user=bucketfs_user, + bucketfs_name=bucketfs_name, + bucket=bucket, + bucketfs_password=bucketfs_password, + bucketfs_use_https=bucketfs_use_https, + path_in_bucket=path_in_bucket, + release_goal=release_goal, + release_name=release_name, + force_rebuild=force_rebuild, + force_rebuild_from=force_rebuild_from, + force_pull=force_pull, + output_directory=output_directory, + temporary_base_directory=temporary_base_directory, + log_build_context_content=log_build_context_content, + cache_directory=cache_directory, + build_name=build_name, + source_docker_repository_name=source_docker_repository_name, + source_docker_tag_prefix=source_docker_tag_prefix, + source_docker_username=source_docker_username, + source_docker_password=source_docker_password, + target_docker_repository_name=target_docker_repository_name, + target_docker_tag_prefix=target_docker_tag_prefix, + target_docker_username=target_docker_username, + target_docker_password=target_docker_password, + workers=workers, + task_dependencies_dot_file=task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ssl_cert_path=ssl_cert_path, + use_ssl_cert_validation=use_ssl_cert_validation, + ) + with result.open("r") as f: + print(f.read()) diff --git a/exasol_script_languages_container_tool/cli/commands/upload.py b/exasol_script_languages_container_tool/cli/commands/upload.py index b527352..22d3e9d 100644 --- a/exasol_script_languages_container_tool/cli/commands/upload.py +++ b/exasol_script_languages_container_tool/cli/commands/upload.py @@ -26,7 +26,9 @@ from exasol_script_languages_container_tool.lib import api -@cli.command(short_help="Uploads the script-language-container to the database.") +@cli.command( + short_help="Uploads the script-language-container to the database.Deprecated." +) @add_options(flavor_options) @click.option("--database-host", type=str, required=True) @click.option("--bucketfs-port", type=int, required=True) @@ -83,6 +85,7 @@ def upload( This command uploads the whole script-language-container package of the flavor to the database. If the stages or the packaged container do not exists locally, the system will build, pull or export them before the upload. + This function is deprecated. Use `deploy` instead. """ with TerminationHandler(): result = api.upload( diff --git a/exasol_script_languages_container_tool/lib/api/__init__.py b/exasol_script_languages_container_tool/lib/api/__init__.py index da578de..7711a3a 100644 --- a/exasol_script_languages_container_tool/lib/api/__init__.py +++ b/exasol_script_languages_container_tool/lib/api/__init__.py @@ -1,6 +1,7 @@ from .build import build from .build_test_container import build_test_container from .clean import clean_all_images, clean_flavor_images +from .deploy import deploy from .export import export from .generate_language_activation import generate_language_activation from .install_starter_scripts import install_starter_scripts diff --git a/exasol_script_languages_container_tool/lib/api/deploy.py b/exasol_script_languages_container_tool/lib/api/deploy.py new file mode 100644 index 0000000..20b7d39 --- /dev/null +++ b/exasol_script_languages_container_tool/lib/api/deploy.py @@ -0,0 +1,121 @@ +import getpass +from typing import Optional, Tuple + +import luigi +from exasol_integration_test_docker_environment.lib.api.common import ( + cli_function, + generate_root_task, + import_build_steps, + run_task, + set_build_config, + set_docker_repository_config, +) +from exasol_integration_test_docker_environment.lib.base.dependency_logger_base_task import ( + DependencyLoggerBaseTask, +) + +from exasol_script_languages_container_tool.lib.tasks.upload.upload_containers import ( + UploadContainers, +) + + +@cli_function +def deploy( + flavor_path: Tuple[str, ...], + bucketfs_host: str, + bucketfs_port: int, + bucketfs_use_https: str, + bucketfs_user: str, + bucketfs_name: str, + bucket: str, + bucketfs_password: str, + path_in_bucket: str, + release_goal: Tuple[str, ...], + release_name: Optional[str], + force_rebuild: bool, + force_rebuild_from: Tuple[str, ...], + force_pull: bool, + output_directory: str, + temporary_base_directory: str, + log_build_context_content: bool, + cache_directory: Optional[str], + build_name: Optional[str], + source_docker_repository_name: str, + source_docker_tag_prefix: str, + source_docker_username: Optional[str], + source_docker_password: Optional[str], + target_docker_repository_name: str, + target_docker_tag_prefix: str, + target_docker_username: Optional[str], + target_docker_password: Optional[str], + workers: int, + task_dependencies_dot_file: Optional[str], + log_level: Optional[str], + use_job_specific_log_file: bool, + ssl_cert_path: str, + use_ssl_cert_validation: bool, +) -> luigi.LocalTarget: + """ + This command uploads the whole script-language-container package of the flavor to the database. + If the stages or the packaged container do not exists locally, the system will build, pull or + export them before the upload. + :raises api_errors.TaskFailureError: if operation is not successful. + :return: Path to resulting report file. + """ + import_build_steps(flavor_path) + set_build_config( + force_rebuild, + force_rebuild_from, + force_pull, + log_build_context_content, + output_directory, + temporary_base_directory, + cache_directory, + build_name, + ) + set_docker_repository_config( + source_docker_password, + source_docker_repository_name, + source_docker_username, + source_docker_tag_prefix, + "source", + ) + set_docker_repository_config( + target_docker_password, + target_docker_repository_name, + target_docker_username, + target_docker_tag_prefix, + "target", + ) + if bucketfs_password is None: + bucketfs_password = getpass.getpass( + "BucketFS Password for BucketFS {} and User {}:".format( + bucketfs_name, bucketfs_user + ) + ) + + def root_task_generator() -> DependencyLoggerBaseTask: + return generate_root_task( + task_class=UploadContainers, + flavor_paths=list(flavor_path), + release_goals=list(release_goal), + database_host=bucketfs_host, + bucketfs_port=bucketfs_port, + bucketfs_username=bucketfs_user, + bucketfs_password=bucketfs_password, + bucket_name=bucket, + path_in_bucket=path_in_bucket, + bucketfs_https=bucketfs_use_https, + release_name=release_name, + bucketfs_name=bucketfs_name, + ssl_cert_path=ssl_cert_path, + use_ssl_cert_validation=use_ssl_cert_validation, + ) + + return run_task( + root_task_generator, + workers=workers, + task_dependencies_dot_file=task_dependencies_dot_file, + log_level=log_level, + use_job_specific_log_file=use_job_specific_log_file, + ) diff --git a/exasol_script_languages_container_tool/lib/api/upload.py b/exasol_script_languages_container_tool/lib/api/upload.py index 9b12b5c..a63737c 100644 --- a/exasol_script_languages_container_tool/lib/api/upload.py +++ b/exasol_script_languages_container_tool/lib/api/upload.py @@ -59,6 +59,7 @@ def upload( This command uploads the whole script-language-container package of the flavor to the database. If the stages or the packaged container do not exists locally, the system will build, pull or export them before the upload. + This function is deprecated. Use `deploy` instead. :raises api_errors.TaskFailureError: if operation is not successful. :return: Path to resulting report file. """ diff --git a/test/test_docker_deploy.py b/test/test_docker_deploy.py new file mode 100644 index 0000000..9e0a5f0 --- /dev/null +++ b/test/test_docker_deploy.py @@ -0,0 +1,140 @@ +import os +import subprocess +import unittest + +import utils as exaslct_utils # type: ignore # pylint: disable=import-error +from exasol_integration_test_docker_environment.testing import utils # type: ignore + + +class DockerDeployTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + print(f"SetUpClass: {cls.__name__}") + cls.test_environment = exaslct_utils.ExaslctTestEnvironmentWithCleanUp( + cls, exaslct_utils.EXASLCT_DEFAULT_BIN + ) + cls.test_environment.clean_images() + cls.docker_environment_name = cls.__name__ + cls.docker_environments = cls.test_environment.spawn_docker_test_environments( + cls.docker_environment_name + ) + if "GOOGLE_CLOUD_BUILD" in os.environ: + cls.docker_environment = cls.docker_environments.google_cloud_environment + else: + cls.docker_environment = cls.docker_environments.on_host_docker_environment + + @classmethod + def tearDownClass(cls): + utils.close_environments(cls.docker_environments, cls.test_environment) + + def test_dockerdeploy_with_path_in_bucket(self): + self.path_in_bucket = "test" + self.release_name = "TEST" + self.bucketfs_name = "bfsdefault" + self.bucket_name = "default" + arguments = " ".join( + [ + f"--bucketfs-host {self.docker_environment.database_host}", + f"--bucketfs-port {self.docker_environment.ports.bucketfs}", + f"--bucketfs-user {self.docker_environment.bucketfs_username}", + f"--bucketfs-password {self.docker_environment.bucketfs_password}", + f"--bucketfs-name {self.bucketfs_name}", + f"--bucket {self.bucket_name}", + f"--path-in-bucket {self.path_in_bucket}", + f"--bucketfs-use-https 0", + f"--release-name {self.release_name}", + ] + ) + command = f"{self.test_environment.executable} deploy {arguments}" + + completed_process = self.test_environment.run_command( + command, track_task_dependencies=True, capture_output=True + ) + self.assertIn( + f"ALTER SESSION SET SCRIPT_LANGUAGES='PYTHON3_TEST=localzmq+protobuf:///{self.bucketfs_name}/" + f"{self.bucket_name}/{self.path_in_bucket}/test-flavor-release-{self.release_name}?lang=python#buckets/" + f"{self.bucketfs_name}/{self.bucket_name}/{self.path_in_bucket}/test-flavor-release-{self.release_name}/" + f"exaudf/exaudfclient_py3", + completed_process.stdout.decode("UTF-8"), + ) + self.validate_file_on_bucket_fs( + f"{self.path_in_bucket}/test-flavor-release-{self.release_name}.tar.gz" + ) + + def test_dockerdeploy_without_path_in_bucket(self): + self.release_name = "TEST" + self.bucketfs_name = "bfsdefault" + self.bucket_name = "default" + arguments = " ".join( + [ + f"--bucketfs-host {self.docker_environment.database_host}", + f"--bucketfs-port {self.docker_environment.ports.bucketfs}", + f"--bucketfs-user {self.docker_environment.bucketfs_username}", + f"--bucketfs-password {self.docker_environment.bucketfs_password}", + f"--bucketfs-name {self.bucketfs_name}", + f"--bucket {self.bucket_name}", + f"--bucketfs-use-https 0", + f"--release-name {self.release_name}", + ] + ) + command = f"{self.test_environment.executable} deploy {arguments}" + + completed_process = self.test_environment.run_command( + command, track_task_dependencies=True, capture_output=True + ) + self.assertIn( + f"ALTER SESSION SET SCRIPT_LANGUAGES='PYTHON3_TEST=localzmq+protobuf:///{self.bucketfs_name}/" + f"{self.bucket_name}/test-flavor-release-{self.release_name}?lang=python#buckets/" + f"{self.bucketfs_name}/{self.bucket_name}/test-flavor-release-{self.release_name}/exaudf/exaudfclient_py3", + completed_process.stdout.decode("UTF-8"), + ) + self.validate_file_on_bucket_fs( + f"test-flavor-release-{self.release_name}.tar.gz" + ) + + def test_dockerdeploy_fail_path_in_bucket(self): + self.release_name = "TEST" + self.bucketfs_name = "bfsdefault" + self.bucket_name = "default" + arguments = " ".join( + [ + f"--bucketfs-host {self.docker_environment.database_host}", + f"--bucketfs-port {self.docker_environment.ports.bucketfs}", + f"--bucketfs-user {self.docker_environment.bucketfs_username}", + f"--bucketfs-password invalid", + f"--bucketfs-name {self.bucketfs_name}", + f"--bucket {self.bucket_name}", + f"--bucketfs-use-https 0", + f"--release-name {self.release_name}", + ] + ) + command = f"{self.test_environment.executable} deploy {arguments}" + + exception_thrown = False + try: + self.test_environment.run_command(command, track_task_dependencies=True) + except: + exception_thrown = True + assert exception_thrown + + def validate_file_on_bucket_fs(self, expected_file_path: str): + url = "http://w:{password}@{host}:{port}/{bucket}".format( + host=self.docker_environment.database_host, # type: ignore + port=self.docker_environment.ports.bucketfs, # type: ignore + bucket=self.bucket_name, + password=self.docker_environment.bucketfs_password, # type: ignore + ) + cmd = ["curl", "--silent", "--show-error", "--fail", url] + p = subprocess.run(cmd, capture_output=True) + p.check_returncode() + found_lines = [ + line + for line in p.stdout.decode("utf-8").split("\n") + if line == expected_file_path + ] + assert len(found_lines) == 1 + + +if __name__ == "__main__": + unittest.main()