From d50cc31b1c274282e2389afca75f30a4841250c3 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 27 Sep 2022 19:52:45 -0700 Subject: [PATCH 1/5] Update makefile rule generate to create temp backend override --- .../terraform/hooks/prepare/hook.py | 27 ++++++++++++++++++- .../terraform/hooks/prepare/test_hook.py | 22 +++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/samcli/hook_packages/terraform/hooks/prepare/hook.py b/samcli/hook_packages/terraform/hooks/prepare/hook.py index ed158d1de9..4c635f1761 100644 --- a/samcli/hook_packages/terraform/hooks/prepare/hook.py +++ b/samcli/hook_packages/terraform/hooks/prepare/hook.py @@ -15,6 +15,7 @@ import hashlib import logging import shutil +import uuid from samcli.lib.hook.exceptions import PrepareHookException, InvalidSamMetadataPropertiesException from samcli.lib.utils import osutils @@ -918,6 +919,9 @@ def _generate_makefile_rule_for_lambda_resource( """ target = _get_makefile_build_target(logical_id) resource_address = sam_metadata_resource.resource.get("address", "") + override_filename = f"zz_sam_cli_backend_{uuid.uuid4()}_override.tf" + backend_filename = f"{uuid.uuid4()}.tfstate" + init_backend_recipe = _generate_and_format_init_local_backend_recipe(override_filename, backend_filename) apply_command_template = "terraform apply -target {resource_address} -replace {resource_address} -auto-approve" apply_command_recipe = _format_makefile_recipe(apply_command_template.format(resource_address=resource_address)) show_command_recipe = _format_makefile_recipe( @@ -925,7 +929,28 @@ def _generate_makefile_rule_for_lambda_resource( python_command_name, output_dir, resource_address, sam_metadata_resource, terraform_application_dir ) ) - return f"{target}{apply_command_recipe}{show_command_recipe}" + rm_override_file_recipe = _format_makefile_recipe(f"rm {override_filename}") + rm_backend_file_recipe = _format_makefile_recipe(f"rm {backend_filename}") + return ( + f"{target}" + f"{init_backend_recipe}" + f"{apply_command_recipe}" + f"{show_command_recipe}" + f"{rm_override_file_recipe}" + f"{rm_backend_file_recipe}" + ) + + +def _generate_and_format_init_local_backend_recipe(override_filename: str, backend_filename: str) -> str: + command_list = [ + f'echo "terraform {{" > {override_filename}', + f'echo " backend \\"local\\" {{" >> {override_filename}', + f'echo " path = \\"./{backend_filename}\\"" >> {override_filename}', + f'echo " }}" >> {override_filename}', + f'echo "}}" >> {override_filename}', + "terraform init -reconfigure", + ] + return "".join([_format_makefile_recipe(line) for line in command_list]) def _build_show_command( diff --git a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py index 172a67b97a..b4687612fb 100644 --- a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py +++ b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py @@ -1,4 +1,6 @@ """Test Terraform prepare hook""" +import logging + from pathlib import Path from subprocess import CalledProcessError from unittest import TestCase @@ -47,6 +49,9 @@ ) +LOG = logging.getLogger(__name__) + + class TestPrepareHook(TestCase): def setUp(self) -> None: self.output_dir = "/output/dir" @@ -2605,11 +2610,19 @@ def test_get_python_command_name_python_not_found(self, mock_run_side_effect, mo @patch("samcli.hook_packages.terraform.hooks.prepare.hook._format_makefile_recipe") def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, get_build_target_mock): format_recipe_mock.side_effect = [ + '\techo "terraform {" > my_override.tf\n', + '\techo " backend \\"local\\" {" >> my_override.tf\n', + '\techo " path = \\"./temp_backend.tfstate\\"" >> my_override.tf\n', + '\techo " }" >> my_override.tf\n', + '\techo "}" >> my_override.tf\n', + "\tterraform init -reconfigure\n", "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n", "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py --expression " '"|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' '|values|triggers|built_output_path" --directory "$(ARTIFACTS_DIR)" ' '--terraform-project-root "/some/dir/path"\n', + "\trm my_override.tf\n", + "\trm temp_backend.tfstate\n", ] get_build_target_mock.return_value = "build-function_logical_id:\n" sam_metadata_resource = SamMetadataResource( @@ -2624,12 +2637,21 @@ def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, ge ) expected_makefile_rule = ( "build-function_logical_id:\n" + '\techo "terraform {" > my_override.tf\n' + '\techo " backend \\"local\\" {" >> my_override.tf\n' + '\techo " path = \\"./temp_backend.tfstate\\"" >> my_override.tf\n' + '\techo " }" >> my_override.tf\n' + '\techo "}" >> my_override.tf\n' + "\tterraform init -reconfigure\n" "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n" "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py " '--expression "|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' '|values|triggers|built_output_path" --directory "$(ARTIFACTS_DIR)" ' '--terraform-project-root "/some/dir/path"\n' + "\trm my_override.tf\n" + "\trm temp_backend.tfstate\n" ) + self.assertEqual(makefile_rule, expected_makefile_rule) @patch("samcli.hook_packages.terraform.hooks.prepare.hook._build_jpath_string") From 50ce6465fbb484a4320f98f13c8f280427c34f28 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Thu, 29 Sep 2022 15:21:01 -0700 Subject: [PATCH 2/5] Revert "Update makefile rule generate to create temp backend override" This reverts commit d50cc31b1c274282e2389afca75f30a4841250c3. --- .../terraform/hooks/prepare/hook.py | 27 +------------------ .../terraform/hooks/prepare/test_hook.py | 22 --------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/samcli/hook_packages/terraform/hooks/prepare/hook.py b/samcli/hook_packages/terraform/hooks/prepare/hook.py index 4c635f1761..ed158d1de9 100644 --- a/samcli/hook_packages/terraform/hooks/prepare/hook.py +++ b/samcli/hook_packages/terraform/hooks/prepare/hook.py @@ -15,7 +15,6 @@ import hashlib import logging import shutil -import uuid from samcli.lib.hook.exceptions import PrepareHookException, InvalidSamMetadataPropertiesException from samcli.lib.utils import osutils @@ -919,9 +918,6 @@ def _generate_makefile_rule_for_lambda_resource( """ target = _get_makefile_build_target(logical_id) resource_address = sam_metadata_resource.resource.get("address", "") - override_filename = f"zz_sam_cli_backend_{uuid.uuid4()}_override.tf" - backend_filename = f"{uuid.uuid4()}.tfstate" - init_backend_recipe = _generate_and_format_init_local_backend_recipe(override_filename, backend_filename) apply_command_template = "terraform apply -target {resource_address} -replace {resource_address} -auto-approve" apply_command_recipe = _format_makefile_recipe(apply_command_template.format(resource_address=resource_address)) show_command_recipe = _format_makefile_recipe( @@ -929,28 +925,7 @@ def _generate_makefile_rule_for_lambda_resource( python_command_name, output_dir, resource_address, sam_metadata_resource, terraform_application_dir ) ) - rm_override_file_recipe = _format_makefile_recipe(f"rm {override_filename}") - rm_backend_file_recipe = _format_makefile_recipe(f"rm {backend_filename}") - return ( - f"{target}" - f"{init_backend_recipe}" - f"{apply_command_recipe}" - f"{show_command_recipe}" - f"{rm_override_file_recipe}" - f"{rm_backend_file_recipe}" - ) - - -def _generate_and_format_init_local_backend_recipe(override_filename: str, backend_filename: str) -> str: - command_list = [ - f'echo "terraform {{" > {override_filename}', - f'echo " backend \\"local\\" {{" >> {override_filename}', - f'echo " path = \\"./{backend_filename}\\"" >> {override_filename}', - f'echo " }}" >> {override_filename}', - f'echo "}}" >> {override_filename}', - "terraform init -reconfigure", - ] - return "".join([_format_makefile_recipe(line) for line in command_list]) + return f"{target}{apply_command_recipe}{show_command_recipe}" def _build_show_command( diff --git a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py index b4687612fb..172a67b97a 100644 --- a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py +++ b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py @@ -1,6 +1,4 @@ """Test Terraform prepare hook""" -import logging - from pathlib import Path from subprocess import CalledProcessError from unittest import TestCase @@ -49,9 +47,6 @@ ) -LOG = logging.getLogger(__name__) - - class TestPrepareHook(TestCase): def setUp(self) -> None: self.output_dir = "/output/dir" @@ -2610,19 +2605,11 @@ def test_get_python_command_name_python_not_found(self, mock_run_side_effect, mo @patch("samcli.hook_packages.terraform.hooks.prepare.hook._format_makefile_recipe") def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, get_build_target_mock): format_recipe_mock.side_effect = [ - '\techo "terraform {" > my_override.tf\n', - '\techo " backend \\"local\\" {" >> my_override.tf\n', - '\techo " path = \\"./temp_backend.tfstate\\"" >> my_override.tf\n', - '\techo " }" >> my_override.tf\n', - '\techo "}" >> my_override.tf\n', - "\tterraform init -reconfigure\n", "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n", "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py --expression " '"|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' '|values|triggers|built_output_path" --directory "$(ARTIFACTS_DIR)" ' '--terraform-project-root "/some/dir/path"\n', - "\trm my_override.tf\n", - "\trm temp_backend.tfstate\n", ] get_build_target_mock.return_value = "build-function_logical_id:\n" sam_metadata_resource = SamMetadataResource( @@ -2637,21 +2624,12 @@ def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, ge ) expected_makefile_rule = ( "build-function_logical_id:\n" - '\techo "terraform {" > my_override.tf\n' - '\techo " backend \\"local\\" {" >> my_override.tf\n' - '\techo " path = \\"./temp_backend.tfstate\\"" >> my_override.tf\n' - '\techo " }" >> my_override.tf\n' - '\techo "}" >> my_override.tf\n' - "\tterraform init -reconfigure\n" "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n" "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py " '--expression "|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' '|values|triggers|built_output_path" --directory "$(ARTIFACTS_DIR)" ' '--terraform-project-root "/some/dir/path"\n' - "\trm my_override.tf\n" - "\trm temp_backend.tfstate\n" ) - self.assertEqual(makefile_rule, expected_makefile_rule) @patch("samcli.hook_packages.terraform.hooks.prepare.hook._build_jpath_string") From 2a34d17be7b0f2bae65a2fd98aa07ac99c9a66a5 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Thu, 29 Sep 2022 23:27:14 -0700 Subject: [PATCH 3/5] Add generate backend override file and update make rule --- .../terraform/hooks/prepare/hook.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/samcli/hook_packages/terraform/hooks/prepare/hook.py b/samcli/hook_packages/terraform/hooks/prepare/hook.py index ed158d1de9..e1d5c71524 100644 --- a/samcli/hook_packages/terraform/hooks/prepare/hook.py +++ b/samcli/hook_packages/terraform/hooks/prepare/hook.py @@ -15,6 +15,7 @@ import hashlib import logging import shutil +import uuid from samcli.lib.hook.exceptions import PrepareHookException, InvalidSamMetadataPropertiesException from samcli.lib.utils import osutils @@ -46,6 +47,7 @@ PropertyBuilderMapping = Dict[str, PropertyBuilder] TERRAFORM_BUILD_SCRIPT = "copy_terraform_built_artifacts.py" +TF_BACKEND_OVERRIDE_FILENAME = "z_samcli_backend_override.tf" CFN_CODE_PROPERTIES = { CFN_AWS_LAMBDA_FUNCTION: "Code", @@ -801,6 +803,9 @@ def _generate_makefile( if not os.path.exists(output_directory_path): os.makedirs(output_directory_path, exist_ok=True) + # create z_samcli_backend_override.tf in output directory + _generate_backend_override_file(output_directory_path) + # copy copy_terraform_built_artifacts.py script into output directory copy_terraform_built_artifacts_script_path = os.path.join( Path(os.path.dirname(__file__)).parent.parent, TERRAFORM_BUILD_SCRIPT @@ -813,6 +818,22 @@ def _generate_makefile( makefile.writelines(makefile_rules) +def _generate_backend_override_file(output_directory_path: str): + """ + Generates an override tf file to use a temporary backend + + Parameters + ---------- + output_directory_path: str + the output directory path to write the generated makefile + """ + statefile_filename = f"{uuid.uuid4()}.tfstate" + override_content = "terraform {\n" ' backend "local" {\n' f' path = "./{statefile_filename}"\n' " }\n" "}\n" + override_file_path = os.path.join(output_directory_path, TF_BACKEND_OVERRIDE_FILENAME) + with open(override_file_path, "w+") as f: + f.write(override_content) + + def _get_relevant_cfn_resource( sam_metadata_resource: SamMetadataResource, cfn_resources: Dict[str, Dict] ) -> Tuple[Dict, str]: @@ -918,6 +939,12 @@ def _generate_makefile_rule_for_lambda_resource( """ target = _get_makefile_build_target(logical_id) resource_address = sam_metadata_resource.resource.get("address", "") + move_override_recipe = _format_makefile_recipe( + "cp " + f"{Path(output_dir, TF_BACKEND_OVERRIDE_FILENAME).relative_to(terraform_application_dir)} " + f"./{TF_BACKEND_OVERRIDE_FILENAME}" + ) + init_command_recipe = _format_makefile_recipe("terraform init -reconfigure") apply_command_template = "terraform apply -target {resource_address} -replace {resource_address} -auto-approve" apply_command_recipe = _format_makefile_recipe(apply_command_template.format(resource_address=resource_address)) show_command_recipe = _format_makefile_recipe( @@ -925,7 +952,13 @@ def _generate_makefile_rule_for_lambda_resource( python_command_name, output_dir, resource_address, sam_metadata_resource, terraform_application_dir ) ) - return f"{target}{apply_command_recipe}{show_command_recipe}" + return ( + f"{target}" + f"{move_override_recipe}" + f"{init_command_recipe}" + f"{apply_command_recipe}" + f"{show_command_recipe}" + ) def _build_show_command( From b8df59deaf2202c9d802ddf4e7d177733275a67d Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Thu, 29 Sep 2022 23:36:41 -0700 Subject: [PATCH 4/5] Update unit test --- .../terraform/hooks/prepare/test_hook.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py index 172a67b97a..5aa4ef1b91 100644 --- a/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py +++ b/tests/unit/hook_packages/terraform/hooks/prepare/test_hook.py @@ -2499,10 +2499,15 @@ def test_generate_makefile( ): mock_os.path.exists.return_value = output_dir_exists + mock_copy_tf_backend_override_file_path = Mock() mock_copy_terraform_built_artifacts_script_path = Mock() mock_makefile_path = Mock() mock_os.path.dirname.return_value = "" - mock_os.path.join.side_effect = [mock_copy_terraform_built_artifacts_script_path, mock_makefile_path] + mock_os.path.join.side_effect = [ + mock_copy_tf_backend_override_file_path, + mock_copy_terraform_built_artifacts_script_path, + mock_makefile_path, + ] mock_makefile = Mock() mock_open.return_value.__enter__.return_value = mock_makefile @@ -2605,6 +2610,8 @@ def test_get_python_command_name_python_not_found(self, mock_run_side_effect, mo @patch("samcli.hook_packages.terraform.hooks.prepare.hook._format_makefile_recipe") def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, get_build_target_mock): format_recipe_mock.side_effect = [ + "\tcp .aws-sam-iacs/iacs_metadata/z_samcli_backend_override.tf ./z_samcli_backend_override.tf\n", + "\tterraform init -reconfigure\n", "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n", "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py --expression " '"|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' @@ -2624,6 +2631,8 @@ def test_generate_makefile_rule_for_lambda_resource(self, format_recipe_mock, ge ) expected_makefile_rule = ( "build-function_logical_id:\n" + "\tcp .aws-sam-iacs/iacs_metadata/z_samcli_backend_override.tf ./z_samcli_backend_override.tf\n" + "\tterraform init -reconfigure\n" "\tterraform apply -target null_resource.sam_metadata_aws_lambda_function -auto-approve\n" "\tterraform show -json | python3 .aws-sam/iacs_metadata/copy_terraform_built_artifacts.py " '--expression "|values|root_module|resources|[?address=="null_resource.sam_metadata_aws_lambda_function"]' From 672a3138b2ea3cf2f6feb4d47c1885b6e5a12706 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Fri, 30 Sep 2022 09:34:26 -0700 Subject: [PATCH 5/5] Not make it a tf file when creating the override file --- samcli/hook_packages/terraform/hooks/prepare/hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samcli/hook_packages/terraform/hooks/prepare/hook.py b/samcli/hook_packages/terraform/hooks/prepare/hook.py index e1d5c71524..58e74b5b2e 100644 --- a/samcli/hook_packages/terraform/hooks/prepare/hook.py +++ b/samcli/hook_packages/terraform/hooks/prepare/hook.py @@ -47,7 +47,7 @@ PropertyBuilderMapping = Dict[str, PropertyBuilder] TERRAFORM_BUILD_SCRIPT = "copy_terraform_built_artifacts.py" -TF_BACKEND_OVERRIDE_FILENAME = "z_samcli_backend_override.tf" +TF_BACKEND_OVERRIDE_FILENAME = "z_samcli_backend_override" CFN_CODE_PROPERTIES = { CFN_AWS_LAMBDA_FUNCTION: "Code", @@ -942,7 +942,7 @@ def _generate_makefile_rule_for_lambda_resource( move_override_recipe = _format_makefile_recipe( "cp " f"{Path(output_dir, TF_BACKEND_OVERRIDE_FILENAME).relative_to(terraform_application_dir)} " - f"./{TF_BACKEND_OVERRIDE_FILENAME}" + f"./{TF_BACKEND_OVERRIDE_FILENAME}.tf" ) init_command_recipe = _format_makefile_recipe("terraform init -reconfigure") apply_command_template = "terraform apply -target {resource_address} -replace {resource_address} -auto-approve"