Skip to content

Commit

Permalink
feat: enrich image lambda functions
Browse files Browse the repository at this point in the history
  • Loading branch information
@moelasmar
moelasmar committed Aug 26, 2022
1 parent d5e2f6e commit 2de11ae
Show file tree
Hide file tree
Showing 2 changed files with 743 additions and 4 deletions.
282 changes: 278 additions & 4 deletions samcli/hook_packages/terraform/hooks/prepare.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
import os
from pathlib import Path
from subprocess import run, CalledProcessError
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Dict, List, Optional, Tuple
import hashlib
import logging

from samcli.lib.hook.exceptions import PrepareHookException, InvalidSamMetadataPropertiesException
from samcli.lib.utils import osutils
from samcli.lib.utils.hash import str_checksum
from samcli.lib.utils.packagetype import ZIP
from samcli.lib.utils.packagetype import ZIP, IMAGE
from samcli.lib.utils.resources import (
AWS_LAMBDA_FUNCTION as CFN_AWS_LAMBDA_FUNCTION,
)
Expand Down Expand Up @@ -301,6 +301,124 @@ def _validate_referenced_resource_matches_sam_metadata_type(
)


def _get_lambda_function_source_code_path(
sam_metadata_attributes: dict,
sam_metadata_resource_address: str,
project_root_dir: str,
src_code_property_name: str,
property_path_property_name: str,
src_code_attribute_name: str,
) -> str:
"""
Validate that sam metadata resource contains the valid metadata properties to get a lambda function source code.
Parameters
----------
sam_metadata_attributes: dict
The sam metadata properties
sam_metadata_resource_address: str
The sam metadata resource address
project_root_dir: str
the terraform project root directory path
src_code_property_name: str
the sam metadata property name that contains the lambda function source code or docker context path
property_path_property_name: str
the sam metadata property name that contains the property to get the source code value if it was provided
as json string
src_code_attribute_name: str
the lambda function source code or docker context to be used to raise the correct exception
Returns
-------
str
The lambda function source code or docker context paths
"""
LOG.info(
"Extract the %s from the sam metadata resource %s from property %s",
src_code_attribute_name,
sam_metadata_resource_address,
src_code_property_name,
)
source_code = sam_metadata_attributes.get(src_code_property_name)
source_code_property = sam_metadata_attributes.get(property_path_property_name)
LOG.debug(
"The found %s value is %s and property value is %s", src_code_attribute_name, source_code, source_code_property
)
if not source_code:
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain the lambda function "
f"{src_code_attribute_name} in property {src_code_property_name}"
)
if isinstance(source_code, str):
try:
LOG.debug("Try to decode the %s value in case if it is a encoded JSON string.", src_code_attribute_name)
source_code = json.loads(source_code)
LOG.debug("The decoded value of the %s value is %s", src_code_attribute_name, source_code)
except Exception:
LOG.debug("Source code value could not be parsed as a JSON object. Handle it as normal string value")

if isinstance(source_code, dict):
LOG.debug(
"Process the extracted %s as JSON object using the property %s",
src_code_attribute_name,
source_code_property,
)
if not source_code_property:
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain the lambda function "
f"{src_code_attribute_name} property in property {property_path_property_name} as the "
f"{src_code_property_name} value is an object"
)
cfn_source_code_path = source_code.get(source_code_property)
if not cfn_source_code_path:
LOG.error(
"The property %s does not exist in the extracted %s JSON object %s",
source_code_property,
src_code_attribute_name,
source_code,
)
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain a valid lambda function "
f"{src_code_attribute_name} property in property {property_path_property_name} as the "
f"{src_code_property_name} value is an object"
)
elif isinstance(source_code, list):
# SAM CLI does not process multiple paths, so we will handle only the first value in this list
LOG.debug("Process the extracted %s as list, and get the first value", src_code_attribute_name)
if len(source_code) < 1:
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain the lambda function "
f"{src_code_attribute_name} in property {src_code_property_name}, and it should not be an empty list"
)
cfn_source_code_path = source_code[0]
if not cfn_source_code_path:
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain a valid lambda function "
f"{src_code_attribute_name} in property {src_code_property_name}"
)
else:
cfn_source_code_path = source_code

LOG.debug("The %s path value is %s", src_code_attribute_name, cfn_source_code_path)

if not os.path.isabs(cfn_source_code_path):
LOG.debug(
"The %s path value is not absoulte value. Get the absolute value based on the root directory %s",
src_code_attribute_name,
project_root_dir,
)
cfn_source_code_path = os.path.normpath(os.path.join(project_root_dir, cfn_source_code_path))
LOG.debug("The calculated absolute path of %s is %s", src_code_attribute_name, cfn_source_code_path)

if not isinstance(cfn_source_code_path, str) or not os.path.exists(cfn_source_code_path):
LOG.error("The path %s does not exist", cfn_source_code_path)
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain a valid string value for the "
f"lambda function {src_code_attribute_name} path"
)

return cfn_source_code_path


def _enrich_mapped_resources(
sam_metadata_resources: List[SamMetadataResource],
cfn_resources: Dict[str, Dict],
Expand All @@ -323,10 +441,102 @@ def _enrich_mapped_resources(
"""

def _enrich_zip_lambda_function(sam_metadata_resource: Dict, cfn_resource: Dict, cfn_resource_logical_id: str):
pass
sam_metadata_attributes = sam_metadata_resource.get("values", {}).get("triggers", {})
sam_metadata_resource_address = sam_metadata_resource.get("address")
if not sam_metadata_resource_address:
raise PrepareHookException(
"Invalid Terraform plan output. The address property should not be null to any terraform resource."
)

LOG.info(
"Enrich the ZIP lambda function %s using the metadata properties defined in resource %s",
cfn_resource_logical_id,
sam_metadata_resource_address,
)
cfn_resource_properties = cfn_resource.get("Properties", {})

_validate_referenced_resource_matches_sam_metadata_type(
cfn_resource, sam_metadata_attributes, sam_metadata_resource_address, ZIP
)

cfn_source_code_path = _get_lambda_function_source_code_path(
sam_metadata_attributes,
sam_metadata_resource_address,
terraform_application_dir,
"original_source_code",
"source_code_property",
"source code",
)
cfn_resource_properties["Code"] = cfn_source_code_path
if not cfn_resource.get("Metadata", {}):
cfn_resource["Metadata"] = {}
cfn_resource["Metadata"]["SkipBuild"] = False
cfn_resource["Metadata"]["BuildMethod"] = "makefile"
cfn_resource["Metadata"]["ContextPath"] = output_directory_path
cfn_resource["Metadata"]["WorkingDirectory"] = terraform_application_dir
# currently we set the terraform project root directory that contains all the terraform artifacts as the project
# directory till we work on the custom hook properties, and add a property for this value.
cfn_resource["Metadata"]["ProjectRootDirectory"] = terraform_application_dir

def _enrich_image_lambda_function(sam_metadata_resource: Dict, cfn_resource: Dict, cfn_resource_logical_id: str):
pass
sam_metadata_attributes = sam_metadata_resource.get("values", {}).get("triggers", {})
sam_metadata_resource_address = sam_metadata_resource.get("address")
if not sam_metadata_resource_address:
raise PrepareHookException(
"Invalid Terraform plan output. The address property should not be null to any terraform resource."
)
cfn_resource_properties = cfn_resource.get("Properties", {})

LOG.info(
"Enrich the IMAGE lambda function %s using the metadata properties defined in resource %s",
cfn_resource_logical_id,
sam_metadata_resource_address,
)

_validate_referenced_resource_matches_sam_metadata_type(
cfn_resource, sam_metadata_attributes, sam_metadata_resource_address, IMAGE
)

cfn_docker_context_path = _get_lambda_function_source_code_path(
sam_metadata_attributes,
sam_metadata_resource_address,
terraform_application_dir,
"docker_context",
"docker_context_property_path",
"docker context",
)
cfn_docker_file = sam_metadata_attributes.get("docker_file")
cfn_docker_build_args_string = sam_metadata_attributes.get("docker_build_args")
if cfn_docker_build_args_string:
try:
LOG.debug("Parse the docker build args %s", cfn_docker_build_args_string)
cfn_docker_build_args = json.loads(cfn_docker_build_args_string)
if not isinstance(cfn_docker_build_args, dict):
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain a valid json "
f"encoded string for the lambda function docker build arguments."
)
except Exception as exc:
raise InvalidSamMetadataPropertiesException(
f"The sam metadata resource {sam_metadata_resource_address} should contain a valid json encoded "
f"string for the lambda function docker build arguments."
) from exc

cfn_docker_tag = sam_metadata_attributes.get("docker_tag")

if cfn_resource_properties.get("Code"):
cfn_resource_properties.pop("Code")

if not cfn_resource.get("Metadata", {}):
cfn_resource["Metadata"] = {}
cfn_resource["Metadata"]["SkipBuild"] = False
cfn_resource["Metadata"]["DockerContext"] = cfn_docker_context_path
if cfn_docker_file:
cfn_resource["Metadata"]["Dockerfile"] = cfn_docker_file
if cfn_docker_tag:
cfn_resource["Metadata"]["DockerTag"] = cfn_docker_tag
if cfn_docker_build_args:
cfn_resource["Metadata"]["DockerBuildArgs"] = cfn_docker_build_args

resources_types_enrichment_functions = {
"ZIP_LAMBDA_FUNCTION": _enrich_zip_lambda_function,
Expand All @@ -343,6 +553,70 @@ def _enrich_image_lambda_function(sam_metadata_resource: Dict, cfn_resource: Dic
f"is not a correct resource type. The resource type should be one of these values "
f"{resources_types_enrichment_functions.keys()}"
)
cfn_resource, logical_id = _get_relevant_cfn_resource(sam_metadata_resource, cfn_resources)
enrichment_function(sam_metadata_resource.resource, cfn_resource, logical_id)


def _get_relevant_cfn_resource(
sam_metadata_resource: SamMetadataResource, cfn_resources: Dict[str, Dict]
) -> Tuple[Dict, str]:
"""
use the sam metadata resource name property to determine the resource address, and transform the address to logical
id to use it to get the cfn_resource.
Parameters
----------
sam_metadata_resource: SamMetadataResource
sam metadata resource that contain extra information about some resource.
cfn_resources: Dict
CloudFormation resources
Returns
-------
tuple(Dict, str)
The cfn resource that mentioned in the sam metadata resource, and the resource logical id
"""
sam_metadata_resource_address = sam_metadata_resource.resource.get("address")
resource_name = sam_metadata_resource.resource.get("values", {}).get("triggers", {}).get("resource_name")
if not resource_name:
raise InvalidSamMetadataPropertiesException(
f"sam cli expects the sam metadata resource {sam_metadata_resource_address} to contain a resource name "
f"that will be enriched using this metadata resource"
)

# the provided resource name can be the name without the module address, or can be a complete resource address name
# check first if the provided name is without the module address
LOG.info(
"Check if the input source name %s is a postfix to the current module address %s",
resource_name,
sam_metadata_resource.current_module_address,
)
full_resource_address = (
f"{sam_metadata_resource.current_module_address}.{resource_name}"
if sam_metadata_resource.current_module_address
else resource_name
)
LOG.debug("check if the resource address %s has a relevant cfn resource or not", full_resource_address)
logical_id = _build_cfn_logical_id(full_resource_address)
cfn_resource = cfn_resources.get(logical_id)
if cfn_resource:
LOG.info("The CFN resource that match the input resource name %s is %s", resource_name, logical_id)
return cfn_resource, logical_id

LOG.info("Check if the input source name %s is a complete address", resource_name)
# check if the provided name is a complete resource address
if sam_metadata_resource.current_module_address:
full_resource_address = resource_name
LOG.debug("check if the resource address %s has a relevant cfn resource or not", full_resource_address)
logical_id = _build_cfn_logical_id(full_resource_address)
cfn_resource = cfn_resources.get(logical_id)
if cfn_resource:
LOG.info("The CFN resource that match the input resource name %s is %s", resource_name, logical_id)
return cfn_resource, logical_id

raise InvalidSamMetadataPropertiesException(
f"There is no resource found that match the provided resource name " f"{resource_name}"
)


def _translate_properties(tf_properties: dict, property_builder_mapping: PropertyBuilderMapping) -> dict:
Expand Down
Loading

0 comments on commit 2de11ae

Please sign in to comment.