From 09be2487089b2a5a3bc3e2ba5f2a155f2019f596 Mon Sep 17 00:00:00 2001 From: sunnycarter <36891339+sunnycarter@users.noreply.github.com> Date: Fri, 9 Jun 2023 17:38:11 +0100 Subject: [PATCH 1/2] Sunny/choose deploy parameters (#23) * choose-deploy-parameters * optioned deployParameters for CNF * lint * lint2 * docs * docs * lint * 9.82 score * Fix bugs * more useful debug logs * Fix bugs and logging * lint * markups --- src/aosm/HISTORY.rst | 4 + src/aosm/README.md | 18 +- src/aosm/azext_aosm/_client_factory.py | 1 + src/aosm/azext_aosm/_configuration.py | 61 +++-- src/aosm/azext_aosm/_help.py | 1 - src/aosm/azext_aosm/_params.py | 18 +- src/aosm/azext_aosm/custom.py | 54 ++-- src/aosm/azext_aosm/delete/delete.py | 3 +- src/aosm/azext_aosm/deploy/artifact.py | 21 +- .../azext_aosm/deploy/artifact_manifest.py | 21 +- src/aosm/azext_aosm/deploy/deploy_with_arm.py | 30 ++- src/aosm/azext_aosm/deploy/pre_deploy.py | 9 +- .../generate_nfd/cnf_nfd_generator.py | 253 +++++++++++++++--- .../generate_nfd/nfd_generator_base.py | 2 +- .../generate_nfd/vnf_nfd_generator.py | 166 +++++++++--- .../azext_aosm/generate_nsd/nsd_generator.py | 20 +- .../tests/latest/test_aosm_scenario.py | 3 +- src/aosm/azext_aosm/util/constants.py | 18 +- .../azext_aosm/util/management_clients.py | 3 +- 19 files changed, 530 insertions(+), 176 deletions(-) diff --git a/src/aosm/HISTORY.rst b/src/aosm/HISTORY.rst index e5bcecabf4b..93107646473 100644 --- a/src/aosm/HISTORY.rst +++ b/src/aosm/HISTORY.rst @@ -3,6 +3,10 @@ Release History =============== +unreleased +++++++++++ +* `az aosm nfd build` options `--order-params` and `--interactive` to help users choose which NF parameters to expose as deployParameters. Feature added that allows CNF value mappings file to be generated if none is supplied. + 0.2.0 ++++++ Breaking change to commands - now use `nfd` instead of `definition`. Publish option removed from build. diff --git a/src/aosm/README.md b/src/aosm/README.md index a4df5c59b07..2ebeb4d1680 100644 --- a/src/aosm/README.md +++ b/src/aosm/README.md @@ -67,7 +67,10 @@ image that would be used for the VNF Virtual Machine. #### CNFs -For CNFs, you must provide helm packages with an associated schema. When filling in the input.json file, you must list helm packages in the order they are to be deployed. For example, if A must be deployed before B, your input.json should look something like this: +For CNFs, you must provide helm packages with an associated schema. +Optionally, you can provide a file path_to_mappings which is a copy of values.yaml with your chosen values replaced by deployment parameters, thus exposing them as parameters to the CNF. You can get this file auto-generated by leaving the value as a blank string, either having every value as +a deployment parameter, or using --interactive to interactively choose. +When filling in the input.json file, you must list helm packages in the order they are to be deployed. For example, if A must be deployed before B, your input.json should look something like this: "helm_packages": [ { @@ -115,6 +118,17 @@ Build an nfd definition locally `az aosm nfd build --config-file input.json` +More options on building an nfd definition locally: + +Choose which of the VNF ARM template parameters you want to expose as NFD deploymentParameters, with the option of interactively choosing each one. + +`az aosm nfd build --config-file input.json --definition_type vnf --order_params` +`az aosm nfd build --config-file input.json --definition_type vnf --order_params --interactive` + +Choose which of the CNF Helm values parameters you want to expose as NFD deploymentParameters. + +`az aosm nfd build --config-file input.json --definition_type cnf [--interactive]` + Publish a pre-built definition `az aosm nfd publish --config-file input.json` @@ -157,4 +171,4 @@ Delete a published design Delete a published design and the publisher, artifact stores and NSD group -`az aosm nsd delete --config-file input.json --clean` \ No newline at end of file +`az aosm nsd delete --config-file input.json --clean` diff --git a/src/aosm/azext_aosm/_client_factory.py b/src/aosm/azext_aosm/_client_factory.py index e55e6142d25..939880b240f 100644 --- a/src/aosm/azext_aosm/_client_factory.py +++ b/src/aosm/azext_aosm/_client_factory.py @@ -5,6 +5,7 @@ from azure.cli.core.commands.client_factory import get_mgmt_service_client from azure.cli.core.profiles import ResourceType + from .vendored_sdks import HybridNetworkManagementClient diff --git a/src/aosm/azext_aosm/_configuration.py b/src/aosm/azext_aosm/_configuration.py index f63e8001aa4..a26a3af3a77 100644 --- a/src/aosm/azext_aosm/_configuration.py +++ b/src/aosm/azext_aosm/_configuration.py @@ -1,28 +1,29 @@ ## Disabling as every if statement in validate in NSConfig class has this condition # pylint: disable=simplifiable-condition +import os from dataclasses import dataclass, field -from typing import Dict, Optional, Any, List from pathlib import Path -import os -from azure.cli.core.azclierror import ValidationError, InvalidArgumentValueError +from typing import Any, Dict, List, Optional + +from azure.cli.core.azclierror import InvalidArgumentValueError, ValidationError + from azext_aosm.util.constants import ( - DEFINITION_OUTPUT_BICEP_PREFIX, - VNF, CNF, + DEFINITION_OUTPUT_BICEP_PREFIX, + NF_DEFINITION_JSON_FILE, NSD, NSD_DEFINITION_OUTPUT_BICEP_PREFIX, - NF_DEFINITION_JSON_FILE, + VNF, ) - DESCRIPTION_MAP: Dict[str, str] = { "publisher_resource_group_name": - "Resource group for the Publisher resource. Will be created if it does not exist." - , + "Resource group for the Publisher resource. " + "Will be created if it does not exist.", "publisher_name": - "Name of the Publisher resource you want your definition published to. Will be created if it does not exist." - , + "Name of the Publisher resource you want your definition published to. " + "Will be created if it does not exist.", "publisher_name_nsd": "Name of the Publisher resource you want your design published to. " "This should be the same as the publisher used for your NFDVs" @@ -33,7 +34,8 @@ "acr_artifact_store_name": "Name of the ACR Artifact Store resource. Will be created if it does not exist.", "location": "Azure location to use when creating resources.", "blob_artifact_store_name": - "Name of the storage account Artifact Store resource. Will be created if it does not exist.", + "Name of the storage account Artifact Store resource. Will be created if it " + "does not exist.", "artifact_name": "Name of the artifact", "file_path": "Optional. File path of the artifact you wish to upload from your local disk. " @@ -60,7 +62,12 @@ "path_to_chart": "File path of Helm Chart on local disk. Accepts .tgz, .tar or .tar.gz", "path_to_mappings": - "File path of value mappings on local disk. Accepts .yaml or .yml", + "File path of value mappings on local disk where chosen values are replaced " + "with deploymentParameter placeholders. Accepts .yaml or .yml. If left as a " + "blank string, a value mappings file will be generated with every value " + "mapped to a deployment parameter. Use a blank string and --interactive on " + "the build command to interactively choose which values to map." + , "helm_depends_on": "Names of the Helm packages this package depends on. " "Leave as an empty array if no dependencies", @@ -121,24 +128,33 @@ class NSConfiguration: nsdv_description: str = DESCRIPTION_MAP["nsdv_description"] def validate(self): - """ Validate that all of the configuration parameters are set """ + """Validate that all of the configuration parameters are set.""" if self.location == DESCRIPTION_MAP["location"] or "": raise ValueError("Location must be set") if self.publisher_name == DESCRIPTION_MAP["publisher_name_nsd"] or "": raise ValueError("Publisher name must be set") - if self.publisher_resource_group_name == DESCRIPTION_MAP["publisher_resource_group_name_nsd"] or "": + if ( + self.publisher_resource_group_name + == DESCRIPTION_MAP["publisher_resource_group_name_nsd"] + or "" + ): raise ValueError("Publisher resource group name must be set") - if self.acr_artifact_store_name == DESCRIPTION_MAP["acr_artifact_store_name"] or "": + if ( + self.acr_artifact_store_name == DESCRIPTION_MAP["acr_artifact_store_name"] + or "" + ): raise ValueError("ACR Artifact Store name must be set") if ( self.network_function_definition_group_name - == DESCRIPTION_MAP["network_function_definition_group_name"] or "" + == DESCRIPTION_MAP["network_function_definition_group_name"] + or "" ): raise ValueError("Network Function Definition Group name must be set") if ( - self.network_function_definition_version_name == - DESCRIPTION_MAP["network_function_definition_version_name"] or "" + self.network_function_definition_version_name + == DESCRIPTION_MAP["network_function_definition_version_name"] + or "" ): raise ValueError("Network Function Definition Version name must be set") if ( @@ -173,8 +189,7 @@ def network_function_name(self) -> str: @property def acr_manifest_name(self) -> str: """Return the ACR manifest name from the NFD name.""" - return \ - f"{self.network_function_name.lower().replace('_', '-')}-acr-manifest-{self.nsd_version.replace('.', '-')}" + return f"{self.network_function_name.lower().replace('_', '-')}-acr-manifest-{self.nsd_version.replace('.', '-')}" @property def nfvi_site_name(self) -> str: @@ -195,10 +210,10 @@ def arm_template(self) -> ArtifactConfig: self.build_output_folder_name, NF_DEFINITION_JSON_FILE ) return artifact - + @property def arm_template_artifact_name(self) -> str: - """Return the artifact name for the ARM template""" + """Return the artifact name for the ARM template.""" return f"{self.network_function_definition_group_name}_nfd_artifact" diff --git a/src/aosm/azext_aosm/_help.py b/src/aosm/azext_aosm/_help.py index 2a9a3013fd9..b11bedb4b53 100644 --- a/src/aosm/azext_aosm/_help.py +++ b/src/aosm/azext_aosm/_help.py @@ -6,7 +6,6 @@ from knack.help_files import helps # pylint: disable=unused-import - helps[ "aosm" ] = """ diff --git a/src/aosm/azext_aosm/_params.py b/src/aosm/azext_aosm/_params.py index 840dda18b6e..b7e15796e0f 100644 --- a/src/aosm/azext_aosm/_params.py +++ b/src/aosm/azext_aosm/_params.py @@ -7,7 +7,7 @@ from argcomplete.completers import FilesCompleter from azure.cli.core import AzCommandsLoader -from .util.constants import VNF, CNF, NSD +from .util.constants import CNF, VNF def load_arguments(self: AzCommandsLoader, _): @@ -52,6 +52,22 @@ def load_arguments(self: AzCommandsLoader, _): completer=FilesCompleter(allowednames="*.bicep"), help="Optional path to a bicep file to publish. Use to override publish of the built design with an alternative file.", ) + c.argument( + "order_params", + arg_type=get_three_state_flag(), + help="VNF definition_type only - ignored for CNF." + " Order deploymentParameters schema and configMappings to have the " + "parameters without default values at the top and those with default " + "values at the bottom. Can make it easier to remove those with defaults " + "which you do not want to expose as NFD parameters.", + ) + c.argument( + "interactive", + options_list=["--interactive", "-i"], + arg_type=get_three_state_flag(), + help="Prompt user to choose every parameter to expose as an NFD parameter." + " Those without defaults are automatically included.", + ) c.argument( "parameters_json_file", options_list=["--parameters-file", "-p"], diff --git a/src/aosm/azext_aosm/custom.py b/src/aosm/azext_aosm/custom.py index ed8e708e9d6..06c454235c4 100644 --- a/src/aosm/azext_aosm/custom.py +++ b/src/aosm/azext_aosm/custom.py @@ -8,29 +8,31 @@ import shutil from dataclasses import asdict from typing import Optional -from knack.log import get_logger + from azure.cli.core.azclierror import ( CLIInternalError, InvalidArgumentValueError, UnclassifiedUserFault, ) +from knack.log import get_logger -from azext_aosm.generate_nfd.cnf_nfd_generator import CnfNfdGenerator -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator -from azext_aosm.generate_nsd.nsd_generator import NSDGenerator -from azext_aosm.generate_nfd.vnf_nfd_generator import VnfNfdGenerator -from azext_aosm.delete.delete import ResourceDeleter -from azext_aosm.deploy.deploy_with_arm import DeployerViaArm -from azext_aosm.util.constants import VNF, CNF, NSD -from azext_aosm.util.management_clients import ApiClients -from azext_aosm.vendored_sdks import HybridNetworkManagementClient from azext_aosm._client_factory import cf_resources from azext_aosm._configuration import ( - get_configuration, + CNFConfiguration, NFConfiguration, NSConfiguration, + VNFConfiguration, + get_configuration, ) - +from azext_aosm.delete.delete import ResourceDeleter +from azext_aosm.deploy.deploy_with_arm import DeployerViaArm +from azext_aosm.generate_nfd.cnf_nfd_generator import CnfNfdGenerator +from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator +from azext_aosm.generate_nfd.vnf_nfd_generator import VnfNfdGenerator +from azext_aosm.generate_nsd.nsd_generator import NSDGenerator +from azext_aosm.util.constants import CNF, NSD, VNF +from azext_aosm.util.management_clients import ApiClients +from azext_aosm.vendored_sdks import HybridNetworkManagementClient logger = get_logger(__name__) @@ -38,6 +40,8 @@ def build_definition( definition_type: str, config_file: str, + order_params: bool = False, + interactive: bool = False, ): """ Build a definition. @@ -54,7 +58,12 @@ def build_definition( ) # Generate the NFD and the artifact manifest. - _generate_nfd(definition_type=definition_type, config=config) + _generate_nfd( + definition_type=definition_type, + config=config, + order_params=order_params, + interactive=interactive, + ) def generate_definition_config(definition_type: str, output_file: str = "input.json"): @@ -90,13 +99,17 @@ def _get_config_from_file( return config -def _generate_nfd(definition_type, config): +def _generate_nfd( + definition_type: str, config: NFConfiguration, order_params: bool, interactive: bool +): """Generate a Network Function Definition for the given type and config.""" nfd_generator: NFDGenerator if definition_type == VNF: - nfd_generator = VnfNfdGenerator(config) + assert isinstance(config, VNFConfiguration) + nfd_generator = VnfNfdGenerator(config, order_params, interactive) elif definition_type == CNF: - nfd_generator = CnfNfdGenerator(config) + assert isinstance(config, CNFConfiguration) + nfd_generator = CnfNfdGenerator(config, interactive) else: raise CLIInternalError( "Generate NFD called for unrecognised definition_type. Only VNF and CNF have been implemented." @@ -215,6 +228,7 @@ def delete_published_definition( def generate_design_config(output_file: str = "input.json"): """ Generate an example config file for building a NSD. + :param output_file: path to output config file, defaults to "input.json" :type output_file: str, optional """ @@ -224,6 +238,7 @@ def generate_design_config(output_file: str = "input.json"): def _generate_config(configuration_type: str, output_file: str = "input.json"): """ Generic generate config function for NFDs and NSDs. + :param configuration_type: CNF, VNF or NSD :param output_file: path to output config file, defaults to "input.json" :type output_file: str, optional @@ -240,7 +255,7 @@ def _generate_config(configuration_type: str, output_file: str = "input.json"): with open(output_file, "w", encoding="utf-8") as f: f.write(config_as_dict) - if configuration_type == CNF or configuration_type == VNF: + if configuration_type in (CNF,VNF): prtName = "definition" else: prtName = "design" @@ -251,6 +266,7 @@ def _generate_config(configuration_type: str, output_file: str = "input.json"): def build_design(cmd, client: HybridNetworkManagementClient, config_file: str): """ Build a Network Service Design. + :param cmd: :type cmd: _type_ :param client: @@ -282,6 +298,7 @@ def delete_published_design( ): """ Delete a published NSD. + :param config_file: Path to the config file :param clean: if True, will delete the NSDG, artifact stores and publisher too. Defaults to False. Only works if no resources have those as a parent. @@ -308,6 +325,7 @@ def publish_design( ): """ Publish a generated design. + :param cmd: :param client: :type client: HybridNetworkManagementClient @@ -347,8 +365,6 @@ def _generate_nsd(config: NSDGenerator, api_clients): if config: nsd_generator = NSDGenerator(config) else: - from azure.cli.core.azclierror import CLIInternalError - raise CLIInternalError("Generate NSD called without a config file") deploy_parameters = _get_nfdv_deployment_parameters(config, api_clients) diff --git a/src/aosm/azext_aosm/delete/delete.py b/src/aosm/azext_aosm/delete/delete.py index 5952a164d0e..7cb01d8ce9e 100644 --- a/src/aosm/azext_aosm/delete/delete.py +++ b/src/aosm/azext_aosm/delete/delete.py @@ -5,11 +5,10 @@ """Contains class for deploying generated definitions using the Python SDK.""" from knack.log import get_logger +from azext_aosm._configuration import NFConfiguration, NSConfiguration, VNFConfiguration from azext_aosm.util.management_clients import ApiClients -from azext_aosm._configuration import NFConfiguration, VNFConfiguration, NSConfiguration from azext_aosm.util.utils import input_ack - logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/deploy/artifact.py b/src/aosm/azext_aosm/deploy/artifact.py index 8ee00fc9b7f..b1d8a17857a 100644 --- a/src/aosm/azext_aosm/deploy/artifact.py +++ b/src/aosm/azext_aosm/deploy/artifact.py @@ -3,15 +3,14 @@ # pylint: disable=unidiomatic-typecheck """A module to handle interacting with artifacts.""" -from typing import Union from dataclasses import dataclass -from knack.log import get_logger +from typing import Union from azure.storage.blob import BlobClient, BlobType -from azext_aosm._configuration import ArtifactConfig +from knack.log import get_logger from oras.client import OrasClient -from azext_aosm._configuration import ArtifactConfig +from azext_aosm._configuration import ArtifactConfig logger = get_logger(__name__) @@ -72,9 +71,13 @@ def _upload_to_storage_account(self, artifact_config: ArtifactConfig) -> None: if artifact_config.file_path: logger.info("Upload to blob store") with open(artifact_config.file_path, "rb") as artifact: - self.artifact_client.upload_blob(artifact, overwrite=True, blob_type=BlobType.PAGEBLOB) + self.artifact_client.upload_blob( + artifact, overwrite=True, blob_type=BlobType.PAGEBLOB + ) logger.info( - "Successfully uploaded %s to %s", artifact_config.file_path, self.artifact_client.account_name + "Successfully uploaded %s to %s", + artifact_config.file_path, + self.artifact_client.account_name, ) else: logger.info("Copy from SAS URL to blob store") @@ -84,8 +87,10 @@ def _upload_to_storage_account(self, artifact_config: ArtifactConfig) -> None: logger.debug(source_blob.url) self.artifact_client.start_copy_from_url(source_blob.url) logger.info( - "Successfully copied %s from %s to %s", - source_blob.blob_name, source_blob.account_name, self.artifact_client.account_name + "Successfully copied %s from %s to %s", + source_blob.blob_name, + source_blob.account_name, + self.artifact_client.account_name, ) else: raise RuntimeError( diff --git a/src/aosm/azext_aosm/deploy/artifact_manifest.py b/src/aosm/azext_aosm/deploy/artifact_manifest.py index 13ba5824c2d..168021e8519 100644 --- a/src/aosm/azext_aosm/deploy/artifact_manifest.py +++ b/src/aosm/azext_aosm/deploy/artifact_manifest.py @@ -4,27 +4,28 @@ from functools import cached_property, lru_cache from typing import Any, List, Union -from knack.log import get_logger -from oras.client import OrasClient from azure.cli.core.azclierror import AzCLIError from azure.storage.blob import BlobClient +from knack.log import get_logger +from oras.client import OrasClient -from azext_aosm.deploy.artifact import Artifact from azext_aosm._configuration import NFConfiguration, NSConfiguration +from azext_aosm.deploy.artifact import Artifact +from azext_aosm.util.management_clients import ApiClients from azext_aosm.vendored_sdks.models import ( ArtifactManifest, - ManifestArtifactFormat, - CredentialType, ArtifactType, + CredentialType, + ManifestArtifactFormat, ) -from azext_aosm.util.management_clients import ApiClients logger = get_logger(__name__) class ArtifactManifestOperator: """ArtifactManifest class.""" + # pylint: disable=too-few-public-methods def __init__( self, @@ -123,7 +124,8 @@ def _get_artifact_client( # Check we have the required artifact types for this credential. Indicates # a coding error if we hit this but worth checking. if not ( - artifact.artifact_type in (ArtifactType.IMAGE_FILE, ArtifactType.VHD_IMAGE_FILE) + artifact.artifact_type + in (ArtifactType.IMAGE_FILE, ArtifactType.VHD_IMAGE_FILE) ): raise AzCLIError( f"Cannot upload artifact {artifact.artifact_name}." @@ -142,7 +144,7 @@ def _get_artifact_client( blob_name = container_name logger.debug("container name: %s, blob name: %s", container_name, blob_name) - + blob_url = self._get_blob_url(container_name, blob_name) return BlobClient.from_blob_url(blob_url) return self._oras_client(self._manifest_credentials["acr_server_url"]) @@ -159,9 +161,8 @@ def _get_blob_url(self, container_name: str, blob_name: str) -> str: sas_uri = str(container_credential["container_sas_uri"]) sas_uri_prefix = sas_uri.split("?")[0] # pylint: disable=use-maxsplit-arg sas_uri_token = sas_uri.split("?")[1] - + blob_url = f"{sas_uri_prefix}/{blob_name}?{sas_uri_token}" - logger.debug("Blob URL: %s", blob_url) return blob_url diff --git a/src/aosm/azext_aosm/deploy/deploy_with_arm.py b/src/aosm/azext_aosm/deploy/deploy_with_arm.py index f8a3de1e1b9..8ecdcb6c293 100644 --- a/src/aosm/azext_aosm/deploy/deploy_with_arm.py +++ b/src/aosm/azext_aosm/deploy/deploy_with_arm.py @@ -7,26 +7,26 @@ import os import shutil import subprocess # noqa -from typing import Any, Dict, Optional import tempfile import time +from typing import Any, Dict, Optional -from knack.log import get_logger from azure.mgmt.resource.resources.models import DeploymentExtended +from knack.log import get_logger +from azext_aosm._configuration import NFConfiguration, NSConfiguration, VNFConfiguration from azext_aosm.deploy.artifact_manifest import ArtifactManifestOperator -from azext_aosm.util.management_clients import ApiClients from azext_aosm.deploy.pre_deploy import PreDeployerViaSDK -from azext_aosm._configuration import NFConfiguration, NSConfiguration, VNFConfiguration from azext_aosm.util.constants import ( - NSD_DEFINITION_BICEP_FILE, - NSD_ARTIFACT_MANIFEST_BICEP_FILE, NF_DEFINITION_BICEP_FILE, - VNF_DEFINITION_BICEP_TEMPLATE, - VNF_MANIFEST_BICEP_TEMPLATE, NSD, + NSD_ARTIFACT_MANIFEST_BICEP_FILE, + NSD_DEFINITION_BICEP_FILE, VNF, + VNF_DEFINITION_BICEP_TEMPLATE, + VNF_MANIFEST_BICEP_TEMPLATE, ) +from azext_aosm.util.management_clients import ApiClients logger = get_logger(__name__) @@ -193,7 +193,7 @@ def construct_manifest_parameters(self) -> Dict[str, Any]: "saArtifactStoreName": {"value": self.config.blob_artifact_store_name}, "acrManifestName": {"value": self.config.acr_manifest_name}, "saManifestName": {"value": self.config.sa_manifest_name}, - 'nfName': {"value": self.config.nf_name}, + "nfName": {"value": self.config.nf_name}, "vhdVersion": {"value": self.config.vhd.version}, "armTemplateVersion": {"value": self.config.arm_template.version}, } @@ -427,11 +427,15 @@ def validate_and_deploy_arm_template( # Validation failed so don't even try to deploy logger.error( "Template for resource group %s has failed validation. The message was: %s.\ - See logs for additional details.", resource_group, validation_res.error.message + See logs for additional details.", + resource_group, + validation_res.error.message, ) logger.debug( "Template for resource group %s failed validation. \ - Full error details: %s", resource_group, validation_res.error + Full error details: %s", + resource_group, + validation_res.error, ) raise RuntimeError("Azure template validation failed.") @@ -470,7 +474,9 @@ def validate_and_deploy_arm_template( f"\nAborting" ) logger.debug( - "Provisioning state of deployment %s : %s", resource_group, depl_props.provisioning_state + "Provisioning state of deployment %s : %s", + resource_group, + depl_props.provisioning_state, ) return depl_props.outputs diff --git a/src/aosm/azext_aosm/deploy/pre_deploy.py b/src/aosm/azext_aosm/deploy/pre_deploy.py index 47e83ff4e81..59ae3e8f2b3 100644 --- a/src/aosm/azext_aosm/deploy/pre_deploy.py +++ b/src/aosm/azext_aosm/deploy/pre_deploy.py @@ -4,22 +4,21 @@ # -------------------------------------------------------------------------------------- """Contains class for deploying resources required by NFDs/NSDs via the SDK.""" -from knack.log import get_logger - -from azure.core import exceptions as azure_exceptions from azure.cli.core.azclierror import AzCLIError +from azure.core import exceptions as azure_exceptions from azure.mgmt.resource.resources.models import ResourceGroup +from knack.log import get_logger +from azext_aosm._configuration import NFConfiguration, NSConfiguration, VNFConfiguration from azext_aosm.util.management_clients import ApiClients from azext_aosm.vendored_sdks.models import ( ArtifactStore, ArtifactStoreType, NetworkFunctionDefinitionGroup, NetworkServiceDesignGroup, - Publisher, ProvisioningState, + Publisher, ) -from azext_aosm._configuration import NFConfiguration, VNFConfiguration, NSConfiguration logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py index deb06343879..8d8ce10f1d0 100644 --- a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py +++ b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py @@ -8,30 +8,31 @@ import re import shutil import tarfile -from typing import Dict, List, Any, Tuple, Optional, Iterator - import tempfile +from typing import Any, Dict, Iterator, List, Optional, Tuple + import yaml -from jinja2 import Template, StrictUndefined -from azure.cli.core.azclierror import InvalidTemplateError, FileOperationError +from azure.cli.core.azclierror import FileOperationError, InvalidTemplateError +from jinja2 import StrictUndefined, Template from knack.log import get_logger -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator from azext_aosm._configuration import CNFConfiguration, HelmPackageConfig +from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator from azext_aosm.util.constants import ( CNF_DEFINITION_BICEP_TEMPLATE, CNF_DEFINITION_JINJA2_SOURCE_TEMPLATE, CNF_MANIFEST_BICEP_TEMPLATE, CNF_MANIFEST_JINJA2_SOURCE_TEMPLATE, + CONFIG_MAPPINGS, DEPLOYMENT_PARAMETER_MAPPING_REGEX, + DEPLOYMENT_PARAMETERS, + GENERATED_VALUES_MAPPINGS, IMAGE_LINE_REGEX, IMAGE_PULL_SECRET_LINE_REGEX, - CONFIG_MAPPINGS, - SCHEMAS, SCHEMA_PREFIX, - DEPLOYMENT_PARAMETERS, + SCHEMAS, ) - +from azext_aosm.util.utils import input_ack logger = get_logger(__name__) @@ -47,8 +48,14 @@ class CnfNfdGenerator(NFDGenerator): # pylint: disable=too-many-instance-attrib - A bicep file for the Artifact manifests """ - def __init__(self, config: CNFConfiguration): - """Create a new CNF NFD Generator.""" + def __init__(self, config: CNFConfiguration, interactive: bool = False): + """ + Create a new CNF NFD Generator. + + Interactive parameter is only used if the user wants to generate the values + mapping file from the values.yaml in the helm package, and also requires the + mapping file in config to be blank. + """ super(NFDGenerator, self).__init__() self.config = config self.nfd_jinja2_template_path = os.path.join( @@ -70,7 +77,8 @@ def __init__(self, config: CNFConfiguration): self._bicep_path = os.path.join( self.output_folder_name, CNF_DEFINITION_BICEP_TEMPLATE ) - self._tmp_folder_name = '' + self.interactive = interactive + self._tmp_folder_name = "" def generate_nfd(self) -> None: """Generate a CNF NFD which comprises a group, an Artifact Manifest and an NFDV.""" @@ -88,6 +96,10 @@ def generate_nfd(self) -> None: # TODO: Validate charts + # Create a chart mapping schema if none has been passed in. + if not helm_package.path_to_mappings: + self._generate_chart_value_mappings(helm_package) + # Get schema for each chart # (extract mappings and take the schema bits we need from values.schema.json) # + Add that schema to the big schema. @@ -154,7 +166,7 @@ def _extract_chart(self, path: str) -> None: logger.debug("Extracting helm package %s", path) (_, ext) = os.path.splitext(path) - if ext in ('.gz', '.tgz'): + if ext in (".gz", ".tgz"): with tarfile.open(path, "r:gz") as tar: tar.extractall(path=self._tmp_folder_name) @@ -168,6 +180,64 @@ def _extract_chart(self, path: str) -> None: Please fix this and run the command again." ) + def _generate_chart_value_mappings(self, helm_package: HelmPackageConfig) -> None: + """ + Optional function to create a chart value mappings file with every value being a deployParameter. + + Expected use when a helm chart is very simple and user wants every value to be a + deployment parameter. + """ + logger.debug( + "Creating chart value mappings file for %s", helm_package.path_to_chart + ) + print("Creating chart value mappings file for %s", helm_package.path_to_chart) + + # Get all the values files in the chart + top_level_values_yaml = self._read_top_level_values_yaml(helm_package) + + mapping_to_write = self._replace_values_with_deploy_params( + top_level_values_yaml, {} + ) + + # Write the mapping to a file + folder_name = os.path.join(self._tmp_folder_name, GENERATED_VALUES_MAPPINGS) + os.makedirs(folder_name, exist_ok=True) + mapping_filepath = os.path.join( + self._tmp_folder_name, + GENERATED_VALUES_MAPPINGS, + f"{helm_package.name}-generated-mapping.yaml", + ) + with open(mapping_filepath, "w", encoding="UTF-8") as mapping_file: + yaml.dump(mapping_to_write, mapping_file) + + # Update the config that points to the mapping file + helm_package.path_to_mappings = mapping_filepath + + def _read_top_level_values_yaml( + self, helm_package: HelmPackageConfig + ) -> Dict[str, Any]: + """Return a dictionary of the values.yaml|yml read from the root of the helm package. + + :param helm_package: The helm package to look in + :type helm_package: HelmPackageConfig + :raises FileOperationError: if no values.yaml|yml found + :return: A dictionary of the yaml read from the file + :rtype: Dict[str, Any] + """ + for file in os.listdir(os.path.join(self._tmp_folder_name, helm_package.name)): + if file in ("values.yaml", "values.yml"): + with open( + os.path.join(self._tmp_folder_name, helm_package.name, file), + "r", + encoding="UTF-8", + ) as values_file: + values_yaml = yaml.safe_load(values_file) + return values_yaml + + raise FileOperationError( + "Cannot find top level values.yaml/.yml file in Helm package." + ) + def write_manifest_bicep_file(self) -> None: """Write the bicep file for the Artifact Manifest.""" with open(self.manifest_jinja2_template_path, "r", encoding="UTF-8") as f: @@ -224,6 +294,7 @@ def copy_to_output_folder(self) -> None: os.mkdir(self.output_folder_name) os.mkdir(os.path.join(self.output_folder_name, SCHEMAS)) + # Copy the nfd and the manifest bicep files to the output folder tmp_nfd_bicep_path = os.path.join( self._tmp_folder_name, CNF_DEFINITION_BICEP_TEMPLATE ) @@ -233,7 +304,23 @@ def copy_to_output_folder(self) -> None: self._tmp_folder_name, CNF_MANIFEST_BICEP_TEMPLATE ) shutil.copy(tmp_manifest_bicep_path, self.output_folder_name) + + # Copy any generated values mappings YAML files to the corresponding folder in + # the output directory so that the user can edit them and re-run the build if + # required + if os.path.exists( + os.path.join(self._tmp_folder_name, GENERATED_VALUES_MAPPINGS) + ): + generated_mappings_path = os.path.join( + self.output_folder_name, GENERATED_VALUES_MAPPINGS + ) + shutil.copytree( + os.path.join(self._tmp_folder_name, GENERATED_VALUES_MAPPINGS), + generated_mappings_path, + ) + # Copy the JSON config mappings and deploymentParameters schema that are used + # for the NFD to the output folder tmp_config_mappings_path = os.path.join(self._tmp_folder_name, CONFIG_MAPPINGS) output_config_mappings_path = os.path.join( self.output_folder_name, CONFIG_MAPPINGS @@ -273,7 +360,7 @@ def generate_nf_application_config( "dependsOnProfile": helm_package.depends_on, "registryValuesPaths": list(registryValuesPaths), "imagePullSecretsValuesPaths": list(imagePullSecretsValuesPaths), - "valueMappingsPath": self.generate_parameter_mappings(helm_package), + "valueMappingsPath": self.jsonify_value_mappings(helm_package), } def _find_yaml_files(self, directory) -> Iterator[str]: @@ -293,8 +380,8 @@ def find_pattern_matches_in_chart( """ Find pattern matches in Helm chart, using provided REGEX pattern. - param helm_package: The helm package config. - param pattern: The regex pattern to match. + param helm_package: The helm package config. param pattern: The regex pattern to + match. """ chart_dir = os.path.join(self._tmp_folder_name, helm_package.name) matches = [] @@ -314,8 +401,8 @@ def get_artifact_list( """ Get the list of artifacts for the chart. - param helm_package: The helm package config. - param image_line_matches: The list of image line matches. + param helm_package: The helm package config. param image_line_matches: The list + of image line matches. """ artifact_list = [] (chart_name, chart_version) = self.get_chart_name_and_version(helm_package) @@ -355,7 +442,7 @@ def get_chart_mapping_schema( if not os.path.exists(mappings_path): raise InvalidTemplateError( f"ERROR: The helm package '{helm_package.name}' does not have a valid values mappings file. \ - The file at '{helm_package.path_to_mappings}' does not exist. \ + The file at '{helm_package.path_to_mappings}' does not exist.\n\ Please fix this and run the command again." ) if not os.path.exists(values_schema): @@ -385,34 +472,134 @@ def get_chart_mapping_schema( def find_deploy_params( self, nested_dict, schema_nested_dict, final_schema ) -> Dict[Any, Any]: - """Find the deploy parameters in the values.mappings.yaml file and add them to the schema.""" + """ + Create a schema of types of only those values in the values.mappings.yaml file which have a deployParameters mapping. + + Finds the relevant part of the full schema of the values file and finds the + type of the parameter name, then adds that to the final schema, with no nesting. + + Returns a schema of form: + { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "DeployParametersSchema", + "type": "object", + "properties": { + "": { + "type": "" + }, + "": { + "type": "" + }, + + nested_dict: the dictionary of the values mappings yaml which contains + deployParameters mapping placeholders + schema_nested_dict: the properties section of the full schema (or sub-object in + schema) + final_schema: Blank dictionary if this is the top level starting point, + otherwise the final_schema as far as we have got. + """ original_schema_nested_dict = schema_nested_dict for k, v in nested_dict.items(): # if value is a string and contains deployParameters. if isinstance(v, str) and re.search(DEPLOYMENT_PARAMETER_MAPPING_REGEX, v): - # only add the parameter name (e.g. from {deployParameter.zone} only param = zone) + logger.debug( + "Found string deploy parameter for key %s, value %s. Find schema type", + k, + v, + ) + # only add the parameter name (e.g. from {deployParameter.zone} only + # param = zone) param = v.split(".", 1)[1] param = param.split("}", 1)[0] - # add the schema for k (from the big schema) to the (smaller) schema - final_schema.update( - {param: {"type": schema_nested_dict["properties"][k]["type"]}} - ) - + # add the schema for k (from the full schema) to the (new) schema + if "properties" in schema_nested_dict.keys(): + # Occurs if top level item in schema properties is an object with + # properties itself + final_schema.update( + {param: {"type": schema_nested_dict["properties"][k]["type"]}} + ) + else: + # Occurs if top level schema item in schema properties are objects + # with no "properties" - but can have "type". + final_schema.update( + {param: {"type": schema_nested_dict[k]["type"]}} + ) # else if value is a (non-empty) dictionary (i.e another layer of nesting) elif hasattr(v, "items") and v.items(): - # handling schema having properties which doesn't map directly to the values file nesting + logger.debug("Found dict value for key %s. Find schema type", k) + # handling schema having properties which doesn't map directly to the + # values file nesting if "properties" in schema_nested_dict.keys(): schema_nested_dict = schema_nested_dict["properties"][k] else: schema_nested_dict = schema_nested_dict[k] # recursively call function with values (i.e the nested dictionary) self.find_deploy_params(v, schema_nested_dict, final_schema) - # reset the schema dict to its original value (once finished with that level of recursion) + # reset the schema dict to its original value (once finished with that + # level of recursion) schema_nested_dict = original_schema_nested_dict return final_schema + def _replace_values_with_deploy_params( + self, + values_yaml_dict, + param_prefix: Optional[str] = None, + ) -> Dict[Any, Any]: + """ + Given the yaml dictionary read from values.yaml, replace all the values with {deploymentParameter.keyname}. + + Thus creating a values mapping file if the user has not provided one in config. + """ + logger.debug("Replacing values with deploy parameters") + final_values_mapping_dict: Dict[Any, Any] = {} + for k, v in values_yaml_dict.items(): + # if value is a string and contains deployParameters. + logger.debug("Processing key %s", k) + param_name = k if param_prefix is None else f"{param_prefix}_{k}" + if isinstance(v, (str, int, bool)): + # Replace the parameter with {deploymentParameter.keyname} + if self.interactive: + # Interactive mode. Prompt user to include or exclude parameters + # This requires the enter key after the y/n input which isn't ideal + if not input_ack("y", f"Expose parameter {param_name}? y/n "): + logger.debug("Excluding parameter %s", param_name) + final_values_mapping_dict.update({k: v}) + continue + replacement_value = f"{{deployParameters.{param_name}}}" + + # add the schema for k (from the big schema) to the (smaller) schema + final_values_mapping_dict.update({k: replacement_value}) + elif isinstance(v, dict): + + final_values_mapping_dict[k] = self._replace_values_with_deploy_params( + v, param_name + ) + elif isinstance(v, list): + final_values_mapping_dict[k] = [] + for index, item in enumerate(v): + param_name = f"{param_prefix}_{k}_{index}" if param_prefix else f"{k})_{index}" + if isinstance(item, dict): + final_values_mapping_dict[k].append( + self._replace_values_with_deploy_params( + item, param_name + ) + ) + elif isinstance(v, (str, int, bool)): + replacement_value = f"{{deployParameters.{param_name}}}" + final_values_mapping_dict[k].append(replacement_value) + else: + raise ValueError( + f"Found an unexpected type {type(v)} of key {k} in " + "values.yaml, cannot generate values mapping file.") + else: + raise ValueError( + f"Found an unexpected type {type(v)} of key {k} in values.yaml, " + "cannot generate values mapping file.") + + return final_values_mapping_dict + def get_chart_name_and_version( self, helm_package: HelmPackageConfig ) -> Tuple[str, str]: @@ -438,11 +625,9 @@ def get_chart_name_and_version( return (chart_name, chart_version) - def generate_parameter_mappings(self, helm_package: HelmPackageConfig) -> str: - """Generate parameter mappings for the given helm package.""" - values = os.path.join( - self._tmp_folder_name, helm_package.name, "values.mappings.yaml" - ) + def jsonify_value_mappings(self, helm_package: HelmPackageConfig) -> str: + """Yaml->JSON values mapping file, then return path to it.""" + mappings_yaml = helm_package.path_to_mappings mappings_folder_path = os.path.join(self._tmp_folder_name, CONFIG_MAPPINGS) mappings_filename = f"{helm_package.name}-mappings.json" @@ -452,7 +637,7 @@ def generate_parameter_mappings(self, helm_package: HelmPackageConfig) -> str: mapping_file_path = os.path.join(mappings_folder_path, mappings_filename) - with open(values, "r", encoding="utf-8") as f: + with open(mappings_yaml, "r", encoding="utf-8") as f: data = yaml.load(f, Loader=yaml.FullLoader) with open(mapping_file_path, "w", encoding="utf-8") as file: diff --git a/src/aosm/azext_aosm/generate_nfd/nfd_generator_base.py b/src/aosm/azext_aosm/generate_nfd/nfd_generator_base.py index 4428bdf45d1..3072f62394e 100644 --- a/src/aosm/azext_aosm/generate_nfd/nfd_generator_base.py +++ b/src/aosm/azext_aosm/generate_nfd/nfd_generator_base.py @@ -5,12 +5,12 @@ """Contains a base class for generating NFDs.""" from knack.log import get_logger - logger = get_logger(__name__) class NFDGenerator: """A class for generating an NFD from a config file.""" + # pylint: disable=too-few-public-methods def __init__( self, diff --git a/src/aosm/azext_aosm/generate_nfd/vnf_nfd_generator.py b/src/aosm/azext_aosm/generate_nfd/vnf_nfd_generator.py index 37e81e009b2..7d33fab1016 100644 --- a/src/aosm/azext_aosm/generate_nfd/vnf_nfd_generator.py +++ b/src/aosm/azext_aosm/generate_nfd/vnf_nfd_generator.py @@ -8,26 +8,38 @@ import os import shutil import tempfile - from functools import cached_property from typing import Any, Dict, Optional -from knack.log import get_logger -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator +from knack.log import get_logger from azext_aosm._configuration import VNFConfiguration +from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator from azext_aosm.util.constants import ( - VNF_DEFINITION_BICEP_TEMPLATE, - VNF_MANIFEST_BICEP_TEMPLATE, CONFIG_MAPPINGS, - SCHEMAS, - SCHEMA_PREFIX, DEPLOYMENT_PARAMETERS, + OPTIONAL_DEPLOYMENT_PARAMETERS_FILE, + OPTIONAL_DEPLOYMENT_PARAMETERS_HEADING, + SCHEMA_PREFIX, + SCHEMAS, + TEMPLATE_PARAMETERS, + VHD_PARAMETERS, + VNF_DEFINITION_BICEP_TEMPLATE, + VNF_MANIFEST_BICEP_TEMPLATE, ) - +from azext_aosm.util.utils import input_ack logger = get_logger(__name__) +# Different types are used in ARM templates and NFDs. The list accepted by NFDs is +# documented in the AOSM meta-schema. This will be published in the future but for now +# can be found in +# https://microsoft.sharepoint.com/:w:/t/NSODevTeam/Ec7ovdKroSRIv5tumQnWIE0BE-B2LykRcll2Qb9JwfVFMQ +ARM_TO_JSON_PARAM_TYPES: Dict[str, str] = { + "int": "integer", + "secureString": "string", +} + class VnfNfdGenerator(NFDGenerator): # pylint: disable=too-many-instance-attributes @@ -39,9 +51,16 @@ class VnfNfdGenerator(NFDGenerator): - Parameters files that are used by the NFDV bicep file, these are the deployParameters and the mapping profiles of those deploy parameters - A bicep file for the Artifact manifests + + @param order_params: whether to order the deployment and template output parameters + with those without a default first, then those with a default. + Those without a default will definitely be required to be + exposed, those with a default may not be. + @param interactive: whether to prompt the user to confirm the parameters to be + exposed. """ - def __init__(self, config: VNFConfiguration): + def __init__(self, config: VNFConfiguration, order_params: bool, interactive: bool): super(NFDGenerator, self).__init__() self.config = config self.bicep_template_name = VNF_DEFINITION_BICEP_TEMPLATE @@ -56,11 +75,14 @@ def __init__(self, config: VNFConfiguration): self._manifest_path = os.path.join( self.output_folder_name, self.manifest_template_name ) - self.tmp_folder_name = '' + self.order_params = order_params + self.interactive = interactive + self.tmp_folder_name = "" def generate_nfd(self) -> None: """ Generate a VNF NFD which comprises an group, an Artifact Manifest and a NFDV. + Create a bicep template for an NFD from the ARM template for the VNF. """ # Create temporary folder. @@ -100,12 +122,35 @@ def vm_parameters(self) -> Dict[str, Any]: parameters: Dict[str, Any] = data["parameters"] else: print( - "No parameters found in the template provided. Your schema will have no properties" + "No parameters found in the template provided. " + "Your NFD will have no deployParameters" ) parameters = {} return parameters + @property + def vm_parameters_ordered(self) -> Dict[str, Any]: + """The parameters from the VM ARM template, ordered as those without defaults then those with.""" + vm_parameters_no_default: Dict[str, Any] = {} + vm_parameters_with_default: Dict[str, Any] = {} + has_default_field: bool = False + has_default: bool = False + + for key in self.vm_parameters: + # Order parameters into those with and without defaults + has_default_field = "defaultValue" in self.vm_parameters[key] + has_default = ( + has_default_field and not self.vm_parameters[key]["defaultValue"] == "" + ) + + if has_default: + vm_parameters_with_default[key] = self.vm_parameters[key] + else: + vm_parameters_no_default[key] = self.vm_parameters[key] + + return {**vm_parameters_no_default, **vm_parameters_with_default} + def create_parameter_files(self) -> None: """Create the Deployment and Template json parameter files.""" schemas_folder_path = os.path.join(self.tmp_folder_name, SCHEMAS) @@ -126,16 +171,42 @@ def write_deployment_parameters(self, folder_path: str) -> None: logger.debug("Create deploymentParameters.json") nfd_parameters = {} + nfd_parameters_with_default = {} + vm_parameters_to_exclude = [] - for key in self.vm_parameters: - # ARM templates allow int and secureString but we do not currently accept them in AOSM - # This may change, but for now we should change them to accepted types integer and string - if self.vm_parameters[key]["type"] == "int": - nfd_parameters[key] = {"type": "integer"} - elif self.vm_parameters[key]["type"] == "secureString": - nfd_parameters[key] = {"type": "string"} - else: - nfd_parameters[key] = {"type": self.vm_parameters[key]["type"]} + vm_parameters = ( + self.vm_parameters_ordered if self.order_params else self.vm_parameters + ) + + for key in vm_parameters: + # Order parameters into those without and then with defaults + has_default_field = "defaultValue" in self.vm_parameters[key] + has_default = ( + has_default_field and not self.vm_parameters[key]["defaultValue"] == "" + ) + + if self.interactive and has_default: + # Interactive mode. Prompt user to include or exclude parameters + # This requires the enter key after the y/n input which isn't ideal + if not input_ack("y", f"Expose parameter {key}? y/n "): + logger.debug("Excluding parameter %s", key) + vm_parameters_to_exclude.append(key) + continue + + # Map ARM parameter types to JSON parameter types accepted by AOSM + arm_type = self.vm_parameters[key]["type"] + json_type = ARM_TO_JSON_PARAM_TYPES.get(arm_type, arm_type) + + if has_default: + nfd_parameters_with_default[key] = {"type": json_type} + + nfd_parameters[key] = {"type": json_type} + + # Now we are out of the vm_parameters loop, we can remove the excluded + # parameters so they don't get included in templateParameters.json + # Remove from both ordered and unordered dicts + for key in vm_parameters_to_exclude: + self.vm_parameters.pop(key, None) deployment_parameters_path = os.path.join(folder_path, DEPLOYMENT_PARAMETERS) @@ -147,6 +218,28 @@ def write_deployment_parameters(self, folder_path: str) -> None: _file.write(json.dumps(deploy_parameters_full, indent=4)) logger.debug("%s created", deployment_parameters_path) + if self.order_params: + print( + "Deployment parameters for the generated NFDV are ordered by those " + "without defaults first to make it easier to choose which to expose." + ) + + # Extra output file to help the user know which parameters are optional + if not self.interactive: + if nfd_parameters_with_default: + optional_deployment_parameters_path = os.path.join( + folder_path, OPTIONAL_DEPLOYMENT_PARAMETERS_FILE + ) + with open( + optional_deployment_parameters_path, "w", encoding="utf-8" + ) as _file: + _file.write(OPTIONAL_DEPLOYMENT_PARAMETERS_HEADING) + _file.write(json.dumps(nfd_parameters_with_default, indent=4)) + print( + "Optional ARM parameters detected. Created " + f"{OPTIONAL_DEPLOYMENT_PARAMETERS_FILE} to help you choose which " + "to expose." + ) def write_template_parameters(self, folder_path: str) -> None: """ @@ -154,12 +247,15 @@ def write_template_parameters(self, folder_path: str) -> None: :param folder_path: The folder to put this file in. """ - logger.debug("Create templateParameters.json") + logger.debug("Create %s", TEMPLATE_PARAMETERS) + vm_parameters = ( + self.vm_parameters_ordered if self.order_params else self.vm_parameters + ) template_parameters = { - key: f"{{deployParameters.{key}}}" for key in self.vm_parameters + key: f"{{deployParameters.{key}}}" for key in vm_parameters } - template_parameters_path = os.path.join(folder_path, "templateParameters.json") + template_parameters_path = os.path.join(folder_path, TEMPLATE_PARAMETERS) with open(template_parameters_path, "w", encoding="utf-8") as _file: _file.write(json.dumps(template_parameters, indent=4)) @@ -186,7 +282,7 @@ def write_vhd_parameters(self, folder_path: str) -> None: "azureDeployLocation": azureDeployLocation, } - vhd_parameters_path = os.path.join(folder_path, "vhdParameters.json") + vhd_parameters_path = os.path.join(folder_path, VHD_PARAMETERS) with open(vhd_parameters_path, "w", encoding="utf-8") as _file: _file.write(json.dumps(vhd_parameters, indent=4)) @@ -204,26 +300,10 @@ def copy_to_output_folder(self) -> None: manifest_path = os.path.join(code_dir, "templates", self.manifest_template_name) shutil.copy(manifest_path, self.output_folder_name) - - os.mkdir(os.path.join(self.output_folder_name, SCHEMAS)) - tmp_schema_path = os.path.join( - self.tmp_folder_name, SCHEMAS, DEPLOYMENT_PARAMETERS - ) - output_schema_path = os.path.join( - self.output_folder_name, SCHEMAS, DEPLOYMENT_PARAMETERS - ) - shutil.copy( - tmp_schema_path, - output_schema_path, - ) - - tmp_config_mappings_path = os.path.join(self.tmp_folder_name, CONFIG_MAPPINGS) - output_config_mappings_path = os.path.join( - self.output_folder_name, CONFIG_MAPPINGS - ) + # Copy everything in the temp folder to the output folder shutil.copytree( - tmp_config_mappings_path, - output_config_mappings_path, + self.tmp_folder_name, + self.output_folder_name, dirs_exist_ok=True, ) diff --git a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py index a4318752a10..5651072eb84 100644 --- a/src/aosm/azext_aosm/generate_nsd/nsd_generator.py +++ b/src/aosm/azext_aosm/generate_nsd/nsd_generator.py @@ -3,35 +3,33 @@ # License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------- """Contains a class for generating VNF NFDs and associated resources.""" -from knack.log import get_logger import json import logging import os import shutil +import tempfile from functools import cached_property from pathlib import Path from typing import Any, Dict, Optional -import tempfile -from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator +from jinja2 import Template +from knack.log import get_logger from azext_aosm._configuration import NSConfiguration +from azext_aosm.generate_nfd.nfd_generator_base import NFDGenerator from azext_aosm.util.constants import ( - NSD_DEFINITION_BICEP_SOURCE_TEMPLATE, - NSD_DEFINITION_BICEP_FILE, - NF_TEMPLATE_BICEP_FILE, + CONFIG_MAPPINGS, NF_DEFINITION_BICEP_FILE, + NF_TEMPLATE_BICEP_FILE, NSD_ARTIFACT_MANIFEST_BICEP_FILE, + NSD_ARTIFACT_MANIFEST_SOURCE_TEMPLATE, NSD_CONFIG_MAPPING_FILE, + NSD_DEFINITION_BICEP_FILE, + NSD_DEFINITION_BICEP_SOURCE_TEMPLATE, SCHEMAS, - CONFIG_MAPPINGS, - NSD_ARTIFACT_MANIFEST_SOURCE_TEMPLATE, TEMPLATES, ) -from jinja2 import Template - - logger = get_logger(__name__) diff --git a/src/aosm/azext_aosm/tests/latest/test_aosm_scenario.py b/src/aosm/azext_aosm/tests/latest/test_aosm_scenario.py index 0bc37d2e16e..e1ff05f0ff8 100644 --- a/src/aosm/azext_aosm/tests/latest/test_aosm_scenario.py +++ b/src/aosm/azext_aosm/tests/latest/test_aosm_scenario.py @@ -7,8 +7,7 @@ import unittest # from azure_devtools.scenario_tests import AllowLargeResponse -from azure.cli.testsdk import ScenarioTest, ResourceGroupPreparer - +from azure.cli.testsdk import ResourceGroupPreparer, ScenarioTest TEST_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), "..")) diff --git a/src/aosm/azext_aosm/util/constants.py b/src/aosm/azext_aosm/util/constants.py index 3ea1f7c25fe..2fcdb0e6cca 100644 --- a/src/aosm/azext_aosm/util/constants.py +++ b/src/aosm/azext_aosm/util/constants.py @@ -33,11 +33,27 @@ CNF_DEFINITION_BICEP_TEMPLATE = "cnfdefinition.bicep" CNF_MANIFEST_BICEP_TEMPLATE = "cnfartifactmanifest.bicep" -DEPLOYMENT_PARAMETERS = "deploymentParameters.json" + # Names of folder used in the repo CONFIG_MAPPINGS = "configMappings" SCHEMAS = "schemas" TEMPLATES = "templates" +GENERATED_VALUES_MAPPINGS = "generatedValuesMappings" + +# Names of files when building NFDs/NSDs +DEPLOYMENT_PARAMETERS = "deploymentParameters.json" +OPTIONAL_DEPLOYMENT_PARAMETERS_FILE = "optionalDeploymentParameters.txt" +TEMPLATE_PARAMETERS = "templateParameters.json" +VHD_PARAMETERS = "vhdParameters.json" +OPTIONAL_DEPLOYMENT_PARAMETERS_HEADING = ( + "# The following parameters are optional as they have default values.\n" + "# If you do not wish to expose them in the NFD, find and remove them from both\n" + f"# {DEPLOYMENT_PARAMETERS} and {TEMPLATE_PARAMETERS} (and {VHD_PARAMETERS} if\n" + "they are there)\n" + "# You can re-run the build command with the --order-params flag to order those\n" + "# files with the optional parameters at the end of the file, and with the \n" + "# --interactive flag to interactively choose y/n for each parameter to expose.\n\n" +) # Deployment Schema diff --git a/src/aosm/azext_aosm/util/management_clients.py b/src/aosm/azext_aosm/util/management_clients.py index 65ea9aa4afb..11712b4e894 100644 --- a/src/aosm/azext_aosm/util/management_clients.py +++ b/src/aosm/azext_aosm/util/management_clients.py @@ -4,8 +4,9 @@ # -------------------------------------------------------------------------------------------- """Clients for the python SDK along with useful caches.""" -from knack.log import get_logger from azure.mgmt.resource import ResourceManagementClient +from knack.log import get_logger + from azext_aosm.vendored_sdks import HybridNetworkManagementClient logger = get_logger(__name__) From f6fb54d6031f00bdb682042cd1166e871a9c3163 Mon Sep 17 00:00:00 2001 From: Sunny Carter Date: Mon, 12 Jun 2023 12:32:59 +0100 Subject: [PATCH 2/2] comment out breaking line --- src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py index 8d8ce10f1d0..c09b125dafa 100644 --- a/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py +++ b/src/aosm/azext_aosm/generate_nfd/cnf_nfd_generator.py @@ -89,7 +89,8 @@ def generate_nfd(self) -> None: try: for helm_package in self.config.helm_packages: # Turn Any type into HelmPackageConfig, to access properties on the object - helm_package = HelmPackageConfig(**helm_package) + # @TODO - commented this out because it was throwing an exception + #helm_package = HelmPackageConfig(**helm_package) # Unpack the chart into the tmp folder self._extract_chart(helm_package.path_to_chart)