From ab8487a2713c1ca64bbd6b4e886cd64f51655381 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Wed, 23 Jun 2021 11:00:53 -0700 Subject: [PATCH 01/88] inital commit --- src/safety/safety_check_v2.py | 110 +++++++++++++ .../test_and_generate_python_safety_report.py | 147 ++++++++++++++++++ tensorflow/buildspec.yml | 3 + .../training/docker/2.4/py3/Dockerfile.cpu | 11 ++ .../inference/test_eks_mxnet_inference.py | 3 +- 5 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 src/safety/safety_check_v2.py create mode 100644 src/safety/test_and_generate_python_safety_report.py diff --git a/src/safety/safety_check_v2.py b/src/safety/safety_check_v2.py new file mode 100644 index 000000000000..6b3287f8a58b --- /dev/null +++ b/src/safety/safety_check_v2.py @@ -0,0 +1,110 @@ +""" +Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +import sys +import argparse +from datetime import datetime +import pkg_resources +import safety as Safety +from safety import safety + + +def main(): + """Return json.dumps() string of below list structure. + [ + { + "package": "package", + "affected": "version_spec", + "installed": "version", + "vulnerabilities": [ + { + "vid": "safety_vulnerability_id", + "advisory": "description of the issue" + }, + ... + ] + } + ... + ] + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--key", + required=True, + ) + args = parser.parse_args() + safety_key = args.key + + if Safety.__version__ == "1.8.7": + proxy_dictionary = {} + else: + from safety.util import get_proxy_dict + proxy_dictionary = get_proxy_dict("http", None, 80) + + # Get safety result + packages = [ + d for d in pkg_resources.working_set if d.key not in {"python", "wsgiref", "argparse"} + ] + # return: Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id", "cvssv2", "cvssv3"])) + vulns = safety.check( + packages=packages, + key=safety_key, + db_mirror="", + cached=True, + ignore_ids=[], + proxy=proxy_dictionary, + ) + + # Generate output + vulns_dict = {} + vulns_list = [] + # Report for unsafe packages + for v in vulns: + package = v.name + affected = v.spec + installed = v.version + advisory = v.advisory + vid = v.vuln_id + if package not in vulns_dict: + vulns_dict[package] = { + "package": package, + "affected": affected, + "installed": installed, + "vulnerabilities": [{"vid": vid, "advisory": advisory}], + } + else: + vulns_dict[package]["vulnerabilities"].append({"vid": vid, "advisory": advisory}) + + # Report for safe packages + timestamp = datetime.now().strftime("%d%m%Y") + for pkg in packages: + if pkg.key not in vulns_dict: + vulns_dict[pkg.key] = { + "package": pkg.key, + "affected": f"No known vulnerability found, PASSED_SAFETY_CHECK on {timestamp}.", + "installed": pkg.version, + "vulnerabilities": [{"vid": "N/A", "advisory": "N/A"}], + } + + for (k, v) in vulns_dict.items(): + vulns_list.append(v) + + print(json.dumps(vulns_list)) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + pass \ No newline at end of file diff --git a/src/safety/test_and_generate_python_safety_report.py b/src/safety/test_and_generate_python_safety_report.py new file mode 100644 index 000000000000..0a529245821b --- /dev/null +++ b/src/safety/test_and_generate_python_safety_report.py @@ -0,0 +1,147 @@ +""" +Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +from dataclasses import dataclass +from typing import List +import boto3 +from botocore.exceptions import ClientError +import base64 +import argparse + +from safety_check_v2 import safety_check_v2 + +@dataclass +class SafetyVulnerabilityAdvisory: + vid: str + advisory: str + + +@dataclass +class SafetyPackageVulnerabilityReport: + package: str + affected: str + installed: str + vulnerabilities: List[SafetyVulnerabilityAdvisory] + + def __post_init__(self): + self.vulnerabilities = [SafetyVulnerabilityAdvisory(**i) for i in self.vulnerabilities] + + +@dataclass +class SafetyPythonEnvironmentVulnerabilityReport: + report: List[SafetyPackageVulnerabilityReport] + + def __post_init__(self): + self.report = [SafetyPackageVulnerabilityReport(**i) for i in self.report] + +def get_secret(): + """ + Retrieves safety api key from secrets manager + """ + secret_name = "/codebuild/safety/key" + secret = "" + + # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + # We rethrow the exception by default. + + try: + secrets_manger_client = boto3.client("secretsmanager", region_name="us-west-2") + get_secret_value_response = secrets_manger_client.get_secret_value(SecretId=secret_name) + except ClientError as e: + if e.response["Error"]["Code"] == "DecryptionFailureException": + # Secrets Manager can't decrypt the protected secret text using the provided KMS key. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response["Error"]["Code"] == "InternalServiceErrorException": + # An error occurred on the server side. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response["Error"]["Code"] == "InvalidParameterException": + # You provided an invalid value for a parameter. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response["Error"]["Code"] == "InvalidRequestException": + # You provided a parameter value that is not valid for the current state of the resource. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + elif e.response["Error"]["Code"] == "ResourceNotFoundException": + # We can't find the resource that you asked for. + # Deal with the exception here, and/or rethrow at your discretion. + raise e + else: + # Decrypts secret using the associated KMS CMK. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if "SecretString" in get_secret_value_response: + secret = get_secret_value_response["SecretString"] + else: + secret = base64.b64decode(get_secret_value_response["SecretBinary"]) + + return secret + + + +parser = argparse.ArgumentParser() +parser.add_argument('--report-path', type=str, help='Safety report path') +parser.add_argument('--ignored-packages', type=str, help='Packages to be ignored') +args = parser.parse_args() + +report_path = args.report_path +ignored_packages = args.ignored_packages + +# run safety check +scan_results = [] + +safety_api_key = get_secret() + +raw_scan_result = json.loads( + f"{safety_check_v2} --key {safety_api_key} | jq '.' | tee {report_path}", +) +scan_results.append( + SafetyPythonEnvironmentVulnerabilityReport( + report=raw_scan_result + ) +) + +# processing safety reports +report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [affected: {affected}] [reason: {reason}] [env: {env}]" +failed_count = 0 +for result in scan_results: + for report_item in result.report: + if "PASSED_SAFETY_CHECK" not in report_item.affected: + if (report_item.package not in ignored_packages) or ( + report_item.package in ignored_packages + and report_item.installed != ignored_packages[report_item.package] + ): + failed_count += 1 + print( + report_log_template.format( + status="FAILED", + pkg=report_item.package, + installed=report_item.installed, + affected=report_item.affected, + reason=None, + env=result.environment, + ) + ) + +if failed_count > 0: + print(f"ERROR: {failed_count} packages") + print("Please check the test output and patch vulnerable packages.") + raise Exception("Safety check failed!") +else: + print("Passed safety check.") + diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index 99fbe6c980ac..ae5aa88c11c2 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -26,6 +26,9 @@ context: deep_learning_container: source: ../../src/deep_learning_container.py target: deep_learning_container.py + safety_files: + source: ../../src/safety + target: safety inference_context: &INFERENCE_CONTEXT sagemaker_package_name: source: docker/build_artifacts/sagemaker diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 27485aa2bea4..6382ad06f1b7 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -167,6 +167,17 @@ RUN HOME_DIR=/root \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* +COPY safety ${HOME_DIR}/safety +RUN ${PIP} install --no-cache-dir -U safety +RUN ${PYTHON} safety/test_and_generate_safety_report.py \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + --ignored-packages { \ + "pycrypto": "2.6.1", \ + "httplib2": "0.9.2" \ + } +RUN rm -rf ${HOME_DIR}/safety +RUN ${PIP} uninstall -y safety + RUN curl https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-2.4/license.txt -o /license.txt CMD ["/bin/bash"] diff --git a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py index 295f432d27b1..0f78975242c5 100644 --- a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py +++ b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py @@ -46,7 +46,8 @@ def test_eks_mxnet_neuron_inference(mxnet_inference, neuron_only): if eks_utils.is_service_running(selector_name): eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8080") - assert test_utils.request_mxnet_inference(port=port_to_forward, model="mxnet-resnet50") + + assert test_utils.request_mxnet_inference(port=port_to_forward, model="mxnet-resnet-neuron") except ValueError as excp: eks_utils.LOGGER.error("Service is not running: %s", excp) finally: From d385f595260d0438400231f2a9d0fe9416d97069 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 12:35:02 -0700 Subject: [PATCH 02/88] Add safety report to docker image --- buildspec.yml | 3 + src/image_builder.py | 5 ++ .../test_and_generate_python_safety_report.py | 59 ++----------------- tensorflow/buildspec.yml | 29 +++------ .../training/docker/2.4/py3/Dockerfile.cpu | 2 + 5 files changed, 24 insertions(+), 74 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 75126adf836f..1cf78b72ccb0 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -1,4 +1,7 @@ version: 0.2 +env: + secrets-manager: + SAFETY_KEY: "/codebuild/safety/key" #TODO: Move all constatns to environment variables phases: diff --git a/src/image_builder.py b/src/image_builder.py index 702f54af93e4..e3028a8aebb0 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -63,6 +63,11 @@ def image_builder(buildspec): extra_build_args = {} labels = {} + safety_key = os.getenv("SAFETY_KEY") + + if safety_key: + extra_build_args["SAFETY_KEY"] = safety_key + if image_config.get("version") is not None: if BUILDSPEC["version"] != image_config.get("version"): continue diff --git a/src/safety/test_and_generate_python_safety_report.py b/src/safety/test_and_generate_python_safety_report.py index 0a529245821b..b3bd458d7c5d 100644 --- a/src/safety/test_and_generate_python_safety_report.py +++ b/src/safety/test_and_generate_python_safety_report.py @@ -20,8 +20,7 @@ from botocore.exceptions import ClientError import base64 import argparse - -from safety_check_v2 import safety_check_v2 +import subprocess @dataclass class SafetyVulnerabilityAdvisory: @@ -47,69 +46,21 @@ class SafetyPythonEnvironmentVulnerabilityReport: def __post_init__(self): self.report = [SafetyPackageVulnerabilityReport(**i) for i in self.report] -def get_secret(): - """ - Retrieves safety api key from secrets manager - """ - secret_name = "/codebuild/safety/key" - secret = "" - - # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. - # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html - # We rethrow the exception by default. - - try: - secrets_manger_client = boto3.client("secretsmanager", region_name="us-west-2") - get_secret_value_response = secrets_manger_client.get_secret_value(SecretId=secret_name) - except ClientError as e: - if e.response["Error"]["Code"] == "DecryptionFailureException": - # Secrets Manager can't decrypt the protected secret text using the provided KMS key. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response["Error"]["Code"] == "InternalServiceErrorException": - # An error occurred on the server side. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response["Error"]["Code"] == "InvalidParameterException": - # You provided an invalid value for a parameter. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response["Error"]["Code"] == "InvalidRequestException": - # You provided a parameter value that is not valid for the current state of the resource. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - elif e.response["Error"]["Code"] == "ResourceNotFoundException": - # We can't find the resource that you asked for. - # Deal with the exception here, and/or rethrow at your discretion. - raise e - else: - # Decrypts secret using the associated KMS CMK. - # Depending on whether the secret is a string or binary, one of these fields will be populated. - if "SecretString" in get_secret_value_response: - secret = get_secret_value_response["SecretString"] - else: - secret = base64.b64decode(get_secret_value_response["SecretBinary"]) - - return secret - - - parser = argparse.ArgumentParser() +parser.add_argument('--safety-key', type=str, help='Safety key') parser.add_argument('--report-path', type=str, help='Safety report path') parser.add_argument('--ignored-packages', type=str, help='Packages to be ignored') args = parser.parse_args() report_path = args.report_path ignored_packages = args.ignored_packages +safety_api_key = args.safety_key # run safety check scan_results = [] -safety_api_key = get_secret() - -raw_scan_result = json.loads( - f"{safety_check_v2} --key {safety_api_key} | jq '.' | tee {report_path}", -) +output = subprocess.check_output(f"python3 safety_check_v2.py --key '{safety_api_key}' | jq '.' | tee '{report_path}'", shell=True, executable="/bin/bash") +raw_scan_result = json.loads(output) scan_results.append( SafetyPythonEnvironmentVulnerabilityReport( report=raw_scan_result diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index ae5aa88c11c2..0a814013c9ed 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -44,28 +44,17 @@ context: target: deep_learning_container.py images: - BuildTensorflowCPUInferencePy3DockerImage: - <<: *INFERENCE_REPOSITORY - build: &TENSORFLOW_CPU_INFERENCE_PY3 false - image_size_baseline: 4899 + BuildTensorflowCpuPy37TrainingDockerImage: + <<: *TRAINING_REPOSITORY + build: &TENSORFLOW_CPU_TRAINING_PY3 false + image_size_baseline: &IMAGE_SIZE_BASELINE 4489 device_type: &DEVICE_TYPE cpu python_version: &DOCKER_PYTHON_VERSION py3 tag_python_version: &TAG_PYTHON_VERSION py37 os_version: &OS_VERSION ubuntu18.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION + ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., + *DEVICE_TYPE ] context: - <<: *INFERENCE_CONTEXT - BuildTensorflowGPUInferencePy3DockerImage: - <<: *INFERENCE_REPOSITORY - build: &TENSORFLOW_GPU_INFERENCE_PY3 false - image_size_baseline: 7738 - device_type: &DEVICE_TYPE gpu - python_version: &DOCKER_PYTHON_VERSION py3 - tag_python_version: &TAG_PYTHON_VERSION py37 - cuda_version: &CUDA_VERSION cu112 - os_version: &OS_VERSION ubuntu18.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] - context: - <<: *INFERENCE_CONTEXT + <<: *TRAINING_CONTEXT diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 6382ad06f1b7..94efcd06e1bc 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -22,6 +22,7 @@ ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 +ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -170,6 +171,7 @@ RUN HOME_DIR=/root \ COPY safety ${HOME_DIR}/safety RUN ${PIP} install --no-cache-dir -U safety RUN ${PYTHON} safety/test_and_generate_safety_report.py \ + --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ --ignored-packages { \ "pycrypto": "2.6.1", \ From e38004f79d121761dd25d249edaadb7f6f610fc9 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 12:46:07 -0700 Subject: [PATCH 03/88] adapt build logic --- src/config/build_config.py | 2 +- src/config/test_config.py | 12 ++++++------ tensorflow/buildspec.yml | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/config/build_config.py b/src/config/build_config.py index 0b7e2581a102..fa4d7e52f4e5 100644 --- a/src/config/build_config.py +++ b/src/config/build_config.py @@ -5,7 +5,7 @@ # Do remember to revert it back to False before merging any PR (including NEURON dedicated PR) ENABLE_NEURON_MODE = False # Frameworks for which you want to disable both builds and tests -DISABLE_FRAMEWORK_TESTS = [] +DISABLE_FRAMEWORK_TESTS = ["mxnet","pytorch"] # Disable new builds or build without datetime tag DISABLE_DATETIME_TAG = False # Note: Need to build the images at least once with DISABLE_DATETIME_TAG = True diff --git a/src/config/test_config.py b/src/config/test_config.py index 3f8ce5f60f0b..26f738f7c2da 100644 --- a/src/config/test_config.py +++ b/src/config/test_config.py @@ -5,11 +5,11 @@ # Disable the test codebuild jobs to be run # It is recommended to set DISABLE_EFA_TESTS to True to disable EFA tests if there is no change to EFA installer version or Frameworks. -DISABLE_EFA_TESTS = False +DISABLE_EFA_TESTS = True -DISABLE_SANITY_TESTS = False -DISABLE_SAGEMAKER_TESTS = False -DISABLE_ECS_TESTS = False -DISABLE_EKS_TESTS = False -DISABLE_EC2_TESTS = False +DISABLE_SANITY_TESTS = True +DISABLE_SAGEMAKER_TESTS = True +DISABLE_ECS_TESTS = True +DISABLE_EKS_TESTS = True +DISABLE_EC2_TESTS = True USE_SCHEDULER = False diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index 0a814013c9ed..08e64f52312c 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -1,8 +1,8 @@ account_id: &ACCOUNT_ID region: ®ION framework: &FRAMEWORK tensorflow -version: &VERSION 2.5.1 -short_version: &SHORT_VERSION 2.5 +version: &VERSION 2.4.1 +short_version: &SHORT_VERSION 2.4 repository_info: training_repository: &TRAINING_REPOSITORY From e2868532b01c7ba8fc683a48a5c7a93c76ee5ab3 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 13:16:00 -0700 Subject: [PATCH 04/88] fix safety dir --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 94efcd06e1bc..f5659c318e27 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -170,7 +170,7 @@ RUN HOME_DIR=/root \ COPY safety ${HOME_DIR}/safety RUN ${PIP} install --no-cache-dir -U safety -RUN ${PYTHON} safety/test_and_generate_safety_report.py \ +RUN ${PYTHON} ${HOME_DIR}/safety/test_and_generate_safety_report.py \ --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ --ignored-packages { \ From f3df287786641f85aee220aabec175eccb0c8be6 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 13:52:05 -0700 Subject: [PATCH 05/88] address file strcuture --- .../training/docker/2.4/py3/Dockerfile.cpu | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index f5659c318e27..6a8cef20cf65 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -155,6 +155,19 @@ RUN ${PIP} install --no-cache-dir -U \ smdebug==${SMDEBUG_VERSION} \ smclarify +RUN cp -r safety /tmp/safety \ + && ${PIP} install --no-cache-dir -U safety \ + && mkdir /opt/aws/dlc/info \ + && ${PYTHON} /tmp/safety/test_and_generate_safety_report.py \ + --safety-key ${SAFETY_KEY} \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + --ignored-packages { \ + "pycrypto": "2.6.1", \ + "httplib2": "0.9.2" \ + } \ + && rm -rf /tmp/safety \ + && ${PIP} uninstall -y safety + COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py @@ -168,18 +181,6 @@ RUN HOME_DIR=/root \ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ && rm -rf ${HOME_DIR}/oss_compliance* -COPY safety ${HOME_DIR}/safety -RUN ${PIP} install --no-cache-dir -U safety -RUN ${PYTHON} ${HOME_DIR}/safety/test_and_generate_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - --ignored-packages { \ - "pycrypto": "2.6.1", \ - "httplib2": "0.9.2" \ - } -RUN rm -rf ${HOME_DIR}/safety -RUN ${PIP} uninstall -y safety - RUN curl https://aws-dlc-licenses.s3.amazonaws.com/tensorflow-2.4/license.txt -o /license.txt CMD ["/bin/bash"] From 8400170e6bac6631736d098cf7bc8571e575a67e Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 14:07:47 -0700 Subject: [PATCH 06/88] add copy cmd --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 6a8cef20cf65..53b30fd35a99 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -155,9 +155,9 @@ RUN ${PIP} install --no-cache-dir -U \ smdebug==${SMDEBUG_VERSION} \ smclarify -RUN cp -r safety /tmp/safety \ - && ${PIP} install --no-cache-dir -U safety \ - && mkdir /opt/aws/dlc/info \ +COPY safety /tmp/safety +RUN mkdir /opt/aws/dlc/info +RUN ${PIP} install --no-cache-dir -U safety \ && ${PYTHON} /tmp/safety/test_and_generate_safety_report.py \ --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ From 821a4e2dbbc83e37733399c895960d2e25dafa58 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 14:23:53 -0700 Subject: [PATCH 07/88] add bash option --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 53b30fd35a99..d38cb1fea3d2 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -156,7 +156,7 @@ RUN ${PIP} install --no-cache-dir -U \ smclarify COPY safety /tmp/safety -RUN mkdir /opt/aws/dlc/info +RUN mkdir -p /opt/aws/dlc/info RUN ${PIP} install --no-cache-dir -U safety \ && ${PYTHON} /tmp/safety/test_and_generate_safety_report.py \ --safety-key ${SAFETY_KEY} \ From f610d4561776b31898f7378f244070ec2f45f4f2 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 14:43:33 -0700 Subject: [PATCH 08/88] fix file name --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index d38cb1fea3d2..7931c0d23aa0 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -158,7 +158,7 @@ RUN ${PIP} install --no-cache-dir -U \ COPY safety /tmp/safety RUN mkdir -p /opt/aws/dlc/info RUN ${PIP} install --no-cache-dir -U safety \ - && ${PYTHON} /tmp/safety/test_and_generate_safety_report.py \ + && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ --ignored-packages { \ @@ -167,7 +167,7 @@ RUN ${PIP} install --no-cache-dir -U safety \ } \ && rm -rf /tmp/safety \ && ${PIP} uninstall -y safety - + COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py From af82f00eabccc335f701551d68ed2d8a6c617c0f Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Fri, 25 Jun 2021 15:54:53 -0700 Subject: [PATCH 09/88] add quotes --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 7931c0d23aa0..d5d50f4d3fa0 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -161,10 +161,10 @@ RUN ${PIP} install --no-cache-dir -U safety \ && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - --ignored-packages { \ + --ignored-packages '{ \ "pycrypto": "2.6.1", \ "httplib2": "0.9.2" \ - } \ + }' \ && rm -rf /tmp/safety \ && ${PIP} uninstall -y safety From ea87368d273df8cade870c847fd741f0badb6174 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Mon, 28 Jun 2021 11:18:01 -0700 Subject: [PATCH 10/88] install jq --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index d5d50f4d3fa0..2f432e1438dd 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -45,6 +45,7 @@ RUN apt-get update \ curl \ emacs \ git \ + jq \ libtemplate-perl \ libssl1.1 \ openssl \ @@ -158,6 +159,7 @@ RUN ${PIP} install --no-cache-dir -U \ COPY safety /tmp/safety RUN mkdir -p /opt/aws/dlc/info RUN ${PIP} install --no-cache-dir -U safety \ + && cd /tmp/safety \ && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ --safety-key ${SAFETY_KEY} \ --report-path "/opt/aws/dlc/info/safety_test_report.json" \ From e20b379835c6fba478df8ab06088ba95fb16f86a Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Mon, 28 Jun 2021 11:53:18 -0700 Subject: [PATCH 11/88] remove safety pacakge --- tensorflow/training/docker/2.4/py3/Dockerfile.cpu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index 2f432e1438dd..ed2a71393910 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -167,8 +167,8 @@ RUN ${PIP} install --no-cache-dir -U safety \ "pycrypto": "2.6.1", \ "httplib2": "0.9.2" \ }' \ - && rm -rf /tmp/safety \ - && ${PIP} uninstall -y safety + && ${PIP} uninstall -y safety \ + && rm -rf /tmp/safety COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py From 67b969a8384e3b5e127ea3156e70486f01603c0e Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Mon, 28 Jun 2021 16:39:06 -0700 Subject: [PATCH 12/88] modify buildspec --- tensorflow/buildspec.yml | 64 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index 08e64f52312c..ee3341c3c83c 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -1,9 +1,8 @@ account_id: &ACCOUNT_ID region: ®ION framework: &FRAMEWORK tensorflow -version: &VERSION 2.4.1 -short_version: &SHORT_VERSION 2.4 - +version: &VERSION 2.5.0 +short_version: &SHORT_VERSION 2.5 repository_info: training_repository: &TRAINING_REPOSITORY image_type: &TRAINING_IMAGE_TYPE training @@ -17,7 +16,6 @@ repository_info: repository_name: &REPOSITORY_NAME !join [pr, "-", *FRAMEWORK, "-", *INFERENCE_IMAGE_TYPE] repository: &REPOSITORY !join [ *ACCOUNT_ID, .dkr.ecr., *REGION, .amazonaws.com/, *REPOSITORY_NAME ] - context: training_context: &TRAINING_CONTEXT dockerd-entrypoint: @@ -58,3 +56,61 @@ images: *DEVICE_TYPE ] context: <<: *TRAINING_CONTEXT + BuildTensorflowGpuPy37Cu112TrainingDockerImage: + <<: *TRAINING_REPOSITORY + build: &TENSORFLOW_GPU_TRAINING_PY3 false + image_size_baseline: &IMAGE_SIZE_BASELINE 7738 + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu112 + os_version: &OS_VERSION ubuntu18.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, + "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, + /Dockerfile., *DEVICE_TYPE ] + context: + <<: *TRAINING_CONTEXT + BuildTensorflowExampleGpuPy37Cu112TrainingDockerImage: + <<: *TRAINING_REPOSITORY + build: &TENSORFLOW_GPU_TRAINING_PY3 false + image_size_baseline: &IMAGE_SIZE_BASELINE 7738 + base_image_name: BuildTensorflowGpuPy37Cu112TrainingDockerImage + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu112 + os_version: &OS_VERSION ubuntu18.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, + "-", *OS_VERSION, "-example" ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example, + /Dockerfile., *DEVICE_TYPE ] + context: + <<: *TRAINING_CONTEXT + BuildTensorflowCPUInferencePy3DockerImage: + <<: *INFERENCE_REPOSITORY + build: &TENSORFLOW_CPU_INFERENCE_PY3 false + image_size_baseline: 4899 + device_type: &DEVICE_TYPE cpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + os_version: &OS_VERSION ubuntu18.04 + # TODO: To be reverted once TF Training 2.5.1 is released + tag: !join [ "2.5.1", "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *INFERENCE_CONTEXT + BuildTensorflowGPUInferencePy3DockerImage: + <<: *INFERENCE_REPOSITORY + build: &TENSORFLOW_GPU_INFERENCE_PY3 false + image_size_baseline: 7738 + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu112 + os_version: &OS_VERSION ubuntu18.04 + # TODO: To be reverted once TF Training 2.5.1 is released + tag: !join [ "2.5.1", "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *INFERENCE_CONTEXT From 7143bd820d74604a46a9415e813228735f93bddd Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Mon, 28 Jun 2021 17:11:09 -0700 Subject: [PATCH 13/88] add safety for TF2.5 --- .../test_and_generate_python_safety_report.py | 2 +- tensorflow/inference/docker/2.5/py3/Dockerfile.cpu | 13 +++++++++++++ .../inference/docker/2.5/py3/cu112/Dockerfile.gpu | 13 +++++++++++++ tensorflow/training/docker/2.5/py3/Dockerfile.cpu | 13 +++++++++++++ .../training/docker/2.5/py3/cu112/Dockerfile.gpu | 12 ++++++++++++ 5 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/safety/test_and_generate_python_safety_report.py b/src/safety/test_and_generate_python_safety_report.py index b3bd458d7c5d..826a04549ff8 100644 --- a/src/safety/test_and_generate_python_safety_report.py +++ b/src/safety/test_and_generate_python_safety_report.py @@ -53,7 +53,7 @@ def __post_init__(self): args = parser.parse_args() report_path = args.report_path -ignored_packages = args.ignored_packages +ignored_packages = args.ignored_packages if "ignored_packages" in args else None safety_api_key = args.safety_key # run safety check diff --git a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu index c103e04bf4f1..9311726ea21b 100644 --- a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu @@ -38,6 +38,7 @@ RUN apt-get update \ unzip \ wget \ vim \ + jq \ libbz2-dev \ liblzma-dev \ libffi-dev \ @@ -112,6 +113,18 @@ RUN echo '#!/bin/bash \n\n' > /usr/bin/tf_serving_entrypoint.sh \ && echo '/usr/bin/tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@"' >> /usr/bin/tf_serving_entrypoint.sh \ && chmod +x /usr/bin/tf_serving_entrypoint.sh +COPY safety /tmp/safety + +RUN mkdir -p /opt/aws/dlc/info + +RUN ${PIP} install --no-cache-dir -U safety \ + && cd /tmp/safety \ + && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ + --safety-key ${SAFETY_KEY} \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + && ${PIP} uninstall -y safety \ + && rm -rf /tmp/safety + COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu index 584e7439d373..3edfaa1b0463 100644 --- a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu @@ -60,6 +60,7 @@ RUN apt-get update \ wget \ unzip \ vim \ + jq \ build-essential \ zlib1g-dev \ openssl \ @@ -145,6 +146,18 @@ RUN echo '#!/bin/bash \n\n' > /usr/bin/tf_serving_entrypoint.sh \ && echo '/usr/bin/tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@"' >> /usr/bin/tf_serving_entrypoint.sh \ && chmod +x /usr/bin/tf_serving_entrypoint.sh +COPY safety /tmp/safety + +RUN mkdir -p /opt/aws/dlc/info + +RUN ${PIP} install --no-cache-dir -U safety \ + && cd /tmp/safety \ + && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ + --safety-key ${SAFETY_KEY} \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + && ${PIP} uninstall -y safety \ + && rm -rf /tmp/safety + COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu index eff5d7a90b25..0e5f67f8387a 100644 --- a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu @@ -102,6 +102,7 @@ RUN apt-get update \ ffmpeg \ libsm6 \ libxext6 \ + jq \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean @@ -171,6 +172,18 @@ RUN ${PIP} install --no-cache-dir -U \ smdebug==${SMDEBUG_VERSION} \ smclarify +COPY safety /tmp/safety + +RUN mkdir -p /opt/aws/dlc/info + +RUN ${PIP} install --no-cache-dir -U safety \ + && cd /tmp/safety \ + && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ + --safety-key ${SAFETY_KEY} \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + && ${PIP} uninstall -y safety \ + && rm -rf /tmp/safety + ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu index 1e889bbc29c2..498757f9ddff 100644 --- a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu @@ -301,6 +301,18 @@ RUN SMDATAPARALLEL_TF=1 ${PIP} install --no-cache-dir ${SMDATAPARALLEL_BINARY} ENV LD_LIBRARY_PATH="/usr/local/lib/python3.7/site-packages/smdistributed/dataparallel/lib:$LD_LIBRARY_PATH" +COPY safety /tmp/safety + +RUN mkdir -p /opt/aws/dlc/info + +RUN ${PIP} install --no-cache-dir -U safety \ + && cd /tmp/safety \ + && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ + --safety-key ${SAFETY_KEY} \ + --report-path "/opt/aws/dlc/info/safety_test_report.json" \ + && ${PIP} uninstall -y safety \ + && rm -rf /tmp/safety + ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py From 2f23d6ddfb0e069c34f2ffcd8fdf5c5dc2e69e4d Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Mon, 28 Jun 2021 22:41:30 -0700 Subject: [PATCH 14/88] support safety for TF2.5 --- src/safety/test_and_generate_python_safety_report.py | 4 ++-- tensorflow/buildspec.yml | 3 +++ tensorflow/inference/docker/2.5/py3/Dockerfile.cpu | 1 + tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu | 1 + tensorflow/training/docker/2.5/py3/Dockerfile.cpu | 1 + tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu | 1 + 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/safety/test_and_generate_python_safety_report.py b/src/safety/test_and_generate_python_safety_report.py index 826a04549ff8..4daeb5256bae 100644 --- a/src/safety/test_and_generate_python_safety_report.py +++ b/src/safety/test_and_generate_python_safety_report.py @@ -49,11 +49,11 @@ def __post_init__(self): parser = argparse.ArgumentParser() parser.add_argument('--safety-key', type=str, help='Safety key') parser.add_argument('--report-path', type=str, help='Safety report path') -parser.add_argument('--ignored-packages', type=str, help='Packages to be ignored') +parser.add_argument('--ignored-packages', type=str, default=None, help='Packages to be ignored') args = parser.parse_args() report_path = args.report_path -ignored_packages = args.ignored_packages if "ignored_packages" in args else None +ignored_packages = args.ignored_packages safety_api_key = args.safety_key # run safety check diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index ee3341c3c83c..cd9997b3c5cb 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -40,6 +40,9 @@ context: deep_learning_container: source: ../../src/deep_learning_container.py target: deep_learning_container.py + safety_files: + source: ../../src/safety + target: safety images: BuildTensorflowCpuPy37TrainingDockerImage: diff --git a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu index 9311726ea21b..d2f7a68006ba 100644 --- a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu @@ -7,6 +7,7 @@ LABEL dlc_major_version="1" LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true LABEL com.amazonaws.sagemaker.capabilities.multi-models=true +ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 diff --git a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu index 3edfaa1b0463..6d080d36911d 100644 --- a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu @@ -6,6 +6,7 @@ LABEL dlc_major_version="1" # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true +ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 diff --git a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu index 0e5f67f8387a..d5541067d69c 100644 --- a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu @@ -22,6 +22,7 @@ ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 +ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 diff --git a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu index 498757f9ddff..639ad02d84a2 100644 --- a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu @@ -22,6 +22,7 @@ ENV KMP_BLOCKTIME=1 ENV KMP_SETTINGS=0 ENV RDMAV_FORK_SAFE=1 +ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 From f1f36c64a18e7febc77f49607099d8683624055f Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 29 Jun 2021 10:13:08 -0700 Subject: [PATCH 15/88] adapt the new safety workflow on the sanity test --- test/dlc_tests/sanity/test_safety_check.py | 40 +++++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py index c39e9460e48e..661f15c9c33e 100644 --- a/test/dlc_tests/sanity/test_safety_check.py +++ b/test/dlc_tests/sanity/test_safety_check.py @@ -159,6 +159,7 @@ def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. + The function checks for the safety report to validate the safety run on the build job. If the report does not exist, the safety check is executed. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() @@ -178,22 +179,27 @@ def test_safety(image): f"--entrypoint='/bin/bash' " f"{image}", hide=True) try: - run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) - json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) - safety_result = json.loads(json_str_safety_result) - for vulnerability in safety_result: - package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] - # Get the latest version of the package with vulnerability - latest_version = _get_latest_package_version(package) - # If the latest version of the package is also affected, ignore this vulnerability - if Version(latest_version) in SpecifierSet(affected_versions): - # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. - # Comparing two versions as a string has some edge cases which require us to write more code. - # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and - # gives an object that can be easily compared against a Version object. - # https://packaging.pypa.io/en/latest/specifiers/ - ignore_str += f" -i {vulnerability_id}" - assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ - f"Safety test failed for {image}" + SAFETY_FILE = "/opt/aws/dlc/info/safety_test_report.json" + safety_file_check = run(f"{docker_exec_cmd} test -e {SAFETY_FILE}", warn=True, hide=True) + if safety_file_check.return_code != 0: + run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) + json_str_safety_result = "safety_check.run_safety_check_on_container(docker_exec_cmd)" + safety_result = json.loads(json_str_safety_result) + for vulnerability in safety_result: + package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] + # Get the latest version of the package with vulnerability + latest_version = _get_latest_package_version(package) + # If the latest version of the package is also affected, ignore this vulnerability + if Version(latest_version) in SpecifierSet(affected_versions): + # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. + # Comparing two versions as a string has some edge cases which require us to write more code. + # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and + # gives an object that can be easily compared against a Version object. + # https://packaging.pypa.io/en/latest/specifiers/ + ignore_str += f" -i {vulnerability_id}" + assert ("safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0"), \ + f"Safety test failed for {image}" + else: + LOGGER.info(f"Safety check is complete as a part of docker build and report exist at {SAFETY_FILE}") finally: run(f"docker rm -f {container_name}", hide=True) From 3ca89e583c86bf14d38a96aeca05cc6be57ae8ac Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 29 Jun 2021 11:13:31 -0700 Subject: [PATCH 16/88] run sanity test --- src/config/build_config.py | 2 +- src/config/test_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/build_config.py b/src/config/build_config.py index fa4d7e52f4e5..891a995cfcf0 100644 --- a/src/config/build_config.py +++ b/src/config/build_config.py @@ -7,7 +7,7 @@ # Frameworks for which you want to disable both builds and tests DISABLE_FRAMEWORK_TESTS = ["mxnet","pytorch"] # Disable new builds or build without datetime tag -DISABLE_DATETIME_TAG = False +DISABLE_DATETIME_TAG = True # Note: Need to build the images at least once with DISABLE_DATETIME_TAG = True # before disabling new builds or tests will fail DISABLE_NEW_BUILDS = False diff --git a/src/config/test_config.py b/src/config/test_config.py index 26f738f7c2da..bcd78e8ed3b4 100644 --- a/src/config/test_config.py +++ b/src/config/test_config.py @@ -7,7 +7,7 @@ # It is recommended to set DISABLE_EFA_TESTS to True to disable EFA tests if there is no change to EFA installer version or Frameworks. DISABLE_EFA_TESTS = True -DISABLE_SANITY_TESTS = True +DISABLE_SANITY_TESTS = False DISABLE_SAGEMAKER_TESTS = True DISABLE_ECS_TESTS = True DISABLE_EKS_TESTS = True From 7603d3064c56f6b1c9b1efb833f5cd0bd50982dd Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 29 Jun 2021 12:27:25 -0700 Subject: [PATCH 17/88] run safety on PR --- src/config/build_config.py | 2 +- src/safety/safety_check_v2.py | 2 +- .../eks/mxnet/inference/test_eks_mxnet_inference.py | 1 - test/dlc_tests/sanity/test_safety_check.py | 7 ------- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/config/build_config.py b/src/config/build_config.py index 891a995cfcf0..2df61418f64f 100644 --- a/src/config/build_config.py +++ b/src/config/build_config.py @@ -10,4 +10,4 @@ DISABLE_DATETIME_TAG = True # Note: Need to build the images at least once with DISABLE_DATETIME_TAG = True # before disabling new builds or tests will fail -DISABLE_NEW_BUILDS = False +DISABLE_NEW_BUILDS = True diff --git a/src/safety/safety_check_v2.py b/src/safety/safety_check_v2.py index 6b3287f8a58b..cc763294581f 100644 --- a/src/safety/safety_check_v2.py +++ b/src/safety/safety_check_v2.py @@ -107,4 +107,4 @@ def main(): try: sys.exit(main()) except KeyboardInterrupt: - pass \ No newline at end of file + pass diff --git a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py index 0f78975242c5..2cefd74e093a 100644 --- a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py +++ b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py @@ -46,7 +46,6 @@ def test_eks_mxnet_neuron_inference(mxnet_inference, neuron_only): if eks_utils.is_service_running(selector_name): eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8080") - assert test_utils.request_mxnet_inference(port=port_to_forward, model="mxnet-resnet-neuron") except ValueError as excp: eks_utils.LOGGER.error("Service is not running: %s", excp) diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py index 661f15c9c33e..10cf11175edf 100644 --- a/test/dlc_tests/sanity/test_safety_check.py +++ b/test/dlc_tests/sanity/test_safety_check.py @@ -148,13 +148,6 @@ def _get_latest_package_version(package): @pytest.mark.model("N/A") @pytest.mark.canary("Run safety tests regularly on production images") @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") -@pytest.mark.skipif( - not (is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), - reason=( - "Skipping the test to decrease the number of calls to the Safety Check DB. " - "Test will be executed in the 'mainline' pipeline and canaries pipeline." - ) -) def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise From d887fb1695af1ca4904cd67797f91bfc85c10af5 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 29 Jun 2021 12:28:55 -0700 Subject: [PATCH 18/88] revert unrelated change --- test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py index 2cefd74e093a..295f432d27b1 100644 --- a/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py +++ b/test/dlc_tests/eks/mxnet/inference/test_eks_mxnet_inference.py @@ -46,7 +46,7 @@ def test_eks_mxnet_neuron_inference(mxnet_inference, neuron_only): if eks_utils.is_service_running(selector_name): eks_utils.eks_forward_port_between_host_and_container(selector_name, port_to_forward, "8080") - assert test_utils.request_mxnet_inference(port=port_to_forward, model="mxnet-resnet-neuron") + assert test_utils.request_mxnet_inference(port=port_to_forward, model="mxnet-resnet50") except ValueError as excp: eks_utils.LOGGER.error("Service is not running: %s", excp) finally: From 8d6982e80e30a385833ee2d48eab625fd2160c52 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 29 Jun 2021 13:07:27 -0700 Subject: [PATCH 19/88] revert temp changes --- test/dlc_tests/sanity/test_safety_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py index 10cf11175edf..cdb84424a0c2 100644 --- a/test/dlc_tests/sanity/test_safety_check.py +++ b/test/dlc_tests/sanity/test_safety_check.py @@ -176,7 +176,7 @@ def test_safety(image): safety_file_check = run(f"{docker_exec_cmd} test -e {SAFETY_FILE}", warn=True, hide=True) if safety_file_check.return_code != 0: run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) - json_str_safety_result = "safety_check.run_safety_check_on_container(docker_exec_cmd)" + json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) safety_result = json.loads(json_str_safety_result) for vulnerability in safety_result: package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] @@ -190,7 +190,7 @@ def test_safety(image): # gives an object that can be easily compared against a Version object. # https://packaging.pypa.io/en/latest/specifiers/ ignore_str += f" -i {vulnerability_id}" - assert ("safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0"), \ + assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ f"Safety test failed for {image}" else: LOGGER.info(f"Safety check is complete as a part of docker build and report exist at {SAFETY_FILE}") From cb6c347fd56db8d2c7db7ffdf15ec3fbae025b34 Mon Sep 17 00:00:00 2001 From: tejaschumbalkar Date: Tue, 27 Jul 2021 15:51:44 -0700 Subject: [PATCH 20/88] multi stage docker build --- buildspec.yml | 3 - src/Dockerfile.multipart | 7 + src/constants.py | 4 + src/image.py | 120 +++++---- src/image_builder.py | 238 +++++++++++------- src/{safety => }/safety_check_v2.py | 0 src/safety_report.py | 190 ++++++++++++++ .../test_and_generate_python_safety_report.py | 0 tensorflow/buildspec.yml | 6 - .../inference/docker/2.5/py3/Dockerfile.cpu | 14 -- .../docker/2.5/py3/cu112/Dockerfile.gpu | 14 -- .../training/docker/2.4/py3/Dockerfile.cpu | 15 -- .../training/docker/2.5/py3/Dockerfile.cpu | 13 - .../docker/2.5/py3/cu112/Dockerfile.gpu | 13 - 14 files changed, 428 insertions(+), 209 deletions(-) create mode 100644 src/Dockerfile.multipart rename src/{safety => }/safety_check_v2.py (100%) create mode 100644 src/safety_report.py rename src/{safety => }/test_and_generate_python_safety_report.py (100%) diff --git a/buildspec.yml b/buildspec.yml index 1cf78b72ccb0..75126adf836f 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -1,7 +1,4 @@ version: 0.2 -env: - secrets-manager: - SAFETY_KEY: "/codebuild/safety/key" #TODO: Move all constatns to environment variables phases: diff --git a/src/Dockerfile.multipart b/src/Dockerfile.multipart new file mode 100644 index 000000000000..539b1cda4aef --- /dev/null +++ b/src/Dockerfile.multipart @@ -0,0 +1,7 @@ +# Use the Deep Learning Container as a base Image +ARG FIRST_STAGE_IMAGE="" + +FROM $FIRST_STAGE_IMAGE + +# Add any script or repo as required +COPY safety_report.json /var/safety_report.json diff --git a/src/constants.py b/src/constants.py index c4a10e646d91..18f1a676a1b7 100644 --- a/src/constants.py +++ b/src/constants.py @@ -31,6 +31,10 @@ # Left and right padding between text and margins in output PADDING = 1 +#Docker build stages +FIRST_STAGE="first" +SECOND_STAGE="second" + # Docker connections DOCKER_URL = "unix://var/run/docker.sock" diff --git a/src/image.py b/src/image.py index 0c76d4385b43..3440dbc26d23 100644 --- a/src/image.py +++ b/src/image.py @@ -26,7 +26,7 @@ class DockerImage: """ def __init__( - self, info, dockerfile, repository, tag, to_build, context=None, + self, info, dockerfile, repository, tag, to_build, stage, context=None, ): # Meta-data about the image should go to info. @@ -36,6 +36,7 @@ def __init__( self.summary = {} self.build_args = {} self.labels = {} + self.stage = stage self.dockerfile = dockerfile self.context = context @@ -69,11 +70,7 @@ def collect_installed_packages_information(self): docker_client.containers.prune() return command_responses - def build(self): - """ - The build function builds the specified docker image - """ - self.summary["start_time"] = datetime.now() + def pre_build_configuration(self): if not self.to_build: self.log = ["Not built"] @@ -84,19 +81,42 @@ def build(self): if self.info.get("base_image_uri"): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] + if self.ecr_url: + self.build_args["FIRST_STAGE_IMAGE"] = self.ecr_url + if self.info.get("extra_build_args"): self.build_args.update(self.info.get("extra_build_args")) - + if self.info.get("labels"): self.labels.update(self.info.get("labels")) + + print(f"self.build_args {self.build_args}") + print(f"self.labels {self.labels}") - with open(self.context.context_path, "rb") as context_file: - response = [] - - for line in self.client.build( - fileobj=context_file, + def build(self): + """ + The build function builds the specified docker image + """ + self.summary["start_time"] = datetime.now() + self.pre_build_configuration() + print(f"self.context {self.context}") + if self.context: + with open(self.context.context_path, "rb") as context_file: + print("within context") + self.docker_build(fileobj=context_file, custom_context=True) + self.context.remove() + else: + print("out of context") + self.docker_build() + #check the size after image is built. + self.image_size_check() + + def docker_build(self, fileobj=None, custom_context=False): + response = [] + for line in self.client.build( + fileobj=fileobj, path=self.dockerfile, - custom_context=True, + custom_context=custom_context, rm=True, decode=True, tag=self.ecr_url, @@ -121,39 +141,51 @@ def build(self): else: response.append(str(line)) - self.context.remove() + self.log = response + print(f"self.log {self.log}") + self.build_status = constants.SUCCESS + #TODO: return required? + return self.build_status - self.summary["image_size"] = int( + + def image_size_check(self): + response = [] + self.summary["image_size"] = int( self.client.inspect_image(self.ecr_url)["Size"] ) / (1024 * 1024) - if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: - response.append("Image size baseline exceeded") - response.append(f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}") - response += self.collect_installed_packages_information() - self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT - else: - self.build_status = constants.SUCCESS - - for line in self.client.push( - self.repository, self.tag, stream=True, decode=True - ): - if line.get("error") is not None: - response.append(line["error"]) - - self.log = response - self.build_status = constants.FAIL - self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] - self.summary["end_time"] = datetime.now() - - return self.build_status - if line.get("stream") is not None: - response.append(line["stream"]) - else: - response.append(str(line)) + if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: + response.append("Image size baseline exceeded") + response.append(f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}") + response += self.collect_installed_packages_information() + self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT + else: + self.build_status = constants.SUCCESS + self.log = response + print(f"self.log {self.log}") + #TODO: return required? + return self.build_status + + def push_image(self): + + for line in self.client.push(self.repository, self.tag, stream=True, decode=True): + response = [] + if line.get("error") is not None: + response.append(line["error"]) - self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] - self.summary["end_time"] = datetime.now() - self.summary["ecr_url"] = self.ecr_url - self.log = response + self.log = response + self.build_status = constants.FAIL + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + self.summary["end_time"] = datetime.now() - return self.build_status + return self.build_status + if line.get("stream") is not None: + response.append(line["stream"]) + else: + response.append(str(line)) + + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + self.summary["end_time"] = datetime.now() + self.summary["ecr_url"] = self.ecr_url + self.log = response + #TODO: return required? + return self.build_status diff --git a/src/image_builder.py b/src/image_builder.py index e3028a8aebb0..7cd6a3482e40 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -29,6 +29,8 @@ from output import OutputFormatter from config import build_config +FORMATTER = OutputFormatter(constants.PADDING) +build_context = os.getenv("BUILD_CONTEXT") def _find_image_object(images_list, image_name): """ @@ -47,11 +49,11 @@ def _find_image_object(images_list, image_name): # TODO: Abstract away to ImageBuilder class def image_builder(buildspec): - FORMATTER = OutputFormatter(constants.PADDING) BUILDSPEC = Buildspec() BUILDSPEC.load(buildspec) - IMAGES = [] + FIRST_STAGES_IMAGES = [] + SECOND_STAGES_IMAGES = [] if "huggingface" in str(BUILDSPEC["framework"]): os.system("echo login into public ECR") @@ -63,11 +65,6 @@ def image_builder(buildspec): extra_build_args = {} labels = {} - safety_key = os.getenv("SAFETY_KEY") - - if safety_key: - extra_build_args["SAFETY_KEY"] = safety_key - if image_config.get("version") is not None: if BUILDSPEC["version"] != image_config.get("version"): continue @@ -75,7 +72,6 @@ def image_builder(buildspec): if image_config.get("context") is not None: ARTIFACTS.update(image_config["context"]) - build_context = os.getenv("BUILD_CONTEXT") image_tag = ( tag_image_with_pr_number(image_config["tag"]) if build_context == "PR" @@ -90,7 +86,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(FIRST_STAGES_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -158,109 +154,177 @@ def image_builder(buildspec): "labels": labels, "extra_build_args": extra_build_args } - - image_object = DockerImage( + + #Create first stage docker object + first_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], + stage=constants.FIRST_STAGE, context=context, ) - IMAGES.append(image_object) + #Create second stage docker object + if "example" not in image_name.lower() and build_context == "MAINLINE": + second_stage_image_object = DockerImage( + info=info, + dockerfile=os.path.join(os.sep, "Dockerfile.multipart"), + repository=image_repo_uri, + tag=image_tag, + to_build=image_config["build"], + stage=constants.SECOND_STAGE, + context=None, + ) - FORMATTER.banner("DLC") - FORMATTER.title("Status") + FORMATTER.separator() - THREADS = {} + FIRST_STAGES_IMAGES.append(first_stage_image_object) + SECOND_STAGES_IMAGES.append(second_stage_image_object) - # In the context of the ThreadPoolExecutor each instance of image.build submitted - # to it is executed concurrently in a separate thread. - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - # Standard images must be built before example images - # Example images will use standard images as base - standard_images = [image for image in IMAGES if "example" not in image.name.lower()] - example_images = [image for image in IMAGES if "example" in image.name.lower()] + FORMATTER.banner("DLC") + #FORMATTER.title("Status") + + # Standard images must be built before example images + # Example images will use standard images as base + first_stage_standard_images = [image for image in FIRST_STAGES_IMAGES if "example" not in image.name.lower()] + second_stage_standard_images = [image for image in SECOND_STAGES_IMAGES] + + example_images = [image for image in FIRST_STAGES_IMAGES if "example" in image.name.lower()] + #needs to be reconfigured + #ALL_IMAGES = first_stage_standard_images + second_stage_standard_images + example_images + + #first stage build + FORMATTER.banner("First Stage Build") + build_images(first_stage_standard_images) - for image in standard_images: - THREADS[image.name] = executor.submit(image.build) + """ + Run safety on first stage image and store the ouput file locally + """ - # the FORMATTER.progress(THREADS) function call also waits until all threads have completed - FORMATTER.progress(THREADS) + FORMATTER.banner("Second Stage Build") + build_images(SECOND_STAGES_IMAGES) + + #second stage build + if second_stage_standard_images: + FORMATTER.banner("Second Stage Build") + build_images(second_stage_standard_images) + + push_images(second_stage_standard_images) + + #example image build + build_images(example_images) + push_images(example_images) + + #After the build, display logs/sumary for all the images. + + show_build_logs(ALL_IMAGES) + show_build_summary(ALL_IMAGES) + is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) + + #change logic here. upload metrics only for the second stage image + upload_metrics(IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) + + # Set environment variables to be consumed by test jobs + test_trigger_job = utils.get_codebuild_project_name() + #needs to be configured to use final built images + utils.set_test_env( + IMAGES, + BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), + TEST_TRIGGER=test_trigger_job, + ) + + +def show_build_logs(images): + + FORMATTER.title("Build Logs") + + if not os.path.isdir("logs"): + os.makedirs("logs") + + for image in images: + image_description = f"{image.name}-{image.stage}" + FORMATTER.title(image_description) + FORMATTER.table(image.info.items()) + FORMATTER.separator() + FORMATTER.print_lines(image.log) + with open(f"logs/{image_description}", "w") as fp: + fp.write("/n".join(image.log)) + image.summary["log"] = f"logs/{image_description}" - for image in example_images: - THREADS[image.name] = executor.submit(image.build) +def show_build_summary(images): - # the FORMATTER.progress(THREADS) function call also waits until all threads have completed - FORMATTER.progress(THREADS) + FORMATTER.title("Summary") - FORMATTER.title("Build Logs") + for image in images: + FORMATTER.title(image.name) + FORMATTER.table(image.summary.items()) - if not os.path.isdir("logs"): - os.makedirs("logs") +def show_build_errors(images): + FORMATTER.title("Errors") + is_any_build_failed = False + is_any_build_failed_size_limit = False - for image in IMAGES: + for image in images: + if image.build_status == constants.FAIL: FORMATTER.title(image.name) - FORMATTER.table(image.info.items()) - FORMATTER.separator() - FORMATTER.print_lines(image.log) - with open(f"logs/{image.name}", "w") as fp: - fp.write("/n".join(image.log)) - image.summary["log"] = f"logs/{image.name}" - - FORMATTER.title("Summary") - - for image in IMAGES: - FORMATTER.title(image.name) - FORMATTER.table(image.summary.items()) - - FORMATTER.title("Errors") - is_any_build_failed = False - is_any_build_failed_size_limit = False - for image in IMAGES: - if image.build_status == constants.FAIL: - FORMATTER.title(image.name) - FORMATTER.print_lines(image.log[-10:]) - is_any_build_failed = True - else: - if image.build_status == constants.FAIL_IMAGE_SIZE_LIMIT: - is_any_build_failed_size_limit = True - if is_any_build_failed: - raise Exception("Build failed") + FORMATTER.print_lines(image.log[-10:]) + is_any_build_failed = True + else: + if image.build_status == constants.FAIL_IMAGE_SIZE_LIMIT: + is_any_build_failed_size_limit = True + if is_any_build_failed: + raise Exception("Build failed") + else: + if is_any_build_failed_size_limit: + FORMATTER.print("Build failed. Image size limit breached.") else: - if is_any_build_failed_size_limit: - FORMATTER.print("Build failed. Image size limit breached.") + FORMATTER.print("No errors") + return is_any_build_failed, is_any_build_failed_size_limit + +def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit): + + FORMATTER.title("Uploading Metrics") + is_any_build_failed = False + is_any_build_failed_size_limit = False + metrics = Metrics( + context=constants.BUILD_CONTEXT, + region=BUILDSPEC["region"], + namespace=constants.METRICS_NAMESPACE, + ) + for image in images: + try: + metrics.push_image_metrics(image) + except Exception as e: + if is_any_build_failed or is_any_build_failed_size_limit: + raise Exception(f"Build failed.{e}") else: - FORMATTER.print("No errors") + raise Exception(f"Build passed. {e}") - FORMATTER.title("Uploading Metrics") - metrics = Metrics( - context=constants.BUILD_CONTEXT, - region=BUILDSPEC["region"], - namespace=constants.METRICS_NAMESPACE, - ) - for image in IMAGES: - try: - metrics.push_image_metrics(image) - except Exception as e: - if is_any_build_failed or is_any_build_failed_size_limit: - raise Exception(f"Build failed.{e}") - else: - raise Exception(f"Build passed. {e}") + if is_any_build_failed_size_limit: + raise Exception("Build failed because of file limit") - if is_any_build_failed_size_limit: - raise Exception("Build failed because of file limit") + FORMATTER.separator() - FORMATTER.separator() +def build_images(images): + THREADS = {} + # In the context of the ThreadPoolExecutor each instance of image.build submitted + # to it is executed concurrently in a separate thread. + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + for image in images: + FORMATTER.print(f"image_object.context {image.context}") + THREADS[image.name] = executor.submit(image.build) + # the FORMATTER.progress(THREADS) function call also waits until all threads have completed + FORMATTER.progress(THREADS) + +def push_images(images): + THREADS = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + for image in images: + THREADS[image.name] = executor.submit(image.push_image) + FORMATTER.progress(THREADS) - # Set environment variables to be consumed by test jobs - test_trigger_job = utils.get_codebuild_project_name() - utils.set_test_env( - IMAGES, - BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), - TEST_TRIGGER=test_trigger_job, - ) def tag_image_with_pr_number(image_tag): diff --git a/src/safety/safety_check_v2.py b/src/safety_check_v2.py similarity index 100% rename from src/safety/safety_check_v2.py rename to src/safety_check_v2.py diff --git a/src/safety_report.py b/src/safety_report.py new file mode 100644 index 000000000000..503510852832 --- /dev/null +++ b/src/safety_report.py @@ -0,0 +1,190 @@ +#this file can be removed if the safety test setup is done somewhere else +import json +import logging +import os +import sys + +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +import pytest +import requests + +from invoke import run + +from test.test_utils import ( + CONTAINER_TESTS_PREFIX, is_dlc_cicd_context, is_canary_context, is_mainline_context, is_time_for_canary_safety_scan +) + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) +LOGGER.addHandler(logging.StreamHandler(sys.stderr)) + +# List of safety check vulnerability IDs to ignore. To get the ID, run safety check on the container, +# and copy paste the ID given by the safety check for a package. +# Note:- 1. This ONLY needs to be done if a package version exists that resolves this safety issue, but that version +# cannot be used because of incompatibilities. +# 2. Ensure that IGNORE_SAFETY_IDS is always as small/empty as possible. +IGNORE_SAFETY_IDS = { + "tensorflow": { + "training": { + "py2": [ + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452', + # for shipping pycrypto<=2.6.1 - the last available version for py2 + '35015' + ], + "py3": [] + }, + "inference": { + "py2": [ + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "inference-eia": { + "py2": [ + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "inference-neuron": { + "py3": [ + # TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches + '39409', '39408', '39407', '39406' + ], + }, + }, + "mxnet": { + "inference-eia": { + "py2": [ + # numpy<=1.16.0 -- This has to only be here while we publish MXNet 1.4.1 EI DLC v1.0 + '36810', + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "inference": { + "py2": [ + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "training": { + "py2": [ + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "inference-neuron": { + "py3": [ + # for shipping tensorflow 1.15.5 + "40673", "40675", "40676", "40794", "40795", "40796" + ] + } + }, + "pytorch": { + "training": { + "py2": [ + # astropy<3.0.1 + '35810', + # for shipping pillow<=6.2.2 - the last available version for py2 + '38449', '38450', '38451', '38452' + ], + "py3": [] + }, + "inference": { + "py3": [] + }, + "inference-eia": { + "py3": [] + }, + "inference-neuron": { + "py3": [ + # 39409, 39408, 39407, 39406: TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches + '39409', '39408', '39407', '39406' + ] + } + } +} + + +def _get_safety_ignore_list(image_uri): + """ + Get a list of known safety check issue IDs to ignore, if specified in IGNORE_LISTS. + :param image_uri: + :return: list of safety check IDs to ignore + """ + framework = ("mxnet" if "mxnet" in image_uri else + "pytorch" if "pytorch" in image_uri else + "tensorflow") + job_type = ("training" if "training" in image_uri else + "inference-eia" if "eia" in image_uri else + "inference-neuron" if "neuron" in image_uri else + "inference") + python_version = "py2" if "py2" in image_uri else "py3" + + return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, []) + + +def _get_latest_package_version(package): + """ + Get the latest package version available on pypi for a package. + It is retried multiple times in case there are transient failures in executing the command. + + :param package: str Name of the package whose latest version must be retrieved + :return: tuple(command_success: bool, latest_version_value: str) + """ + pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") + data = json.loads(pypi_package_info.text) + versions = data["releases"].keys() + return str(max(Version(v) for v in versions)) + + +def test_safety(image): + """ + Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise + error if an issue is fixable. + """ + from dlc.safety_check import SafetyCheck + safety_check = SafetyCheck() + + repo_name, image_tag = image.split('/')[-1].split(':') + ignore_ids_list = _get_safety_ignore_list(image) + sep = " -i " + ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}" + + container_name = f"{repo_name}-{image_tag}-safety" + docker_exec_cmd = f"docker exec -i {container_name}" + test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety") + # Add null entrypoint to ensure command exits immediately + run(f"docker run -id " + f"--name {container_name} " + f"--mount type=bind,src=$(pwd)/container_tests,target=/test " + f"--entrypoint='/bin/bash' " + f"{image}", hide=True) + try: + run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) + json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) + safety_result = json.loads(json_str_safety_result) + for vulnerability in safety_result: + package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] + # Get the latest version of the package with vulnerability + latest_version = _get_latest_package_version(package) + # If the latest version of the package is also affected, ignore this vulnerability + if Version(latest_version) in SpecifierSet(affected_versions): + # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. + # Comparing two versions as a string has some edge cases which require us to write more code. + # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and + # gives an object that can be easily compared against a Version object. + # https://packaging.pypa.io/en/latest/specifiers/ + ignore_str += f" -i {vulnerability_id}" + assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ + f"Safety test failed for {image}" + finally: + run(f"docker rm -f {container_name}", hide=True) diff --git a/src/safety/test_and_generate_python_safety_report.py b/src/test_and_generate_python_safety_report.py similarity index 100% rename from src/safety/test_and_generate_python_safety_report.py rename to src/test_and_generate_python_safety_report.py diff --git a/tensorflow/buildspec.yml b/tensorflow/buildspec.yml index cd9997b3c5cb..8f58c182ee12 100644 --- a/tensorflow/buildspec.yml +++ b/tensorflow/buildspec.yml @@ -24,9 +24,6 @@ context: deep_learning_container: source: ../../src/deep_learning_container.py target: deep_learning_container.py - safety_files: - source: ../../src/safety - target: safety inference_context: &INFERENCE_CONTEXT sagemaker_package_name: source: docker/build_artifacts/sagemaker @@ -40,9 +37,6 @@ context: deep_learning_container: source: ../../src/deep_learning_container.py target: deep_learning_container.py - safety_files: - source: ../../src/safety - target: safety images: BuildTensorflowCpuPy37TrainingDockerImage: diff --git a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu index d2f7a68006ba..c103e04bf4f1 100644 --- a/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/inference/docker/2.5/py3/Dockerfile.cpu @@ -7,7 +7,6 @@ LABEL dlc_major_version="1" LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true LABEL com.amazonaws.sagemaker.capabilities.multi-models=true -ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -39,7 +38,6 @@ RUN apt-get update \ unzip \ wget \ vim \ - jq \ libbz2-dev \ liblzma-dev \ libffi-dev \ @@ -114,18 +112,6 @@ RUN echo '#!/bin/bash \n\n' > /usr/bin/tf_serving_entrypoint.sh \ && echo '/usr/bin/tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@"' >> /usr/bin/tf_serving_entrypoint.sh \ && chmod +x /usr/bin/tf_serving_entrypoint.sh -COPY safety /tmp/safety - -RUN mkdir -p /opt/aws/dlc/info - -RUN ${PIP} install --no-cache-dir -U safety \ - && cd /tmp/safety \ - && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - && ${PIP} uninstall -y safety \ - && rm -rf /tmp/safety - COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu index 6d080d36911d..584e7439d373 100644 --- a/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/inference/docker/2.5/py3/cu112/Dockerfile.gpu @@ -6,7 +6,6 @@ LABEL dlc_major_version="1" # https://docs.aws.amazon.com/sagemaker/latest/dg/inference-pipeline-real-time.html LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true -ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -61,7 +60,6 @@ RUN apt-get update \ wget \ unzip \ vim \ - jq \ build-essential \ zlib1g-dev \ openssl \ @@ -147,18 +145,6 @@ RUN echo '#!/bin/bash \n\n' > /usr/bin/tf_serving_entrypoint.sh \ && echo '/usr/bin/tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=${MODEL_NAME} --model_base_path=${MODEL_BASE_PATH}/${MODEL_NAME} "$@"' >> /usr/bin/tf_serving_entrypoint.sh \ && chmod +x /usr/bin/tf_serving_entrypoint.sh -COPY safety /tmp/safety - -RUN mkdir -p /opt/aws/dlc/info - -RUN ${PIP} install --no-cache-dir -U safety \ - && cd /tmp/safety \ - && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - && ${PIP} uninstall -y safety \ - && rm -rf /tmp/safety - COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu index ed2a71393910..36eac407afa5 100644 --- a/tensorflow/training/docker/2.4/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.4/py3/Dockerfile.cpu @@ -22,7 +22,6 @@ ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 -ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -45,7 +44,6 @@ RUN apt-get update \ curl \ emacs \ git \ - jq \ libtemplate-perl \ libssl1.1 \ openssl \ @@ -156,19 +154,6 @@ RUN ${PIP} install --no-cache-dir -U \ smdebug==${SMDEBUG_VERSION} \ smclarify -COPY safety /tmp/safety -RUN mkdir -p /opt/aws/dlc/info -RUN ${PIP} install --no-cache-dir -U safety \ - && cd /tmp/safety \ - && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - --ignored-packages '{ \ - "pycrypto": "2.6.1", \ - "httplib2": "0.9.2" \ - }' \ - && ${PIP} uninstall -y safety \ - && rm -rf /tmp/safety COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu index d5541067d69c..cffa6afdf5f4 100644 --- a/tensorflow/training/docker/2.5/py3/Dockerfile.cpu +++ b/tensorflow/training/docker/2.5/py3/Dockerfile.cpu @@ -22,7 +22,6 @@ ENV PYTHONIOENCODING=UTF-8 ENV LANG=C.UTF-8 ENV LC_ALL=C.UTF-8 -ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -103,7 +102,6 @@ RUN apt-get update \ ffmpeg \ libsm6 \ libxext6 \ - jq \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean @@ -173,17 +171,6 @@ RUN ${PIP} install --no-cache-dir -U \ smdebug==${SMDEBUG_VERSION} \ smclarify -COPY safety /tmp/safety - -RUN mkdir -p /opt/aws/dlc/info - -RUN ${PIP} install --no-cache-dir -U safety \ - && cd /tmp/safety \ - && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - && ${PIP} uninstall -y safety \ - && rm -rf /tmp/safety ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py diff --git a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu index 639ad02d84a2..1e889bbc29c2 100644 --- a/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu +++ b/tensorflow/training/docker/2.5/py3/cu112/Dockerfile.gpu @@ -22,7 +22,6 @@ ENV KMP_BLOCKTIME=1 ENV KMP_SETTINGS=0 ENV RDMAV_FORK_SAFE=1 -ARG SAFETY_KEY ARG PYTHON=python3.7 ARG PYTHON_PIP=python3-pip ARG PIP=pip3 @@ -302,18 +301,6 @@ RUN SMDATAPARALLEL_TF=1 ${PIP} install --no-cache-dir ${SMDATAPARALLEL_BINARY} ENV LD_LIBRARY_PATH="/usr/local/lib/python3.7/site-packages/smdistributed/dataparallel/lib:$LD_LIBRARY_PATH" -COPY safety /tmp/safety - -RUN mkdir -p /opt/aws/dlc/info - -RUN ${PIP} install --no-cache-dir -U safety \ - && cd /tmp/safety \ - && ${PYTHON} /tmp/safety/test_and_generate_python_safety_report.py \ - --safety-key ${SAFETY_KEY} \ - --report-path "/opt/aws/dlc/info/safety_test_report.json" \ - && ${PIP} uninstall -y safety \ - && rm -rf /tmp/safety - ADD https://raw.githubusercontent.com/aws/deep-learning-containers/master/src/deep_learning_container.py /usr/local/bin/deep_learning_container.py RUN chmod +x /usr/local/bin/deep_learning_container.py From 61641ec35a78939e6e83e2360bbf76e2ad1f8adb Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 12 Aug 2021 03:48:04 +0000 Subject: [PATCH 21/88] Fixed a few issues. Edited Dockerfiles to allow running the script faster in local. Will revert. --- mxnet/buildspec.yml | 54 +-- mxnet/training/docker/1.8/py3/Dockerfile.cpu | 366 +++++++++---------- src/image.py | 8 +- src/image_builder.py | 38 +- src/test_build.py | 12 + 5 files changed, 223 insertions(+), 255 deletions(-) create mode 100644 src/test_build.py diff --git a/mxnet/buildspec.yml b/mxnet/buildspec.yml index ed97a377ea75..bf279d2de50a 100644 --- a/mxnet/buildspec.yml +++ b/mxnet/buildspec.yml @@ -49,56 +49,4 @@ images: docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] context: <<: *TRAINING_CONTEXT - BuildMXNetGPUTrainPy3DockerImage: - <<: *TRAINING_REPOSITORY - build: &MXNET_GPU_TRAINING_PY3 false - image_size_baseline: 7159 - device_type: &DEVICE_TYPE gpu - python_version: &DOCKER_PYTHON_VERSION py3 - tag_python_version: &TAG_PYTHON_VERSION py37 - cuda_version: &CUDA_VERSION cu110 - os_version: &OS_VERSION ubuntu16.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] - context: - <<: *TRAINING_CONTEXT - BuildMXNetCPUInferencePy3DockerImage: - <<: *INFERENCE_REPOSITORY - build: &MXNET_CPU_INFERENCE_PY3 false - image_size_baseline: 1687 - device_type: &DEVICE_TYPE cpu - python_version: &DOCKER_PYTHON_VERSION py3 - tag_python_version: &TAG_PYTHON_VERSION py37 - os_version: &OS_VERSION ubuntu16.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] - context: - <<: *INFERENCE_CONTEXT - BuildMXNetGPUInferencePy3DockerImage: - << : *INFERENCE_REPOSITORY - build: &MXNET_GPU_INFERENCE_PY3 false - image_size_baseline: 5578 - device_type: &DEVICE_TYPE gpu - python_version: &DOCKER_PYTHON_VERSION py3 - tag_python_version: &TAG_PYTHON_VERSION py37 - cuda_version: &CUDA_VERSION cu110 - os_version: &OS_VERSION ubuntu16.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] - context: - <<: *INFERENCE_CONTEXT - BuildMXNetExampleGPUTrainPy3DockerImage: - <<: *TRAINING_REPOSITORY - build: &MXNET_GPU_TRAINING_PY3 false - image_size_baseline: 7324 - base_image_name: BuildMXNetGPUTrainPy3DockerImage - device_type: &DEVICE_TYPE gpu - python_version: &DOCKER_PYTHON_VERSION py3 - tag_python_version: &TAG_PYTHON_VERSION py37 - cuda_version: &CUDA_VERSION cu110 - os_version: &OS_VERSION ubuntu16.04 - tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, - "-example" ] - docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example, /Dockerfile., *DEVICE_TYPE ] - context: - <<: *TRAINING_CONTEXT + diff --git a/mxnet/training/docker/1.8/py3/Dockerfile.cpu b/mxnet/training/docker/1.8/py3/Dockerfile.cpu index 02cb3ef9d946..11e5932ec730 100644 --- a/mxnet/training/docker/1.8/py3/Dockerfile.cpu +++ b/mxnet/training/docker/1.8/py3/Dockerfile.cpu @@ -3,186 +3,186 @@ FROM ubuntu:16.04 LABEL maintainer="Amazon AI" LABEL dlc_major_version="1" -ARG PYTHON=python3 -ARG PIP=pip3 -ARG PYTHON_VERSION=3.7.10 -ARG MX_URL=https://aws-mx-pypi.s3-us-west-2.amazonaws.com/1.8.0/aws_mx-1.8.0-py2.py3-none-manylinux2014_x86_64.whl - -# The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating -# the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. -ARG SMDEBUG_VERSION=0.9.4 -ARG OPENSSL_VERSION=1.1.1k - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ - PYTHONIOENCODING=UTF-8 \ - LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 \ - SAGEMAKER_TRAINING_MODULE=sagemaker_mxnet_container.training:main \ - DGLBACKEND=mxnet - -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - software-properties-common \ - build-essential \ - ca-certificates \ - curl \ - emacs \ - git \ - libopencv-dev \ - openssh-client \ - openssh-server \ - vim \ - wget \ - unzip \ - zlib1g-dev \ - libreadline-gplv2-dev \ - libncursesw5-dev \ - libssl-dev \ - libsqlite3-dev \ - libgdbm-dev \ - libc6-dev \ - libbz2-dev \ - tk-dev \ - libffi-dev \ - cmake \ - # Install dependent library for OpenCV - libgtk2.0-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -########################################################################### -# Horovod dependencies -########################################################################### - -# Install Open MPI -RUN mkdir /tmp/openmpi \ - && cd /tmp/openmpi \ - && wget -q https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.1.tar.gz \ - && tar zxf openmpi-4.0.1.tar.gz \ - && cd openmpi-4.0.1 \ - && ./configure --enable-orterun-prefix-by-default \ - && make -j $(nproc) all \ - && make install \ - && ldconfig \ - && cd /tmp && rm -rf /tmp/openmpi - -# Create a wrapper for OpenMPI to allow running as root by default -RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real \ - && echo '#!/bin/bash' > /usr/local/bin/mpirun \ - && echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun \ - && chmod a+x /usr/local/bin/mpirun \ - && echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf \ - && echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf - -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -ENV PATH=/usr/local/bin:$PATH - -# install OpenSSL -RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ - && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ - && cd openssl-${OPENSSL_VERSION} \ - && ./config && make -j $(nproc) && make install \ - && ldconfig \ - && cd .. && rm -rf openssl-* \ - && rmdir /usr/local/ssl/certs \ - && ln -s /etc/ssl/certs /usr/local/ssl/certs - -# install Python -RUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ - && tar -xzf Python-$PYTHON_VERSION.tgz \ - && cd Python-$PYTHON_VERSION \ - && ./configure --enable-shared --prefix=/usr/local \ - && make -j $(nproc) && make install \ - && cd .. && rm -rf ../Python-$PYTHON_VERSION* \ - && ln -s /usr/local/bin/pip3 /usr/bin/pip \ - && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \ - && ${PIP} --no-cache-dir install --upgrade \ - pip \ - setuptools - -WORKDIR / - -RUN ${PIP} install --no-cache --upgrade \ - keras-mxnet==2.2.4.2 \ - h5py==2.10.0 \ - onnx==1.6.0 \ - numpy==1.19.1 \ - pandas==0.25.1 \ - Pillow \ - requests==2.22.0 \ - scikit-learn==0.20.4 \ - dgl==0.4.* \ - scipy==1.2.2 \ - gluonnlp==0.10.0 \ - gluoncv==0.8.0 \ - # Putting a cap in versions number to avoid potential issues with a new major version - "urllib3>=1.25.10,<1.26.0" \ - # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli - python-dateutil==2.8.0 \ - tqdm==4.39.0 \ - sagemaker-experiments==0.* \ - # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli - "PyYAML>=5.4,<5.5" \ - mpi4py==3.0.2 \ - sagemaker-mxnet-training \ - ${MX_URL} \ - smdebug==${SMDEBUG_VERSION} \ - sagemaker \ - awscli \ - smclarify - -# Install extra packages -RUN pip install --no-cache-dir -U \ - "bokeh>=2.3,<3" \ - "imageio>=2.9,<3" \ - "opencv-python>=4.3,<5" \ - "plotly>=5.1,<6" \ - "seaborn>=0.11,<1" \ - "shap>=0.39,<1" - -# Install Horovod -RUN ${PIP} install --no-cache-dir horovod==0.19.5 - -# Allow OpenSSH to talk to containers without asking for confirmation -RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ - && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ - && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config - -# OpenSHH config for MPI communication -RUN mkdir -p /var/run/sshd && \ - sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -RUN rm -rf /root/.ssh/ && \ - mkdir -p /root/.ssh/ && \ - ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ - cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ - && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config - -# "channels first" is recommended for keras-mxnet -# https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn -RUN mkdir /root/.keras \ - && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json - -# This is here to make our installed version of OpenCV work. -# https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 -# TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? -RUN ln -s /dev/null /dev/raw1394 - -COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py - -RUN chmod +x /usr/local/bin/deep_learning_container.py - -RUN HOME_DIR=/root \ - && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ - && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ - && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ - && chmod +x /usr/local/bin/testOSSCompliance \ - && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ - && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ - && rm -rf ${HOME_DIR}/oss_compliance* - -RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt - -CMD ["/bin/bash"] +# ARG PYTHON=python3 +# ARG PIP=pip3 +# ARG PYTHON_VERSION=3.7.10 +# ARG MX_URL=https://aws-mx-pypi.s3-us-west-2.amazonaws.com/1.8.0/aws_mx-1.8.0-py2.py3-none-manylinux2014_x86_64.whl + +# # The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating +# # the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. +# ARG SMDEBUG_VERSION=0.9.4 +# ARG OPENSSL_VERSION=1.1.1k + +# ENV PYTHONDONTWRITEBYTECODE=1 \ +# PYTHONUNBUFFERED=1 \ +# LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ +# PYTHONIOENCODING=UTF-8 \ +# LANG=C.UTF-8 \ +# LC_ALL=C.UTF-8 \ +# SAGEMAKER_TRAINING_MODULE=sagemaker_mxnet_container.training:main \ +# DGLBACKEND=mxnet + +# RUN apt-get update \ +# && apt-get install -y --no-install-recommends \ +# software-properties-common \ +# build-essential \ +# ca-certificates \ +# curl \ +# emacs \ +# git \ +# libopencv-dev \ +# openssh-client \ +# openssh-server \ +# vim \ +# wget \ +# unzip \ +# zlib1g-dev \ +# libreadline-gplv2-dev \ +# libncursesw5-dev \ +# libssl-dev \ +# libsqlite3-dev \ +# libgdbm-dev \ +# libc6-dev \ +# libbz2-dev \ +# tk-dev \ +# libffi-dev \ +# cmake \ +# # Install dependent library for OpenCV +# libgtk2.0-dev \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/* + +# ########################################################################### +# # Horovod dependencies +# ########################################################################### + +# # Install Open MPI +# RUN mkdir /tmp/openmpi \ +# && cd /tmp/openmpi \ +# && wget -q https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.1.tar.gz \ +# && tar zxf openmpi-4.0.1.tar.gz \ +# && cd openmpi-4.0.1 \ +# && ./configure --enable-orterun-prefix-by-default \ +# && make -j $(nproc) all \ +# && make install \ +# && ldconfig \ +# && cd /tmp && rm -rf /tmp/openmpi + +# # Create a wrapper for OpenMPI to allow running as root by default +# RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real \ +# && echo '#!/bin/bash' > /usr/local/bin/mpirun \ +# && echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun \ +# && chmod a+x /usr/local/bin/mpirun \ +# && echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf \ +# && echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf + +# ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +# ENV PATH=/usr/local/bin:$PATH + +# # install OpenSSL +# RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ +# && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ +# && cd openssl-${OPENSSL_VERSION} \ +# && ./config && make -j $(nproc) && make install \ +# && ldconfig \ +# && cd .. && rm -rf openssl-* \ +# && rmdir /usr/local/ssl/certs \ +# && ln -s /etc/ssl/certs /usr/local/ssl/certs + +# # install Python +# RUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ +# && tar -xzf Python-$PYTHON_VERSION.tgz \ +# && cd Python-$PYTHON_VERSION \ +# && ./configure --enable-shared --prefix=/usr/local \ +# && make -j $(nproc) && make install \ +# && cd .. && rm -rf ../Python-$PYTHON_VERSION* \ +# && ln -s /usr/local/bin/pip3 /usr/bin/pip \ +# && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \ +# && ${PIP} --no-cache-dir install --upgrade \ +# pip \ +# setuptools + +# WORKDIR / + +# RUN ${PIP} install --no-cache --upgrade \ +# keras-mxnet==2.2.4.2 \ +# h5py==2.10.0 \ +# onnx==1.6.0 \ +# numpy==1.19.1 \ +# pandas==0.25.1 \ +# Pillow \ +# requests==2.22.0 \ +# scikit-learn==0.20.4 \ +# dgl==0.4.* \ +# scipy==1.2.2 \ +# gluonnlp==0.10.0 \ +# gluoncv==0.8.0 \ +# # Putting a cap in versions number to avoid potential issues with a new major version +# "urllib3>=1.25.10,<1.26.0" \ +# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli +# python-dateutil==2.8.0 \ +# tqdm==4.39.0 \ +# sagemaker-experiments==0.* \ +# # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli +# "PyYAML>=5.4,<5.5" \ +# mpi4py==3.0.2 \ +# sagemaker-mxnet-training \ +# ${MX_URL} \ +# smdebug==${SMDEBUG_VERSION} \ +# sagemaker \ +# awscli \ +# smclarify + +# # Install extra packages +# RUN pip install --no-cache-dir -U \ +# "bokeh>=2.3,<3" \ +# "imageio>=2.9,<3" \ +# "opencv-python>=4.3,<5" \ +# "plotly>=5.1,<6" \ +# "seaborn>=0.11,<1" \ +# "shap>=0.39,<1" + +# # Install Horovod +# RUN ${PIP} install --no-cache-dir horovod==0.19.5 + +# # Allow OpenSSH to talk to containers without asking for confirmation +# RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ +# && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ +# && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config + +# # OpenSHH config for MPI communication +# RUN mkdir -p /var/run/sshd && \ +# sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# RUN rm -rf /root/.ssh/ && \ +# mkdir -p /root/.ssh/ && \ +# ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ +# cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ +# && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config + +# # "channels first" is recommended for keras-mxnet +# # https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn +# RUN mkdir /root/.keras \ +# && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json + +# # This is here to make our installed version of OpenCV work. +# # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 +# # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? +# RUN ln -s /dev/null /dev/raw1394 + +# COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py + +# RUN chmod +x /usr/local/bin/deep_learning_container.py + +# RUN HOME_DIR=/root \ +# && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ +# && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ +# && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ +# && chmod +x /usr/local/bin/testOSSCompliance \ +# && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ +# && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ +# && rm -rf ${HOME_DIR}/oss_compliance* + +# RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt + +CMD ["echo hi"] diff --git a/src/image.py b/src/image.py index 3440dbc26d23..a0cd91a1614f 100644 --- a/src/image.py +++ b/src/image.py @@ -105,11 +105,16 @@ def build(self): print("within context") self.docker_build(fileobj=context_file, custom_context=True) self.context.remove() + elif self.stage == constants.SECOND_STAGE: + with open(self.dockerfile, "rb") as dockerfile_obj: + self.docker_build(fileobj=dockerfile_obj) else: print("out of context") self.docker_build() #check the size after image is built. self.image_size_check() + ## This return is necessary. Otherwise formatter fails while displaying the status. + return self.build_status def docker_build(self, fileobj=None, custom_context=False): response = [] @@ -124,7 +129,8 @@ def docker_build(self, fileobj=None, custom_context=False): labels=self.labels ): if line.get("error") is not None: - self.context.remove() + if self.context: + self.context.remove() response.append(line["error"]) self.log = response diff --git a/src/image_builder.py b/src/image_builder.py index 979e2aee8107..749385b8e571 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -157,6 +157,7 @@ def image_builder(buildspec): } #Create first stage docker object + print(f"[SHAN_TRIP] Image Config Dockerfile {image_config['docker_file']}") first_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], @@ -168,10 +169,13 @@ def image_builder(buildspec): ) #Create second stage docker object - if "example" not in image_name.lower() and build_context == "MAINLINE": + second_stage_image_object = None + # if "example" not in image_name.lower() and build_context == "MAINLINE": + ###### UNDO THIS CHANGE ######## + if "example" not in image_name.lower(): second_stage_image_object = DockerImage( info=info, - dockerfile=os.path.join(os.sep, "Dockerfile.multipart"), + dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], @@ -182,7 +186,8 @@ def image_builder(buildspec): FORMATTER.separator() FIRST_STAGES_IMAGES.append(first_stage_image_object) - SECOND_STAGES_IMAGES.append(second_stage_image_object) + if second_stage_image_object is not None: + SECOND_STAGES_IMAGES.append(second_stage_image_object) FORMATTER.banner("DLC") #FORMATTER.title("Status") @@ -194,7 +199,7 @@ def image_builder(buildspec): example_images = [image for image in FIRST_STAGES_IMAGES if "example" in image.name.lower()] #needs to be reconfigured - #ALL_IMAGES = first_stage_standard_images + second_stage_standard_images + example_images + ALL_IMAGES = list(set(first_stage_standard_images + second_stage_standard_images + example_images)) #first stage build FORMATTER.banner("First Stage Build") @@ -203,20 +208,17 @@ def image_builder(buildspec): """ Run safety on first stage image and store the ouput file locally """ - - FORMATTER.banner("Second Stage Build") - build_images(SECOND_STAGES_IMAGES) - + #second stage build - if second_stage_standard_images: + if len(second_stage_standard_images) > 0: FORMATTER.banner("Second Stage Build") build_images(second_stage_standard_images) - push_images(second_stage_standard_images) + # push_images(second_stage_standard_images) #example image build build_images(example_images) - push_images(example_images) + # push_images(example_images) #After the build, display logs/sumary for all the images. @@ -225,16 +227,16 @@ def image_builder(buildspec): is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) #change logic here. upload metrics only for the second stage image - upload_metrics(IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) + # upload_metrics(ALL_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) # Set environment variables to be consumed by test jobs - test_trigger_job = utils.get_codebuild_project_name() + # test_trigger_job = utils.get_codebuild_project_name() #needs to be configured to use final built images - utils.set_test_env( - IMAGES, - BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), - TEST_TRIGGER=test_trigger_job, - ) + # utils.set_test_env( + # ALL_IMAGES, + # BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), + # TEST_TRIGGER=test_trigger_job, + # ) def show_build_logs(images): diff --git a/src/test_build.py b/src/test_build.py new file mode 100644 index 000000000000..958fd7b0c1a3 --- /dev/null +++ b/src/test_build.py @@ -0,0 +1,12 @@ +from docker import APIClient +from docker import DockerClient + +client = APIClient(base_url="unix://var/run/docker.sock") + +folder_path = "/home/ubuntu/deep-learning-containers/src/" +dock_path = "/home/ubuntu/deep-learning-containers/src/Dockerfile.multipart" +build_args = {'FIRST_STAGE_IMAGE': '669063966089.dkr.ecr.us-west-2.amazonaws.com/beta-mxnet-training:1.8.0-cpu-py37-ubuntu16.04-2021-08-12-03-20-03'} + +with open(dock_path, "rb") as dockerfile_obj: + for line in client.build(fileobj=dockerfile_obj, path=folder_path, buildargs=build_args): + print(line) \ No newline at end of file From c5a20a26ceac599826699e5eabac1866a4acb176 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 12 Aug 2021 22:44:54 +0000 Subject: [PATCH 22/88] Made Context to properly run conclude stage Dockerfile --- mxnet/training/docker/1.8/py3/Dockerfile.cpu | 2 +- src/image_builder.py | 24 +++++++++++++++-- src/test_build.py | 27 +++++++++++++++++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/mxnet/training/docker/1.8/py3/Dockerfile.cpu b/mxnet/training/docker/1.8/py3/Dockerfile.cpu index 11e5932ec730..db5f9da7b9c5 100644 --- a/mxnet/training/docker/1.8/py3/Dockerfile.cpu +++ b/mxnet/training/docker/1.8/py3/Dockerfile.cpu @@ -185,4 +185,4 @@ LABEL dlc_major_version="1" # RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt -CMD ["echo hi"] +# CMD ["echo hi"] diff --git a/src/image_builder.py b/src/image_builder.py index 749385b8e571..9d2e01a73b54 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -157,7 +157,6 @@ def image_builder(buildspec): } #Create first stage docker object - print(f"[SHAN_TRIP] Image Config Dockerfile {image_config['docker_file']}") first_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], @@ -173,6 +172,7 @@ def image_builder(buildspec): # if "example" not in image_name.lower() and build_context == "MAINLINE": ###### UNDO THIS CHANGE ######## if "example" not in image_name.lower(): + conclude_stage_context = get_conclude_stage_context() second_stage_image_object = DockerImage( info=info, dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), @@ -180,7 +180,7 @@ def image_builder(buildspec): tag=image_tag, to_build=image_config["build"], stage=constants.SECOND_STAGE, - context=None, + context=conclude_stage_context, ) FORMATTER.separator() @@ -237,7 +237,27 @@ def image_builder(buildspec): # BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), # TEST_TRIGGER=test_trigger_job, # ) + +def get_conclude_stage_context(): + ARTIFACTS = {} + ARTIFACTS.update( + { + "safety_report": { + "source": f"safety_report.json", + "target": "safety_report.json" + } + }) + ARTIFACTS.update( + { + "dockerfile": { + "source": f"Dockerfile.multipart", + "target": "Dockerfile", + } + } + ) + artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" + return Context(ARTIFACTS, context_path=f'build/safety-json-file.tar.gz',artifact_root=artifact_root) def show_build_logs(images): diff --git a/src/test_build.py b/src/test_build.py index 958fd7b0c1a3..690e3cd3c4c9 100644 --- a/src/test_build.py +++ b/src/test_build.py @@ -9,4 +9,29 @@ with open(dock_path, "rb") as dockerfile_obj: for line in client.build(fileobj=dockerfile_obj, path=folder_path, buildargs=build_args): - print(line) \ No newline at end of file + print(line) + +docker_multipart_path = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart") +safety_report_path = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "safety_report.json") +os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" + +# import os +# from context import Context +# ARTIFACTS = {} +# ARTIFACTS.update({ +# "safety_report": { +# "source": f"safety_report.json", +# "target": "safety_report.json" +# } +# }) + +# ARTIFACTS.update( +# { +# "dockerfile": { +# "source": f"Dockerfile.multipart", +# "target": "Dockerfile", +# } +# } +# ) +# artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" +# Context(ARTIFACTS, context_path=f'build/safety-json-file.tar.gz',artifact_root=artifact_root) \ No newline at end of file From a5d4fb7427cabb0d6f538d7cbec7308012e74cc2 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 13 Aug 2021 00:10:15 +0000 Subject: [PATCH 23/88] Renaming to Conclusion_Stage --- src/constants.py | 2 +- src/image.py | 6 +----- src/image_builder.py | 40 ++++++++++++++++++++-------------------- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/constants.py b/src/constants.py index 18f1a676a1b7..99942d545350 100644 --- a/src/constants.py +++ b/src/constants.py @@ -33,7 +33,7 @@ #Docker build stages FIRST_STAGE="first" -SECOND_STAGE="second" +CONCLUSION_STAGE="second" # Docker connections DOCKER_URL = "unix://var/run/docker.sock" diff --git a/src/image.py b/src/image.py index a0cd91a1614f..ec723d714bd1 100644 --- a/src/image.py +++ b/src/image.py @@ -105,9 +105,6 @@ def build(self): print("within context") self.docker_build(fileobj=context_file, custom_context=True) self.context.remove() - elif self.stage == constants.SECOND_STAGE: - with open(self.dockerfile, "rb") as dockerfile_obj: - self.docker_build(fileobj=dockerfile_obj) else: print("out of context") self.docker_build() @@ -129,8 +126,7 @@ def docker_build(self, fileobj=None, custom_context=False): labels=self.labels ): if line.get("error") is not None: - if self.context: - self.context.remove() + self.context.remove() response.append(line["error"]) self.log = response diff --git a/src/image_builder.py b/src/image_builder.py index 9d2e01a73b54..b1d4ba10c969 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -52,8 +52,8 @@ def image_builder(buildspec): BUILDSPEC = Buildspec() BUILDSPEC.load(buildspec) - FIRST_STAGES_IMAGES = [] - SECOND_STAGES_IMAGES = [] + FIRST_STAGE_IMAGES = [] + CONCLUSION_STAGE_IMAGES = [] if "huggingface" in str(BUILDSPEC["framework"]): os.system("echo login into public ECR") @@ -87,7 +87,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(FIRST_STAGES_IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(FIRST_STAGE_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -167,39 +167,39 @@ def image_builder(buildspec): context=context, ) - #Create second stage docker object - second_stage_image_object = None + #Create Conclusion stage docker object + conclusion_stage_image_object = None # if "example" not in image_name.lower() and build_context == "MAINLINE": ###### UNDO THIS CHANGE ######## if "example" not in image_name.lower(): conclude_stage_context = get_conclude_stage_context() - second_stage_image_object = DockerImage( + conclusion_stage_image_object = DockerImage( info=info, dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], - stage=constants.SECOND_STAGE, + stage=constants.CONCLUSION_STAGE, context=conclude_stage_context, ) FORMATTER.separator() - FIRST_STAGES_IMAGES.append(first_stage_image_object) - if second_stage_image_object is not None: - SECOND_STAGES_IMAGES.append(second_stage_image_object) + FIRST_STAGE_IMAGES.append(first_stage_image_object) + if conclusion_stage_image_object is not None: + CONCLUSION_STAGE_IMAGES.append(conclusion_stage_image_object) FORMATTER.banner("DLC") #FORMATTER.title("Status") # Standard images must be built before example images # Example images will use standard images as base - first_stage_standard_images = [image for image in FIRST_STAGES_IMAGES if "example" not in image.name.lower()] - second_stage_standard_images = [image for image in SECOND_STAGES_IMAGES] + first_stage_standard_images = [image for image in FIRST_STAGE_IMAGES if "example" not in image.name.lower()] + conclusion_stage_standard_images = [image for image in CONCLUSION_STAGE_IMAGES] - example_images = [image for image in FIRST_STAGES_IMAGES if "example" in image.name.lower()] + example_images = [image for image in FIRST_STAGE_IMAGES if "example" in image.name.lower()] #needs to be reconfigured - ALL_IMAGES = list(set(first_stage_standard_images + second_stage_standard_images + example_images)) + ALL_IMAGES = list(set(first_stage_standard_images + conclusion_stage_standard_images + example_images)) #first stage build FORMATTER.banner("First Stage Build") @@ -209,12 +209,12 @@ def image_builder(buildspec): Run safety on first stage image and store the ouput file locally """ - #second stage build - if len(second_stage_standard_images) > 0: - FORMATTER.banner("Second Stage Build") - build_images(second_stage_standard_images) + #Conclusion stage build + if len(conclusion_stage_standard_images) > 0: + FORMATTER.banner("Conclusion Stage Build") + build_images(conclusion_stage_standard_images) - # push_images(second_stage_standard_images) + # push_images(conclusion_stage_standard_images) #example image build build_images(example_images) @@ -226,7 +226,7 @@ def image_builder(buildspec): show_build_summary(ALL_IMAGES) is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) - #change logic here. upload metrics only for the second stage image + #change logic here. upload metrics only for the Conclusion stage image # upload_metrics(ALL_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) # Set environment variables to be consumed by test jobs From b091ea0c8230a28d8cb521216c9ee4cf8704b573 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Mon, 16 Aug 2021 07:47:47 +0000 Subject: [PATCH 24/88] Created Safety Scan Reports --- safety_data.json | 1135 +++++++++++++++++++++++++++++++++++++++++++++ src/test_build.py | 37 -- 2 files changed, 1135 insertions(+), 37 deletions(-) create mode 100644 safety_data.json delete mode 100644 src/test_build.py diff --git a/safety_data.json b/safety_data.json new file mode 100644 index 000000000000..d9596d120ac0 --- /dev/null +++ b/safety_data.json @@ -0,0 +1,1135 @@ +[ + { + "package": "zope.interface", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "5.4.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "zope.event", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.5.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "zipp", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.5.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "wheel", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.36.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "werkzeug", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.0.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "wcwidth", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2.5", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "urllib3", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.26.6", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "typing-extensions", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.10.0.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "traitlets", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "5.0.5", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "tqdm", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.62.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "torchvision", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.10.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "torch", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.9.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "toml", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.10.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "threadpoolctl", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "tabulate", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.8.9", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "smdebug", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.0.9", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "smdebug-rulesconfig", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.0.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "smclarify", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "sklearn", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "six", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.16.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "setuptools", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "49.6.0.post20210108", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "scipy", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.7.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "scikit-learn", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.24.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "sagemaker", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.51.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "sagemaker-training", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.9.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "sagemaker-pytorch-training", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.4.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "sagemaker-experiments", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.1.34", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "safety", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.10.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "s3transfer", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.5.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "s3fs", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.4.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "ruamel-yaml-conda", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.15.100", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "rsa", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.7.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "retrying", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.3.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "requests", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.25.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyyaml", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "5.4.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pytz", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2021.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "python-dateutil", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.8.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pysocks", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.7.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyparsing", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.4.7", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyopenssl", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "19.1.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pynacl", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.4.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyinstrument", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.0.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pygments", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.7.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyfunctional", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.4.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pycparser", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.20", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pycosat", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.6.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyasn1", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.4.8", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pyarrow", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "5.0.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "ptyprocess", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.6.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "psutil", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "5.6.7", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "protobuf3-to-dict", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.1.5", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "protobuf", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.17.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "prompt-toolkit", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.0.8", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "ppft", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.6.6.4", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pox", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.3.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pip", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "21.2.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pillow", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "8.3.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pickleshare", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.7.5", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pexpect", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.8.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pathos", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2.8", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "parso", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.8.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "paramiko", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.7.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "pandas", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.2.4", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "packaging", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "21.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "numpy", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.19.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "networkx", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.6.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "multiprocess", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.70.12.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "joblib", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.0.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "jmespath", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.10.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "jedi", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.18.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "ipython", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "7.18.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "ipython-genutils", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "inotify-simple", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.2.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "importlib-metadata", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.6.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "idna", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2.10", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "h5py", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.2.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "greenlet", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.1.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "google-pasta", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "gevent", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "21.1.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "fsspec", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2021.7.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "dparse", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.5.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "docutils", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.15.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "dill", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.3.4", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "dgl", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.6.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "decorator", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.4.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "cython", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.29.21", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "cryptography", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.4.7", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "conda", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.10.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "conda-package-handling", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.7.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "colorama", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.4.3", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "click", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "8.0.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "chardet", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "4.0.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "cffi", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.14.6", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "certifi", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "2021.5.30", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "cached-property", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.5.2", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "brotlipy", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.7.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "botocore", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.21.14", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "boto3", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.18.14", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "bcrypt", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "3.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "backcall", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "awsio", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "0.0.1", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "awscli", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "1.20.14", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + }, + { + "package": "attrs", + "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", + "installed": "21.2.0", + "vulnerabilities": [ + { + "vid": "N/A", + "advisory": "N/A" + } + ] + } +] \ No newline at end of file diff --git a/src/test_build.py b/src/test_build.py deleted file mode 100644 index 690e3cd3c4c9..000000000000 --- a/src/test_build.py +++ /dev/null @@ -1,37 +0,0 @@ -from docker import APIClient -from docker import DockerClient - -client = APIClient(base_url="unix://var/run/docker.sock") - -folder_path = "/home/ubuntu/deep-learning-containers/src/" -dock_path = "/home/ubuntu/deep-learning-containers/src/Dockerfile.multipart" -build_args = {'FIRST_STAGE_IMAGE': '669063966089.dkr.ecr.us-west-2.amazonaws.com/beta-mxnet-training:1.8.0-cpu-py37-ubuntu16.04-2021-08-12-03-20-03'} - -with open(dock_path, "rb") as dockerfile_obj: - for line in client.build(fileobj=dockerfile_obj, path=folder_path, buildargs=build_args): - print(line) - -docker_multipart_path = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart") -safety_report_path = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "safety_report.json") -os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" - -# import os -# from context import Context -# ARTIFACTS = {} -# ARTIFACTS.update({ -# "safety_report": { -# "source": f"safety_report.json", -# "target": "safety_report.json" -# } -# }) - -# ARTIFACTS.update( -# { -# "dockerfile": { -# "source": f"Dockerfile.multipart", -# "target": "Dockerfile", -# } -# } -# ) -# artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" -# Context(ARTIFACTS, context_path=f'build/safety-json-file.tar.gz',artifact_root=artifact_root) \ No newline at end of file From 9674ebc5797a4c4a3a3dafa831a97bc1c9e5bd9e Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Mon, 16 Aug 2021 22:49:19 +0000 Subject: [PATCH 25/88] Added the generate_safety_report_for_image function in utils.py --- src/utils.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/utils.py b/src/utils.py index 15f9871a144c..a0c062570c88 100644 --- a/src/utils.py +++ b/src/utils.py @@ -23,6 +23,7 @@ from config import parse_dlc_developer_configs, is_build_enabled from invoke.context import Context from botocore.exceptions import ClientError +from dlc.safety_check import SafetyCheck LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) @@ -486,3 +487,38 @@ def set_test_env(images, images_env="DLC_IMAGES", **kwargs): def get_codebuild_project_name(): # Default value for codebuild project name is "local_test" when run outside of CodeBuild return os.getenv("CODEBUILD_BUILD_ID", "local_test").split(":")[0] + +def generate_safety_report_for_image(image_name, storage_file_path=None): + """ + Genereate safety scan reports for an image and store it at the location specified + :param image_name: str that consists of f"{image_repo}:{image_tag}" + :param storage_file_path: str that looks like "storage_location.json" + :return: safety report that looks like following + [ + { + "package": "package", + "affected": "version_spec", + "installed": "version", + "vulnerabilities": [ + { + "vid": "safety_vulnerability_id", + "advisory": "description of the issue" + }, + ... + ] + } + ... + ] + """ + ctx = Context() + docker_run_cmd = f"docker run -itd -v $PYTHONPATH/src:/src {image_name}" + container_id = ctx.run(f"{docker_run_cmd}").stdout.strip() + install_safety_cmd = "pip install safety" + ctx.run(f"docker exec {container_id} {install_safety_cmd}") + docker_exec_cmd = f"docker exec -i {container_id}" + run_output = SafetyCheck().run_safety_check_script_on_container(docker_exec_cmd) + json_formatted_output = json.loads(run_output.strip()) + if storage_file_path: + with open(storage_file_path, 'w', encoding='utf-8') as f: + json.dump(json_formatted_output, f, ensure_ascii=False, indent=4) + return json_formatted_output \ No newline at end of file From 2c43fb11586c5d81f59eb482535a4902990cca7b Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Tue, 17 Aug 2021 01:48:59 +0000 Subject: [PATCH 26/88] Added the report generation functionality in image class --- mxnet/training/docker/1.8/py3/Dockerfile.cpu | 170 ++++++++++--------- src/image.py | 37 ++++ 2 files changed, 123 insertions(+), 84 deletions(-) diff --git a/mxnet/training/docker/1.8/py3/Dockerfile.cpu b/mxnet/training/docker/1.8/py3/Dockerfile.cpu index db5f9da7b9c5..fdfe63d399b2 100644 --- a/mxnet/training/docker/1.8/py3/Dockerfile.cpu +++ b/mxnet/training/docker/1.8/py3/Dockerfile.cpu @@ -1,7 +1,9 @@ FROM ubuntu:16.04 +FROM python:3 -LABEL maintainer="Amazon AI" -LABEL dlc_major_version="1" + +# LABEL maintainer="Amazon AI" +# LABEL dlc_major_version="1" # ARG PYTHON=python3 # ARG PIP=pip3 @@ -104,85 +106,85 @@ LABEL dlc_major_version="1" # WORKDIR / -# RUN ${PIP} install --no-cache --upgrade \ -# keras-mxnet==2.2.4.2 \ -# h5py==2.10.0 \ -# onnx==1.6.0 \ -# numpy==1.19.1 \ -# pandas==0.25.1 \ -# Pillow \ -# requests==2.22.0 \ -# scikit-learn==0.20.4 \ -# dgl==0.4.* \ -# scipy==1.2.2 \ -# gluonnlp==0.10.0 \ -# gluoncv==0.8.0 \ -# # Putting a cap in versions number to avoid potential issues with a new major version -# "urllib3>=1.25.10,<1.26.0" \ -# # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli -# python-dateutil==2.8.0 \ -# tqdm==4.39.0 \ -# sagemaker-experiments==0.* \ -# # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli -# "PyYAML>=5.4,<5.5" \ -# mpi4py==3.0.2 \ -# sagemaker-mxnet-training \ -# ${MX_URL} \ -# smdebug==${SMDEBUG_VERSION} \ -# sagemaker \ -# awscli \ -# smclarify - -# # Install extra packages -# RUN pip install --no-cache-dir -U \ -# "bokeh>=2.3,<3" \ -# "imageio>=2.9,<3" \ -# "opencv-python>=4.3,<5" \ -# "plotly>=5.1,<6" \ -# "seaborn>=0.11,<1" \ -# "shap>=0.39,<1" - -# # Install Horovod -# RUN ${PIP} install --no-cache-dir horovod==0.19.5 - -# # Allow OpenSSH to talk to containers without asking for confirmation -# RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ -# && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ -# && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config - -# # OpenSHH config for MPI communication -# RUN mkdir -p /var/run/sshd && \ -# sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# RUN rm -rf /root/.ssh/ && \ -# mkdir -p /root/.ssh/ && \ -# ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ -# cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ -# && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config - -# # "channels first" is recommended for keras-mxnet -# # https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn -# RUN mkdir /root/.keras \ -# && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json - -# # This is here to make our installed version of OpenCV work. -# # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 -# # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? -# RUN ln -s /dev/null /dev/raw1394 - -# COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py - -# RUN chmod +x /usr/local/bin/deep_learning_container.py - -# RUN HOME_DIR=/root \ -# && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ -# && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ -# && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ -# && chmod +x /usr/local/bin/testOSSCompliance \ -# && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ -# && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ -# && rm -rf ${HOME_DIR}/oss_compliance* - -# RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt - -# CMD ["echo hi"] +# # RUN ${PIP} install --no-cache --upgrade \ +# # keras-mxnet==2.2.4.2 \ +# # h5py==2.10.0 \ +# # onnx==1.6.0 \ +# # numpy==1.19.1 \ +# # pandas==0.25.1 \ +# # Pillow \ +# # requests==2.22.0 \ +# # scikit-learn==0.20.4 \ +# # dgl==0.4.* \ +# # scipy==1.2.2 \ +# # gluonnlp==0.10.0 \ +# # gluoncv==0.8.0 \ +# # # Putting a cap in versions number to avoid potential issues with a new major version +# # "urllib3>=1.25.10,<1.26.0" \ +# # # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli +# # python-dateutil==2.8.0 \ +# # tqdm==4.39.0 \ +# # sagemaker-experiments==0.* \ +# # # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli +# # "PyYAML>=5.4,<5.5" \ +# # mpi4py==3.0.2 \ +# # sagemaker-mxnet-training \ +# # ${MX_URL} \ +# # smdebug==${SMDEBUG_VERSION} \ +# # sagemaker \ +# # awscli \ +# # smclarify + +# # # Install extra packages +# # RUN pip install --no-cache-dir -U \ +# # "bokeh>=2.3,<3" \ +# # "imageio>=2.9,<3" \ +# # "opencv-python>=4.3,<5" \ +# # "plotly>=5.1,<6" \ +# # "seaborn>=0.11,<1" \ +# # "shap>=0.39,<1" + +# # # Install Horovod +# # RUN ${PIP} install --no-cache-dir horovod==0.19.5 + +# # # Allow OpenSSH to talk to containers without asking for confirmation +# # RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ +# # && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ +# # && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config + +# # # OpenSHH config for MPI communication +# # RUN mkdir -p /var/run/sshd && \ +# # sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# # RUN rm -rf /root/.ssh/ && \ +# # mkdir -p /root/.ssh/ && \ +# # ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ +# # cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ +# # && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config + +# # # "channels first" is recommended for keras-mxnet +# # # https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn +# # RUN mkdir /root/.keras \ +# # && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json + +# # # This is here to make our installed version of OpenCV work. +# # # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 +# # # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? +# # RUN ln -s /dev/null /dev/raw1394 + +# # COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py + +# # RUN chmod +x /usr/local/bin/deep_learning_container.py + +# # RUN HOME_DIR=/root \ +# # && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ +# # && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ +# # && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ +# # && chmod +x /usr/local/bin/testOSSCompliance \ +# # && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ +# # && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ +# # && rm -rf ${HOME_DIR}/oss_compliance* + +# # RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt + +# # CMD ["echo hi"] diff --git a/src/image.py b/src/image.py index ec723d714bd1..ad5c7aeb4f15 100644 --- a/src/image.py +++ b/src/image.py @@ -17,7 +17,11 @@ from docker import APIClient from docker import DockerClient +from utils import generate_safety_report_for_image +from context import Context + import constants +import os class DockerImage: @@ -70,6 +74,36 @@ def collect_installed_packages_information(self): docker_client.containers.prune() return command_responses + def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conclusion-stage-file'): + ARTIFACTS = {} + ARTIFACTS.update( + { + "safety_report": { + "source": safety_report_path, + "target": "safety_report.json" + } + }) + ARTIFACTS.update( + { + "dockerfile": { + "source": f"Dockerfile.multipart", + "target": "Dockerfile", + } + } + ) + + artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" + return Context(ARTIFACTS, context_path=f'build/{tarfile_name}.tar.gz',artifact_root=artifact_root) + + + def pre_build_configuration_for_conclsion_stage(self): + ## Generate safety scan report for the first stage image and add the file to artifacts + first_stage_image_uri = self.build_args['FIRST_STAGE_IMAGE'] + processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') + storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{processed_image_uri}_safety_report.json" + generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) + self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=processed_image_uri) + def pre_build_configuration(self): if not self.to_build: @@ -89,6 +123,9 @@ def pre_build_configuration(self): if self.info.get("labels"): self.labels.update(self.info.get("labels")) + + if self.stage == constants.CONCLUSION_STAGE: + self.pre_build_configuration_for_conclsion_stage() print(f"self.build_args {self.build_args}") print(f"self.labels {self.labels}") From a83067d7ab552b639de41bcf119ffefc003e25b0 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Tue, 17 Aug 2021 08:26:01 +0000 Subject: [PATCH 27/88] Added ConclusionStage class --- src/conclusion_stage_image.py | 66 +++++++++++++++++++++++++++++++++++ src/image.py | 35 ------------------- src/image_builder.py | 26 ++------------ 3 files changed, 69 insertions(+), 58 deletions(-) create mode 100644 src/conclusion_stage_image.py diff --git a/src/conclusion_stage_image.py b/src/conclusion_stage_image.py new file mode 100644 index 000000000000..bfe3807d4226 --- /dev/null +++ b/src/conclusion_stage_image.py @@ -0,0 +1,66 @@ +""" +Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"). You +may not use this file except in compliance with the License. A copy of +the License is located at + + http://aws.amazon.com/apache2.0/ + +or in the "license" file accompanying this file. This file is +distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +ANY KIND, either express or implied. See the License for the specific +language governing permissions and limitations under the License. +""" + +from image import DockerImage +from context import Context +from utils import generate_safety_report_for_image + +import os + +class ConclusionStageImage(DockerImage): + """ + Class designed to handle the ConclusionStageImages + """ + + def pre_build_configuration(self): + """ + Conducts all the pre-build configurations from the parent class and then conducts + Safety Scan on the images generated in previous stage builds. The safety scan generates + the safety_report which is then baked into the image. + """ + ## Call the pre_build_configuration steps from the parent class + super(ConclusionStageImage, self).pre_build_configuration() + ## Generate safety scan report for the first stage image and add the file to artifacts + first_stage_image_uri = self.build_args['FIRST_STAGE_IMAGE'] + processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') + storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{processed_image_uri}_safety_report.json" + generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) + self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=processed_image_uri) + + def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conclusion-stage-file'): + """ + For ConclusionStageImage, build context is built once the safety report is generated. This is because + the Dockerfile.multipart uses this safety report to COPY the report into the image. + """ + ARTIFACTS = {} + ARTIFACTS.update( + { + "safety_report": { + "source": safety_report_path, + "target": "safety_report.json" + } + }) + ARTIFACTS.update( + { + "dockerfile": { + "source": f"Dockerfile.multipart", + "target": "Dockerfile", + } + } + ) + + artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" + return Context(ARTIFACTS, context_path=f'build/{tarfile_name}.tar.gz',artifact_root=artifact_root) + diff --git a/src/image.py b/src/image.py index ad5c7aeb4f15..24fbbe805be3 100644 --- a/src/image.py +++ b/src/image.py @@ -17,11 +17,8 @@ from docker import APIClient from docker import DockerClient -from utils import generate_safety_report_for_image -from context import Context import constants -import os class DockerImage: @@ -74,36 +71,6 @@ def collect_installed_packages_information(self): docker_client.containers.prune() return command_responses - def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conclusion-stage-file'): - ARTIFACTS = {} - ARTIFACTS.update( - { - "safety_report": { - "source": safety_report_path, - "target": "safety_report.json" - } - }) - ARTIFACTS.update( - { - "dockerfile": { - "source": f"Dockerfile.multipart", - "target": "Dockerfile", - } - } - ) - - artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" - return Context(ARTIFACTS, context_path=f'build/{tarfile_name}.tar.gz',artifact_root=artifact_root) - - - def pre_build_configuration_for_conclsion_stage(self): - ## Generate safety scan report for the first stage image and add the file to artifacts - first_stage_image_uri = self.build_args['FIRST_STAGE_IMAGE'] - processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') - storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{processed_image_uri}_safety_report.json" - generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) - self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=processed_image_uri) - def pre_build_configuration(self): if not self.to_build: @@ -124,8 +91,6 @@ def pre_build_configuration(self): if self.info.get("labels"): self.labels.update(self.info.get("labels")) - if self.stage == constants.CONCLUSION_STAGE: - self.pre_build_configuration_for_conclsion_stage() print(f"self.build_args {self.build_args}") print(f"self.labels {self.labels}") diff --git a/src/image_builder.py b/src/image_builder.py index b1d4ba10c969..3bb45bd54750 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -25,6 +25,7 @@ from context import Context from metrics import Metrics from image import DockerImage +from conclusion_stage_image import ConclusionStageImage from buildspec import Buildspec from output import OutputFormatter from config import parse_dlc_developer_configs @@ -172,15 +173,14 @@ def image_builder(buildspec): # if "example" not in image_name.lower() and build_context == "MAINLINE": ###### UNDO THIS CHANGE ######## if "example" not in image_name.lower(): - conclude_stage_context = get_conclude_stage_context() - conclusion_stage_image_object = DockerImage( + conclusion_stage_image_object = ConclusionStageImage( info=info, dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], stage=constants.CONCLUSION_STAGE, - context=conclude_stage_context, + context=None, ) FORMATTER.separator() @@ -238,26 +238,6 @@ def image_builder(buildspec): # TEST_TRIGGER=test_trigger_job, # ) -def get_conclude_stage_context(): - ARTIFACTS = {} - ARTIFACTS.update( - { - "safety_report": { - "source": f"safety_report.json", - "target": "safety_report.json" - } - }) - ARTIFACTS.update( - { - "dockerfile": { - "source": f"Dockerfile.multipart", - "target": "Dockerfile", - } - } - ) - - artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" - return Context(ARTIFACTS, context_path=f'build/safety-json-file.tar.gz',artifact_root=artifact_root) def show_build_logs(images): From bbf00601ede6f5e58ea22fd258639e68e6fe0c02 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 18 Aug 2021 01:41:33 +0000 Subject: [PATCH 28/88] Added logic to first genereate conclusion and then example. Also added dummy boto client creation. --- mxnet/buildspec.yml | 29 ++++++++++++++++++++++++++++- src/image_builder.py | 16 +++++++++++++--- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/mxnet/buildspec.yml b/mxnet/buildspec.yml index bf279d2de50a..54732d4ec79b 100644 --- a/mxnet/buildspec.yml +++ b/mxnet/buildspec.yml @@ -49,4 +49,31 @@ images: docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] context: <<: *TRAINING_CONTEXT - + BuildMXNetGPUTrainPy3DockerImage: + <<: *TRAINING_REPOSITORY + build: &MXNET_GPU_TRAINING_PY3 false + image_size_baseline: 7159 + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu110 + os_version: &OS_VERSION ubuntu16.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *TRAINING_CONTEXT + BuildMXNetExampleGPUTrainPy3DockerImage: + <<: *TRAINING_REPOSITORY + build: &MXNET_GPU_TRAINING_PY3 false + image_size_baseline: 7324 + base_image_name: BuildMXNetGPUTrainPy3DockerImage + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu110 + os_version: &OS_VERSION ubuntu16.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION, + "-example" ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /example, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *TRAINING_CONTEXT diff --git a/src/image_builder.py b/src/image_builder.py index 3bb45bd54750..61410a8623f8 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -88,7 +88,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(FIRST_STAGE_IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(CONCLUSION_STAGE_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -212,7 +212,7 @@ def image_builder(buildspec): #Conclusion stage build if len(conclusion_stage_standard_images) > 0: FORMATTER.banner("Conclusion Stage Build") - build_images(conclusion_stage_standard_images) + build_images(conclusion_stage_standard_images, make_dummy_boto_client=True) # push_images(conclusion_stage_standard_images) @@ -310,17 +310,27 @@ def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_s FORMATTER.separator() -def build_images(images): +def build_images(images, make_dummy_boto_client=False): THREADS = {} # In the context of the ThreadPoolExecutor each instance of image.build submitted # to it is executed concurrently in a separate thread. with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + if make_dummy_boto_client: + get_dummy_boto_client() for image in images: FORMATTER.print(f"image_object.context {image.context}") THREADS[image.name] = executor.submit(image.build) # the FORMATTER.progress(THREADS) function call also waits until all threads have completed FORMATTER.progress(THREADS) +def get_dummy_boto_client(): + # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. + # If this function is not added, boto3 fails because boto3 sessions are not thread safe. + # However, once a dummy client is created, it is ensured that the calls are thread safe. + # If the boto3 call made in the dlc package is changed to boto3.Session().client(), even then this issue can be resolved. + import boto3 + return boto3.client("secretsmanager", region_name=os.getenv('REGION')) + def push_images(images): THREADS = {} with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: From 9c168d5f070bfa91b16b784db118bf9b6cba0a16 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 18 Aug 2021 08:06:05 +0000 Subject: [PATCH 29/88] Added logic to build conclusion images at the very end --- src/image.py | 3 ++- src/image_builder.py | 60 ++++++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/image.py b/src/image.py index 24fbbe805be3..519139dd231d 100644 --- a/src/image.py +++ b/src/image.py @@ -27,7 +27,7 @@ class DockerImage: """ def __init__( - self, info, dockerfile, repository, tag, to_build, stage, context=None, + self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True ): # Meta-data about the image should go to info. @@ -41,6 +41,7 @@ def __init__( self.dockerfile = dockerfile self.context = context + self.to_push = to_push # TODO: Add ability to tag image with multiple tags self.repository = repository diff --git a/src/image_builder.py b/src/image_builder.py index 61410a8623f8..9e1228f834d9 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -88,7 +88,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(CONCLUSION_STAGE_IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(FIRST_STAGE_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -168,20 +168,13 @@ def image_builder(buildspec): context=context, ) - #Create Conclusion stage docker object - conclusion_stage_image_object = None + ##### Create Conclusion stage docker object ##### + # If we create a conclusion stage image, then we do not push the first stage image to ECR + # The only image that gets pushed is the conclusion stage image # if "example" not in image_name.lower() and build_context == "MAINLINE": ###### UNDO THIS CHANGE ######## - if "example" not in image_name.lower(): - conclusion_stage_image_object = ConclusionStageImage( - info=info, - dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), - repository=image_repo_uri, - tag=image_tag, - to_build=image_config["build"], - stage=constants.CONCLUSION_STAGE, - context=None, - ) + conclusion_stage_image_object = get_conclusion_stage_image_object(first_stage_image_object) + first_stage_image_object.to_push = False FORMATTER.separator() @@ -194,30 +187,30 @@ def image_builder(buildspec): # Standard images must be built before example images # Example images will use standard images as base - first_stage_standard_images = [image for image in FIRST_STAGE_IMAGES if "example" not in image.name.lower()] - conclusion_stage_standard_images = [image for image in CONCLUSION_STAGE_IMAGES] - + # Conclusion images must be built at the end as they will consume respective standard and example images + standard_images = [image for image in FIRST_STAGE_IMAGES if "example" not in image.name.lower()] example_images = [image for image in FIRST_STAGE_IMAGES if "example" in image.name.lower()] - #needs to be reconfigured - ALL_IMAGES = list(set(first_stage_standard_images + conclusion_stage_standard_images + example_images)) + conclusion_stage_images = [image for image in CONCLUSION_STAGE_IMAGES] + ALL_IMAGES = FIRST_STAGE_IMAGES + CONCLUSION_STAGE_IMAGES + IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push] #first stage build - FORMATTER.banner("First Stage Build") - build_images(first_stage_standard_images) + FORMATTER.banner("Standard Images Build") + build_images(standard_images) - """ - Run safety on first stage image and store the ouput file locally - """ + #example image build + FORMATTER.banner("Example Images Build") + build_images(example_images) #Conclusion stage build - if len(conclusion_stage_standard_images) > 0: + if len(conclusion_stage_images) > 0: FORMATTER.banner("Conclusion Stage Build") - build_images(conclusion_stage_standard_images, make_dummy_boto_client=True) + build_images(conclusion_stage_images, make_dummy_boto_client=True) # push_images(conclusion_stage_standard_images) #example image build - build_images(example_images) + # build_images(example_images) # push_images(example_images) #After the build, display logs/sumary for all the images. @@ -238,6 +231,17 @@ def image_builder(buildspec): # TEST_TRIGGER=test_trigger_job, # ) +def get_conclusion_stage_image_object(first_stage_image_object): + conclusion_stage_image_object = None + conclusion_stage_image_object = ConclusionStageImage( + info=first_stage_image_object.info, + dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), + repository=first_stage_image_object.repository, + tag=first_stage_image_object.tag, + to_build=first_stage_image_object.to_build, + stage=constants.CONCLUSION_STAGE, + ) + return conclusion_stage_image_object def show_build_logs(images): @@ -315,7 +319,8 @@ def build_images(images, make_dummy_boto_client=False): # In the context of the ThreadPoolExecutor each instance of image.build submitted # to it is executed concurrently in a separate thread. with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - if make_dummy_boto_client: + #### TODO: Remove this entire if block when https://github.com/boto/boto3/issues/1592 is resolved #### + if make_dummy_boto_client: get_dummy_boto_client() for image in images: FORMATTER.print(f"image_object.context {image.context}") @@ -323,6 +328,7 @@ def build_images(images, make_dummy_boto_client=False): # the FORMATTER.progress(THREADS) function call also waits until all threads have completed FORMATTER.progress(THREADS) +#### TODO: Remove this entire method when https://github.com/boto/boto3/issues/1592 is resolved #### def get_dummy_boto_client(): # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. # If this function is not added, boto3 fails because boto3 sessions are not thread safe. From 7eeac6f129f4e426f7643c4c3a1a707af926dcd2 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 18 Aug 2021 17:23:41 +0000 Subject: [PATCH 30/88] Fixed the pre build step --- src/image.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/image.py b/src/image.py index 519139dd231d..02345fb97900 100644 --- a/src/image.py +++ b/src/image.py @@ -74,12 +74,6 @@ def collect_installed_packages_information(self): def pre_build_configuration(self): - if not self.to_build: - self.log = ["Not built"] - self.build_status = constants.NOT_BUILT - self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] - return self.build_status - if self.info.get("base_image_uri"): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] @@ -91,7 +85,6 @@ def pre_build_configuration(self): if self.info.get("labels"): self.labels.update(self.info.get("labels")) - print(f"self.build_args {self.build_args}") print(f"self.labels {self.labels}") @@ -101,8 +94,19 @@ def build(self): The build function builds the specified docker image """ self.summary["start_time"] = datetime.now() + + ## Confirm if building the image is required or not + if not self.to_build: + self.log = ["Not built"] + self.build_status = constants.NOT_BUILT + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + return self.build_status + + ## Conduct some preprocessing before building the image self.pre_build_configuration() print(f"self.context {self.context}") + + ## Start building the image if self.context: with open(self.context.context_path, "rb") as context_file: print("within context") @@ -111,9 +115,11 @@ def build(self): else: print("out of context") self.docker_build() + #check the size after image is built. self.image_size_check() - ## This return is necessary. Otherwise formatter fails while displaying the status. + + ## This return is necessary. Otherwise FORMATTER fails while displaying the status. return self.build_status def docker_build(self, fileobj=None, custom_context=False): From e4ddf98adbdc51faf1263df6d9ff4272ff44f481 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 18 Aug 2021 20:27:59 +0000 Subject: [PATCH 31/88] Refactored First Stage to Initial Stage --- src/Dockerfile.multipart | 4 ++-- src/conclusion_stage_image.py | 2 +- src/constants.py | 4 ++-- src/image.py | 2 +- src/image_builder.py | 38 +++++++++++++++++------------------ 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/Dockerfile.multipart b/src/Dockerfile.multipart index 539b1cda4aef..3409c1cff874 100644 --- a/src/Dockerfile.multipart +++ b/src/Dockerfile.multipart @@ -1,7 +1,7 @@ # Use the Deep Learning Container as a base Image -ARG FIRST_STAGE_IMAGE="" +ARG INITIAL_STAGE_IMAGE="" -FROM $FIRST_STAGE_IMAGE +FROM $INITIAL_STAGE_IMAGE # Add any script or repo as required COPY safety_report.json /var/safety_report.json diff --git a/src/conclusion_stage_image.py b/src/conclusion_stage_image.py index bfe3807d4226..40497c3402a9 100644 --- a/src/conclusion_stage_image.py +++ b/src/conclusion_stage_image.py @@ -33,7 +33,7 @@ def pre_build_configuration(self): ## Call the pre_build_configuration steps from the parent class super(ConclusionStageImage, self).pre_build_configuration() ## Generate safety scan report for the first stage image and add the file to artifacts - first_stage_image_uri = self.build_args['FIRST_STAGE_IMAGE'] + first_stage_image_uri = self.build_args['INITIAL_STAGE_IMAGE'] processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{processed_image_uri}_safety_report.json" generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) diff --git a/src/constants.py b/src/constants.py index 99942d545350..aa046467220e 100644 --- a/src/constants.py +++ b/src/constants.py @@ -32,8 +32,8 @@ PADDING = 1 #Docker build stages -FIRST_STAGE="first" -CONCLUSION_STAGE="second" +INITIAL_STAGE="initial" +CONCLUSION_STAGE="conclusion" # Docker connections DOCKER_URL = "unix://var/run/docker.sock" diff --git a/src/image.py b/src/image.py index 02345fb97900..9c17dd35a604 100644 --- a/src/image.py +++ b/src/image.py @@ -78,7 +78,7 @@ def pre_build_configuration(self): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] if self.ecr_url: - self.build_args["FIRST_STAGE_IMAGE"] = self.ecr_url + self.build_args["INITIAL_STAGE_IMAGE"] = self.ecr_url if self.info.get("extra_build_args"): self.build_args.update(self.info.get("extra_build_args")) diff --git a/src/image_builder.py b/src/image_builder.py index 9e1228f834d9..4a68fa6c8808 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -53,7 +53,7 @@ def image_builder(buildspec): BUILDSPEC = Buildspec() BUILDSPEC.load(buildspec) - FIRST_STAGE_IMAGES = [] + INITIAL_STAGE_IMAGES = [] CONCLUSION_STAGE_IMAGES = [] if "huggingface" in str(BUILDSPEC["framework"]): @@ -88,7 +88,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(FIRST_STAGE_IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(INITIAL_STAGE_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -157,28 +157,28 @@ def image_builder(buildspec): "extra_build_args": extra_build_args } - #Create first stage docker object - first_stage_image_object = DockerImage( + #Create initial stage docker object + initial_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], - stage=constants.FIRST_STAGE, + stage=constants.INITIAL_STAGE, context=context, ) ##### Create Conclusion stage docker object ##### - # If we create a conclusion stage image, then we do not push the first stage image to ECR + # If we create a conclusion stage image, then we do not push the initial stage image to ECR # The only image that gets pushed is the conclusion stage image # if "example" not in image_name.lower() and build_context == "MAINLINE": ###### UNDO THIS CHANGE ######## - conclusion_stage_image_object = get_conclusion_stage_image_object(first_stage_image_object) - first_stage_image_object.to_push = False + conclusion_stage_image_object = get_conclusion_stage_image_object(initial_stage_image_object) + initial_stage_image_object.to_push = False FORMATTER.separator() - FIRST_STAGE_IMAGES.append(first_stage_image_object) + INITIAL_STAGE_IMAGES.append(initial_stage_image_object) if conclusion_stage_image_object is not None: CONCLUSION_STAGE_IMAGES.append(conclusion_stage_image_object) @@ -188,17 +188,17 @@ def image_builder(buildspec): # Standard images must be built before example images # Example images will use standard images as base # Conclusion images must be built at the end as they will consume respective standard and example images - standard_images = [image for image in FIRST_STAGE_IMAGES if "example" not in image.name.lower()] - example_images = [image for image in FIRST_STAGE_IMAGES if "example" in image.name.lower()] + standard_images = [image for image in INITIAL_STAGE_IMAGES if "example" not in image.name.lower()] + example_images = [image for image in INITIAL_STAGE_IMAGES if "example" in image.name.lower()] conclusion_stage_images = [image for image in CONCLUSION_STAGE_IMAGES] - ALL_IMAGES = FIRST_STAGE_IMAGES + CONCLUSION_STAGE_IMAGES + ALL_IMAGES = INITIAL_STAGE_IMAGES + CONCLUSION_STAGE_IMAGES IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push] - #first stage build + #initial stage standard images build FORMATTER.banner("Standard Images Build") build_images(standard_images) - #example image build + #initial stage example images build FORMATTER.banner("Example Images Build") build_images(example_images) @@ -231,14 +231,14 @@ def image_builder(buildspec): # TEST_TRIGGER=test_trigger_job, # ) -def get_conclusion_stage_image_object(first_stage_image_object): +def get_conclusion_stage_image_object(initial_stage_image_object): conclusion_stage_image_object = None conclusion_stage_image_object = ConclusionStageImage( - info=first_stage_image_object.info, + info=initial_stage_image_object.info, dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), - repository=first_stage_image_object.repository, - tag=first_stage_image_object.tag, - to_build=first_stage_image_object.to_build, + repository=initial_stage_image_object.repository, + tag=initial_stage_image_object.tag, + to_build=initial_stage_image_object.to_build, stage=constants.CONCLUSION_STAGE, ) return conclusion_stage_image_object From 096ff19dc0ef636c831f7c71b210318f752810ce Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 18 Aug 2021 21:36:21 +0000 Subject: [PATCH 32/88] Added the image push logic --- src/image_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/image_builder.py b/src/image_builder.py index 4a68fa6c8808..4f32560e4e1f 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -192,7 +192,7 @@ def image_builder(buildspec): example_images = [image for image in INITIAL_STAGE_IMAGES if "example" in image.name.lower()] conclusion_stage_images = [image for image in CONCLUSION_STAGE_IMAGES] ALL_IMAGES = INITIAL_STAGE_IMAGES + CONCLUSION_STAGE_IMAGES - IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push] + IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push and image.to_build] #initial stage standard images build FORMATTER.banner("Standard Images Build") @@ -207,7 +207,7 @@ def image_builder(buildspec): FORMATTER.banner("Conclusion Stage Build") build_images(conclusion_stage_images, make_dummy_boto_client=True) - # push_images(conclusion_stage_standard_images) + push_images(IMAGES_TO_PUSH) #example image build # build_images(example_images) From c629e9b40ca792ad05b780780f277f2a517b5c5f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 19 Aug 2021 01:15:42 +0000 Subject: [PATCH 33/88] Added the working for upload metrics --- src/image.py | 5 +++++ src/image_builder.py | 20 +++++++++----------- src/metrics.py | 2 ++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/image.py b/src/image.py index 9c17dd35a604..e475ec7ebcc2 100644 --- a/src/image.py +++ b/src/image.py @@ -116,6 +116,11 @@ def build(self): print("out of context") self.docker_build() + if not self.to_push: + ## If this image is not supposed to be pushed, in that case, we are already done + ## with building the image and do not need to conduct any further processing. + self.summary["end_time"] = datetime.now() + #check the size after image is built. self.image_size_check() diff --git a/src/image_builder.py b/src/image_builder.py index 4f32560e4e1f..795edfd3b11e 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -195,32 +195,30 @@ def image_builder(buildspec): IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push and image.to_build] #initial stage standard images build - FORMATTER.banner("Standard Images Build") + FORMATTER.banner("Standard Build") build_images(standard_images) #initial stage example images build - FORMATTER.banner("Example Images Build") + FORMATTER.banner("Example Build") build_images(example_images) #Conclusion stage build if len(conclusion_stage_images) > 0: - FORMATTER.banner("Conclusion Stage Build") + FORMATTER.banner("Conclusion Build") build_images(conclusion_stage_images, make_dummy_boto_client=True) push_images(IMAGES_TO_PUSH) - #example image build - # build_images(example_images) - # push_images(example_images) - - #After the build, display logs/sumary for all the images. - + #After the build, display logs/summary for all the images. show_build_logs(ALL_IMAGES) show_build_summary(ALL_IMAGES) is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) - #change logic here. upload metrics only for the Conclusion stage image - # upload_metrics(ALL_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) + #From all images, filter the images that were supposed to be built and upload their metrics + BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] + + # change logic here. upload metrics only for the Conclusion stage image + upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) # Set environment variables to be consumed by test jobs # test_trigger_job = utils.get_codebuild_project_name() diff --git a/src/metrics.py b/src/metrics.py index c9141c2ecd0d..7741a94fa11d 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -40,6 +40,8 @@ def push_image_metrics(self, image): "device_type": image.device_type, "python_version": image.python_version, "image_type": image.image_type, + "image_stage": image.stage, + "push_required": str(image.to_push) } if image.build_status == constants.NOT_BUILT: return None From 549d1da61b101ee40633bb7940ea610cee0ad211 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 19 Aug 2021 08:36:24 +0000 Subject: [PATCH 34/88] Added the set_test_env portion --- src/image_builder.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/image_builder.py b/src/image_builder.py index 795edfd3b11e..82b15a1aba1e 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -169,12 +169,10 @@ def image_builder(buildspec): ) ##### Create Conclusion stage docker object ##### - # If we create a conclusion stage image, then we do not push the initial stage image to ECR - # The only image that gets pushed is the conclusion stage image - # if "example" not in image_name.lower() and build_context == "MAINLINE": - ###### UNDO THIS CHANGE ######## + # If for an initial stage image we create a conclusion stage image, then we do not push the initial stage image + # to the repository. Instead, we just push its conclusion stage image to the repository. Therefore, + # inside function get_conclusion_stage_image_object we make initial_stage_image_object non pushable. conclusion_stage_image_object = get_conclusion_stage_image_object(initial_stage_image_object) - initial_stage_image_object.to_push = False FORMATTER.separator() @@ -221,15 +219,17 @@ def image_builder(buildspec): upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) # Set environment variables to be consumed by test jobs - # test_trigger_job = utils.get_codebuild_project_name() - #needs to be configured to use final built images - # utils.set_test_env( - # ALL_IMAGES, - # BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), - # TEST_TRIGGER=test_trigger_job, - # ) + test_trigger_job = utils.get_codebuild_project_name() + # Tests should only run on images that were pushed to the repository + utils.set_test_env( + IMAGES_TO_PUSH, + BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), + TEST_TRIGGER=test_trigger_job, + ) def get_conclusion_stage_image_object(initial_stage_image_object): + # Check if this is only required for mainline + # if build_context == "MAINLINE": conclusion_stage_image_object = None conclusion_stage_image_object = ConclusionStageImage( info=initial_stage_image_object.info, @@ -239,6 +239,7 @@ def get_conclusion_stage_image_object(initial_stage_image_object): to_build=initial_stage_image_object.to_build, stage=constants.CONCLUSION_STAGE, ) + initial_stage_image_object.to_push = False return conclusion_stage_image_object def show_build_logs(images): From 4313a84e2923fb6e87dbf051c8db47a8360d9df9 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 19 Aug 2021 18:28:43 +0000 Subject: [PATCH 35/88] Resolved the same name tarfile conflict --- src/conclusion_stage_image.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/conclusion_stage_image.py b/src/conclusion_stage_image.py index 40497c3402a9..56b93034c5fd 100644 --- a/src/conclusion_stage_image.py +++ b/src/conclusion_stage_image.py @@ -35,9 +35,11 @@ def pre_build_configuration(self): ## Generate safety scan report for the first stage image and add the file to artifacts first_stage_image_uri = self.build_args['INITIAL_STAGE_IMAGE'] processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') - storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{processed_image_uri}_safety_report.json" + image_name = self.name + tarfile_name_for_context = f"{processed_image_uri}-{image_name}" + storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{tarfile_name_for_context}_safety_report.json" generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) - self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=processed_image_uri) + self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conclusion-stage-file'): """ From 372bfae9f1e4209e63aacd2852cf9e029436ad99 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 20 Aug 2021 05:28:39 +0000 Subject: [PATCH 36/88] Added the safety tests --- test/dlc_tests/sanity/test_safety_check_v2.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 test/dlc_tests/sanity/test_safety_check_v2.py diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py new file mode 100644 index 000000000000..a3076611755c --- /dev/null +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -0,0 +1,102 @@ +import json +import logging +import sys + +import pytest + +from invoke import run +from dataclasses import dataclass +from typing import List + +from test.test_utils import ( + CONTAINER_TESTS_PREFIX, is_dlc_cicd_context, is_canary_context, is_mainline_context, is_time_for_canary_safety_scan +) + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) +LOGGER.addHandler(logging.StreamHandler(sys.stderr)) + +SAFETY_FILE = '/var/safety_report.json' +SAFETY_FILE_EXISTS = 0 + + +@dataclass +class SafetyVulnerabilityAdvisory: + vid: str + advisory: str + + +@dataclass +class SafetyPackageVulnerabilityReport: + package: str + affected: str + installed: str + vulnerabilities: List[SafetyVulnerabilityAdvisory] + + def __post_init__(self): + self.vulnerabilities = [SafetyVulnerabilityAdvisory(**i) for i in self.vulnerabilities] + + +@dataclass +class SafetyPythonEnvironmentVulnerabilityReport: + report: List[SafetyPackageVulnerabilityReport] + + def __post_init__(self): + self.report = [SafetyPackageVulnerabilityReport(**i) for i in self.report] + + + +@pytest.mark.model("N/A") +@pytest.mark.canary("Run safety tests regularly on production images") +# @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") +def test_safety_file_exists(image): + repo_name, image_tag = image.split('/')[-1].split(':') + container_name = f"{repo_name}-{image_tag}-safety" + # Add null entrypoint to ensure command exits immediately + run(f"docker run -id " + f"--name {container_name} " + f"--entrypoint='/bin/bash' " + f"{image}", hide=True) + + try: + # Check if file exists + docker_exec_cmd = f"docker exec -i {container_name}" + safety_file_check = run(f"{docker_exec_cmd} test -f {SAFETY_FILE}", warn=True, hide=True) + assert safety_file_check.return_code == SAFETY_FILE_EXISTS, f"Safety file existence test failed for {image}" + + file_content = run(f"{docker_exec_cmd} cat {SAFETY_FILE}", warn=True, hide=True) + raw_scan_result = json.loads(file_content.stdout) + scan_results = [] + scan_results.append( + SafetyPythonEnvironmentVulnerabilityReport( + report=raw_scan_result + ) + ) + + ignored_packages = [] + + # processing safety reports + report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [affected: {affected}] [reason: {reason}] [env: {env}]" + failed_count = 0 + for result in scan_results: + for report_item in result.report: + if "PASSED_SAFETY_CHECK" not in report_item.affected: + if (report_item.package not in ignored_packages) or ( + report_item.package in ignored_packages + and report_item.installed != ignored_packages[report_item.package] + ): + failed_count += 1 + print( + report_log_template.format( + status="FAILED", + pkg=report_item.package, + installed=report_item.installed, + affected=report_item.affected, + reason=None, + env=result.environment, + ) + ) + assert failed_count == 0, f"Found {failed_count} vulnerabilities. Safety check failed! for {image}" + LOGGER.info(f"Safety check is complete as a part of docker build and report exist at {SAFETY_FILE}") + finally: + run(f"docker rm -f {container_name}", hide=True) From cc79be73f4e6259d59ddb70b7d9e0a3a7cfad1a1 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 20 Aug 2021 18:22:28 +0000 Subject: [PATCH 37/88] Added reason to ignore field in safety_check_v2 --- src/safety_check_v2.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/safety_check_v2.py b/src/safety_check_v2.py index cc763294581f..6d03ab451c58 100644 --- a/src/safety_check_v2.py +++ b/src/safety_check_v2.py @@ -41,10 +41,19 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( "--key", - required=True, + required=False, + ) + parser.add_argument( + "--ignore_dict_str", + required=False, ) args = parser.parse_args() + + safety_key = args.key + ignore_dict = {} + if args.ignore_dict_str is not None: + ignore_dict = json.loads(args.ignore_dict_str) if Safety.__version__ == "1.8.7": proxy_dictionary = {} @@ -76,15 +85,18 @@ def main(): installed = v.version advisory = v.advisory vid = v.vuln_id + vulnerability_details = {"vid": vid, "advisory": advisory, "reason_to_ignore":"N/A"} + if vid in ignore_dict: + vulnerability_details["reason_to_ignore"] = ignore_dict[vid] if package not in vulns_dict: vulns_dict[package] = { "package": package, "affected": affected, "installed": installed, - "vulnerabilities": [{"vid": vid, "advisory": advisory}], + "vulnerabilities": [vulnerability_details], } else: - vulns_dict[package]["vulnerabilities"].append({"vid": vid, "advisory": advisory}) + vulns_dict[package]["vulnerabilities"].append(vulnerability_details) # Report for safe packages timestamp = datetime.now().strftime("%d%m%Y") @@ -94,7 +106,7 @@ def main(): "package": pkg.key, "affected": f"No known vulnerability found, PASSED_SAFETY_CHECK on {timestamp}.", "installed": pkg.version, - "vulnerabilities": [{"vid": "N/A", "advisory": "N/A"}], + "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A"}], } for (k, v) in vulns_dict.items(): From 36d822e1a089f1a66cc68527aa6d6ed01bcb668b Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 20 Aug 2021 20:13:58 +0000 Subject: [PATCH 38/88] Added the new safety_check_v2 script --- src/safety_check_v2.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/safety_check_v2.py b/src/safety_check_v2.py index 6d03ab451c58..91bab1e5b746 100644 --- a/src/safety_check_v2.py +++ b/src/safety_check_v2.py @@ -19,6 +19,7 @@ import safety as Safety from safety import safety +TO_BE_DECIDED="To Be Decided" def main(): """Return json.dumps() string of below list structure. @@ -78,20 +79,28 @@ def main(): # Generate output vulns_dict = {} vulns_list = [] + ignored_vulnerability_count = {} # Report for unsafe packages + # "FAILED", "SUCCEEDED" , "IGNORED" for v in vulns: package = v.name - affected = v.spec + spec = v.spec installed = v.version advisory = v.advisory vid = v.vuln_id - vulnerability_details = {"vid": vid, "advisory": advisory, "reason_to_ignore":"N/A"} + vulnerability_details = {"vid": vid, "advisory": advisory, "spec": spec, "reason_to_ignore":"N/A"} + + if package not in ignored_vulnerability_count: + ignored_vulnerability_count[package] = 0 + if vid in ignore_dict: vulnerability_details["reason_to_ignore"] = ignore_dict[vid] + ignored_vulnerability_count[package] += 1 + if package not in vulns_dict: vulns_dict[package] = { "package": package, - "affected": affected, + "scan_status": TO_BE_DECIDED, "installed": installed, "vulnerabilities": [vulnerability_details], } @@ -104,12 +113,17 @@ def main(): if pkg.key not in vulns_dict: vulns_dict[pkg.key] = { "package": pkg.key, - "affected": f"No known vulnerability found, PASSED_SAFETY_CHECK on {timestamp}.", + "scan_status": "SUCCEEDED", "installed": pkg.version, - "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A"}], + "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A", "spec":"N/A"}], } - + for (k, v) in vulns_dict.items(): + if v["scan_status"] == TO_BE_DECIDED: + if len(v["vulnerabilities"]) == ignored_vulnerability_count[k]: + v["scan_status"] = "IGNORED" + else: + v["scan_status"] = "FAILED" vulns_list.append(v) print(json.dumps(vulns_list)) From 4c6d3795c53428a56c1f273adeda7cc95edb8f22 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 20 Aug 2021 23:19:31 +0000 Subject: [PATCH 39/88] Modified the tests to run on new safety report --- test/dlc_tests/sanity/test_safety_check_v2.py | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index a3076611755c..9f0ad96e8ff9 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -24,12 +24,14 @@ class SafetyVulnerabilityAdvisory: vid: str advisory: str + reason_to_ignore: str + spec: str @dataclass class SafetyPackageVulnerabilityReport: package: str - affected: str + scan_status: str installed: str vulnerabilities: List[SafetyVulnerabilityAdvisory] @@ -73,30 +75,22 @@ def test_safety_file_exists(image): ) ) - ignored_packages = [] - # processing safety reports - report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [affected: {affected}] [reason: {reason}] [env: {env}]" + report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [vulnerabilities: {vulnerabilities}]" failed_count = 0 for result in scan_results: for report_item in result.report: - if "PASSED_SAFETY_CHECK" not in report_item.affected: - if (report_item.package not in ignored_packages) or ( - report_item.package in ignored_packages - and report_item.installed != ignored_packages[report_item.package] - ): + if report_item.scan_status == "FAILED": failed_count += 1 - print( + LOGGER.info( report_log_template.format( status="FAILED", pkg=report_item.package, installed=report_item.installed, - affected=report_item.affected, - reason=None, - env=result.environment, + vulnerabilities = report_item.vulnerabilities, ) ) - assert failed_count == 0, f"Found {failed_count} vulnerabilities. Safety check failed! for {image}" - LOGGER.info(f"Safety check is complete as a part of docker build and report exist at {SAFETY_FILE}") + assert failed_count == 0, f"{failed_count} package/s failed safety test for {image} !!!" + LOGGER.info(f"Safety check is successfully complete and report exists at {SAFETY_FILE}") finally: run(f"docker rm -f {container_name}", hide=True) From 89666a649b637920932a094192caf6f1bda2316f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 01:04:52 +0000 Subject: [PATCH 40/88] Added the ignore dict logic with the data --- data/ignore_ids_safety_scan.json | 137 +++++++++++++++++++++++++++++++ src/utils.py | 32 +++++++- 2 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 data/ignore_ids_safety_scan.json diff --git a/data/ignore_ids_safety_scan.json b/data/ignore_ids_safety_scan.json new file mode 100644 index 000000000000..ee60247aef01 --- /dev/null +++ b/data/ignore_ids_safety_scan.json @@ -0,0 +1,137 @@ +{ + "tensorflow": { + "training": { + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2", + "35015":"for shipping pycrypto<=2.6.1 - the last available version for py2" + }, + "py3": { + } + }, + "inference":{ + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2" + }, + "py3": { + } + }, + "inference-eia": { + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2" + }, + "py3": { + } + }, + "inference-neuron":{ + "_comment":"py2 is deprecated", + "py2": { + }, + "py3": { + "39409":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39408":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39407":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39406":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches" + } + } + }, + "mxnet": { + "training": { + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2" + }, + "py3": { + } + }, + "inference": { + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2" + }, + "py3": { + } + }, + "inference-eia": { + "_comment":"py2 is deprecated", + "py2": { + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2", + "36810":"numpy<=1.16.0 -- This has to only be here while we publish MXNet 1.4.1 EI DLC v1.0" + }, + "py3": { + } + }, + "inference-neuron": { + "_comment":"py2 is deprecated", + "py2": { + }, + "py3": { + "40673":"for shipping tensorflow 1.15.5", + "40675":"for shipping tensorflow 1.15.5", + "40676":"for shipping tensorflow 1.15.5", + "40794":"for shipping tensorflow 1.15.5", + "40795":"for shipping tensorflow 1.15.5", + "40796":"for shipping tensorflow 1.15.5" + } + } + }, + "pytorch": { + "training": { + "_comment":"py2 is deprecated", + "py2": { + "35810":"for astropy<3.0.1", + "38449":"for shipping pillow<=6.2.2 - the last available version for py2", + "38450":"for shipping pillow<=6.2.2 - the last available version for py2", + "38451":"for shipping pillow<=6.2.2 - the last available version for py2", + "38452":"for shipping pillow<=6.2.2 - the last available version for py2" + }, + "py3": { + } + }, + "inference": { + "_comment":"py2 is deprecated", + "py2": { + }, + "py3": { + } + }, + "inference-eia": { + "_comment":"py2 is deprecated", + "py2": { + }, + "py3": { + } + }, + "inference-neuron": { + "_comment":"py2 is deprecated", + "py2": { + }, + "py3": { + "39409":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39408":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39407":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches", + "39406":"TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches" + } + } + } +} \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index a0c062570c88..88999680f93c 100644 --- a/src/utils.py +++ b/src/utils.py @@ -488,7 +488,30 @@ def get_codebuild_project_name(): # Default value for codebuild project name is "local_test" when run outside of CodeBuild return os.getenv("CODEBUILD_BUILD_ID", "local_test").split(":")[0] -def generate_safety_report_for_image(image_name, storage_file_path=None): +def get_safety_ignore_dict(image_uri): + """ + Get a dict of known safety check issue IDs to ignore, if specified in file ../data/ignore_ids_safety_scan.json. + :param image_uri: + :return: list of safety check IDs to ignore + """ + framework = ("mxnet" if "mxnet" in image_uri else + "pytorch" if "pytorch" in image_uri else + "tensorflow") + job_type = ("training" if "training" in image_uri else + "inference-eia" if "eia" in image_uri else + "inference-neuron" if "neuron" in image_uri else + "inference") + python_version = "py2" if "py2" in image_uri else "py3" + + IGNORE_SAFETY_IDS = {} + with open(f'{os.getenv("PYTHONPATH")}/data/ignore_ids_safety_scan.json') as f: + IGNORE_SAFETY_IDS = json.load(f) + + return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, {}) + + + +def generate_safety_report_for_image(image_uri, storage_file_path=None): """ Genereate safety scan reports for an image and store it at the location specified :param image_name: str that consists of f"{image_repo}:{image_tag}" @@ -511,14 +534,15 @@ def generate_safety_report_for_image(image_name, storage_file_path=None): ] """ ctx = Context() - docker_run_cmd = f"docker run -itd -v $PYTHONPATH/src:/src {image_name}" + docker_run_cmd = f"docker run -itd -v $PYTHONPATH/src:/src {image_uri}" container_id = ctx.run(f"{docker_run_cmd}").stdout.strip() install_safety_cmd = "pip install safety" ctx.run(f"docker exec {container_id} {install_safety_cmd}") docker_exec_cmd = f"docker exec -i {container_id}" - run_output = SafetyCheck().run_safety_check_script_on_container(docker_exec_cmd) + ignore_dict = get_safety_ignore_dict(image_uri) + run_output = SafetyCheck().run_safety_check_script_on_container(docker_exec_cmd, ignore_dict_str=json.dumps(ignore_dict)) json_formatted_output = json.loads(run_output.strip()) if storage_file_path: with open(storage_file_path, 'w', encoding='utf-8') as f: - json.dump(json_formatted_output, f, ensure_ascii=False, indent=4) + json.dump(json_formatted_output, f, indent=4) return json_formatted_output \ No newline at end of file From 800a5b878efa2d3cf83bfdc03730141a817cb98a Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 01:20:16 +0000 Subject: [PATCH 41/88] Refactored PYTHONPATH to ROOT_FOLDER_PATH --- buildspec.yml | 1 + src/conclusion_stage_image.py | 4 ++-- src/image_builder.py | 2 +- src/utils.py | 10 ++++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index fbfe7483ae54..8585fb32a055 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -12,6 +12,7 @@ phases: - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) - pip install -r src/requirements.txt - bash src/setup.sh $FRAMEWORK + - export ROOT_FOLDER_PATH=$(pwd) - python src/parse_partner_developers.py build: commands: diff --git a/src/conclusion_stage_image.py b/src/conclusion_stage_image.py index 56b93034c5fd..ad03407fdbdb 100644 --- a/src/conclusion_stage_image.py +++ b/src/conclusion_stage_image.py @@ -37,7 +37,7 @@ def pre_build_configuration(self): processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" - storage_file_path = f"{os.getenv('PYTHONPATH')}/src/{tarfile_name_for_context}_safety_report.json" + storage_file_path = f"{os.getenv('ROOT_FOLDER_PATH')}/src/{tarfile_name_for_context}_safety_report.json" generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) @@ -63,6 +63,6 @@ def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conc } ) - artifact_root = os.path.join(os.sep, os.getenv("PYTHONPATH"), "src") + "/" + artifact_root = os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src") + "/" return Context(ARTIFACTS, context_path=f'build/{tarfile_name}.tar.gz',artifact_root=artifact_root) diff --git a/src/image_builder.py b/src/image_builder.py index 82b15a1aba1e..4b494a1f1e23 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -233,7 +233,7 @@ def get_conclusion_stage_image_object(initial_stage_image_object): conclusion_stage_image_object = None conclusion_stage_image_object = ConclusionStageImage( info=initial_stage_image_object.info, - dockerfile=os.path.join(os.sep, os.getenv("PYTHONPATH"), "src", "Dockerfile.multipart"), + dockerfile=os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src", "Dockerfile.multipart"), repository=initial_stage_image_object.repository, tag=initial_stage_image_object.tag, to_build=initial_stage_image_object.to_build, diff --git a/src/utils.py b/src/utils.py index 88999680f93c..001b408758ed 100644 --- a/src/utils.py +++ b/src/utils.py @@ -504,7 +504,7 @@ def get_safety_ignore_dict(image_uri): python_version = "py2" if "py2" in image_uri else "py3" IGNORE_SAFETY_IDS = {} - with open(f'{os.getenv("PYTHONPATH")}/data/ignore_ids_safety_scan.json') as f: + with open(f'{os.getenv("ROOT_FOLDER_PATH")}/data/ignore_ids_safety_scan.json') as f: IGNORE_SAFETY_IDS = json.load(f) return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, {}) @@ -520,12 +520,14 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): [ { "package": "package", - "affected": "version_spec", + "scan_status": "SUCCEEDED/FAILED/IGNORED", "installed": "version", "vulnerabilities": [ { "vid": "safety_vulnerability_id", - "advisory": "description of the issue" + "advisory": "description of the issue", + "spec": "version_spec", + "reason_to_ignore": "Either N/A or the reason fetched from data/ignore_ids_safety_scan.json" }, ... ] @@ -534,7 +536,7 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): ] """ ctx = Context() - docker_run_cmd = f"docker run -itd -v $PYTHONPATH/src:/src {image_uri}" + docker_run_cmd = f"docker run -itd -v $ROOT_FOLDER_PATH/src:/src {image_uri}" container_id = ctx.run(f"{docker_run_cmd}").stdout.strip() install_safety_cmd = "pip install safety" ctx.run(f"docker exec {container_id} {install_safety_cmd}") From a8cea0092002c07b3580350efa85886f2dfce06e Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:16:37 +0000 Subject: [PATCH 42/88] Refactored comments in safety_check_v2.py --- src/safety_check_v2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/safety_check_v2.py b/src/safety_check_v2.py index 91bab1e5b746..fa4ade7e69ab 100644 --- a/src/safety_check_v2.py +++ b/src/safety_check_v2.py @@ -26,12 +26,14 @@ def main(): [ { "package": "package", - "affected": "version_spec", + "scan_status": "SUCCEEDED/FAILED/IGNORED", "installed": "version", "vulnerabilities": [ { "vid": "safety_vulnerability_id", - "advisory": "description of the issue" + "advisory": "description of the issue", + "reason_to_ignore":"reason to ignore the vid", + "spec": "version_spec" }, ... ] From db5f9a4b4694f8de360eea4cebde1a009e8f2b32 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:19:25 +0000 Subject: [PATCH 43/88] Deleted src/test_and_generate_python_safety_report.py --- src/test_and_generate_python_safety_report.py | 98 ------------------- 1 file changed, 98 deletions(-) delete mode 100644 src/test_and_generate_python_safety_report.py diff --git a/src/test_and_generate_python_safety_report.py b/src/test_and_generate_python_safety_report.py deleted file mode 100644 index 4daeb5256bae..000000000000 --- a/src/test_and_generate_python_safety_report.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import json -from dataclasses import dataclass -from typing import List -import boto3 -from botocore.exceptions import ClientError -import base64 -import argparse -import subprocess - -@dataclass -class SafetyVulnerabilityAdvisory: - vid: str - advisory: str - - -@dataclass -class SafetyPackageVulnerabilityReport: - package: str - affected: str - installed: str - vulnerabilities: List[SafetyVulnerabilityAdvisory] - - def __post_init__(self): - self.vulnerabilities = [SafetyVulnerabilityAdvisory(**i) for i in self.vulnerabilities] - - -@dataclass -class SafetyPythonEnvironmentVulnerabilityReport: - report: List[SafetyPackageVulnerabilityReport] - - def __post_init__(self): - self.report = [SafetyPackageVulnerabilityReport(**i) for i in self.report] - -parser = argparse.ArgumentParser() -parser.add_argument('--safety-key', type=str, help='Safety key') -parser.add_argument('--report-path', type=str, help='Safety report path') -parser.add_argument('--ignored-packages', type=str, default=None, help='Packages to be ignored') -args = parser.parse_args() - -report_path = args.report_path -ignored_packages = args.ignored_packages -safety_api_key = args.safety_key - -# run safety check -scan_results = [] - -output = subprocess.check_output(f"python3 safety_check_v2.py --key '{safety_api_key}' | jq '.' | tee '{report_path}'", shell=True, executable="/bin/bash") -raw_scan_result = json.loads(output) -scan_results.append( - SafetyPythonEnvironmentVulnerabilityReport( - report=raw_scan_result - ) -) - -# processing safety reports -report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [affected: {affected}] [reason: {reason}] [env: {env}]" -failed_count = 0 -for result in scan_results: - for report_item in result.report: - if "PASSED_SAFETY_CHECK" not in report_item.affected: - if (report_item.package not in ignored_packages) or ( - report_item.package in ignored_packages - and report_item.installed != ignored_packages[report_item.package] - ): - failed_count += 1 - print( - report_log_template.format( - status="FAILED", - pkg=report_item.package, - installed=report_item.installed, - affected=report_item.affected, - reason=None, - env=result.environment, - ) - ) - -if failed_count > 0: - print(f"ERROR: {failed_count} packages") - print("Please check the test output and patch vulnerable packages.") - raise Exception("Safety check failed!") -else: - print("Passed safety check.") - From 50adb3861425b22cc6bfc52c7046399a89ed7ab0 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:37:17 +0000 Subject: [PATCH 44/88] Resolved mxnet Dockerfile cpu conflict --- mxnet/training/docker/1.8/py3/Dockerfile.cpu | 376 +++++++++---------- 1 file changed, 187 insertions(+), 189 deletions(-) diff --git a/mxnet/training/docker/1.8/py3/Dockerfile.cpu b/mxnet/training/docker/1.8/py3/Dockerfile.cpu index fdfe63d399b2..c683f89b6681 100644 --- a/mxnet/training/docker/1.8/py3/Dockerfile.cpu +++ b/mxnet/training/docker/1.8/py3/Dockerfile.cpu @@ -1,190 +1,188 @@ FROM ubuntu:16.04 -FROM python:3 - - -# LABEL maintainer="Amazon AI" -# LABEL dlc_major_version="1" - -# ARG PYTHON=python3 -# ARG PIP=pip3 -# ARG PYTHON_VERSION=3.7.10 -# ARG MX_URL=https://aws-mx-pypi.s3-us-west-2.amazonaws.com/1.8.0/aws_mx-1.8.0-py2.py3-none-manylinux2014_x86_64.whl - -# # The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating -# # the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. -# ARG SMDEBUG_VERSION=0.9.4 -# ARG OPENSSL_VERSION=1.1.1k - -# ENV PYTHONDONTWRITEBYTECODE=1 \ -# PYTHONUNBUFFERED=1 \ -# LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ -# PYTHONIOENCODING=UTF-8 \ -# LANG=C.UTF-8 \ -# LC_ALL=C.UTF-8 \ -# SAGEMAKER_TRAINING_MODULE=sagemaker_mxnet_container.training:main \ -# DGLBACKEND=mxnet - -# RUN apt-get update \ -# && apt-get install -y --no-install-recommends \ -# software-properties-common \ -# build-essential \ -# ca-certificates \ -# curl \ -# emacs \ -# git \ -# libopencv-dev \ -# openssh-client \ -# openssh-server \ -# vim \ -# wget \ -# unzip \ -# zlib1g-dev \ -# libreadline-gplv2-dev \ -# libncursesw5-dev \ -# libssl-dev \ -# libsqlite3-dev \ -# libgdbm-dev \ -# libc6-dev \ -# libbz2-dev \ -# tk-dev \ -# libffi-dev \ -# cmake \ -# # Install dependent library for OpenCV -# libgtk2.0-dev \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* - -# ########################################################################### -# # Horovod dependencies -# ########################################################################### - -# # Install Open MPI -# RUN mkdir /tmp/openmpi \ -# && cd /tmp/openmpi \ -# && wget -q https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.1.tar.gz \ -# && tar zxf openmpi-4.0.1.tar.gz \ -# && cd openmpi-4.0.1 \ -# && ./configure --enable-orterun-prefix-by-default \ -# && make -j $(nproc) all \ -# && make install \ -# && ldconfig \ -# && cd /tmp && rm -rf /tmp/openmpi - -# # Create a wrapper for OpenMPI to allow running as root by default -# RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real \ -# && echo '#!/bin/bash' > /usr/local/bin/mpirun \ -# && echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun \ -# && chmod a+x /usr/local/bin/mpirun \ -# && echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf \ -# && echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf - -# ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH -# ENV PATH=/usr/local/bin:$PATH - -# # install OpenSSL -# RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ -# && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ -# && cd openssl-${OPENSSL_VERSION} \ -# && ./config && make -j $(nproc) && make install \ -# && ldconfig \ -# && cd .. && rm -rf openssl-* \ -# && rmdir /usr/local/ssl/certs \ -# && ln -s /etc/ssl/certs /usr/local/ssl/certs - -# # install Python -# RUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ -# && tar -xzf Python-$PYTHON_VERSION.tgz \ -# && cd Python-$PYTHON_VERSION \ -# && ./configure --enable-shared --prefix=/usr/local \ -# && make -j $(nproc) && make install \ -# && cd .. && rm -rf ../Python-$PYTHON_VERSION* \ -# && ln -s /usr/local/bin/pip3 /usr/bin/pip \ -# && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \ -# && ${PIP} --no-cache-dir install --upgrade \ -# pip \ -# setuptools - -# WORKDIR / - -# # RUN ${PIP} install --no-cache --upgrade \ -# # keras-mxnet==2.2.4.2 \ -# # h5py==2.10.0 \ -# # onnx==1.6.0 \ -# # numpy==1.19.1 \ -# # pandas==0.25.1 \ -# # Pillow \ -# # requests==2.22.0 \ -# # scikit-learn==0.20.4 \ -# # dgl==0.4.* \ -# # scipy==1.2.2 \ -# # gluonnlp==0.10.0 \ -# # gluoncv==0.8.0 \ -# # # Putting a cap in versions number to avoid potential issues with a new major version -# # "urllib3>=1.25.10,<1.26.0" \ -# # # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli -# # python-dateutil==2.8.0 \ -# # tqdm==4.39.0 \ -# # sagemaker-experiments==0.* \ -# # # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli -# # "PyYAML>=5.4,<5.5" \ -# # mpi4py==3.0.2 \ -# # sagemaker-mxnet-training \ -# # ${MX_URL} \ -# # smdebug==${SMDEBUG_VERSION} \ -# # sagemaker \ -# # awscli \ -# # smclarify - -# # # Install extra packages -# # RUN pip install --no-cache-dir -U \ -# # "bokeh>=2.3,<3" \ -# # "imageio>=2.9,<3" \ -# # "opencv-python>=4.3,<5" \ -# # "plotly>=5.1,<6" \ -# # "seaborn>=0.11,<1" \ -# # "shap>=0.39,<1" - -# # # Install Horovod -# # RUN ${PIP} install --no-cache-dir horovod==0.19.5 - -# # # Allow OpenSSH to talk to containers without asking for confirmation -# # RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ -# # && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ -# # && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config - -# # # OpenSHH config for MPI communication -# # RUN mkdir -p /var/run/sshd && \ -# # sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -# # RUN rm -rf /root/.ssh/ && \ -# # mkdir -p /root/.ssh/ && \ -# # ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ -# # cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ -# # && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config - -# # # "channels first" is recommended for keras-mxnet -# # # https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn -# # RUN mkdir /root/.keras \ -# # && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json - -# # # This is here to make our installed version of OpenCV work. -# # # https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 -# # # TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? -# # RUN ln -s /dev/null /dev/raw1394 - -# # COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py - -# # RUN chmod +x /usr/local/bin/deep_learning_container.py - -# # RUN HOME_DIR=/root \ -# # && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ -# # && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ -# # && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ -# # && chmod +x /usr/local/bin/testOSSCompliance \ -# # && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ -# # && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ -# # && rm -rf ${HOME_DIR}/oss_compliance* - -# # RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt - -# # CMD ["echo hi"] + +LABEL maintainer="Amazon AI" +LABEL dlc_major_version="1" + +ARG PYTHON=python3 +ARG PIP=pip3 +ARG PYTHON_VERSION=3.7.10 +ARG MX_URL=https://aws-mx-pypi.s3-us-west-2.amazonaws.com/1.8.0/aws_mx-1.8.0-py2.py3-none-manylinux2014_x86_64.whl + +# The smdebug pipeline relies for following format to perform string replace and trigger DLC pipeline for validating +# the nightly builds. Therefore, while updating the smdebug version, please ensure that the format is not disturbed. +ARG SMDEBUG_VERSION=0.9.4 +ARG OPENSSL_VERSION=1.1.1k + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib" \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + SAGEMAKER_TRAINING_MODULE=sagemaker_mxnet_container.training:main \ + DGLBACKEND=mxnet + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + software-properties-common \ + build-essential \ + ca-certificates \ + curl \ + emacs \ + git \ + libopencv-dev \ + openssh-client \ + openssh-server \ + vim \ + wget \ + unzip \ + zlib1g-dev \ + libreadline-gplv2-dev \ + libncursesw5-dev \ + libssl-dev \ + libsqlite3-dev \ + libgdbm-dev \ + libc6-dev \ + libbz2-dev \ + tk-dev \ + libffi-dev \ + cmake \ + # Install dependent library for OpenCV + libgtk2.0-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +########################################################################### +# Horovod dependencies +########################################################################### + +# Install Open MPI +RUN mkdir /tmp/openmpi \ + && cd /tmp/openmpi \ + && wget -q https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.1.tar.gz \ + && tar zxf openmpi-4.0.1.tar.gz \ + && cd openmpi-4.0.1 \ + && ./configure --enable-orterun-prefix-by-default \ + && make -j $(nproc) all \ + && make install \ + && ldconfig \ + && cd /tmp && rm -rf /tmp/openmpi + +# Create a wrapper for OpenMPI to allow running as root by default +RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real \ + && echo '#!/bin/bash' > /usr/local/bin/mpirun \ + && echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun \ + && chmod a+x /usr/local/bin/mpirun \ + && echo "hwloc_base_binding_policy = none" >> /usr/local/etc/openmpi-mca-params.conf \ + && echo "rmaps_base_mapping_policy = slot" >> /usr/local/etc/openmpi-mca-params.conf + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH +ENV PATH=/usr/local/bin:$PATH + +# install OpenSSL +RUN wget -q https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ + && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ + && cd openssl-${OPENSSL_VERSION} \ + && ./config && make -j $(nproc) && make install \ + && ldconfig \ + && cd .. && rm -rf openssl-* \ + && rmdir /usr/local/ssl/certs \ + && ln -s /etc/ssl/certs /usr/local/ssl/certs + +# install Python +RUN wget -q https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz \ + && tar -xzf Python-$PYTHON_VERSION.tgz \ + && cd Python-$PYTHON_VERSION \ + && ./configure --enable-shared --prefix=/usr/local \ + && make -j $(nproc) && make install \ + && cd .. && rm -rf ../Python-$PYTHON_VERSION* \ + && ln -s /usr/local/bin/pip3 /usr/bin/pip \ + && ln -s /usr/local/bin/$PYTHON /usr/local/bin/python \ + && ${PIP} --no-cache-dir install --upgrade \ + pip \ + setuptools + +WORKDIR / + +RUN ${PIP} install --no-cache --upgrade \ + keras-mxnet==2.2.4.2 \ + "h5py<3" \ + "onnx>=1.7.0,<1.9.0" \ + "numpy>1.16.0,<=1.19.1" \ + pandas \ + Pillow \ + "requests>=2.18.4,<=2.22.0" \ + scikit-learn \ + dgl==0.4.* \ + "scipy>=1.2.2,<=1.4.1" \ + gluonnlp==0.10.0 \ + gluoncv==0.8.0 \ + # Putting a cap in versions number to avoid potential issues with a new major version + "urllib3>=1.25.10,<1.26.0" \ + # python-dateutil==2.8.0 to satisfy botocore associated with latest awscli + python-dateutil==2.8.0 \ + tqdm==4.39.0 \ + sagemaker-experiments==0.* \ + # install PyYAML>=5.4,<5.5 to avoid conflict with latest awscli + "PyYAML>=5.4,<5.5" \ + mpi4py==3.0.2 \ + sagemaker-mxnet-training \ + ${MX_URL} \ + smdebug==${SMDEBUG_VERSION} \ + sagemaker \ + awscli \ + smclarify + +# Install extra packages +RUN pip install --no-cache-dir -U \ + "bokeh>=2.3,<3" \ + "imageio>=2.9,<3" \ + "opencv-python>=4.3,<5" \ + "plotly>=5.1,<6" \ + "seaborn>=0.11,<1" \ + "shap>=0.39,<1" + +# Install Horovod +RUN ${PIP} install --no-cache-dir horovod==0.19.5 + +# Allow OpenSSH to talk to containers without asking for confirmation +RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ + && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ + && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config + +# OpenSHH config for MPI communication +RUN mkdir -p /var/run/sshd && \ + sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +RUN rm -rf /root/.ssh/ && \ + mkdir -p /root/.ssh/ && \ + ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && \ + cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config + +# "channels first" is recommended for keras-mxnet +# https://github.com/awslabs/keras-apache-mxnet/blob/master/docs/mxnet_backend/performance_guide.md#channels-first-image-data-format-for-cnn +RUN mkdir /root/.keras \ + && echo '{"image_data_format": "channels_first"}' > /root/.keras/keras.json + +# This is here to make our installed version of OpenCV work. +# https://stackoverflow.com/questions/29274638/opencv-libdc1394-error-failed-to-initialize-libdc1394 +# TODO: Should we be installing OpenCV in our image like this? Is there another way we can fix this? +RUN ln -s /dev/null /dev/raw1394 + +COPY deep_learning_container.py /usr/local/bin/deep_learning_container.py + +RUN chmod +x /usr/local/bin/deep_learning_container.py + +RUN HOME_DIR=/root \ + && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \ + && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \ + && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \ + && chmod +x /usr/local/bin/testOSSCompliance \ + && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \ + && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \ + && rm -rf ${HOME_DIR}/oss_compliance* + +RUN curl -o /license.txt https://aws-dlc-licenses.s3.amazonaws.com/aws-mx-1.8.0/license.txt + +CMD ["/bin/bash"] From ef62fa9dcd366951fae3df5ca1cea77cf13f6d3b Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:39:55 +0000 Subject: [PATCH 45/88] Resolved mxnet buildpsec --- mxnet/buildspec.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mxnet/buildspec.yml b/mxnet/buildspec.yml index 54732d4ec79b..ed97a377ea75 100644 --- a/mxnet/buildspec.yml +++ b/mxnet/buildspec.yml @@ -62,6 +62,31 @@ images: docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] context: <<: *TRAINING_CONTEXT + BuildMXNetCPUInferencePy3DockerImage: + <<: *INFERENCE_REPOSITORY + build: &MXNET_CPU_INFERENCE_PY3 false + image_size_baseline: 1687 + device_type: &DEVICE_TYPE cpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + os_version: &OS_VERSION ubuntu16.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *INFERENCE_CONTEXT + BuildMXNetGPUInferencePy3DockerImage: + << : *INFERENCE_REPOSITORY + build: &MXNET_GPU_INFERENCE_PY3 false + image_size_baseline: 5578 + device_type: &DEVICE_TYPE gpu + python_version: &DOCKER_PYTHON_VERSION py3 + tag_python_version: &TAG_PYTHON_VERSION py37 + cuda_version: &CUDA_VERSION cu110 + os_version: &OS_VERSION ubuntu16.04 + tag: !join [ *VERSION, "-", *DEVICE_TYPE, "-", *TAG_PYTHON_VERSION, "-", *CUDA_VERSION, "-", *OS_VERSION ] + docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *CUDA_VERSION, /Dockerfile., *DEVICE_TYPE ] + context: + <<: *INFERENCE_CONTEXT BuildMXNetExampleGPUTrainPy3DockerImage: <<: *TRAINING_REPOSITORY build: &MXNET_GPU_TRAINING_PY3 false From 41edfbdd74c0ea40fb537496df78b1dc27c3dd3d Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:51:16 +0000 Subject: [PATCH 46/88] Reverted test_safety_check.py to master --- test/dlc_tests/sanity/test_safety_check.py | 47 +++++++++++----------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py index cdb84424a0c2..c39e9460e48e 100644 --- a/test/dlc_tests/sanity/test_safety_check.py +++ b/test/dlc_tests/sanity/test_safety_check.py @@ -148,11 +148,17 @@ def _get_latest_package_version(package): @pytest.mark.model("N/A") @pytest.mark.canary("Run safety tests regularly on production images") @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") +@pytest.mark.skipif( + not (is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), + reason=( + "Skipping the test to decrease the number of calls to the Safety Check DB. " + "Test will be executed in the 'mainline' pipeline and canaries pipeline." + ) +) def test_safety(image): """ Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise error if an issue is fixable. - The function checks for the safety report to validate the safety run on the build job. If the report does not exist, the safety check is executed. """ from dlc.safety_check import SafetyCheck safety_check = SafetyCheck() @@ -172,27 +178,22 @@ def test_safety(image): f"--entrypoint='/bin/bash' " f"{image}", hide=True) try: - SAFETY_FILE = "/opt/aws/dlc/info/safety_test_report.json" - safety_file_check = run(f"{docker_exec_cmd} test -e {SAFETY_FILE}", warn=True, hide=True) - if safety_file_check.return_code != 0: - run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) - json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) - safety_result = json.loads(json_str_safety_result) - for vulnerability in safety_result: - package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] - # Get the latest version of the package with vulnerability - latest_version = _get_latest_package_version(package) - # If the latest version of the package is also affected, ignore this vulnerability - if Version(latest_version) in SpecifierSet(affected_versions): - # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. - # Comparing two versions as a string has some edge cases which require us to write more code. - # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and - # gives an object that can be easily compared against a Version object. - # https://packaging.pypa.io/en/latest/specifiers/ - ignore_str += f" -i {vulnerability_id}" - assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ - f"Safety test failed for {image}" - else: - LOGGER.info(f"Safety check is complete as a part of docker build and report exist at {SAFETY_FILE}") + run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) + json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) + safety_result = json.loads(json_str_safety_result) + for vulnerability in safety_result: + package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] + # Get the latest version of the package with vulnerability + latest_version = _get_latest_package_version(package) + # If the latest version of the package is also affected, ignore this vulnerability + if Version(latest_version) in SpecifierSet(affected_versions): + # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. + # Comparing two versions as a string has some edge cases which require us to write more code. + # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and + # gives an object that can be easily compared against a Version object. + # https://packaging.pypa.io/en/latest/specifiers/ + ignore_str += f" -i {vulnerability_id}" + assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ + f"Safety test failed for {image}" finally: run(f"docker rm -f {container_name}", hide=True) From 8d2c4e3103e32014efe070be1346503faab1b4df Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 21 Aug 2021 02:57:57 +0000 Subject: [PATCH 47/88] Deleted safety_report.py --- src/safety_report.py | 190 ------------------------------------------- 1 file changed, 190 deletions(-) delete mode 100644 src/safety_report.py diff --git a/src/safety_report.py b/src/safety_report.py deleted file mode 100644 index 503510852832..000000000000 --- a/src/safety_report.py +++ /dev/null @@ -1,190 +0,0 @@ -#this file can be removed if the safety test setup is done somewhere else -import json -import logging -import os -import sys - -from packaging.specifiers import SpecifierSet -from packaging.version import Version - -import pytest -import requests - -from invoke import run - -from test.test_utils import ( - CONTAINER_TESTS_PREFIX, is_dlc_cicd_context, is_canary_context, is_mainline_context, is_time_for_canary_safety_scan -) - -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.INFO) -LOGGER.addHandler(logging.StreamHandler(sys.stderr)) - -# List of safety check vulnerability IDs to ignore. To get the ID, run safety check on the container, -# and copy paste the ID given by the safety check for a package. -# Note:- 1. This ONLY needs to be done if a package version exists that resolves this safety issue, but that version -# cannot be used because of incompatibilities. -# 2. Ensure that IGNORE_SAFETY_IDS is always as small/empty as possible. -IGNORE_SAFETY_IDS = { - "tensorflow": { - "training": { - "py2": [ - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452', - # for shipping pycrypto<=2.6.1 - the last available version for py2 - '35015' - ], - "py3": [] - }, - "inference": { - "py2": [ - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "inference-eia": { - "py2": [ - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "inference-neuron": { - "py3": [ - # TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches - '39409', '39408', '39407', '39406' - ], - }, - }, - "mxnet": { - "inference-eia": { - "py2": [ - # numpy<=1.16.0 -- This has to only be here while we publish MXNet 1.4.1 EI DLC v1.0 - '36810', - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "inference": { - "py2": [ - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "training": { - "py2": [ - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "inference-neuron": { - "py3": [ - # for shipping tensorflow 1.15.5 - "40673", "40675", "40676", "40794", "40795", "40796" - ] - } - }, - "pytorch": { - "training": { - "py2": [ - # astropy<3.0.1 - '35810', - # for shipping pillow<=6.2.2 - the last available version for py2 - '38449', '38450', '38451', '38452' - ], - "py3": [] - }, - "inference": { - "py3": [] - }, - "inference-eia": { - "py3": [] - }, - "inference-neuron": { - "py3": [ - # 39409, 39408, 39407, 39406: TF 1.15.5 is on par with TF 2.0.4, 2.1.3, 2.2.2, 2.3.2 in security patches - '39409', '39408', '39407', '39406' - ] - } - } -} - - -def _get_safety_ignore_list(image_uri): - """ - Get a list of known safety check issue IDs to ignore, if specified in IGNORE_LISTS. - :param image_uri: - :return: list of safety check IDs to ignore - """ - framework = ("mxnet" if "mxnet" in image_uri else - "pytorch" if "pytorch" in image_uri else - "tensorflow") - job_type = ("training" if "training" in image_uri else - "inference-eia" if "eia" in image_uri else - "inference-neuron" if "neuron" in image_uri else - "inference") - python_version = "py2" if "py2" in image_uri else "py3" - - return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, []) - - -def _get_latest_package_version(package): - """ - Get the latest package version available on pypi for a package. - It is retried multiple times in case there are transient failures in executing the command. - - :param package: str Name of the package whose latest version must be retrieved - :return: tuple(command_success: bool, latest_version_value: str) - """ - pypi_package_info = requests.get(f"https://pypi.org/pypi/{package}/json") - data = json.loads(pypi_package_info.text) - versions = data["releases"].keys() - return str(max(Version(v) for v in versions)) - - -def test_safety(image): - """ - Runs safety check on a container with the capability to ignore safety issues that cannot be fixed, and only raise - error if an issue is fixable. - """ - from dlc.safety_check import SafetyCheck - safety_check = SafetyCheck() - - repo_name, image_tag = image.split('/')[-1].split(':') - ignore_ids_list = _get_safety_ignore_list(image) - sep = " -i " - ignore_str = "" if not ignore_ids_list else f"{sep}{sep.join(ignore_ids_list)}" - - container_name = f"{repo_name}-{image_tag}-safety" - docker_exec_cmd = f"docker exec -i {container_name}" - test_file_path = os.path.join(CONTAINER_TESTS_PREFIX, "testSafety") - # Add null entrypoint to ensure command exits immediately - run(f"docker run -id " - f"--name {container_name} " - f"--mount type=bind,src=$(pwd)/container_tests,target=/test " - f"--entrypoint='/bin/bash' " - f"{image}", hide=True) - try: - run(f"{docker_exec_cmd} pip install safety yolk3k ", hide=True) - json_str_safety_result = safety_check.run_safety_check_on_container(docker_exec_cmd) - safety_result = json.loads(json_str_safety_result) - for vulnerability in safety_result: - package, affected_versions, curr_version, _, vulnerability_id = vulnerability[:5] - # Get the latest version of the package with vulnerability - latest_version = _get_latest_package_version(package) - # If the latest version of the package is also affected, ignore this vulnerability - if Version(latest_version) in SpecifierSet(affected_versions): - # Version(x) gives an object that can be easily compared with another version, or with a SpecifierSet. - # Comparing two versions as a string has some edge cases which require us to write more code. - # SpecifierSet(x) takes a version constraint, such as "<=4.5.6", ">1.2.3", or ">=1.2,<3.4.5", and - # gives an object that can be easily compared against a Version object. - # https://packaging.pypa.io/en/latest/specifiers/ - ignore_str += f" -i {vulnerability_id}" - assert (safety_check.run_safety_check_with_ignore_list(docker_exec_cmd, ignore_str) == 0), \ - f"Safety test failed for {image}" - finally: - run(f"docker rm -f {container_name}", hide=True) From 94c07335cc91e482846041024e6e00ac7892c5f2 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Mon, 23 Aug 2021 07:32:10 +0000 Subject: [PATCH 48/88] Added container removal logic --- src/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils.py b/src/utils.py index 001b408758ed..b819be065f41 100644 --- a/src/utils.py +++ b/src/utils.py @@ -543,6 +543,7 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): docker_exec_cmd = f"docker exec -i {container_id}" ignore_dict = get_safety_ignore_dict(image_uri) run_output = SafetyCheck().run_safety_check_script_on_container(docker_exec_cmd, ignore_dict_str=json.dumps(ignore_dict)) + ctx.run(f"docker rm -f {container_id}") json_formatted_output = json.loads(run_output.strip()) if storage_file_path: with open(storage_file_path, 'w', encoding='utf-8') as f: From dd5c6bdae98eeadde7ad0880e538491e02d4b623 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 03:24:59 +0000 Subject: [PATCH 49/88] Added safety report generator class --- src/safety_report_generator.py | 122 +++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 src/safety_report_generator.py diff --git a/src/safety_report_generator.py b/src/safety_report_generator.py new file mode 100644 index 000000000000..eff506f640cc --- /dev/null +++ b/src/safety_report_generator.py @@ -0,0 +1,122 @@ +from dlc.safety_check import SafetyCheck +from invoke.context import Context +from datetime import datetime + +import json + +TO_BE_DECIDED="To Be Decided" + +class SafetyReportGenerator: + """ + The SafetyReportGenerator class deals with the functionality of generating safety reports for running containers. + The safety report takes the following format: + [ + { + "package": "package", + "scan_status": "SUCCEEDED/FAILED/IGNORED", + "installed": "version", + "vulnerabilities": [ + { + "vid": "safety_vulnerability_id", + "advisory": "description of the issue", + "reason_to_ignore":"reason to ignore the vid", + "spec": "version_spec" + }, + ... + ] + "date": + } + ... + ] + """ + + def __init__(self, container_id, ignore_dict = {}): + self.container_id = container_id + self.vulns_dict = {} + self.vulns_list = [] + self.ignore_dict = ignore_dict + self.ignored_vulnerability_count = {} + self.ctx = Context() + self.docker_exec_cmd = f"docker exec -i {container_id}" + self.safety_check_output = None + + def insert_vulnerabilites_into_report(self, vulns): + for v in vulns: + package = v[0] + spec = v[1] + installed = v[2] + advisory = v[3] + vid = v[4] + vulnerability_details = {"vid": vid, "advisory": advisory, "spec": spec, "reason_to_ignore":"N/A"} + + if package not in self.ignored_vulnerability_count: + self.ignored_vulnerability_count[package] = 0 + + if vid in self.ignore_dict: + vulnerability_details["reason_to_ignore"] = self.ignore_dict[vid] + self.ignored_vulnerability_count[package] += 1 + + if package not in self.vulns_dict: + self.vulns_dict[package] = { + "package": package, + "scan_status": TO_BE_DECIDED, + "installed": installed, + "vulnerabilities": [vulnerability_details], + "date":self.timestamp, + } + else: + self.vulns_dict[package]["vulnerabilities"].append(vulnerability_details) + + def get_package_set_from_container(self): + python_cmd_to_extract_package_set = """ python -c "import pkg_resources; \ + import json; \ + print(json.dumps([{'key':d.key, 'version':d.version} for d in pkg_resources.working_set]))" """ + + run_output = self.ctx.run(f"{self.docker_exec_cmd} {python_cmd_to_extract_package_set}", hide=True, warn=True) + if run_output.exited != 0: + raise Exception('Package set cannot be retrieved from the container.') + + return json.loads(run_output.stdout) + + + def insert_safe_packages_into_report(self, packages): + for pkg in packages: + if pkg['key'] not in self.vulns_dict: + self.vulns_dict[pkg['key']] = { + "package": pkg['key'], + "scan_status": "SUCCEEDED", + "installed": pkg['version'], + "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A", "spec":"N/A"}], + "date":self.timestamp + } + + def process_report(self): + for (k, v) in self.vulns_dict.items(): + if v["scan_status"] == TO_BE_DECIDED: + if len(v["vulnerabilities"]) == self.ignored_vulnerability_count[k]: + v["scan_status"] = "IGNORED" + else: + v["scan_status"] = "FAILED" + self.vulns_list.append(v) + + + + def generate(self): + self.timestamp = datetime.now().strftime("%d-%m-%Y") + # If cicd condition to be added + self.safety_check_output = SafetyCheck().run_safety_check_on_container(self.docker_exec_cmd) + vulns = json.loads(self.safety_check_output) + self.insert_vulnerabilites_into_report(vulns) + packages = self.get_package_set_from_container() + self.insert_safe_packages_into_report(packages) + self.process_report() + return self.vulns_list + + + + + + + + + From 5dce6c78915e90f818d60cf7203d0f0484bfec60 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 04:02:45 +0000 Subject: [PATCH 50/88] Changes the utils generate_safety_report_for_image function --- src/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/utils.py b/src/utils.py index b819be065f41..2394140e6428 100644 --- a/src/utils.py +++ b/src/utils.py @@ -23,6 +23,7 @@ from config import parse_dlc_developer_configs, is_build_enabled from invoke.context import Context from botocore.exceptions import ClientError +from safety_report_generator import SafetyReportGenerator from dlc.safety_check import SafetyCheck LOGGER = logging.getLogger(__name__) @@ -536,16 +537,15 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): ] """ ctx = Context() - docker_run_cmd = f"docker run -itd -v $ROOT_FOLDER_PATH/src:/src {image_uri}" - container_id = ctx.run(f"{docker_run_cmd}").stdout.strip() + docker_run_cmd = f"docker run -itd {image_uri}" + container_id = ctx.run(f"{docker_run_cmd}", hide=True, warn=True).stdout.strip() install_safety_cmd = "pip install safety" - ctx.run(f"docker exec {container_id} {install_safety_cmd}") + ctx.run(f"docker exec {container_id} {install_safety_cmd}", hide=True, warn=True) docker_exec_cmd = f"docker exec -i {container_id}" ignore_dict = get_safety_ignore_dict(image_uri) - run_output = SafetyCheck().run_safety_check_script_on_container(docker_exec_cmd, ignore_dict_str=json.dumps(ignore_dict)) - ctx.run(f"docker rm -f {container_id}") - json_formatted_output = json.loads(run_output.strip()) + safety_scan_output = SafetyReportGenerator(container_id, ignore_dict=ignore_dict).generate() + ctx.run(f"docker rm -f {container_id}", hide=True, warn=True) if storage_file_path: with open(storage_file_path, 'w', encoding='utf-8') as f: - json.dump(json_formatted_output, f, indent=4) - return json_formatted_output \ No newline at end of file + json.dump(safety_scan_output, f, indent=4) + return safety_scan_output From ed247247ace87bf4257a7a5b2a34c65df23a6c0a Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 17:06:15 +0000 Subject: [PATCH 51/88] Added Formatter messages --- src/image_builder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/image_builder.py b/src/image_builder.py index 4b494a1f1e23..8b68eb32c15c 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -181,7 +181,7 @@ def image_builder(buildspec): CONCLUSION_STAGE_IMAGES.append(conclusion_stage_image_object) FORMATTER.banner("DLC") - #FORMATTER.title("Status") + FORMATTER.title("Status") # Standard images must be built before example images # Example images will use standard images as base @@ -205,8 +205,10 @@ def image_builder(buildspec): FORMATTER.banner("Conclusion Build") build_images(conclusion_stage_images, make_dummy_boto_client=True) + FORMATTER.banner("Push Started") push_images(IMAGES_TO_PUSH) + FORMATTER.banner("Log Display") #After the build, display logs/summary for all the images. show_build_logs(ALL_IMAGES) show_build_summary(ALL_IMAGES) @@ -215,9 +217,11 @@ def image_builder(buildspec): #From all images, filter the images that were supposed to be built and upload their metrics BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] + FORMATTER.banner("Upload Metrics") # change logic here. upload metrics only for the Conclusion stage image upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) + FORMATTER.banner("Setting Test Env") # Set environment variables to be consumed by test jobs test_trigger_job = utils.get_codebuild_project_name() # Tests should only run on images that were pushed to the repository From 8181a3862b0809bcee30436bbaa1d189dd501629 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 18:10:19 +0000 Subject: [PATCH 52/88] Add null entrypoint while running docker images --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 2394140e6428..70a318e73054 100644 --- a/src/utils.py +++ b/src/utils.py @@ -537,7 +537,7 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): ] """ ctx = Context() - docker_run_cmd = f"docker run -itd {image_uri}" + docker_run_cmd = f"docker run -id --entrypoint='/bin/bash' {image_uri} " container_id = ctx.run(f"{docker_run_cmd}", hide=True, warn=True).stdout.strip() install_safety_cmd = "pip install safety" ctx.run(f"docker exec {container_id} {install_safety_cmd}", hide=True, warn=True) From 71dce0e3cacaa4d083abf7c5af6f5aa82d1b64a6 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 19:13:51 +0000 Subject: [PATCH 53/88] Added extended timeouts --- buildspec.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildspec.yml b/buildspec.yml index 8585fb32a055..02e40f0c38f1 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -13,6 +13,8 @@ phases: - pip install -r src/requirements.txt - bash src/setup.sh $FRAMEWORK - export ROOT_FOLDER_PATH=$(pwd) + - export DOCKER_CLIENT_TIMEOUT=720 + - export COMPOSE_HTTP_TIMEOUT=720 - python src/parse_partner_developers.py build: commands: From a1fb7e47b7774da4604ae63d10abc0f14ed2c94e Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 20:17:20 +0000 Subject: [PATCH 54/88] Added debug print lines --- src/image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/image.py b/src/image.py index e475ec7ebcc2..68f6dd4e97f9 100644 --- a/src/image.py +++ b/src/image.py @@ -185,6 +185,7 @@ def push_image(self): for line in self.client.push(self.repository, self.tag, stream=True, decode=True): response = [] + print(f'*** [{self.repository}:{self.tag}] {line} ***') if line.get("error") is not None: response.append(line["error"]) From 73bd4a64779730a7bb546717be2138372bda98be Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 25 Aug 2021 21:45:52 +0000 Subject: [PATCH 55/88] Pushing images sequentially --- src/image_builder.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/image_builder.py b/src/image_builder.py index 8b68eb32c15c..b95ec867d26f 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -342,10 +342,14 @@ def get_dummy_boto_client(): def push_images(images): THREADS = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - for image in images: - THREADS[image.name] = executor.submit(image.push_image) - FORMATTER.progress(THREADS) + for image in images: + THREADS[image.name] = image.push_image() + FORMATTER.title("Completed PUSH") + # FORMATTER.progress(THREADS) + # with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + # for image in images: + # THREADS[image.name] = executor.submit(image.push_image) + # FORMATTER.progress(THREADS) From 1f48f08bf69ac511b7ecebafe1bd4be6a13d12d8 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 00:37:07 +0000 Subject: [PATCH 56/88] Added client timeout in constants with parallelization --- src/constants.py | 2 ++ src/image.py | 4 ++-- src/image_builder.py | 12 ++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/constants.py b/src/constants.py index aa046467220e..8d3e690a79d4 100644 --- a/src/constants.py +++ b/src/constants.py @@ -75,3 +75,5 @@ ECS_TESTS = "ecs" EKS_TESTS = "eks" ALL_TESTS = ["sagemaker", "ec2", "eks", "ecs"] + +API_CLIENT_TIMEOUT = 600 \ No newline at end of file diff --git a/src/image.py b/src/image.py index 68f6dd4e97f9..9a64919484da 100644 --- a/src/image.py +++ b/src/image.py @@ -53,7 +53,7 @@ def __init__( self.to_build = to_build self.build_status = None - self.client = APIClient(base_url=constants.DOCKER_URL) + self.client = APIClient(base_url=constants.DOCKER_URL, timeout=constants.API_CLIENT_TIMEOUT) self.log = [] def __getattr__(self, name): @@ -182,7 +182,7 @@ def image_size_check(self): return self.build_status def push_image(self): - + print(f'####### CLIENT_TIMEOUT {self.client.timeout} ####') for line in self.client.push(self.repository, self.tag, stream=True, decode=True): response = [] print(f'*** [{self.repository}:{self.tag}] {line} ***') diff --git a/src/image_builder.py b/src/image_builder.py index b95ec867d26f..8b68eb32c15c 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -342,14 +342,10 @@ def get_dummy_boto_client(): def push_images(images): THREADS = {} - for image in images: - THREADS[image.name] = image.push_image() - FORMATTER.title("Completed PUSH") - # FORMATTER.progress(THREADS) - # with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - # for image in images: - # THREADS[image.name] = executor.submit(image.push_image) - # FORMATTER.progress(THREADS) + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + for image in images: + THREADS[image.name] = executor.submit(image.push_image) + FORMATTER.progress(THREADS) From 49a6e2af3bf89fdbd7b15bcf38d841dd4e1941f9 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 01:25:20 +0000 Subject: [PATCH 57/88] Changed the test. Removed the safety_check_v2.py --- src/constants.py | 2 +- src/safety_check_v2.py | 138 ------------------ test/dlc_tests/sanity/test_safety_check_v2.py | 1 + 3 files changed, 2 insertions(+), 139 deletions(-) delete mode 100644 src/safety_check_v2.py diff --git a/src/constants.py b/src/constants.py index 8d3e690a79d4..d0b32a621931 100644 --- a/src/constants.py +++ b/src/constants.py @@ -76,4 +76,4 @@ EKS_TESTS = "eks" ALL_TESTS = ["sagemaker", "ec2", "eks", "ecs"] -API_CLIENT_TIMEOUT = 600 \ No newline at end of file +API_CLIENT_TIMEOUT = 600 diff --git a/src/safety_check_v2.py b/src/safety_check_v2.py deleted file mode 100644 index fa4ade7e69ab..000000000000 --- a/src/safety_check_v2.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import json -import sys -import argparse -from datetime import datetime -import pkg_resources -import safety as Safety -from safety import safety - -TO_BE_DECIDED="To Be Decided" - -def main(): - """Return json.dumps() string of below list structure. - [ - { - "package": "package", - "scan_status": "SUCCEEDED/FAILED/IGNORED", - "installed": "version", - "vulnerabilities": [ - { - "vid": "safety_vulnerability_id", - "advisory": "description of the issue", - "reason_to_ignore":"reason to ignore the vid", - "spec": "version_spec" - }, - ... - ] - } - ... - ] - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--key", - required=False, - ) - parser.add_argument( - "--ignore_dict_str", - required=False, - ) - args = parser.parse_args() - - - safety_key = args.key - ignore_dict = {} - if args.ignore_dict_str is not None: - ignore_dict = json.loads(args.ignore_dict_str) - - if Safety.__version__ == "1.8.7": - proxy_dictionary = {} - else: - from safety.util import get_proxy_dict - proxy_dictionary = get_proxy_dict("http", None, 80) - - # Get safety result - packages = [ - d for d in pkg_resources.working_set if d.key not in {"python", "wsgiref", "argparse"} - ] - # return: Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id", "cvssv2", "cvssv3"])) - vulns = safety.check( - packages=packages, - key=safety_key, - db_mirror="", - cached=True, - ignore_ids=[], - proxy=proxy_dictionary, - ) - - # Generate output - vulns_dict = {} - vulns_list = [] - ignored_vulnerability_count = {} - # Report for unsafe packages - # "FAILED", "SUCCEEDED" , "IGNORED" - for v in vulns: - package = v.name - spec = v.spec - installed = v.version - advisory = v.advisory - vid = v.vuln_id - vulnerability_details = {"vid": vid, "advisory": advisory, "spec": spec, "reason_to_ignore":"N/A"} - - if package not in ignored_vulnerability_count: - ignored_vulnerability_count[package] = 0 - - if vid in ignore_dict: - vulnerability_details["reason_to_ignore"] = ignore_dict[vid] - ignored_vulnerability_count[package] += 1 - - if package not in vulns_dict: - vulns_dict[package] = { - "package": package, - "scan_status": TO_BE_DECIDED, - "installed": installed, - "vulnerabilities": [vulnerability_details], - } - else: - vulns_dict[package]["vulnerabilities"].append(vulnerability_details) - - # Report for safe packages - timestamp = datetime.now().strftime("%d%m%Y") - for pkg in packages: - if pkg.key not in vulns_dict: - vulns_dict[pkg.key] = { - "package": pkg.key, - "scan_status": "SUCCEEDED", - "installed": pkg.version, - "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A", "spec":"N/A"}], - } - - for (k, v) in vulns_dict.items(): - if v["scan_status"] == TO_BE_DECIDED: - if len(v["vulnerabilities"]) == ignored_vulnerability_count[k]: - v["scan_status"] = "IGNORED" - else: - v["scan_status"] = "FAILED" - vulns_list.append(v) - - print(json.dumps(vulns_list)) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except KeyboardInterrupt: - pass diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index 9f0ad96e8ff9..5a9f6c1e5051 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -34,6 +34,7 @@ class SafetyPackageVulnerabilityReport: scan_status: str installed: str vulnerabilities: List[SafetyVulnerabilityAdvisory] + date: str def __post_init__(self): self.vulnerabilities = [SafetyVulnerabilityAdvisory(**i) for i in self.vulnerabilities] From 6286f8edd7a4b2d40245eaabaa708032e9798c37 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 02:48:53 +0000 Subject: [PATCH 58/88] Reverted print and debug statements --- buildspec.yml | 2 -- src/image.py | 2 -- src/image_builder.py | 6 +++--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/buildspec.yml b/buildspec.yml index 02e40f0c38f1..8585fb32a055 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -13,8 +13,6 @@ phases: - pip install -r src/requirements.txt - bash src/setup.sh $FRAMEWORK - export ROOT_FOLDER_PATH=$(pwd) - - export DOCKER_CLIENT_TIMEOUT=720 - - export COMPOSE_HTTP_TIMEOUT=720 - python src/parse_partner_developers.py build: commands: diff --git a/src/image.py b/src/image.py index 9a64919484da..d2504a190acf 100644 --- a/src/image.py +++ b/src/image.py @@ -182,10 +182,8 @@ def image_size_check(self): return self.build_status def push_image(self): - print(f'####### CLIENT_TIMEOUT {self.client.timeout} ####') for line in self.client.push(self.repository, self.tag, stream=True, decode=True): response = [] - print(f'*** [{self.repository}:{self.tag}] {line} ***') if line.get("error") is not None: response.append(line["error"]) diff --git a/src/image_builder.py b/src/image_builder.py index 8b68eb32c15c..fc119d3ac481 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -208,7 +208,7 @@ def image_builder(buildspec): FORMATTER.banner("Push Started") push_images(IMAGES_TO_PUSH) - FORMATTER.banner("Log Display") + FORMATTER.title("Log Display") #After the build, display logs/summary for all the images. show_build_logs(ALL_IMAGES) show_build_summary(ALL_IMAGES) @@ -217,11 +217,11 @@ def image_builder(buildspec): #From all images, filter the images that were supposed to be built and upload their metrics BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] - FORMATTER.banner("Upload Metrics") + FORMATTER.title("Upload Metrics") # change logic here. upload metrics only for the Conclusion stage image upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) - FORMATTER.banner("Setting Test Env") + FORMATTER.title("Setting Test Env") # Set environment variables to be consumed by test jobs test_trigger_job = utils.get_codebuild_project_name() # Tests should only run on images that were pushed to the repository From 9cdfaaaf22162fc03fc6c311efbb18a798fb9855 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 20:30:55 +0000 Subject: [PATCH 59/88] Restricted worker count for pushing images --- src/constants.py | 1 + src/image_builder.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/constants.py b/src/constants.py index d0b32a621931..68b0d111769b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -77,3 +77,4 @@ ALL_TESTS = ["sagemaker", "ec2", "eks", "ecs"] API_CLIENT_TIMEOUT = 600 +MAX_WORKER_COUNT_FOR_PUSHING_IMAGES = 3 diff --git a/src/image_builder.py b/src/image_builder.py index fc119d3ac481..e0bd262814f0 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -336,13 +336,14 @@ def get_dummy_boto_client(): # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. # If this function is not added, boto3 fails because boto3 sessions are not thread safe. # However, once a dummy client is created, it is ensured that the calls are thread safe. - # If the boto3 call made in the dlc package is changed to boto3.Session().client(), even then this issue can be resolved. import boto3 return boto3.client("secretsmanager", region_name=os.getenv('REGION')) def push_images(images): THREADS = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + with concurrent.futures.ThreadPoolExecutor( + max_workers=constants.MAX_WORKER_COUNT_FOR_PUSHING_IMAGES + ) as executor: for image in images: THREADS[image.name] = executor.submit(image.push_image) FORMATTER.progress(THREADS) From d576ae9b680f9c0f24903236660dbf669b97a1f7 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 23:37:43 +0000 Subject: [PATCH 60/88] Reverted modified Dockerfile from pull request --- mxnet/training/docker/1.8/py3/Dockerfile.cpu | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mxnet/training/docker/1.8/py3/Dockerfile.cpu b/mxnet/training/docker/1.8/py3/Dockerfile.cpu index c683f89b6681..02cb3ef9d946 100644 --- a/mxnet/training/docker/1.8/py3/Dockerfile.cpu +++ b/mxnet/training/docker/1.8/py3/Dockerfile.cpu @@ -106,15 +106,15 @@ WORKDIR / RUN ${PIP} install --no-cache --upgrade \ keras-mxnet==2.2.4.2 \ - "h5py<3" \ - "onnx>=1.7.0,<1.9.0" \ - "numpy>1.16.0,<=1.19.1" \ - pandas \ + h5py==2.10.0 \ + onnx==1.6.0 \ + numpy==1.19.1 \ + pandas==0.25.1 \ Pillow \ - "requests>=2.18.4,<=2.22.0" \ - scikit-learn \ + requests==2.22.0 \ + scikit-learn==0.20.4 \ dgl==0.4.* \ - "scipy>=1.2.2,<=1.4.1" \ + scipy==1.2.2 \ gluonnlp==0.10.0 \ gluoncv==0.8.0 \ # Putting a cap in versions number to avoid potential issues with a new major version From 3923a68ead62a1c3a17890317065d96633407a5c Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 26 Aug 2021 23:41:24 +0000 Subject: [PATCH 61/88] removed Safety Data Json --- safety_data.json | 1135 ---------------------------------------------- 1 file changed, 1135 deletions(-) delete mode 100644 safety_data.json diff --git a/safety_data.json b/safety_data.json deleted file mode 100644 index d9596d120ac0..000000000000 --- a/safety_data.json +++ /dev/null @@ -1,1135 +0,0 @@ -[ - { - "package": "zope.interface", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "5.4.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "zope.event", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.5.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "zipp", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.5.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "wheel", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.36.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "werkzeug", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.0.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "wcwidth", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2.5", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "urllib3", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.26.6", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "typing-extensions", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.10.0.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "traitlets", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "5.0.5", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "tqdm", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.62.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "torchvision", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.10.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "torch", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.9.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "toml", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.10.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "threadpoolctl", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "tabulate", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.8.9", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "smdebug", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.0.9", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "smdebug-rulesconfig", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.0.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "smclarify", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "sklearn", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "six", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.16.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "setuptools", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "49.6.0.post20210108", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "scipy", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.7.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "scikit-learn", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.24.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "sagemaker", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.51.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "sagemaker-training", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.9.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "sagemaker-pytorch-training", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.4.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "sagemaker-experiments", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.1.34", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "safety", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.10.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "s3transfer", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.5.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "s3fs", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.4.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "ruamel-yaml-conda", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.15.100", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "rsa", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.7.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "retrying", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.3.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "requests", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.25.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyyaml", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "5.4.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pytz", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2021.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "python-dateutil", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.8.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pysocks", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.7.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyparsing", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.4.7", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyopenssl", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "19.1.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pynacl", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.4.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyinstrument", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.0.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pygments", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.7.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyfunctional", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.4.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pycparser", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.20", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pycosat", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.6.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyasn1", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.4.8", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pyarrow", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "5.0.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "ptyprocess", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.6.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "psutil", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "5.6.7", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "protobuf3-to-dict", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.1.5", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "protobuf", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.17.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "prompt-toolkit", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.0.8", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "ppft", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.6.6.4", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pox", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.3.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pip", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "21.2.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pillow", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "8.3.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pickleshare", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.7.5", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pexpect", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.8.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pathos", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2.8", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "parso", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.8.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "paramiko", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.7.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "pandas", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.2.4", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "packaging", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "21.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "numpy", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.19.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "networkx", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.6.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "multiprocess", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.70.12.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "joblib", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.0.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "jmespath", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.10.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "jedi", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.18.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "ipython", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "7.18.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "ipython-genutils", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "inotify-simple", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.2.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "importlib-metadata", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.6.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "idna", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2.10", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "h5py", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.2.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "greenlet", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.1.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "google-pasta", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "gevent", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "21.1.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "fsspec", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2021.7.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "dparse", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.5.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "docutils", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.15.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "dill", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.3.4", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "dgl", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.6.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "decorator", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.4.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "cython", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.29.21", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "cryptography", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.4.7", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "conda", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.10.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "conda-package-handling", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.7.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "colorama", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.4.3", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "click", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "8.0.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "chardet", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "4.0.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "cffi", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.14.6", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "certifi", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "2021.5.30", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "cached-property", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.5.2", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "brotlipy", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.7.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "botocore", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.21.14", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "boto3", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.18.14", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "bcrypt", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "3.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "backcall", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "awsio", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "0.0.1", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "awscli", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "1.20.14", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - }, - { - "package": "attrs", - "affected": "No known vulnerability found, PASSED_SAFETY_CHECK on 16082021.", - "installed": "21.2.0", - "vulnerabilities": [ - { - "vid": "N/A", - "advisory": "N/A" - } - ] - } -] \ No newline at end of file From 301c66322ebc822a2ec81ece5c0af2d2f4556e98 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Mon, 30 Aug 2021 20:08:00 +0000 Subject: [PATCH 62/88] Made changes to build images in non Codebuild Context --- src/safety_report_generator.py | 25 +++++++++++++++++++++---- src/utils.py | 3 +-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/safety_report_generator.py b/src/safety_report_generator.py index eff506f640cc..a8262fd3099b 100644 --- a/src/safety_report_generator.py +++ b/src/safety_report_generator.py @@ -1,8 +1,8 @@ -from dlc.safety_check import SafetyCheck from invoke.context import Context from datetime import datetime import json +import os TO_BE_DECIDED="To Be Decided" @@ -99,12 +99,29 @@ def process_report(self): v["scan_status"] = "FAILED" self.vulns_list.append(v) - + def run_safety_check_in_non_cb_context(self): + print('Running in Non CodeBuild Context') + safety_check_command = f"{self.docker_exec_cmd} safety check --json" + run_out = self.ctx.run(safety_check_command, warn=True, hide=True) + if run_out.return_code != 0: + print( + f"safety check command returned non-zero error code. stderr printed for logging: {run_out.stderr}" + ) + return run_out.stdout + + def run_safety_check_in_cb_context(self): + print('Running in CodeBuild Context') + from dlc.safety_check import SafetyCheck + return SafetyCheck().run_safety_check_on_container(self.docker_exec_cmd) def generate(self): self.timestamp = datetime.now().strftime("%d-%m-%Y") - # If cicd condition to be added - self.safety_check_output = SafetyCheck().run_safety_check_on_container(self.docker_exec_cmd) + if os.getenv('IS_CODEBUILD_IMAGE') is None: + self.safety_check_output = self.run_safety_check_in_non_cb_context() + elif os.getenv('IS_CODEBUILD_IMAGE').upper() == 'TRUE': + self.safety_check_output = self.run_safety_check_in_cb_context() + # In case of errors, json.loads command will fail. We want the failure to occur to ensure that + # build process fails in case the safety report cannot be generated properly. vulns = json.loads(self.safety_check_output) self.insert_vulnerabilites_into_report(vulns) packages = self.get_package_set_from_container() diff --git a/src/utils.py b/src/utils.py index f7972864897f..532c95b2140f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -24,7 +24,6 @@ from invoke.context import Context from botocore.exceptions import ClientError from safety_report_generator import SafetyReportGenerator -from dlc.safety_check import SafetyCheck LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.DEBUG) @@ -540,8 +539,8 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): docker_run_cmd = f"docker run -id --entrypoint='/bin/bash' {image_uri} " container_id = ctx.run(f"{docker_run_cmd}", hide=True, warn=True).stdout.strip() install_safety_cmd = "pip install safety" - ctx.run(f"docker exec {container_id} {install_safety_cmd}", hide=True, warn=True) docker_exec_cmd = f"docker exec -i {container_id}" + ctx.run(f"{docker_exec_cmd} {install_safety_cmd}", hide=True, warn=True) ignore_dict = get_safety_ignore_dict(image_uri) safety_scan_output = SafetyReportGenerator(container_id, ignore_dict=ignore_dict).generate() ctx.run(f"docker rm -f {container_id}", hide=True, warn=True) From 5ec5776355d1fc9718981fde057042ff04d75c1b Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Mon, 30 Aug 2021 21:42:22 +0000 Subject: [PATCH 63/88] Changed the test files to run on desired contexts --- dlc_developer_config.toml | 6 +++--- test/dlc_tests/sanity/test_safety_check.py | 4 ++-- test/dlc_tests/sanity/test_safety_check_v2.py | 10 ++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dlc_developer_config.toml b/dlc_developer_config.toml index 07e7b6ad4d5c..f8304f8fbf6c 100644 --- a/dlc_developer_config.toml +++ b/dlc_developer_config.toml @@ -26,9 +26,9 @@ do_build = true [test] ### On by default sanity_tests = true -ecs_tests = true -eks_tests = true -ec2_tests = true +ecs_tests = false +eks_tests = false +ec2_tests = false ### Off by default efa_tests = false diff --git a/test/dlc_tests/sanity/test_safety_check.py b/test/dlc_tests/sanity/test_safety_check.py index c39e9460e48e..f40b439f1a2e 100644 --- a/test/dlc_tests/sanity/test_safety_check.py +++ b/test/dlc_tests/sanity/test_safety_check.py @@ -149,10 +149,10 @@ def _get_latest_package_version(package): @pytest.mark.canary("Run safety tests regularly on production images") @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") @pytest.mark.skipif( - not (is_mainline_context() or (is_canary_context() and is_time_for_canary_safety_scan())), + not (is_canary_context() and is_time_for_canary_safety_scan()), reason=( "Skipping the test to decrease the number of calls to the Safety Check DB. " - "Test will be executed in the 'mainline' pipeline and canaries pipeline." + "Test will be executed only in canaries pipeline." ) ) def test_safety(image): diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index 5a9f6c1e5051..5c0c597ede27 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -9,7 +9,7 @@ from typing import List from test.test_utils import ( - CONTAINER_TESTS_PREFIX, is_dlc_cicd_context, is_canary_context, is_mainline_context, is_time_for_canary_safety_scan + is_canary_context ) LOGGER = logging.getLogger(__name__) @@ -50,9 +50,11 @@ def __post_init__(self): @pytest.mark.model("N/A") -@pytest.mark.canary("Run safety tests regularly on production images") -# @pytest.mark.skipif(not is_dlc_cicd_context(), reason="Skipping test because it is not running in dlc cicd infra") -def test_safety_file_exists(image): +@pytest.mark.skipif( + is_canary_context(), + reason="Skipping test because it is not required to run it on canaries. test_safety_check.py runs on canaries." +) +def test_safety_file_exists_and_is_valid(image): repo_name, image_tag = image.split('/')[-1].split(':') container_name = f"{repo_name}-{image_tag}-safety" # Add null entrypoint to ensure command exits immediately From ddd25aa65a8b27c7355170b4a4ecf32e6368bc6d Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 2 Sep 2021 07:47:01 +0000 Subject: [PATCH 64/88] Changing dummy client --- src/image_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image_builder.py b/src/image_builder.py index e0bd262814f0..3596e7056295 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -337,7 +337,7 @@ def get_dummy_boto_client(): # If this function is not added, boto3 fails because boto3 sessions are not thread safe. # However, once a dummy client is created, it is ensured that the calls are thread safe. import boto3 - return boto3.client("secretsmanager", region_name=os.getenv('REGION')) + return boto3.client("sts", region_name=os.getenv('REGION')) def push_images(images): THREADS = {} From d9db7903c1035de986258a3c5bfb560cd204271b Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 2 Sep 2021 17:44:38 +0000 Subject: [PATCH 65/88] Handling context deletion --- src/image.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/image.py b/src/image.py index d2504a190acf..9b36607f4a45 100644 --- a/src/image.py +++ b/src/image.py @@ -111,7 +111,10 @@ def build(self): with open(self.context.context_path, "rb") as context_file: print("within context") self.docker_build(fileobj=context_file, custom_context=True) - self.context.remove() + try: + self.context.remove() + except: + print("Context was already deleted") else: print("out of context") self.docker_build() From 9a4af96b8cbf494707ed0b2cf23bf34cb2ab0a4c Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 2 Sep 2021 19:25:35 +0000 Subject: [PATCH 66/88] Changed the context remove --- src/image.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/image.py b/src/image.py index 9b36607f4a45..57a7e5eebac1 100644 --- a/src/image.py +++ b/src/image.py @@ -111,10 +111,7 @@ def build(self): with open(self.context.context_path, "rb") as context_file: print("within context") self.docker_build(fileobj=context_file, custom_context=True) - try: - self.context.remove() - except: - print("Context was already deleted") + self.context.remove() else: print("out of context") self.docker_build() @@ -143,13 +140,14 @@ def docker_build(self, fileobj=None, custom_context=False): labels=self.labels ): if line.get("error") is not None: - self.context.remove() response.append(line["error"]) self.log = response self.build_status = constants.FAIL self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() + print("******** ERROR during Docker BUILD ********") + print(f"Error message received for {self.dockerfile} while docker build: {line}") return self.build_status @@ -161,7 +159,7 @@ def docker_build(self, fileobj=None, custom_context=False): response.append(str(line)) self.log = response - print(f"self.log {self.log}") + # print(f"self.log {self.log}") self.build_status = constants.SUCCESS #TODO: return required? return self.build_status From a5ca249bc35372f34b8f61bd95425f179dba5e3f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 2 Sep 2021 22:24:04 +0000 Subject: [PATCH 67/88] Refactoring image, conclusion, misc_docker --- .../Dockerfile.common | 2 +- src/conclusion_stage_image.py | 45 +++--- src/image.py | 132 ++++++++++-------- test/dlc_tests/sanity/test_safety_check_v2.py | 2 +- 4 files changed, 94 insertions(+), 87 deletions(-) rename src/Dockerfile.multipart => miscellaneous_dockerfiles/Dockerfile.common (69%) diff --git a/src/Dockerfile.multipart b/miscellaneous_dockerfiles/Dockerfile.common similarity index 69% rename from src/Dockerfile.multipart rename to miscellaneous_dockerfiles/Dockerfile.common index 3409c1cff874..7d9405a041f7 100644 --- a/src/Dockerfile.multipart +++ b/miscellaneous_dockerfiles/Dockerfile.common @@ -4,4 +4,4 @@ ARG INITIAL_STAGE_IMAGE="" FROM $INITIAL_STAGE_IMAGE # Add any script or repo as required -COPY safety_report.json /var/safety_report.json +COPY safety_report.json /opt/aws/dlc/info/safety_report.json diff --git a/src/conclusion_stage_image.py b/src/conclusion_stage_image.py index ad03407fdbdb..1a0118868dde 100644 --- a/src/conclusion_stage_image.py +++ b/src/conclusion_stage_image.py @@ -19,50 +19,41 @@ import os + class ConclusionStageImage(DockerImage): """ Class designed to handle the ConclusionStageImages """ - def pre_build_configuration(self): + def update_pre_build_configuration(self): """ Conducts all the pre-build configurations from the parent class and then conducts Safety Scan on the images generated in previous stage builds. The safety scan generates the safety_report which is then baked into the image. """ - ## Call the pre_build_configuration steps from the parent class - super(ConclusionStageImage, self).pre_build_configuration() - ## Generate safety scan report for the first stage image and add the file to artifacts - first_stage_image_uri = self.build_args['INITIAL_STAGE_IMAGE'] - processed_image_uri = first_stage_image_uri.replace('.','-').replace('/','-').replace(':','-') + # Call the update_pre_build_configuration steps from the parent class + super(ConclusionStageImage, self).update_pre_build_configuration() + # Generate safety scan report for the first stage image and add the file to artifacts + first_stage_image_uri = self.build_args["INITIAL_STAGE_IMAGE"] + processed_image_uri = first_stage_image_uri.replace(".", "-").replace("/", "-").replace(":", "-") image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" storage_file_path = f"{os.getenv('ROOT_FOLDER_PATH')}/src/{tarfile_name_for_context}_safety_report.json" generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) - def generate_conclude_stage_context(self, safety_report_path, tarfile_name='conclusion-stage-file'): + def generate_conclude_stage_context(self, safety_report_path, tarfile_name="conclusion-stage-file"): """ For ConclusionStageImage, build context is built once the safety report is generated. This is because the Dockerfile.multipart uses this safety report to COPY the report into the image. """ - ARTIFACTS = {} - ARTIFACTS.update( - { - "safety_report": { - "source": safety_report_path, - "target": "safety_report.json" - } - }) - ARTIFACTS.update( - { - "dockerfile": { - "source": f"Dockerfile.multipart", - "target": "Dockerfile", - } - } - ) - - artifact_root = os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src") + "/" - return Context(ARTIFACTS, context_path=f'build/{tarfile_name}.tar.gz',artifact_root=artifact_root) - + artifacts = { + "safety_report": {"source": safety_report_path, "target": "safety_report.json"}, + "dockerfile": { + "source": f"{os.getenv('ROOT_FOLDER_PATH')}/miscellaneous_dockerfiles/Dockerfile.common", + "target": "Dockerfile", + }, + } + + artifact_root = os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src") + return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root) diff --git a/src/image.py b/src/image.py index 57a7e5eebac1..40c579b650b6 100644 --- a/src/image.py +++ b/src/image.py @@ -19,6 +19,11 @@ import constants +import logging +import json + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.DEBUG) class DockerImage: @@ -26,9 +31,7 @@ class DockerImage: The DockerImage class has the functions and attributes for building the dockerimage """ - def __init__( - self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True - ): + def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True): # Meta-data about the image should go to info. # All keys in info are accessible as attributes @@ -72,7 +75,7 @@ def collect_installed_packages_information(self): docker_client.containers.prune() return command_responses - def pre_build_configuration(self): + def update_pre_build_configuration(self): if self.info.get("base_image_uri"): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] @@ -82,12 +85,12 @@ def pre_build_configuration(self): if self.info.get("extra_build_args"): self.build_args.update(self.info.get("extra_build_args")) - + if self.info.get("labels"): self.labels.update(self.info.get("labels")) - - print(f"self.build_args {self.build_args}") - print(f"self.labels {self.labels}") + + LOGGER.info(f"self.build_args {json.dumps(self.build_args, indent=4)}") + LOGGER.info(f"self.labels {json.dumps(self.labels, indent=4)}") def build(self): """ @@ -95,91 +98,96 @@ def build(self): """ self.summary["start_time"] = datetime.now() - ## Confirm if building the image is required or not + # Confirm if building the image is required or not if not self.to_build: self.log = ["Not built"] self.build_status = constants.NOT_BUILT self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] return self.build_status - - ## Conduct some preprocessing before building the image - self.pre_build_configuration() - print(f"self.context {self.context}") - ## Start building the image + # Conduct some preprocessing before building the image + self.update_pre_build_configuration() + + # Start building the image if self.context: with open(self.context.context_path, "rb") as context_file: - print("within context") self.docker_build(fileobj=context_file, custom_context=True) self.context.remove() else: - print("out of context") self.docker_build() + if self.build_status == constants.FAIL: + return self.build_status + if not self.to_push: - ## If this image is not supposed to be pushed, in that case, we are already done - ## with building the image and do not need to conduct any further processing. + # If this image is not supposed to be pushed, in that case, we are already done + # with building the image and do not need to conduct any further processing. self.summary["end_time"] = datetime.now() - #check the size after image is built. + # check the size after image is built. self.image_size_check() - ## This return is necessary. Otherwise FORMATTER fails while displaying the status. + # This return is necessary. Otherwise FORMATTER fails while displaying the status. return self.build_status - + def docker_build(self, fileobj=None, custom_context=False): response = [] for line in self.client.build( - fileobj=fileobj, - path=self.dockerfile, - custom_context=custom_context, - rm=True, - decode=True, - tag=self.ecr_url, - buildargs=self.build_args, - labels=self.labels - ): - if line.get("error") is not None: - response.append(line["error"]) - - self.log = response - self.build_status = constants.FAIL - self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] - self.summary["end_time"] = datetime.now() - print("******** ERROR during Docker BUILD ********") - print(f"Error message received for {self.dockerfile} while docker build: {line}") - - return self.build_status - - if line.get("stream") is not None: - response.append(line["stream"]) - elif line.get("status") is not None: - response.append(line["status"]) - else: - response.append(str(line)) + fileobj=fileobj, + path=self.dockerfile, + custom_context=custom_context, + rm=True, + decode=True, + tag=self.ecr_url, + buildargs=self.build_args, + labels=self.labels, + ): + if line.get("error") is not None: + response.append(line["error"]) + self.log = response + self.build_status = constants.FAIL + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + self.summary["end_time"] = datetime.now() + + pretty_logs = "\n".join(self.log[:-100]) + LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") + LOGGER.error("******** ERROR during Docker BUILD ********") + LOGGER.error(f"Error message received for {self.dockerfile} while docker build: {line}") + + return self.build_status + + if line.get("stream") is not None: + response.append(line["stream"]) + elif line.get("status") is not None: + response.append(line["status"]) + else: + response.append(str(line)) self.log = response - # print(f"self.log {self.log}") + + pretty_logs = "\n".join(self.log[-10:]) + LOGGER.info(f"Docker Build Logs: {pretty_logs}") + LOGGER.info(f"Completed Build for {self.name}") + self.build_status = constants.SUCCESS - #TODO: return required? return self.build_status - def image_size_check(self): response = [] - self.summary["image_size"] = int( - self.client.inspect_image(self.ecr_url)["Size"] - ) / (1024 * 1024) + self.summary["image_size"] = int(self.client.inspect_image(self.ecr_url)["Size"]) / (1024 * 1024) if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: response.append("Image size baseline exceeded") response.append(f"{self.summary['image_size']} > 1.2 * {self.info['image_size_baseline']}") response += self.collect_installed_packages_information() self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT else: + response.append(f"Image Size Check Succeeded for {self.name}") self.build_status = constants.SUCCESS self.log = response - print(f"self.log {self.log}") - #TODO: return required? + + pretty_log = "\n".join(self.log) + LOGGER.info(f"{pretty_log}") + return self.build_status def push_image(self): @@ -187,11 +195,15 @@ def push_image(self): response = [] if line.get("error") is not None: response.append(line["error"]) - self.log = response self.build_status = constants.FAIL self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() + pretty_logs = "\n".join(self.log[:-100]) + + LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") + LOGGER.error("******** ERROR during Docker PUSH ********") + LOGGER.error(f"Error message received for {self.dockerfile} while docker push: {line}") return self.build_status if line.get("stream") is not None: @@ -203,5 +215,9 @@ def push_image(self): self.summary["end_time"] = datetime.now() self.summary["ecr_url"] = self.ecr_url self.log = response - #TODO: return required? + + pretty_logs = "\n".join(self.log[-10:]) + LOGGER.info(f"Docker Build Logs: {pretty_logs}") + LOGGER.info(f"Completed Build for {self.name}") + return self.build_status diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index 5c0c597ede27..57258b4b3d6e 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -16,7 +16,7 @@ LOGGER.setLevel(logging.INFO) LOGGER.addHandler(logging.StreamHandler(sys.stderr)) -SAFETY_FILE = '/var/safety_report.json' +SAFETY_FILE = '/opt/aws/dlc/info/safety_report.json' SAFETY_FILE_EXISTS = 0 From a15e23cc7d8114f3247bc32a07f81cfa37303a4f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 2 Sep 2021 23:50:43 +0000 Subject: [PATCH 68/88] Refactoring Conclusion to Common and Initial to Pre_push --- miscellaneous_dockerfiles/Dockerfile.common | 4 +- ...n_stage_image.py => common_stage_image.py} | 22 ++--- src/constants.py | 5 +- src/image.py | 2 +- src/image_builder.py | 81 +++++++++---------- 5 files changed, 56 insertions(+), 58 deletions(-) rename src/{conclusion_stage_image.py => common_stage_image.py} (65%) diff --git a/miscellaneous_dockerfiles/Dockerfile.common b/miscellaneous_dockerfiles/Dockerfile.common index 7d9405a041f7..81f7bfa81a79 100644 --- a/miscellaneous_dockerfiles/Dockerfile.common +++ b/miscellaneous_dockerfiles/Dockerfile.common @@ -1,7 +1,7 @@ # Use the Deep Learning Container as a base Image -ARG INITIAL_STAGE_IMAGE="" +ARG PRE_PUSH_IMAGE="" -FROM $INITIAL_STAGE_IMAGE +FROM $PRE_PUSH_IMAGE # Add any script or repo as required COPY safety_report.json /opt/aws/dlc/info/safety_report.json diff --git a/src/conclusion_stage_image.py b/src/common_stage_image.py similarity index 65% rename from src/conclusion_stage_image.py rename to src/common_stage_image.py index 1a0118868dde..2e463ee78be8 100644 --- a/src/conclusion_stage_image.py +++ b/src/common_stage_image.py @@ -20,32 +20,36 @@ import os -class ConclusionStageImage(DockerImage): +class CommonStageImage(DockerImage): """ - Class designed to handle the ConclusionStageImages + This class is especially designed to handle the build process for CommonStageImages. + All the functionality - either safety scan report, ecr scan report, etc. - that is especially + required to run the miscellaneous_dockerfiles/Dockerfile.common should go into this file. As of now, + this class takes care of generating a safety report from a pre_push_image and then uses this + safety report for creating a context for Dockerfile.common """ def update_pre_build_configuration(self): """ Conducts all the pre-build configurations from the parent class and then conducts Safety Scan on the images generated in previous stage builds. The safety scan generates - the safety_report which is then baked into the image. + the safety_report which is then copied into the image. """ # Call the update_pre_build_configuration steps from the parent class - super(ConclusionStageImage, self).update_pre_build_configuration() + super(CommonStageImage, self).update_pre_build_configuration() # Generate safety scan report for the first stage image and add the file to artifacts - first_stage_image_uri = self.build_args["INITIAL_STAGE_IMAGE"] + first_stage_image_uri = self.build_args["PRE_PUSH_IMAGE"] processed_image_uri = first_stage_image_uri.replace(".", "-").replace("/", "-").replace(":", "-") image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" storage_file_path = f"{os.getenv('ROOT_FOLDER_PATH')}/src/{tarfile_name_for_context}_safety_report.json" generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) - self.context = self.generate_conclude_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) + self.context = self.generate_common_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) - def generate_conclude_stage_context(self, safety_report_path, tarfile_name="conclusion-stage-file"): + def generate_common_stage_context(self, safety_report_path, tarfile_name="common-stage-file"): """ - For ConclusionStageImage, build context is built once the safety report is generated. This is because - the Dockerfile.multipart uses this safety report to COPY the report into the image. + For CommonStageImage, build context is built once the safety report is generated. This is because + the Dockerfile.common uses this safety report to COPY the report into the image. """ artifacts = { "safety_report": {"source": safety_report_path, "target": "safety_report.json"}, diff --git a/src/constants.py b/src/constants.py index 1d2b84f541f7..d590a46cf9d0 100644 --- a/src/constants.py +++ b/src/constants.py @@ -32,8 +32,8 @@ PADDING = 1 #Docker build stages -INITIAL_STAGE="initial" -CONCLUSION_STAGE="conclusion" +PRE_PUSH_STAGE = "pre_push" +COMMON_STAGE = "common" # Docker connections DOCKER_URL = "unix://var/run/docker.sock" @@ -76,5 +76,6 @@ EKS_TESTS = "eks" ALL_TESTS = ["sagemaker", "ec2", "eks", "ecs"] +# Timeout in seconds for Docker API client. API_CLIENT_TIMEOUT = 600 MAX_WORKER_COUNT_FOR_PUSHING_IMAGES = 3 diff --git a/src/image.py b/src/image.py index 40c579b650b6..09f926127642 100644 --- a/src/image.py +++ b/src/image.py @@ -81,7 +81,7 @@ def update_pre_build_configuration(self): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] if self.ecr_url: - self.build_args["INITIAL_STAGE_IMAGE"] = self.ecr_url + self.build_args["PRE_PUSH_IMAGE"] = self.ecr_url if self.info.get("extra_build_args"): self.build_args.update(self.info.get("extra_build_args")) diff --git a/src/image_builder.py b/src/image_builder.py index 3596e7056295..fab945a5beaf 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -21,11 +21,12 @@ import constants import utils +import boto3 from context import Context from metrics import Metrics from image import DockerImage -from conclusion_stage_image import ConclusionStageImage +from common_stage_image import CommonStageImage from buildspec import Buildspec from output import OutputFormatter from config import parse_dlc_developer_configs @@ -53,8 +54,8 @@ def image_builder(buildspec): BUILDSPEC = Buildspec() BUILDSPEC.load(buildspec) - INITIAL_STAGE_IMAGES = [] - CONCLUSION_STAGE_IMAGES = [] + PRE_PUSH_STAGE_IMAGES = [] + COMMON_STAGE_IMAGES = [] if "huggingface" in str(BUILDSPEC["framework"]): os.system("echo login into public ECR") @@ -88,7 +89,7 @@ def image_builder(buildspec): ) base_image_uri = None if image_config.get("base_image_name") is not None: - base_image_object = _find_image_object(INITIAL_STAGE_IMAGES, image_config["base_image_name"]) + base_image_object = _find_image_object(PRE_PUSH_STAGE_IMAGES, image_config["base_image_name"]) base_image_uri = base_image_object.ecr_url if image_config.get("download_artifacts") is not None: @@ -157,54 +158,51 @@ def image_builder(buildspec): "extra_build_args": extra_build_args } - #Create initial stage docker object - initial_stage_image_object = DockerImage( + #Create pre_push stage docker object + pre_push_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], repository=image_repo_uri, tag=image_tag, to_build=image_config["build"], - stage=constants.INITIAL_STAGE, + stage=constants.PRE_PUSH_STAGE, context=context, ) - ##### Create Conclusion stage docker object ##### - # If for an initial stage image we create a conclusion stage image, then we do not push the initial stage image - # to the repository. Instead, we just push its conclusion stage image to the repository. Therefore, - # inside function get_conclusion_stage_image_object we make initial_stage_image_object non pushable. - conclusion_stage_image_object = get_conclusion_stage_image_object(initial_stage_image_object) - + ##### Create Common stage docker object ##### + # If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image + # to the repository. Instead, we just push its common stage image to the repository. Therefore, + # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. + common_stage_image_object = get_common_stage_image_object(pre_push_stage_image_object) + + PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) + COMMON_STAGE_IMAGES.append(common_stage_image_object) FORMATTER.separator() - INITIAL_STAGE_IMAGES.append(initial_stage_image_object) - if conclusion_stage_image_object is not None: - CONCLUSION_STAGE_IMAGES.append(conclusion_stage_image_object) - FORMATTER.banner("DLC") FORMATTER.title("Status") # Standard images must be built before example images # Example images will use standard images as base - # Conclusion images must be built at the end as they will consume respective standard and example images - standard_images = [image for image in INITIAL_STAGE_IMAGES if "example" not in image.name.lower()] - example_images = [image for image in INITIAL_STAGE_IMAGES if "example" in image.name.lower()] - conclusion_stage_images = [image for image in CONCLUSION_STAGE_IMAGES] - ALL_IMAGES = INITIAL_STAGE_IMAGES + CONCLUSION_STAGE_IMAGES + # Common images must be built at the end as they will consume respective standard and example images + standard_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" not in image.name.lower()] + example_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" in image.name.lower()] + common_stage_images = [image for image in COMMON_STAGE_IMAGES] + ALL_IMAGES = PRE_PUSH_STAGE_IMAGES + COMMON_STAGE_IMAGES IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push and image.to_build] - #initial stage standard images build + #pre_push stage standard images build FORMATTER.banner("Standard Build") build_images(standard_images) - #initial stage example images build + #pre_push stage example images build FORMATTER.banner("Example Build") build_images(example_images) - - #Conclusion stage build - if len(conclusion_stage_images) > 0: - FORMATTER.banner("Conclusion Build") - build_images(conclusion_stage_images, make_dummy_boto_client=True) - + + #Common stage build + FORMATTER.banner("Common Build") + build_images(common_stage_images, make_dummy_boto_client=True) + FORMATTER.banner("Push Started") push_images(IMAGES_TO_PUSH) @@ -218,7 +216,6 @@ def image_builder(buildspec): BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] FORMATTER.title("Upload Metrics") - # change logic here. upload metrics only for the Conclusion stage image upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) FORMATTER.title("Setting Test Env") @@ -231,20 +228,17 @@ def image_builder(buildspec): TEST_TRIGGER=test_trigger_job, ) -def get_conclusion_stage_image_object(initial_stage_image_object): - # Check if this is only required for mainline - # if build_context == "MAINLINE": - conclusion_stage_image_object = None - conclusion_stage_image_object = ConclusionStageImage( - info=initial_stage_image_object.info, +def get_common_stage_image_object(pre_push_stage_image_object): + common_stage_image_object = CommonStageImage( + info=pre_push_stage_image_object.info, dockerfile=os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src", "Dockerfile.multipart"), - repository=initial_stage_image_object.repository, - tag=initial_stage_image_object.tag, - to_build=initial_stage_image_object.to_build, - stage=constants.CONCLUSION_STAGE, + repository=pre_push_stage_image_object.repository, + tag=pre_push_stage_image_object.tag, + to_build=pre_push_stage_image_object.to_build, + stage=constants.COMMON_STAGE, ) - initial_stage_image_object.to_push = False - return conclusion_stage_image_object + pre_push_stage_image_object.to_push = False + return common_stage_image_object def show_build_logs(images): @@ -336,7 +330,6 @@ def get_dummy_boto_client(): # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. # If this function is not added, boto3 fails because boto3 sessions are not thread safe. # However, once a dummy client is created, it is ensured that the calls are thread safe. - import boto3 return boto3.client("sts", region_name=os.getenv('REGION')) def push_images(images): From c71628c7d9fcb7e16c9d8b38b44911e19e042b09 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 3 Sep 2021 01:19:37 +0000 Subject: [PATCH 69/88] Added the get_root_folder_path method --- src/common_stage_image.py | 12 ++++++++---- src/image_builder.py | 35 +++++++++++++++-------------------- src/utils.py | 8 +++++++- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/common_stage_image.py b/src/common_stage_image.py index 2e463ee78be8..92843a239918 100644 --- a/src/common_stage_image.py +++ b/src/common_stage_image.py @@ -15,7 +15,7 @@ from image import DockerImage from context import Context -from utils import generate_safety_report_for_image +from utils import generate_safety_report_for_image, get_root_folder_path import os @@ -42,7 +42,9 @@ def update_pre_build_configuration(self): processed_image_uri = first_stage_image_uri.replace(".", "-").replace("/", "-").replace(":", "-") image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" - storage_file_path = f"{os.getenv('ROOT_FOLDER_PATH')}/src/{tarfile_name_for_context}_safety_report.json" + storage_file_path = os.path.join( + os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json" + ) generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) self.context = self.generate_common_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) @@ -54,10 +56,12 @@ def generate_common_stage_context(self, safety_report_path, tarfile_name="common artifacts = { "safety_report": {"source": safety_report_path, "target": "safety_report.json"}, "dockerfile": { - "source": f"{os.getenv('ROOT_FOLDER_PATH')}/miscellaneous_dockerfiles/Dockerfile.common", + "source": os.path.join( + os.sep, get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common" + ), "target": "Dockerfile", }, } - artifact_root = os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src") + artifact_root = os.path.join(os.sep, get_root_folder_path(), "src") return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root) diff --git a/src/image_builder.py b/src/image_builder.py index fab945a5beaf..9f0190b21615 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -158,7 +158,7 @@ def image_builder(buildspec): "extra_build_args": extra_build_args } - #Create pre_push stage docker object + # Create pre_push stage docker object pre_push_stage_image_object = DockerImage( info=info, dockerfile=image_config["docker_file"], @@ -174,14 +174,14 @@ def image_builder(buildspec): # to the repository. Instead, we just push its common stage image to the repository. Therefore, # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. common_stage_image_object = get_common_stage_image_object(pre_push_stage_image_object) - + PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) COMMON_STAGE_IMAGES.append(common_stage_image_object) FORMATTER.separator() FORMATTER.banner("DLC") FORMATTER.title("Status") - + # Standard images must be built before example images # Example images will use standard images as base # Common images must be built at the end as they will consume respective standard and example images @@ -191,31 +191,30 @@ def image_builder(buildspec): ALL_IMAGES = PRE_PUSH_STAGE_IMAGES + COMMON_STAGE_IMAGES IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push and image.to_build] - #pre_push stage standard images build + # pre_push stage standard images build FORMATTER.banner("Standard Build") build_images(standard_images) - #pre_push stage example images build + # pre_push stage example images build FORMATTER.banner("Example Build") build_images(example_images) - #Common stage build + # Common stage build FORMATTER.banner("Common Build") build_images(common_stage_images, make_dummy_boto_client=True) - + FORMATTER.banner("Push Started") push_images(IMAGES_TO_PUSH) - FORMATTER.title("Log Display") - #After the build, display logs/summary for all the images. + FORMATTER.banner("Logs/Summary") + # After the build, display logs/summary for all the images. show_build_logs(ALL_IMAGES) show_build_summary(ALL_IMAGES) is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) - #From all images, filter the images that were supposed to be built and upload their metrics + # From all images, filter the images that were supposed to be built and upload their metrics BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] - FORMATTER.title("Upload Metrics") upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) FORMATTER.title("Setting Test Env") @@ -223,15 +222,13 @@ def image_builder(buildspec): test_trigger_job = utils.get_codebuild_project_name() # Tests should only run on images that were pushed to the repository utils.set_test_env( - IMAGES_TO_PUSH, - BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), - TEST_TRIGGER=test_trigger_job, + IMAGES_TO_PUSH, BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), TEST_TRIGGER=test_trigger_job, ) def get_common_stage_image_object(pre_push_stage_image_object): common_stage_image_object = CommonStageImage( info=pre_push_stage_image_object.info, - dockerfile=os.path.join(os.sep, os.getenv("ROOT_FOLDER_PATH"), "src", "Dockerfile.multipart"), + dockerfile=os.path.join(os.sep, utils.get_root_folder_path(), "src", "Dockerfile.multipart"), repository=pre_push_stage_image_object.repository, tag=pre_push_stage_image_object.tag, to_build=pre_push_stage_image_object.to_build, @@ -317,7 +314,7 @@ def build_images(images, make_dummy_boto_client=False): # to it is executed concurrently in a separate thread. with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: #### TODO: Remove this entire if block when https://github.com/boto/boto3/issues/1592 is resolved #### - if make_dummy_boto_client: + if make_dummy_boto_client: get_dummy_boto_client() for image in images: FORMATTER.print(f"image_object.context {image.context}") @@ -330,13 +327,11 @@ def get_dummy_boto_client(): # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. # If this function is not added, boto3 fails because boto3 sessions are not thread safe. # However, once a dummy client is created, it is ensured that the calls are thread safe. - return boto3.client("sts", region_name=os.getenv('REGION')) + return boto3.client("sts", region_name=os.getenv("REGION")) def push_images(images): THREADS = {} - with concurrent.futures.ThreadPoolExecutor( - max_workers=constants.MAX_WORKER_COUNT_FOR_PUSHING_IMAGES - ) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=constants.MAX_WORKER_COUNT_FOR_PUSHING_IMAGES) as executor: for image in images: THREADS[image.name] = executor.submit(image.push_image) FORMATTER.progress(THREADS) diff --git a/src/utils.py b/src/utils.py index 532c95b2140f..9a49a30f30c2 100644 --- a/src/utils.py +++ b/src/utils.py @@ -488,6 +488,11 @@ def get_codebuild_project_name(): # Default value for codebuild project name is "local_test" when run outside of CodeBuild return os.getenv("CODEBUILD_BUILD_ID", "local_test").split(":")[0] +def get_root_folder_path(): + root_dir_pattern = re.compile(r"^(\S+deep-learning-containers)") + pwd = os.getcwd() + return os.getenv("CODEBUILD_SRC_DIR", root_dir_pattern.match(pwd).group(1)) + def get_safety_ignore_dict(image_uri): """ Get a dict of known safety check issue IDs to ignore, if specified in file ../data/ignore_ids_safety_scan.json. @@ -504,7 +509,8 @@ def get_safety_ignore_dict(image_uri): python_version = "py2" if "py2" in image_uri else "py3" IGNORE_SAFETY_IDS = {} - with open(f'{os.getenv("ROOT_FOLDER_PATH")}/data/ignore_ids_safety_scan.json') as f: + ignore_data_file = os.path.join(os.sep, get_root_folder_path(), "data", "ignore_ids_safety_scan.json") + with open(ignore_data_file) as f: IGNORE_SAFETY_IDS = json.load(f) return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, {}) From 8146137000c20cf37e2f8a9a868d718c5db49b23 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 3 Sep 2021 04:34:07 +0000 Subject: [PATCH 70/88] Changed safety report generator --- src/safety_report_generator.py | 147 +++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/src/safety_report_generator.py b/src/safety_report_generator.py index a8262fd3099b..9655444a32a9 100644 --- a/src/safety_report_generator.py +++ b/src/safety_report_generator.py @@ -4,7 +4,6 @@ import json import os -TO_BE_DECIDED="To Be Decided" class SafetyReportGenerator: """ @@ -17,9 +16,9 @@ class SafetyReportGenerator: "installed": "version", "vulnerabilities": [ { - "vid": "safety_vulnerability_id", + "vulnerability_id": "safety_vulnerability_id", "advisory": "description of the issue", - "reason_to_ignore":"reason to ignore the vid", + "reason_to_ignore":"reason to ignore the vulnerability_id", "spec": "version_spec" }, ... @@ -30,110 +29,136 @@ class SafetyReportGenerator: ] """ - def __init__(self, container_id, ignore_dict = {}): + def __init__(self, container_id, ignore_dict={}): self.container_id = container_id - self.vulns_dict = {} - self.vulns_list = [] + self.vulnerability_dict = {} + self.vulnerability_list = [] self.ignore_dict = ignore_dict self.ignored_vulnerability_count = {} self.ctx = Context() self.docker_exec_cmd = f"docker exec -i {container_id}" self.safety_check_output = None - - def insert_vulnerabilites_into_report(self, vulns): - for v in vulns: - package = v[0] - spec = v[1] - installed = v[2] - advisory = v[3] - vid = v[4] - vulnerability_details = {"vid": vid, "advisory": advisory, "spec": spec, "reason_to_ignore":"N/A"} + + def insert_vulnerabilites_into_report(self, scanned_vulnerabilities): + """ + Takes the list of vulnerabilites produced by safety scan as the input and iterates through the list to insert + the vulnerabilites into the vulnerability_dict. + + Parameters: + scanned_vulnerabilities: list[list] + + Returns: + None + """ + for vulnerability in scanned_vulnerabilities: + package, spec, installed, advisory, vulnerability_id = vulnerability[:5] + vulnerability_details = { + "vulnerability_id": vulnerability_id, + "advisory": advisory, + "spec": spec, + "reason_to_ignore": "N/A", + } if package not in self.ignored_vulnerability_count: self.ignored_vulnerability_count[package] = 0 - if vid in self.ignore_dict: - vulnerability_details["reason_to_ignore"] = self.ignore_dict[vid] + if vulnerability_id in self.ignore_dict: + vulnerability_details["reason_to_ignore"] = self.ignore_dict[vulnerability_id] self.ignored_vulnerability_count[package] += 1 - if package not in self.vulns_dict: - self.vulns_dict[package] = { + if package not in self.vulnerability_dict: + self.vulnerability_dict[package] = { "package": package, - "scan_status": TO_BE_DECIDED, + "scan_status": "TBD", "installed": installed, "vulnerabilities": [vulnerability_details], - "date":self.timestamp, + "date": self.timestamp, } else: - self.vulns_dict[package]["vulnerabilities"].append(vulnerability_details) - + self.vulnerability_dict[package]["vulnerabilities"].append(vulnerability_details) + def get_package_set_from_container(self): + """ + Extracts package set of a container. + """ python_cmd_to_extract_package_set = """ python -c "import pkg_resources; \ import json; \ print(json.dumps([{'key':d.key, 'version':d.version} for d in pkg_resources.working_set]))" """ - + run_output = self.ctx.run(f"{self.docker_exec_cmd} {python_cmd_to_extract_package_set}", hide=True, warn=True) if run_output.exited != 0: - raise Exception('Package set cannot be retrieved from the container.') - + raise Exception("Package set cannot be retrieved from the container.") + return json.loads(run_output.stdout) - def insert_safe_packages_into_report(self, packages): + """ + Takes the list of all the packages existing in a container and inserts safe packages into the + vulnerability_dict. + :param packages: list[dict], each dict looks like {"key":package_name, "version":package_version} + """ for pkg in packages: - if pkg['key'] not in self.vulns_dict: - self.vulns_dict[pkg['key']] = { - "package": pkg['key'], + if pkg["key"] not in self.vulnerability_dict: + self.vulnerability_dict[pkg["key"]] = { + "package": pkg["key"], "scan_status": "SUCCEEDED", - "installed": pkg['version'], - "vulnerabilities": [{"vid": "N/A", "advisory": "N/A", "reason_to_ignore":"N/A", "spec":"N/A"}], - "date":self.timestamp + "installed": pkg["version"], + "vulnerabilities": [ + {"vulnerability_id": "N/A", "advisory": "N/A", "reason_to_ignore": "N/A", "spec": "N/A"} + ], + "date": self.timestamp, } - + def process_report(self): - for (k, v) in self.vulns_dict.items(): - if v["scan_status"] == TO_BE_DECIDED: - if len(v["vulnerabilities"]) == self.ignored_vulnerability_count[k]: - v["scan_status"] = "IGNORED" + """ + Once all the packages (safe and unsafe both) have been inserted in the vulnerability_dict, this method is called. + On being called, it processes each package within the vulnerability_dict and appends it to the vulnerability_list. + Before appending it checks if the scan_status is "TBD". If yes, it assigns the correct scan_status to the package. + """ + for (package, package_scan_results) in self.vulnerability_dict.items(): + if package_scan_results["scan_status"] == "TBD": + if len(package_scan_results["vulnerabilities"]) == self.ignored_vulnerability_count[package]: + package_scan_results["scan_status"] = "IGNORED" else: - v["scan_status"] = "FAILED" - self.vulns_list.append(v) + package_scan_results["scan_status"] = "FAILED" + self.vulnerability_list.append(package_scan_results) def run_safety_check_in_non_cb_context(self): - print('Running in Non CodeBuild Context') + """ + Runs the safety check on the container in Non-CodeBuild Context + + :return: string, A JSON formatted string containing vulnerabilities found in the container + """ safety_check_command = f"{self.docker_exec_cmd} safety check --json" run_out = self.ctx.run(safety_check_command, warn=True, hide=True) if run_out.return_code != 0: - print( - f"safety check command returned non-zero error code. stderr printed for logging: {run_out.stderr}" - ) + print(f"safety check command returned non-zero error code. stderr printed for logging: {run_out.stderr}") return run_out.stdout - + def run_safety_check_in_cb_context(self): - print('Running in CodeBuild Context') + """ + Runs the safety check on the container in CodeBuild Context + """ from dlc.safety_check import SafetyCheck + return SafetyCheck().run_safety_check_on_container(self.docker_exec_cmd) - + def generate(self): + """ + Acts as a driver function for this class that initiates the entire process of running safety check and returing + the vulnerability_list + :return: list[dict], the output follows the same format as mentioned in the description of the class + """ self.timestamp = datetime.now().strftime("%d-%m-%Y") - if os.getenv('IS_CODEBUILD_IMAGE') is None: + if os.getenv("IS_CODEBUILD_IMAGE") is None: self.safety_check_output = self.run_safety_check_in_non_cb_context() - elif os.getenv('IS_CODEBUILD_IMAGE').upper() == 'TRUE': + elif os.getenv("IS_CODEBUILD_IMAGE").upper() == "TRUE": self.safety_check_output = self.run_safety_check_in_cb_context() # In case of errors, json.loads command will fail. We want the failure to occur to ensure that # build process fails in case the safety report cannot be generated properly. - vulns = json.loads(self.safety_check_output) - self.insert_vulnerabilites_into_report(vulns) + scanned_vulnerabilities = json.loads(self.safety_check_output) + self.insert_vulnerabilites_into_report(scanned_vulnerabilities) packages = self.get_package_set_from_container() self.insert_safe_packages_into_report(packages) self.process_report() - return self.vulns_list - - - - - - - - - + return self.vulnerability_list From f093cdefaba3496bb0acbba725f9c456058b85c7 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 3 Sep 2021 19:38:46 +0000 Subject: [PATCH 71/88] Changing the key in test file --- src/safety_report_generator.py | 14 ++++++++------ test/dlc_tests/sanity/test_safety_check_v2.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/safety_report_generator.py b/src/safety_report_generator.py index 9655444a32a9..d90be30f5203 100644 --- a/src/safety_report_generator.py +++ b/src/safety_report_generator.py @@ -43,12 +43,8 @@ def insert_vulnerabilites_into_report(self, scanned_vulnerabilities): """ Takes the list of vulnerabilites produced by safety scan as the input and iterates through the list to insert the vulnerabilites into the vulnerability_dict. - - Parameters: - scanned_vulnerabilities: list[list] - - Returns: - None + + :param scanned_vulnerabilities: list[list], consists of a list of Vulnerabilities. Each vulnerability is a list itself. """ for vulnerability in scanned_vulnerabilities: package, spec, installed, advisory, vulnerability_id = vulnerability[:5] @@ -80,6 +76,8 @@ def insert_vulnerabilites_into_report(self, scanned_vulnerabilities): def get_package_set_from_container(self): """ Extracts package set of a container. + + :return: list[dict], each dict is structured like {'key': package_name, 'version':package_version} """ python_cmd_to_extract_package_set = """ python -c "import pkg_resources; \ import json; \ @@ -95,6 +93,7 @@ def insert_safe_packages_into_report(self, packages): """ Takes the list of all the packages existing in a container and inserts safe packages into the vulnerability_dict. + :param packages: list[dict], each dict looks like {"key":package_name, "version":package_version} """ for pkg in packages: @@ -138,6 +137,8 @@ def run_safety_check_in_non_cb_context(self): def run_safety_check_in_cb_context(self): """ Runs the safety check on the container in CodeBuild Context + + :return: string, A JSON formatted string containing vulnerabilities found in the container """ from dlc.safety_check import SafetyCheck @@ -147,6 +148,7 @@ def generate(self): """ Acts as a driver function for this class that initiates the entire process of running safety check and returing the vulnerability_list + :return: list[dict], the output follows the same format as mentioned in the description of the class """ self.timestamp = datetime.now().strftime("%d-%m-%Y") diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index 57258b4b3d6e..860839097a8b 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -22,7 +22,7 @@ @dataclass class SafetyVulnerabilityAdvisory: - vid: str + vulnerability_id: str advisory: str reason_to_ignore: str spec: str From 2bb790ae155a8bc322eeb2eeeac06308538ada36 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 3 Sep 2021 21:33:16 +0000 Subject: [PATCH 72/88] Added new logging logic --- src/image.py | 26 +++++++++++++------------- src/image_builder.py | 41 ++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/image.py b/src/image.py index 09f926127642..954cfe692836 100644 --- a/src/image.py +++ b/src/image.py @@ -100,7 +100,7 @@ def build(self): # Confirm if building the image is required or not if not self.to_build: - self.log = ["Not built"] + self.log.append(["Not built"]) self.build_status = constants.NOT_BUILT self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] return self.build_status @@ -131,7 +131,7 @@ def build(self): return self.build_status def docker_build(self, fileobj=None, custom_context=False): - response = [] + response = [f"Starting the Build Process for {self.name}"] for line in self.client.build( fileobj=fileobj, path=self.dockerfile, @@ -144,12 +144,12 @@ def docker_build(self, fileobj=None, custom_context=False): ): if line.get("error") is not None: response.append(line["error"]) - self.log = response + self.log.append(response) self.build_status = constants.FAIL self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() - pretty_logs = "\n".join(self.log[:-100]) + pretty_logs = "\n".join(self.log[-1][:-100]) LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") LOGGER.error("******** ERROR during Docker BUILD ********") LOGGER.error(f"Error message received for {self.dockerfile} while docker build: {line}") @@ -163,9 +163,9 @@ def docker_build(self, fileobj=None, custom_context=False): else: response.append(str(line)) - self.log = response + self.log.append(response) - pretty_logs = "\n".join(self.log[-10:]) + pretty_logs = "\n".join(self.log[-1][-10:]) LOGGER.info(f"Docker Build Logs: {pretty_logs}") LOGGER.info(f"Completed Build for {self.name}") @@ -173,7 +173,7 @@ def docker_build(self, fileobj=None, custom_context=False): return self.build_status def image_size_check(self): - response = [] + response = [f"Starting image size check for {self.name}"] self.summary["image_size"] = int(self.client.inspect_image(self.ecr_url)["Size"]) / (1024 * 1024) if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: response.append("Image size baseline exceeded") @@ -183,19 +183,19 @@ def image_size_check(self): else: response.append(f"Image Size Check Succeeded for {self.name}") self.build_status = constants.SUCCESS - self.log = response + self.log.append(response) - pretty_log = "\n".join(self.log) + pretty_log = "\n".join(self.log[-1]) LOGGER.info(f"{pretty_log}") return self.build_status def push_image(self): + response = [f"Starting image Push for {self.name}"] for line in self.client.push(self.repository, self.tag, stream=True, decode=True): - response = [] if line.get("error") is not None: response.append(line["error"]) - self.log = response + self.log.append(response) self.build_status = constants.FAIL self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() @@ -214,9 +214,9 @@ def push_image(self): self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() self.summary["ecr_url"] = self.ecr_url - self.log = response + self.log.append(response) - pretty_logs = "\n".join(self.log[-10:]) + pretty_logs = "\n".join(self.log[-1][-10:]) LOGGER.info(f"Docker Build Logs: {pretty_logs}") LOGGER.info(f"Completed Build for {self.name}") diff --git a/src/image_builder.py b/src/image_builder.py index 9f0190b21615..3acf98c03e5d 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -22,6 +22,7 @@ import constants import utils import boto3 +import itertools from context import Context from metrics import Metrics @@ -206,18 +207,23 @@ def image_builder(buildspec): FORMATTER.banner("Push Started") push_images(IMAGES_TO_PUSH) - FORMATTER.banner("Logs/Summary") # After the build, display logs/summary for all the images. + FORMATTER.banner("Build Logs") show_build_logs(ALL_IMAGES) + + FORMATTER.banner("Summary") show_build_summary(ALL_IMAGES) + + FORMATTER.banner("Errors") is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) # From all images, filter the images that were supposed to be built and upload their metrics BUILT_IMAGES = [image for image in ALL_IMAGES if image.to_build] + FORMATTER.banner("Upload Metrics") upload_metrics(BUILT_IMAGES, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit) - FORMATTER.title("Setting Test Env") + FORMATTER.banner("Test Env") # Set environment variables to be consumed by test jobs test_trigger_job = utils.get_codebuild_project_name() # Tests should only run on images that were pushed to the repository @@ -226,9 +232,17 @@ def image_builder(buildspec): ) def get_common_stage_image_object(pre_push_stage_image_object): + """ + Creates a common stage image object for a pre_push stage image. If for a pre_push stage image we create a common + stage image, then we do not push the pre_push stage image to the repository. Instead, we just push its common stage + image to the repository. Therefore, inside the function pre_push_stage_image_object is made NON-PUSHABLE. + + :param pre_push_stage_image_object: DockerImage, an object of class DockerImage + :return: CommonStageImage, an object of class CommonStageImage. CommonStageImage inherits DockerImage. + """ common_stage_image_object = CommonStageImage( info=pre_push_stage_image_object.info, - dockerfile=os.path.join(os.sep, utils.get_root_folder_path(), "src", "Dockerfile.multipart"), + dockerfile=os.path.join(os.sep, utils.get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common"), repository=pre_push_stage_image_object.repository, tag=pre_push_stage_image_object.tag, to_build=pre_push_stage_image_object.to_build, @@ -238,8 +252,11 @@ def get_common_stage_image_object(pre_push_stage_image_object): return common_stage_image_object def show_build_logs(images): + """ + Display and save the build logs for a list of input images. - FORMATTER.title("Build Logs") + :param images: list[DockerImage] + """ if not os.path.isdir("logs"): os.makedirs("logs") @@ -248,29 +265,32 @@ def show_build_logs(images): image_description = f"{image.name}-{image.stage}" FORMATTER.title(image_description) FORMATTER.table(image.info.items()) - FORMATTER.separator() - FORMATTER.print_lines(image.log) + FORMATTER.title(f'Ending Logs for {image_description}') + FORMATTER.print_lines(image.log[-1][-2:]) + flattened_logs = list(itertools.chain(*image.log)) with open(f"logs/{image_description}", "w") as fp: - fp.write("/n".join(image.log)) + fp.write("/n".join(flattened_logs)) image.summary["log"] = f"logs/{image_description}" def show_build_summary(images): + """ + Display and save the build logs for a list of input images. - FORMATTER.title("Summary") + :param images: list[DockerImage] + """ for image in images: FORMATTER.title(image.name) FORMATTER.table(image.summary.items()) def show_build_errors(images): - FORMATTER.title("Errors") is_any_build_failed = False is_any_build_failed_size_limit = False for image in images: if image.build_status == constants.FAIL: FORMATTER.title(image.name) - FORMATTER.print_lines(image.log[-10:]) + FORMATTER.print_lines(image.log[-1][-10:]) is_any_build_failed = True else: if image.build_status == constants.FAIL_IMAGE_SIZE_LIMIT: @@ -286,7 +306,6 @@ def show_build_errors(images): def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit): - FORMATTER.title("Uploading Metrics") is_any_build_failed = False is_any_build_failed_size_limit = False metrics = Metrics( From 3eb309fcf0dd4b89504683ff8b48071c0b209a3f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 3 Sep 2021 22:55:06 +0000 Subject: [PATCH 73/88] Removed extra vars, Added docstring --- src/image.py | 35 ++++++++++++++++++++++++++++------- src/image_builder.py | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/src/image.py b/src/image.py index 954cfe692836..641e78d96dac 100644 --- a/src/image.py +++ b/src/image.py @@ -76,7 +76,9 @@ def collect_installed_packages_information(self): return command_responses def update_pre_build_configuration(self): - + """ + Updates image configuration before the docker client starts building the image. + """ if self.info.get("base_image_uri"): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] @@ -94,7 +96,9 @@ def update_pre_build_configuration(self): def build(self): """ - The build function builds the specified docker image + The build function sets the stage for starting the docker build process for a given image. + + :return: int, Build Status """ self.summary["start_time"] = datetime.now() @@ -131,6 +135,13 @@ def build(self): return self.build_status def docker_build(self, fileobj=None, custom_context=False): + """ + Uses low level Docker API Client to actually start the process of building the image. + + :param fileobj: FileObject, a readable file-like object pointing to the context tarfile. + :param custom_context: bool + :return: int, Build Status + """ response = [f"Starting the Build Process for {self.name}"] for line in self.client.build( fileobj=fileobj, @@ -151,7 +162,7 @@ def docker_build(self, fileobj=None, custom_context=False): pretty_logs = "\n".join(self.log[-1][:-100]) LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") - LOGGER.error("******** ERROR during Docker BUILD ********") + LOGGER.error("ERROR during Docker BUILD") LOGGER.error(f"Error message received for {self.dockerfile} while docker build: {line}") return self.build_status @@ -166,13 +177,18 @@ def docker_build(self, fileobj=None, custom_context=False): self.log.append(response) pretty_logs = "\n".join(self.log[-1][-10:]) - LOGGER.info(f"Docker Build Logs: {pretty_logs}") + LOGGER.info(f"DOCKER BUILD LOGS: \n{pretty_logs}") LOGGER.info(f"Completed Build for {self.name}") self.build_status = constants.SUCCESS return self.build_status def image_size_check(self): + """ + Checks if the size of the image is not greater than the baseline. + + :return: int, Build Status + """ response = [f"Starting image size check for {self.name}"] self.summary["image_size"] = int(self.client.inspect_image(self.ecr_url)["Size"]) / (1024 * 1024) if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: @@ -191,6 +207,11 @@ def image_size_check(self): return self.build_status def push_image(self): + """ + Pushes the Docker image to ECR using Docker low-level API client for docker. + + :return: int, states if the Push was successful or not + """ response = [f"Starting image Push for {self.name}"] for line in self.client.push(self.repository, self.tag, stream=True, decode=True): if line.get("error") is not None: @@ -202,7 +223,7 @@ def push_image(self): pretty_logs = "\n".join(self.log[:-100]) LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") - LOGGER.error("******** ERROR during Docker PUSH ********") + LOGGER.error("ERROR during Docker PUSH") LOGGER.error(f"Error message received for {self.dockerfile} while docker push: {line}") return self.build_status @@ -217,7 +238,7 @@ def push_image(self): self.log.append(response) pretty_logs = "\n".join(self.log[-1][-10:]) - LOGGER.info(f"Docker Build Logs: {pretty_logs}") - LOGGER.info(f"Completed Build for {self.name}") + LOGGER.info(f"DOCKER PUSH LOGS: \n {pretty_logs}") + LOGGER.info(f"Completed Push for {self.name}") return self.build_status diff --git a/src/image_builder.py b/src/image_builder.py index 3acf98c03e5d..73a1d8dce16a 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -181,7 +181,6 @@ def image_builder(buildspec): FORMATTER.separator() FORMATTER.banner("DLC") - FORMATTER.title("Status") # Standard images must be built before example images # Example images will use standard images as base @@ -274,7 +273,7 @@ def show_build_logs(images): def show_build_summary(images): """ - Display and save the build logs for a list of input images. + Display the summary for a list of input images. :param images: list[DockerImage] """ @@ -284,6 +283,12 @@ def show_build_summary(images): FORMATTER.table(image.summary.items()) def show_build_errors(images): + """ + Iterates through each image to check if there is any image that has a failed status. In case + an image with a failed status is found, it raises an exception. + + :param images: list[DockerImage] + """ is_any_build_failed = False is_any_build_failed_size_limit = False @@ -305,9 +310,14 @@ def show_build_errors(images): return is_any_build_failed, is_any_build_failed_size_limit def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit): + """ + Uploads Metrics for a list of images. - is_any_build_failed = False - is_any_build_failed_size_limit = False + :param images: list[DockerImage] + :param BUILDSPEC: Buildspec + :param is_any_build_failed: bool + :param is_any_build_failed_size_limit: bool + """ metrics = Metrics( context=constants.BUILD_CONTEXT, region=BUILDSPEC["region"], @@ -328,27 +338,43 @@ def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_s FORMATTER.separator() def build_images(images, make_dummy_boto_client=False): + """ + Takes a list of images and executes their build process concurrently. + + :param images: list[DockerImage] + :param make_dummy_boto_client: bool, specifies if a dummy client should be declared or not. + + TODO: The parameter make_dummy_boto_client should be removed when get_dummy_boto_client method is removed. + """ THREADS = {} # In the context of the ThreadPoolExecutor each instance of image.build submitted # to it is executed concurrently in a separate thread. with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - #### TODO: Remove this entire if block when https://github.com/boto/boto3/issues/1592 is resolved #### + #### TODO: Remove this entire if block when get_dummy_boto_client is removed #### if make_dummy_boto_client: get_dummy_boto_client() for image in images: - FORMATTER.print(f"image_object.context {image.context}") THREADS[image.name] = executor.submit(image.build) # the FORMATTER.progress(THREADS) function call also waits until all threads have completed FORMATTER.progress(THREADS) #### TODO: Remove this entire method when https://github.com/boto/boto3/issues/1592 is resolved #### def get_dummy_boto_client(): - # In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. - # If this function is not added, boto3 fails because boto3 sessions are not thread safe. - # However, once a dummy client is created, it is ensured that the calls are thread safe. + """ + Makes a dummy boto3 client to ensure that boto3 clients behave in a thread safe manner. + In absence of this method, the behaviour documented in https://github.com/boto/boto3/issues/1592 is observed. + Once https://github.com/boto/boto3/issues/1592 is resolved, this method can be removed. + + :return: BotocoreClientSTS + """ return boto3.client("sts", region_name=os.getenv("REGION")) def push_images(images): + """ + Takes a list of images and PUSHES them to ECR concurrently. + + :param images: list[DockerImage] + """ THREADS = {} with concurrent.futures.ThreadPoolExecutor(max_workers=constants.MAX_WORKER_COUNT_FOR_PUSHING_IMAGES) as executor: for image in images: From e3fccd3eba23abeb73a96cb91934f4408a8bf679 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 4 Sep 2021 02:25:40 +0000 Subject: [PATCH 74/88] Changed test file --- src/image_builder.py | 2 +- src/utils.py | 43 ++++++-------- test/dlc_tests/sanity/test_safety_check_v2.py | 56 ++++++++++--------- 3 files changed, 48 insertions(+), 53 deletions(-) diff --git a/src/image_builder.py b/src/image_builder.py index 73a1d8dce16a..af44a1b410d5 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -335,7 +335,7 @@ def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_s if is_any_build_failed_size_limit: raise Exception("Build failed because of file limit") - FORMATTER.separator() + FORMATTER.print("Metrics Uploaded") def build_images(images, make_dummy_boto_client=False): """ diff --git a/src/utils.py b/src/utils.py index 9a49a30f30c2..adae256fff77 100644 --- a/src/utils.py +++ b/src/utils.py @@ -488,16 +488,24 @@ def get_codebuild_project_name(): # Default value for codebuild project name is "local_test" when run outside of CodeBuild return os.getenv("CODEBUILD_BUILD_ID", "local_test").split(":")[0] + def get_root_folder_path(): + """ + Extract the root folder path for the repository. + + :return: str + """ root_dir_pattern = re.compile(r"^(\S+deep-learning-containers)") pwd = os.getcwd() return os.getenv("CODEBUILD_SRC_DIR", root_dir_pattern.match(pwd).group(1)) + def get_safety_ignore_dict(image_uri): """ Get a dict of known safety check issue IDs to ignore, if specified in file ../data/ignore_ids_safety_scan.json. - :param image_uri: - :return: list of safety check IDs to ignore + + :param image_uri: str, consists of f"{image_repo}:{image_tag}" + :return: dict, key is the ignored vulnerability id and value is the reason to ignore it """ framework = ("mxnet" if "mxnet" in image_uri else "pytorch" if "pytorch" in image_uri else @@ -508,38 +516,21 @@ def get_safety_ignore_dict(image_uri): "inference") python_version = "py2" if "py2" in image_uri else "py3" - IGNORE_SAFETY_IDS = {} + ignore_safety_ids = {} ignore_data_file = os.path.join(os.sep, get_root_folder_path(), "data", "ignore_ids_safety_scan.json") with open(ignore_data_file) as f: - IGNORE_SAFETY_IDS = json.load(f) + ignore_safety_ids = json.load(f) - return IGNORE_SAFETY_IDS.get(framework, {}).get(job_type, {}).get(python_version, {}) - + return ignore_safety_ids.get(framework, {}).get(job_type, {}).get(python_version, {}) def generate_safety_report_for_image(image_uri, storage_file_path=None): """ Genereate safety scan reports for an image and store it at the location specified - :param image_name: str that consists of f"{image_repo}:{image_tag}" - :param storage_file_path: str that looks like "storage_location.json" - :return: safety report that looks like following - [ - { - "package": "package", - "scan_status": "SUCCEEDED/FAILED/IGNORED", - "installed": "version", - "vulnerabilities": [ - { - "vid": "safety_vulnerability_id", - "advisory": "description of the issue", - "spec": "version_spec", - "reason_to_ignore": "Either N/A or the reason fetched from data/ignore_ids_safety_scan.json" - }, - ... - ] - } - ... - ] + + :param image_uri: str, consists of f"{image_repo}:{image_tag}" + :param storage_file_path: str, looks like "storage_location.json" + :return: list[dict], safety report generated by SafetyReportGenerator """ ctx = Context() docker_run_cmd = f"docker run -id --entrypoint='/bin/bash' {image_uri} " diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_check_v2.py index 860839097a8b..dca1dba171c4 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_check_v2.py @@ -17,11 +17,13 @@ LOGGER.addHandler(logging.StreamHandler(sys.stderr)) SAFETY_FILE = '/opt/aws/dlc/info/safety_report.json' -SAFETY_FILE_EXISTS = 0 @dataclass class SafetyVulnerabilityAdvisory: + """ + One of the DataClasses for parsing Safety Report + """ vulnerability_id: str advisory: str reason_to_ignore: str @@ -30,6 +32,9 @@ class SafetyVulnerabilityAdvisory: @dataclass class SafetyPackageVulnerabilityReport: + """ + One of the DataClasses for parsing Safety Report + """ package: str scan_status: str installed: str @@ -42,6 +47,9 @@ def __post_init__(self): @dataclass class SafetyPythonEnvironmentVulnerabilityReport: + """ + One of the DataClasses for parsing Safety Report + """ report: List[SafetyPackageVulnerabilityReport] def __post_init__(self): @@ -50,50 +58,46 @@ def __post_init__(self): @pytest.mark.model("N/A") -@pytest.mark.skipif( - is_canary_context(), - reason="Skipping test because it is not required to run it on canaries. test_safety_check.py runs on canaries." -) def test_safety_file_exists_and_is_valid(image): + """ + Checks if the image has a safety report at the desired location and fails if any of the + packages in the report have failed the safety check. + + :param image: str, image uri + """ repo_name, image_tag = image.split('/')[-1].split(':') container_name = f"{repo_name}-{image_tag}-safety" # Add null entrypoint to ensure command exits immediately run(f"docker run -id " f"--name {container_name} " f"--entrypoint='/bin/bash' " - f"{image}", hide=True) + f"{image}", hide=True, warn=True) try: # Check if file exists docker_exec_cmd = f"docker exec -i {container_name}" safety_file_check = run(f"{docker_exec_cmd} test -f {SAFETY_FILE}", warn=True, hide=True) - assert safety_file_check.return_code == SAFETY_FILE_EXISTS, f"Safety file existence test failed for {image}" + assert safety_file_check.ok, f"Safety file existence test failed for {image}" file_content = run(f"{docker_exec_cmd} cat {SAFETY_FILE}", warn=True, hide=True) raw_scan_result = json.loads(file_content.stdout) - scan_results = [] - scan_results.append( - SafetyPythonEnvironmentVulnerabilityReport( - report=raw_scan_result - ) - ) + safety_report_object = SafetyPythonEnvironmentVulnerabilityReport(report=raw_scan_result) # processing safety reports report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [vulnerabilities: {vulnerabilities}]" failed_count = 0 - for result in scan_results: - for report_item in result.report: - if report_item.scan_status == "FAILED": - failed_count += 1 - LOGGER.info( - report_log_template.format( - status="FAILED", - pkg=report_item.package, - installed=report_item.installed, - vulnerabilities = report_item.vulnerabilities, - ) - ) + for report_item in safety_report_object.report: + if report_item.scan_status == "FAILED": + failed_count += 1 + LOGGER.error( + report_log_template.format( + status="FAILED", + pkg=report_item.package, + installed=report_item.installed, + vulnerabilities = report_item.vulnerabilities, + ) + ) assert failed_count == 0, f"{failed_count} package/s failed safety test for {image} !!!" LOGGER.info(f"Safety check is successfully complete and report exists at {SAFETY_FILE}") finally: - run(f"docker rm -f {container_name}", hide=True) + run(f"docker rm -f {container_name}", hide=True, warn=True) From f62553e1e576a04b2a1db07f41a75c9b6dae45de Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sun, 5 Sep 2021 21:30:45 +0000 Subject: [PATCH 75/88] Renamed test file. Added buildspec --- buildspec.yml | 1 - .../{test_safety_check_v2.py => test_safety_report_file.py} | 3 --- 2 files changed, 4 deletions(-) rename test/dlc_tests/sanity/{test_safety_check_v2.py => test_safety_report_file.py} (98%) diff --git a/buildspec.yml b/buildspec.yml index 64a4063f69f0..e7e71738408e 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -12,7 +12,6 @@ phases: - $(aws ecr get-login --no-include-email --region $AWS_DEFAULT_REGION) - pip install -r src/requirements.txt - bash src/setup.sh $FRAMEWORK - - export ROOT_FOLDER_PATH=$(pwd) - python src/parse_partner_developers.py build: commands: diff --git a/test/dlc_tests/sanity/test_safety_check_v2.py b/test/dlc_tests/sanity/test_safety_report_file.py similarity index 98% rename from test/dlc_tests/sanity/test_safety_check_v2.py rename to test/dlc_tests/sanity/test_safety_report_file.py index dca1dba171c4..c057757ad88f 100644 --- a/test/dlc_tests/sanity/test_safety_check_v2.py +++ b/test/dlc_tests/sanity/test_safety_report_file.py @@ -8,9 +8,6 @@ from dataclasses import dataclass from typing import List -from test.test_utils import ( - is_canary_context -) LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.INFO) From 699cacac802cb42eabaa3268c5e74ff2c66e7193 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sun, 5 Sep 2021 21:50:21 +0000 Subject: [PATCH 76/88] Added logging method to image --- src/image.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/image.py b/src/image.py index 641e78d96dac..f04a879300e9 100644 --- a/src/image.py +++ b/src/image.py @@ -74,6 +74,16 @@ def collect_installed_packages_information(self): command_responses.append(bytes.decode(docker_client.containers.run(self.ecr_url, command))) docker_client.containers.prune() return command_responses + + def get_tail_logs_in_pretty_format(self, number_of_lines=10): + """ + Displays the tail of the logs. + + :param number_of_lines: int, number of ending lines to be printed + :return: str, last number_of_lines of the logs concatenated with a new line + """ + return "\n".join(self.log[-1][-number_of_lines:]) + def update_pre_build_configuration(self): """ @@ -160,8 +170,7 @@ def docker_build(self, fileobj=None, custom_context=False): self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() - pretty_logs = "\n".join(self.log[-1][:-100]) - LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") + LOGGER.info(f"Docker Build Logs: \n {self.get_tail_logs_in_pretty_format(100)}") LOGGER.error("ERROR during Docker BUILD") LOGGER.error(f"Error message received for {self.dockerfile} while docker build: {line}") @@ -176,8 +185,7 @@ def docker_build(self, fileobj=None, custom_context=False): self.log.append(response) - pretty_logs = "\n".join(self.log[-1][-10:]) - LOGGER.info(f"DOCKER BUILD LOGS: \n{pretty_logs}") + LOGGER.info(f"DOCKER BUILD LOGS: \n{self.get_tail_logs_in_pretty_format()}") LOGGER.info(f"Completed Build for {self.name}") self.build_status = constants.SUCCESS @@ -201,8 +209,7 @@ def image_size_check(self): self.build_status = constants.SUCCESS self.log.append(response) - pretty_log = "\n".join(self.log[-1]) - LOGGER.info(f"{pretty_log}") + LOGGER.info(f"{self.get_tail_logs_in_pretty_format()}") return self.build_status @@ -220,9 +227,8 @@ def push_image(self): self.build_status = constants.FAIL self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() - pretty_logs = "\n".join(self.log[:-100]) - LOGGER.info(f"Docker Build Logs: \n {pretty_logs}") + LOGGER.info(f"Docker Build Logs: \n {self.get_tail_logs_in_pretty_format(100)}") LOGGER.error("ERROR during Docker PUSH") LOGGER.error(f"Error message received for {self.dockerfile} while docker push: {line}") @@ -237,8 +243,7 @@ def push_image(self): self.summary["ecr_url"] = self.ecr_url self.log.append(response) - pretty_logs = "\n".join(self.log[-1][-10:]) - LOGGER.info(f"DOCKER PUSH LOGS: \n {pretty_logs}") + LOGGER.info(f"DOCKER PUSH LOGS: \n {self.get_tail_logs_in_pretty_format(2)}") LOGGER.info(f"Completed Push for {self.name}") return self.build_status From 3d0cde0391793466f975aa053880cd9d09120830 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sun, 5 Sep 2021 22:57:48 +0000 Subject: [PATCH 77/88] Ran black -l 120 (https://github.com/psf/black) for following files: 1. common_stage 2. constant.py (Only to this PR's code) 3. image_buider (Only to this PR's code) 4. image 5. test_safety_report_file 6. metrics.py (Only to this PR's code) 7. utils.py (Only to this PR's code) 8. safety_report_generato (Already ran) Not run on: 1. Dockerfile.common 2. Data Json 3. test_safety_check --- src/common_stage_image.py | 8 +++--- src/constants.py | 2 +- src/image.py | 5 ++-- src/image_builder.py | 17 ++++++++----- src/metrics.py | 2 +- src/utils.py | 2 +- .../sanity/test_safety_report_file.py | 25 ++++++++++--------- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/common_stage_image.py b/src/common_stage_image.py index 92843a239918..ac0d857b87a2 100644 --- a/src/common_stage_image.py +++ b/src/common_stage_image.py @@ -43,7 +43,7 @@ def update_pre_build_configuration(self): image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" storage_file_path = os.path.join( - os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json" + os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json", ) generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) self.context = self.generate_common_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) @@ -54,14 +54,14 @@ def generate_common_stage_context(self, safety_report_path, tarfile_name="common the Dockerfile.common uses this safety report to COPY the report into the image. """ artifacts = { - "safety_report": {"source": safety_report_path, "target": "safety_report.json"}, + "safety_report": {"source": safety_report_path, "target": "safety_report.json",}, "dockerfile": { "source": os.path.join( - os.sep, get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common" + os.sep, get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common", ), "target": "Dockerfile", }, } artifact_root = os.path.join(os.sep, get_root_folder_path(), "src") - return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root) + return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root,) diff --git a/src/constants.py b/src/constants.py index d590a46cf9d0..7be641ef288a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -31,7 +31,7 @@ # Left and right padding between text and margins in output PADDING = 1 -#Docker build stages +# Docker build stages PRE_PUSH_STAGE = "pre_push" COMMON_STAGE = "common" diff --git a/src/image.py b/src/image.py index f04a879300e9..7c062c1df4bd 100644 --- a/src/image.py +++ b/src/image.py @@ -74,7 +74,7 @@ def collect_installed_packages_information(self): command_responses.append(bytes.decode(docker_client.containers.run(self.ecr_url, command))) docker_client.containers.prune() return command_responses - + def get_tail_logs_in_pretty_format(self, number_of_lines=10): """ Displays the tail of the logs. @@ -84,7 +84,6 @@ def get_tail_logs_in_pretty_format(self, number_of_lines=10): """ return "\n".join(self.log[-1][-number_of_lines:]) - def update_pre_build_configuration(self): """ Updates image configuration before the docker client starts building the image. @@ -132,7 +131,7 @@ def build(self): if self.build_status == constants.FAIL: return self.build_status - + if not self.to_push: # If this image is not supposed to be pushed, in that case, we are already done # with building the image and do not need to conduct any further processing. diff --git a/src/image_builder.py b/src/image_builder.py index af44a1b410d5..3d76011d20ce 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -230,6 +230,7 @@ def image_builder(buildspec): IMAGES_TO_PUSH, BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), TEST_TRIGGER=test_trigger_job, ) + def get_common_stage_image_object(pre_push_stage_image_object): """ Creates a common stage image object for a pre_push stage image. If for a pre_push stage image we create a common @@ -250,6 +251,7 @@ def get_common_stage_image_object(pre_push_stage_image_object): pre_push_stage_image_object.to_push = False return common_stage_image_object + def show_build_logs(images): """ Display and save the build logs for a list of input images. @@ -264,16 +266,17 @@ def show_build_logs(images): image_description = f"{image.name}-{image.stage}" FORMATTER.title(image_description) FORMATTER.table(image.info.items()) - FORMATTER.title(f'Ending Logs for {image_description}') + FORMATTER.title(f"Ending Logs for {image_description}") FORMATTER.print_lines(image.log[-1][-2:]) flattened_logs = list(itertools.chain(*image.log)) with open(f"logs/{image_description}", "w") as fp: fp.write("/n".join(flattened_logs)) image.summary["log"] = f"logs/{image_description}" + def show_build_summary(images): """ - Display the summary for a list of input images. + Display the build summary for a list of input images. :param images: list[DockerImage] """ @@ -282,6 +285,7 @@ def show_build_summary(images): FORMATTER.title(image.name) FORMATTER.table(image.summary.items()) + def show_build_errors(images): """ Iterates through each image to check if there is any image that has a failed status. In case @@ -309,6 +313,7 @@ def show_build_errors(images): FORMATTER.print("No errors") return is_any_build_failed, is_any_build_failed_size_limit + def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_size_limit): """ Uploads Metrics for a list of images. @@ -319,9 +324,7 @@ def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_s :param is_any_build_failed_size_limit: bool """ metrics = Metrics( - context=constants.BUILD_CONTEXT, - region=BUILDSPEC["region"], - namespace=constants.METRICS_NAMESPACE, + context=constants.BUILD_CONTEXT, region=BUILDSPEC["region"], namespace=constants.METRICS_NAMESPACE, ) for image in images: try: @@ -337,6 +340,7 @@ def upload_metrics(images, BUILDSPEC, is_any_build_failed, is_any_build_failed_s FORMATTER.print("Metrics Uploaded") + def build_images(images, make_dummy_boto_client=False): """ Takes a list of images and executes their build process concurrently. @@ -358,6 +362,7 @@ def build_images(images, make_dummy_boto_client=False): # the FORMATTER.progress(THREADS) function call also waits until all threads have completed FORMATTER.progress(THREADS) + #### TODO: Remove this entire method when https://github.com/boto/boto3/issues/1592 is resolved #### def get_dummy_boto_client(): """ @@ -369,6 +374,7 @@ def get_dummy_boto_client(): """ return boto3.client("sts", region_name=os.getenv("REGION")) + def push_images(images): """ Takes a list of images and PUSHES them to ECR concurrently. @@ -382,7 +388,6 @@ def push_images(images): FORMATTER.progress(THREADS) - def tag_image_with_pr_number(image_tag): pr_number = os.getenv("CODEBUILD_SOURCE_VERSION").replace("/", "-") return f"{image_tag}-{pr_number}" diff --git a/src/metrics.py b/src/metrics.py index 7741a94fa11d..0f6cd7284e9b 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -41,7 +41,7 @@ def push_image_metrics(self, image): "python_version": image.python_version, "image_type": image.image_type, "image_stage": image.stage, - "push_required": str(image.to_push) + "push_required": str(image.to_push), } if image.build_status == constants.NOT_BUILT: return None diff --git a/src/utils.py b/src/utils.py index adae256fff77..b900e94d7be1 100644 --- a/src/utils.py +++ b/src/utils.py @@ -542,6 +542,6 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): safety_scan_output = SafetyReportGenerator(container_id, ignore_dict=ignore_dict).generate() ctx.run(f"docker rm -f {container_id}", hide=True, warn=True) if storage_file_path: - with open(storage_file_path, 'w', encoding='utf-8') as f: + with open(storage_file_path, "w", encoding="utf-8") as f: json.dump(safety_scan_output, f, indent=4) return safety_scan_output diff --git a/test/dlc_tests/sanity/test_safety_report_file.py b/test/dlc_tests/sanity/test_safety_report_file.py index c057757ad88f..6d97f22f467d 100644 --- a/test/dlc_tests/sanity/test_safety_report_file.py +++ b/test/dlc_tests/sanity/test_safety_report_file.py @@ -13,7 +13,7 @@ LOGGER.setLevel(logging.INFO) LOGGER.addHandler(logging.StreamHandler(sys.stderr)) -SAFETY_FILE = '/opt/aws/dlc/info/safety_report.json' +SAFETY_FILE = "/opt/aws/dlc/info/safety_report.json" @dataclass @@ -21,6 +21,7 @@ class SafetyVulnerabilityAdvisory: """ One of the DataClasses for parsing Safety Report """ + vulnerability_id: str advisory: str reason_to_ignore: str @@ -32,6 +33,7 @@ class SafetyPackageVulnerabilityReport: """ One of the DataClasses for parsing Safety Report """ + package: str scan_status: str installed: str @@ -47,13 +49,13 @@ class SafetyPythonEnvironmentVulnerabilityReport: """ One of the DataClasses for parsing Safety Report """ + report: List[SafetyPackageVulnerabilityReport] def __post_init__(self): self.report = [SafetyPackageVulnerabilityReport(**i) for i in self.report] - @pytest.mark.model("N/A") def test_safety_file_exists_and_is_valid(image): """ @@ -62,36 +64,35 @@ def test_safety_file_exists_and_is_valid(image): :param image: str, image uri """ - repo_name, image_tag = image.split('/')[-1].split(':') + repo_name, image_tag = image.split("/")[-1].split(":") container_name = f"{repo_name}-{image_tag}-safety" # Add null entrypoint to ensure command exits immediately - run(f"docker run -id " - f"--name {container_name} " - f"--entrypoint='/bin/bash' " - f"{image}", hide=True, warn=True) - + run(f"docker run -id " f"--name {container_name} " f"--entrypoint='/bin/bash' " f"{image}", hide=True, warn=True) + try: # Check if file exists docker_exec_cmd = f"docker exec -i {container_name}" safety_file_check = run(f"{docker_exec_cmd} test -f {SAFETY_FILE}", warn=True, hide=True) - assert safety_file_check.ok, f"Safety file existence test failed for {image}" + assert safety_file_check.ok, f"Safety file existence test failed for {image}" file_content = run(f"{docker_exec_cmd} cat {SAFETY_FILE}", warn=True, hide=True) raw_scan_result = json.loads(file_content.stdout) safety_report_object = SafetyPythonEnvironmentVulnerabilityReport(report=raw_scan_result) # processing safety reports - report_log_template = "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [vulnerabilities: {vulnerabilities}]" + report_log_template = ( + "SAFETY_REPORT ({status}) [pkg: {pkg}] [installed: {installed}] [vulnerabilities: {vulnerabilities}]" + ) failed_count = 0 for report_item in safety_report_object.report: - if report_item.scan_status == "FAILED": + if report_item.scan_status == "FAILED": failed_count += 1 LOGGER.error( report_log_template.format( status="FAILED", pkg=report_item.package, installed=report_item.installed, - vulnerabilities = report_item.vulnerabilities, + vulnerabilities=report_item.vulnerabilities, ) ) assert failed_count == 0, f"{failed_count} package/s failed safety test for {image} !!!" From 664e9467ed248411e59aec716216f7f0f1f8f3da Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Tue, 7 Sep 2021 03:38:19 +0000 Subject: [PATCH 78/88] Minor changed --- miscellaneous_dockerfiles/Dockerfile.common | 2 +- src/image.py | 4 ---- test/dlc_tests/sanity/test_safety_report_file.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/miscellaneous_dockerfiles/Dockerfile.common b/miscellaneous_dockerfiles/Dockerfile.common index 81f7bfa81a79..9a4968327ed3 100644 --- a/miscellaneous_dockerfiles/Dockerfile.common +++ b/miscellaneous_dockerfiles/Dockerfile.common @@ -3,5 +3,5 @@ ARG PRE_PUSH_IMAGE="" FROM $PRE_PUSH_IMAGE -# Add any script or repo as required +# Copy safety report generated from PRE_PUSH_IMAGE to docker image COPY safety_report.json /opt/aws/dlc/info/safety_report.json diff --git a/src/image.py b/src/image.py index 7c062c1df4bd..5288f98d51da 100644 --- a/src/image.py +++ b/src/image.py @@ -17,7 +17,6 @@ from docker import APIClient from docker import DockerClient - import constants import logging import json @@ -100,9 +99,6 @@ def update_pre_build_configuration(self): if self.info.get("labels"): self.labels.update(self.info.get("labels")) - LOGGER.info(f"self.build_args {json.dumps(self.build_args, indent=4)}") - LOGGER.info(f"self.labels {json.dumps(self.labels, indent=4)}") - def build(self): """ The build function sets the stage for starting the docker build process for a given image. diff --git a/test/dlc_tests/sanity/test_safety_report_file.py b/test/dlc_tests/sanity/test_safety_report_file.py index 6d97f22f467d..af433dd4344e 100644 --- a/test/dlc_tests/sanity/test_safety_report_file.py +++ b/test/dlc_tests/sanity/test_safety_report_file.py @@ -96,6 +96,6 @@ def test_safety_file_exists_and_is_valid(image): ) ) assert failed_count == 0, f"{failed_count} package/s failed safety test for {image} !!!" - LOGGER.info(f"Safety check is successfully complete and report exists at {SAFETY_FILE}") + LOGGER.info(f"Safety report file validation is successfully complete and report exists at {SAFETY_FILE}") finally: run(f"docker rm -f {container_name}", hide=True, warn=True) From 0a0aeebc5710a55c813d6061e0f8edc9c0b3b9ce Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 8 Sep 2021 05:13:16 +0000 Subject: [PATCH 79/88] Add multistage.common tag --- src/image.py | 3 --- src/image_builder.py | 19 +++++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/image.py b/src/image.py index 5288f98d51da..9935c38fd453 100644 --- a/src/image.py +++ b/src/image.py @@ -90,9 +90,6 @@ def update_pre_build_configuration(self): if self.info.get("base_image_uri"): self.build_args["BASE_IMAGE"] = self.info["base_image_uri"] - if self.ecr_url: - self.build_args["PRE_PUSH_IMAGE"] = self.ecr_url - if self.info.get("extra_build_args"): self.build_args.update(self.info.get("extra_build_args")) diff --git a/src/image_builder.py b/src/image_builder.py index 3d76011d20ce..a90162cf349c 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -240,11 +240,15 @@ def get_common_stage_image_object(pre_push_stage_image_object): :param pre_push_stage_image_object: DockerImage, an object of class DockerImage :return: CommonStageImage, an object of class CommonStageImage. CommonStageImage inherits DockerImage. """ + common_stage_info = deepcopy(pre_push_stage_image_object.info) + common_stage_info["extra_build_args"].update( + {"PRE_PUSH_IMAGE": pre_push_stage_image_object.ecr_url,} + ) common_stage_image_object = CommonStageImage( - info=pre_push_stage_image_object.info, + info=common_stage_info, dockerfile=os.path.join(os.sep, utils.get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common"), repository=pre_push_stage_image_object.repository, - tag=pre_push_stage_image_object.tag, + tag=tag_image_with_multistage_common(pre_push_stage_image_object.tag), to_build=pre_push_stage_image_object.to_build, stage=constants.COMMON_STAGE, ) @@ -398,6 +402,17 @@ def tag_image_with_datetime(image_tag): return f"{image_tag}-{datetime_suffix}" +def tag_image_with_multistage_common(image_tag): + """ + Appends multistage-common tag to the image + + :param image_tag: str + :return: str, image tag appended with multistage-common + """ + append_tag = "multistage.common" + return f"{image_tag}-{append_tag}" + + def modify_repository_name_for_context(image_repo_uri, build_context): repo_uri_values = image_repo_uri.split("/") repo_name = repo_uri_values[-1] From 4bf75d9ad530fcc1f97b7939c0cfc600af60f431 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Tue, 21 Sep 2021 20:35:51 +0000 Subject: [PATCH 80/88] Added the process_image function to build and push images. Changed tag name. --- src/image.py | 17 ++++++++++++++ src/image_builder.py | 55 ++++++++++++++++++++++++++++++-------------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/src/image.py b/src/image.py index 04b9a6df7142..ccfa173f7ef1 100644 --- a/src/image.py +++ b/src/image.py @@ -57,6 +57,7 @@ def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=N self.build_status = None self.client = APIClient(base_url=constants.DOCKER_URL, timeout=constants.API_CLIENT_TIMEOUT) self.log = [] + self._corresponding_common_stage_image = None def __getattr__(self, name): return self.info[name] @@ -72,6 +73,22 @@ def is_child_image(self): def is_test_promotion_enabled(self): return bool(self.info.get('enable_test_promotion')) + @property + def corresponding_common_stage_image(self): + """ + Retrieve the corresponding common stage image for a given image. + """ + return self._corresponding_common_stage_image + + @corresponding_common_stage_image.setter + def corresponding_common_stage_image(self, docker_image_object): + """ + Sets the value for the corresponding_common_stage_image variable. + """ + if self.to_push: + raise ValueError("Corresponding common stage image can only exist if the image is non-pushable") + self._corresponding_common_stage_image = docker_image_object + def collect_installed_packages_information(self): """ Returns an array with outcomes of the commands listed in the 'commands' array diff --git a/src/image_builder.py b/src/image_builder.py index ce056b3841ee..ce77ce8d60f7 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -175,7 +175,7 @@ def image_builder(buildspec): # If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image # to the repository. Instead, we just push its common stage image to the repository. Therefore, # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. - common_stage_image_object = get_common_stage_image_object(pre_push_stage_image_object) + common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object) PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) COMMON_STAGE_IMAGES.append(common_stage_image_object) @@ -188,24 +188,14 @@ def image_builder(buildspec): # Common images must be built at the end as they will consume respective standard and example images standard_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" not in image.name.lower()] example_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" in image.name.lower()] - common_stage_images = [image for image in COMMON_STAGE_IMAGES] ALL_IMAGES = PRE_PUSH_STAGE_IMAGES + COMMON_STAGE_IMAGES IMAGES_TO_PUSH = [image for image in ALL_IMAGES if image.to_push and image.to_build] - # pre_push stage standard images build - FORMATTER.banner("Standard Build") - build_images(standard_images) + pushed_images = [] + pushed_images += process_images(standard_images, "Standard") + pushed_images += process_images(example_images, "Example") - # pre_push stage example images build - FORMATTER.banner("Example Build") - build_images(example_images) - - # Common stage build - FORMATTER.banner("Common Build") - build_images(common_stage_images, make_dummy_boto_client=True) - - FORMATTER.banner("Push Started") - push_images(IMAGES_TO_PUSH) + assert all(image in pushed_images for image in IMAGES_TO_PUSH), "Few images could not be pushed." # After the build, display logs/summary for all the images. FORMATTER.banner("Build Logs") @@ -232,7 +222,37 @@ def image_builder(buildspec): ) -def get_common_stage_image_object(pre_push_stage_image_object): +def process_images(pre_push_image_list, pre_push_image_type="Pre-push"): + """ + Handles all the tasks related to a particular type of Pre Push images. It takes in the list of + pre push images and then builds it. After the pre-push images have been built, it extracts the + corresponding common stage images for the pre-push images and builds those common stage images. + After the common stage images have been built, it finds outs the docker images that need to be + pushed and pushes them according. + + :param pre_push_image_list: list[DockerImage], list of pre-push images + :param pre_push_image_type: str, used to display the message on the logs + :return: list[DockerImage], images that were supposed to be pushed. + """ + FORMATTER.banner(f"{pre_push_image_type} Build") + build_images(pre_push_image_list) + + FORMATTER.banner(f"{pre_push_image_type} Common Build") + common_stage_image_list = [ + image.corresponding_common_stage_image + for image in pre_push_image_list + if image.corresponding_common_stage_image is not None + ] + build_images(common_stage_image_list, make_dummy_boto_client=True) + + FORMATTER.banner(f"{pre_push_image_type} Push Images") + all_images = pre_push_image_list + common_stage_image_list + images_to_push = [image for image in all_images if image.to_push and image.to_build] + push_images(images_to_push) + return images_to_push + + +def generate_common_stage_image_object(pre_push_stage_image_object): """ Creates a common stage image object for a pre_push stage image. If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image to the repository. Instead, we just push its common stage @@ -254,6 +274,7 @@ def get_common_stage_image_object(pre_push_stage_image_object): stage=constants.COMMON_STAGE, ) pre_push_stage_image_object.to_push = False + pre_push_stage_image_object.corresponding_common_stage_image = common_stage_image_object return common_stage_image_object @@ -410,7 +431,7 @@ def tag_image_with_multistage_common(image_tag): :param image_tag: str :return: str, image tag appended with multistage-common """ - append_tag = "multistage.common" + append_tag = "multistage-common" return f"{image_tag}-{append_tag}" From 742d1c955012cf804d7081bc7da15e59c3b163a0 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Wed, 22 Sep 2021 00:35:07 +0000 Subject: [PATCH 81/88] Made the get_safety_ignore_dict method bit generic --- src/common_stage_image.py | 8 +++++--- src/metrics.py | 1 - src/utils.py | 24 +++++++++++++----------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/common_stage_image.py b/src/common_stage_image.py index ac0d857b87a2..03a91536a0a4 100644 --- a/src/common_stage_image.py +++ b/src/common_stage_image.py @@ -38,14 +38,16 @@ def update_pre_build_configuration(self): # Call the update_pre_build_configuration steps from the parent class super(CommonStageImage, self).update_pre_build_configuration() # Generate safety scan report for the first stage image and add the file to artifacts - first_stage_image_uri = self.build_args["PRE_PUSH_IMAGE"] - processed_image_uri = first_stage_image_uri.replace(".", "-").replace("/", "-").replace(":", "-") + pre_push_stage_image_uri = self.build_args["PRE_PUSH_IMAGE"] + processed_image_uri = pre_push_stage_image_uri.replace(".", "-").replace("/", "-").replace(":", "-") image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" storage_file_path = os.path.join( os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json", ) - generate_safety_report_for_image(first_stage_image_uri, storage_file_path=storage_file_path) + generate_safety_report_for_image( + pre_push_stage_image_uri, image_info=self.info, storage_file_path=storage_file_path + ) self.context = self.generate_common_stage_context(storage_file_path, tarfile_name=tarfile_name_for_context) def generate_common_stage_context(self, safety_report_path, tarfile_name="common-stage-file"): diff --git a/src/metrics.py b/src/metrics.py index 0f6cd7284e9b..3ac7b8a0d57d 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -41,7 +41,6 @@ def push_image_metrics(self, image): "python_version": image.python_version, "image_type": image.image_type, "image_stage": image.stage, - "push_required": str(image.to_push), } if image.build_status == constants.NOT_BUILT: return None diff --git a/src/utils.py b/src/utils.py index fd841a562f67..a376a7f6390d 100644 --- a/src/utils.py +++ b/src/utils.py @@ -502,21 +502,20 @@ def get_root_folder_path(): return os.getenv("CODEBUILD_SRC_DIR", root_dir_pattern.match(pwd).group(1)) -def get_safety_ignore_dict(image_uri): +def get_safety_ignore_dict(image_uri, framework, python_version, job_type): """ Get a dict of known safety check issue IDs to ignore, if specified in file ../data/ignore_ids_safety_scan.json. :param image_uri: str, consists of f"{image_repo}:{image_tag}" + :param framework: str, framework like tensorflow, mxnet etc. + :param python_version: str, py2 or py3 + :param job_type: str, type of training job. Can be "training"/"inference" :return: dict, key is the ignored vulnerability id and value is the reason to ignore it """ - framework = ("mxnet" if "mxnet" in image_uri else - "pytorch" if "pytorch" in image_uri else - "tensorflow") - job_type = ("training" if "training" in image_uri else - "inference-eia" if "eia" in image_uri else - "inference-neuron" if "neuron" in image_uri else - "inference") - python_version = "py2" if "py2" in image_uri else "py3" + if job_type == "inference": + job_type = ( + "inference-eia" if "eia" in image_uri else "inference-neuron" if "neuron" in image_uri else "inference" + ) ignore_safety_ids = {} ignore_data_file = os.path.join(os.sep, get_root_folder_path(), "data", "ignore_ids_safety_scan.json") @@ -526,11 +525,12 @@ def get_safety_ignore_dict(image_uri): return ignore_safety_ids.get(framework, {}).get(job_type, {}).get(python_version, {}) -def generate_safety_report_for_image(image_uri, storage_file_path=None): +def generate_safety_report_for_image(image_uri, image_info, storage_file_path=None): """ Genereate safety scan reports for an image and store it at the location specified :param image_uri: str, consists of f"{image_repo}:{image_tag}" + :param image_info: dict, should consist of 3 keys - "framework", "python_version" and "image_type". :param storage_file_path: str, looks like "storage_location.json" :return: list[dict], safety report generated by SafetyReportGenerator """ @@ -540,7 +540,9 @@ def generate_safety_report_for_image(image_uri, storage_file_path=None): install_safety_cmd = "pip install safety" docker_exec_cmd = f"docker exec -i {container_id}" ctx.run(f"{docker_exec_cmd} {install_safety_cmd}", hide=True, warn=True) - ignore_dict = get_safety_ignore_dict(image_uri) + ignore_dict = get_safety_ignore_dict( + image_uri, image_info["framework"], image_info["python_version"], image_info["image_type"] + ) safety_scan_output = SafetyReportGenerator(container_id, ignore_dict=ignore_dict).generate() ctx.run(f"docker rm -f {container_id}", hide=True, warn=True) if storage_file_path: From dd1c08c6455430161432f65e85e9616c7bb2290f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Thu, 23 Sep 2021 06:48:11 +0000 Subject: [PATCH 82/88] Removed the if condition for context --- src/image.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/image.py b/src/image.py index ccfa173f7ef1..b0d46508d039 100644 --- a/src/image.py +++ b/src/image.py @@ -143,12 +143,9 @@ def build(self): self.update_pre_build_configuration() # Start building the image - if self.context: - with open(self.context.context_path, "rb") as context_file: - self.docker_build(fileobj=context_file, custom_context=True) - self.context.remove() - else: - self.docker_build() + with open(self.context.context_path, "rb") as context_file: + self.docker_build(fileobj=context_file, custom_context=True) + self.context.remove() if self.build_status == constants.FAIL: return self.build_status From cae71c0758f31563f8504465488cbc797af8184f Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 24 Sep 2021 02:35:20 +0000 Subject: [PATCH 83/88] Addressed the nit changes --- src/common_stage_image.py | 8 ++++---- src/image.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/common_stage_image.py b/src/common_stage_image.py index 03a91536a0a4..bf59541b9cbd 100644 --- a/src/common_stage_image.py +++ b/src/common_stage_image.py @@ -43,7 +43,7 @@ def update_pre_build_configuration(self): image_name = self.name tarfile_name_for_context = f"{processed_image_uri}-{image_name}" storage_file_path = os.path.join( - os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json", + os.sep, get_root_folder_path(), "src", f"{tarfile_name_for_context}_safety_report.json" ) generate_safety_report_for_image( pre_push_stage_image_uri, image_info=self.info, storage_file_path=storage_file_path @@ -56,14 +56,14 @@ def generate_common_stage_context(self, safety_report_path, tarfile_name="common the Dockerfile.common uses this safety report to COPY the report into the image. """ artifacts = { - "safety_report": {"source": safety_report_path, "target": "safety_report.json",}, + "safety_report": {"source": safety_report_path, "target": "safety_report.json"}, "dockerfile": { "source": os.path.join( - os.sep, get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common", + os.sep, get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common" ), "target": "Dockerfile", }, } artifact_root = os.path.join(os.sep, get_root_folder_path(), "src") - return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root,) + return Context(artifacts, context_path=f"build/{tarfile_name}.tar.gz", artifact_root=artifact_root) diff --git a/src/image.py b/src/image.py index b0d46508d039..4412c6170baf 100644 --- a/src/image.py +++ b/src/image.py @@ -83,10 +83,12 @@ def corresponding_common_stage_image(self): @corresponding_common_stage_image.setter def corresponding_common_stage_image(self, docker_image_object): """ - Sets the value for the corresponding_common_stage_image variable. + For a pre-push stage image, it sets the value for the corresponding_common_stage_image variable. """ if self.to_push: - raise ValueError("Corresponding common stage image can only exist if the image is non-pushable") + raise ValueError( + "For any pre-push stage image, corresponding common stage image should only exist if the pre-push stage image is non-pushable." + ) self._corresponding_common_stage_image = docker_image_object def collect_installed_packages_information(self): From 0f7416ceb6484da86db791b43cfff4944f2c61c6 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 24 Sep 2021 17:03:39 +0000 Subject: [PATCH 84/88] Ignoring TF2.6 vulnerability --- data/ignore_ids_safety_scan.json | 1 + 1 file changed, 1 insertion(+) diff --git a/data/ignore_ids_safety_scan.json b/data/ignore_ids_safety_scan.json index ee60247aef01..bad59904bc00 100644 --- a/data/ignore_ids_safety_scan.json +++ b/data/ignore_ids_safety_scan.json @@ -10,6 +10,7 @@ "35015":"for shipping pycrypto<=2.6.1 - the last available version for py2" }, "py3": { + "41161":"TF 2.6.0 is the last version for 2.6 series and does not have a fix yet." } }, "inference":{ From feed6bd20ebe62a4f21f18449a5d010c8b138f28 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 24 Sep 2021 22:19:01 +0000 Subject: [PATCH 85/88] Added the multi-tagging mechanism --- src/image.py | 54 ++++++++++++++++++++++++++++++++++++++------ src/image_builder.py | 35 ++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/image.py b/src/image.py index 4412c6170baf..4c3d71918056 100644 --- a/src/image.py +++ b/src/image.py @@ -30,7 +30,7 @@ class DockerImage: The DockerImage class has the functions and attributes for building the dockerimage """ - def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True): + def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=None, to_push=True, additional_tags=[]): # Meta-data about the image should go to info. # All keys in info are accessible as attributes @@ -48,6 +48,7 @@ def __init__(self, info, dockerfile, repository, tag, to_build, stage, context=N # TODO: Add ability to tag image with multiple tags self.repository = repository self.tag = tag + self.additional_tags = additional_tags self.ecr_url = f"{self.repository}:{self.tag}" if not isinstance(to_build, bool): @@ -149,7 +150,7 @@ def build(self): self.docker_build(fileobj=context_file, custom_context=True) self.context.remove() - if self.build_status == constants.FAIL: + if self.build_status != constants.SUCCESS: return self.build_status if not self.to_push: @@ -232,14 +233,19 @@ def image_size_check(self): return self.build_status - def push_image(self): + def push_image(self, tag_value=None): """ Pushes the Docker image to ECR using Docker low-level API client for docker. + :param tag_value: str, an optional variable to provide a different tag :return: int, states if the Push was successful or not """ - response = [f"Starting image Push for {self.name}"] - for line in self.client.push(self.repository, self.tag, stream=True, decode=True): + tag = tag_value + if tag_value is None: + tag = self.tag + + response = [f"Starting image Push for {self.repository}:{tag}"] + for line in self.client.push(self.repository, tag, stream=True, decode=True): if line.get("error") is not None: response.append(line["error"]) self.log.append(response) @@ -249,7 +255,7 @@ def push_image(self): LOGGER.info(f"Docker Build Logs: \n {self.get_tail_logs_in_pretty_format(100)}") LOGGER.error("ERROR during Docker PUSH") - LOGGER.error(f"Error message received for {self.dockerfile} while docker push: {line}") + LOGGER.error(f"Error message received for {self.repository}:{tag} while docker push: {line}") return self.build_status if line.get("stream") is not None: @@ -260,9 +266,43 @@ def push_image(self): self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] self.summary["end_time"] = datetime.now() self.summary["ecr_url"] = self.ecr_url + if "pushed_uris" not in self.summary: + self.summary["pushed_uris"] = [] + self.summary["pushed_uris"].append(f"{self.repository}:{tag}") + response.append(f"Completed Push for {self.repository}:{tag}") self.log.append(response) LOGGER.info(f"DOCKER PUSH LOGS: \n {self.get_tail_logs_in_pretty_format(2)}") - LOGGER.info(f"Completed Push for {self.name}") + return self.build_status + + def push_image_with_additional_tags(self): + """ + Pushes an already built Docker image by applying additional tags to it. + + :return: int, states if the Push was successful or not + """ + self.log.append([f"Started Tagging for {self.ecr_url}"]) + for additional_tag in self.additional_tags: + response = [f"Tagging {self.ecr_url} as {self.repository}:{additional_tag}"] + tagging_successful = self.client.tag(self.ecr_url, self.repository, additional_tag) + if not tagging_successful: + response.append(f"Tagging {self.ecr_url} with {additional_tag} unsuccessful.") + self.log.append(response) + LOGGER.error("ERROR during Tagging") + LOGGER.error(f"Tagging {self.ecr_url} with {additional_tag} unsuccessful.") + self.build_status = constants.FAIL + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + return self.build_status + response.append(f"Tagged {self.ecr_url} succussefully as {self.repository}:{additional_tag}") + self.log.append(response) + + self.build_status = self.push_image(tag_value=additional_tag) + if self.build_status != constants.SUCCESS: + return self.build_status + + self.summary["status"] = constants.STATUS_MESSAGE[self.build_status] + self.summary["end_time"] = datetime.now() + self.log.append([f"Completed Tagging for {self.ecr_url}"]) + LOGGER.info(f"DOCKER TAG and PUSH LOGS: \n {self.get_tail_logs_in_pretty_format(5)}") return self.build_status diff --git a/src/image_builder.py b/src/image_builder.py index ce77ce8d60f7..a459afe0cfef 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -165,17 +165,18 @@ def image_builder(buildspec): info=info, dockerfile=image_config["docker_file"], repository=image_repo_uri, - tag=image_tag, + tag=append_tag(image_tag, "pre-push"), to_build=image_config["build"], stage=constants.PRE_PUSH_STAGE, context=context, + additional_tags=[image_tag], ) ##### Create Common stage docker object ##### # If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image # to the repository. Instead, we just push its common stage image to the repository. Therefore, # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. - common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object) + common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object, image_tag) PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) COMMON_STAGE_IMAGES.append(common_stage_image_object) @@ -249,10 +250,13 @@ def process_images(pre_push_image_list, pre_push_image_type="Pre-push"): all_images = pre_push_image_list + common_stage_image_list images_to_push = [image for image in all_images if image.to_push and image.to_build] push_images(images_to_push) + + FORMATTER.banner(f"{pre_push_image_type} Retagging") + retag_and_push_images(images_to_push) return images_to_push -def generate_common_stage_image_object(pre_push_stage_image_object): +def generate_common_stage_image_object(pre_push_stage_image_object, image_tag): """ Creates a common stage image object for a pre_push stage image. If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image to the repository. Instead, we just push its common stage @@ -269,9 +273,10 @@ def generate_common_stage_image_object(pre_push_stage_image_object): info=common_stage_info, dockerfile=os.path.join(os.sep, utils.get_root_folder_path(), "miscellaneous_dockerfiles", "Dockerfile.common"), repository=pre_push_stage_image_object.repository, - tag=tag_image_with_multistage_common(pre_push_stage_image_object.tag), + tag=append_tag(image_tag, "multistage-common"), to_build=pre_push_stage_image_object.to_build, stage=constants.COMMON_STAGE, + additional_tags=[image_tag], ) pre_push_stage_image_object.to_push = False pre_push_stage_image_object.corresponding_common_stage_image = common_stage_image_object @@ -413,6 +418,17 @@ def push_images(images): THREADS[image.name] = executor.submit(image.push_image) FORMATTER.progress(THREADS) +def retag_and_push_images(images): + """ + Takes a list of images, retags them and pushes to the repository + + :param images: list[DockerImage] + """ + THREADS = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=constants.MAX_WORKER_COUNT_FOR_PUSHING_IMAGES) as executor: + for image in images: + THREADS[image.name] = executor.submit(image.push_image_with_additional_tags) + FORMATTER.progress(THREADS) def tag_image_with_pr_number(image_tag): pr_number = os.getenv("CODEBUILD_SOURCE_VERSION").replace("/", "-") @@ -424,15 +440,14 @@ def tag_image_with_datetime(image_tag): return f"{image_tag}-{datetime_suffix}" -def tag_image_with_multistage_common(image_tag): +def append_tag(image_tag, append_str): """ - Appends multistage-common tag to the image + Appends image_tag with append_str - :param image_tag: str - :return: str, image tag appended with multistage-common + :param image_tag: str, original image tag + :param append_str: str, string to be appended """ - append_tag = "multistage-common" - return f"{image_tag}-{append_tag}" + return f"{image_tag}-{append_str}" def modify_repository_name_for_context(image_repo_uri, build_context): From 9c192c9a74253302ef53b5528fb3e1d2651bde8a Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Fri, 24 Sep 2021 23:14:33 +0000 Subject: [PATCH 86/88] Configured test env file to dump no appended tags. Other loggin related changes --- src/image.py | 8 +++---- src/image_builder.py | 40 +++++++++++++++------------------- src/safety_report_generator.py | 2 +- src/utils.py | 18 +++++++++------ 4 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/image.py b/src/image.py index 4c3d71918056..dcf0bd3ad85a 100644 --- a/src/image.py +++ b/src/image.py @@ -172,7 +172,7 @@ def docker_build(self, fileobj=None, custom_context=False): :param custom_context: bool :return: int, Build Status """ - response = [f"Starting the Build Process for {self.name}"] + response = [f"Starting the Build Process for {self.repository}:{self.tag}"] for line in self.client.build( fileobj=fileobj, path=self.dockerfile, @@ -206,7 +206,7 @@ def docker_build(self, fileobj=None, custom_context=False): self.log.append(response) LOGGER.info(f"DOCKER BUILD LOGS: \n{self.get_tail_logs_in_pretty_format()}") - LOGGER.info(f"Completed Build for {self.name}") + LOGGER.info(f"Completed Build for {self.repository}:{self.tag}") self.build_status = constants.SUCCESS return self.build_status @@ -217,7 +217,7 @@ def image_size_check(self): :return: int, Build Status """ - response = [f"Starting image size check for {self.name}"] + response = [f"Starting image size check for {self.repository}:{self.tag}"] self.summary["image_size"] = int(self.client.inspect_image(self.ecr_url)["Size"]) / (1024 * 1024) if self.summary["image_size"] > self.info["image_size_baseline"] * 1.20: response.append("Image size baseline exceeded") @@ -225,7 +225,7 @@ def image_size_check(self): response += self.collect_installed_packages_information() self.build_status = constants.FAIL_IMAGE_SIZE_LIMIT else: - response.append(f"Image Size Check Succeeded for {self.name}") + response.append(f"Image Size Check Succeeded for {self.repository}:{self.tag}") self.build_status = constants.SUCCESS self.log.append(response) diff --git a/src/image_builder.py b/src/image_builder.py index a459afe0cfef..f1c715b4a837 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -177,16 +177,15 @@ def image_builder(buildspec): # to the repository. Instead, we just push its common stage image to the repository. Therefore, # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object, image_tag) + COMMON_STAGE_IMAGES.append(common_stage_image_object) PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) - COMMON_STAGE_IMAGES.append(common_stage_image_object) FORMATTER.separator() FORMATTER.banner("DLC") # Standard images must be built before example images # Example images will use standard images as base - # Common images must be built at the end as they will consume respective standard and example images standard_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" not in image.name.lower()] example_images = [image for image in PRE_PUSH_STAGE_IMAGES if "example" in image.name.lower()] ALL_IMAGES = PRE_PUSH_STAGE_IMAGES + COMMON_STAGE_IMAGES @@ -199,11 +198,8 @@ def image_builder(buildspec): assert all(image in pushed_images for image in IMAGES_TO_PUSH), "Few images could not be pushed." # After the build, display logs/summary for all the images. - FORMATTER.banner("Build Logs") - show_build_logs(ALL_IMAGES) - FORMATTER.banner("Summary") - show_build_summary(ALL_IMAGES) + show_build_info(ALL_IMAGES) FORMATTER.banner("Errors") is_any_build_failed, is_any_build_failed_size_limit = show_build_errors(ALL_IMAGES) @@ -219,7 +215,10 @@ def image_builder(buildspec): test_trigger_job = utils.get_codebuild_project_name() # Tests should only run on images that were pushed to the repository utils.set_test_env( - IMAGES_TO_PUSH, BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), TEST_TRIGGER=test_trigger_job, + IMAGES_TO_PUSH, + use_latest_additional_tag=True, + BUILD_CONTEXT=os.getenv("BUILD_CONTEXT"), + TEST_TRIGGER=test_trigger_job ) @@ -229,7 +228,11 @@ def process_images(pre_push_image_list, pre_push_image_type="Pre-push"): pre push images and then builds it. After the pre-push images have been built, it extracts the corresponding common stage images for the pre-push images and builds those common stage images. After the common stage images have been built, it finds outs the docker images that need to be - pushed and pushes them according. + pushed and pushes them accordingly. + + Note that the common stage images should always be built after the pre-push images of a + particular kind. This is because the Common stage images use are built on respctive + Standard and Example images. :param pre_push_image_list: list[DockerImage], list of pre-push images :param pre_push_image_type: str, used to display the message on the logs @@ -283,9 +286,9 @@ def generate_common_stage_image_object(pre_push_stage_image_object, image_tag): return common_stage_image_object -def show_build_logs(images): +def show_build_info(images): """ - Display and save the build logs for a list of input images. + Displays the build info for a list of input images. :param images: list[DockerImage] """ @@ -297,25 +300,16 @@ def show_build_logs(images): image_description = f"{image.name}-{image.stage}" FORMATTER.title(image_description) FORMATTER.table(image.info.items()) - FORMATTER.title(f"Ending Logs for {image_description}") - FORMATTER.print_lines(image.log[-1][-2:]) + flattened_logs = list(itertools.chain(*image.log)) with open(f"logs/{image_description}", "w") as fp: fp.write("/n".join(flattened_logs)) image.summary["log"] = f"logs/{image_description}" - - -def show_build_summary(images): - """ - Display the build summary for a list of input images. - - :param images: list[DockerImage] - """ - - for image in images: - FORMATTER.title(image.name) FORMATTER.table(image.summary.items()) + FORMATTER.title(f"Ending Logs for {image_description}") + FORMATTER.print_lines(image.log[-1][-2:]) + def show_build_errors(images): """ diff --git a/src/safety_report_generator.py b/src/safety_report_generator.py index d90be30f5203..b52c2804fd13 100644 --- a/src/safety_report_generator.py +++ b/src/safety_report_generator.py @@ -131,7 +131,7 @@ def run_safety_check_in_non_cb_context(self): safety_check_command = f"{self.docker_exec_cmd} safety check --json" run_out = self.ctx.run(safety_check_command, warn=True, hide=True) if run_out.return_code != 0: - print(f"safety check command returned non-zero error code. stderr printed for logging: {run_out.stderr}") + print("safety check command returned non-zero error code. This indicates that vulnerabilities might exist.") return run_out.stdout def run_safety_check_in_cb_context(self): diff --git a/src/utils.py b/src/utils.py index a376a7f6390d..5adbbad2d595 100644 --- a/src/utils.py +++ b/src/utils.py @@ -404,7 +404,7 @@ def build_setup(framework, device_types=None, image_types=None, py_versions=None os.environ[env_variable] = "true" -def fetch_dlc_images_for_test_jobs(images): +def fetch_dlc_images_for_test_jobs(images, use_latest_additional_tag=False): """ use the JobParamters.run_test_types values to pass on image ecr urls to each test type. :param images: list @@ -419,8 +419,12 @@ def fetch_dlc_images_for_test_jobs(images): continue use_preexisting_images = (build_disabled and docker_image.build_status == constants.NOT_BUILT) if docker_image.build_status == constants.SUCCESS or use_preexisting_images: + ecr_url_to_test = docker_image.ecr_url + if use_latest_additional_tag and len(docker_image.additional_tags) > 0: + ecr_url_to_test = f"{docker_image.repository}:{docker_image.additional_tags[-1]}" + # Run sanity tests on the all images built - DLC_IMAGES["sanity"].append(docker_image.ecr_url) + DLC_IMAGES["sanity"].append(ecr_url_to_test) image_job_type = docker_image.info.get("image_type") image_device_type = docker_image.info.get("device_type") image_python_version = docker_image.info.get("python_version") @@ -432,12 +436,12 @@ def fetch_dlc_images_for_test_jobs(images): constants.ALL_TESTS if constants.ALL in run_tests else run_tests ) for test in run_tests: - DLC_IMAGES[test].append(docker_image.ecr_url) + DLC_IMAGES[test].append(ecr_url_to_test) # when key is training or inference values can be (ecs, eks, ec2, sagemaker) if image_job_type in JobParameters.image_run_test_types.keys(): run_tests = JobParameters.image_run_test_types.get(image_job_type) for test in run_tests: - DLC_IMAGES[test].append(docker_image.ecr_url) + DLC_IMAGES[test].append(ecr_url_to_test) # when key is image_tag (training-cpu-py3) values can be (ecs, eks, ec2, sagemaker) if image_tag in JobParameters.image_run_test_types.keys(): run_tests = JobParameters.image_run_test_types.get(image_tag) @@ -445,7 +449,7 @@ def fetch_dlc_images_for_test_jobs(images): constants.ALL_TESTS if constants.ALL in run_tests else run_tests ) for test in run_tests: - DLC_IMAGES[test].append(docker_image.ecr_url) + DLC_IMAGES[test].append(ecr_url_to_test) for test_type in DLC_IMAGES.keys(): test_images = DLC_IMAGES[test_type] @@ -459,7 +463,7 @@ def write_to_json_file(file_name, content): json.dump(content, fp) -def set_test_env(images, images_env="DLC_IMAGES", **kwargs): +def set_test_env(images, use_latest_additional_tag=False, images_env="DLC_IMAGES", **kwargs): """ Util function to write a file to be consumed by test env with necessary environment variables @@ -472,7 +476,7 @@ def set_test_env(images, images_env="DLC_IMAGES", **kwargs): """ test_envs = [] - test_images_dict = fetch_dlc_images_for_test_jobs(images) + test_images_dict = fetch_dlc_images_for_test_jobs(images, use_latest_additional_tag=use_latest_additional_tag) # dumping the test_images to dict that can be used in src/start_testbuilds.py write_to_json_file(constants.TEST_TYPE_IMAGES_PATH, test_images_dict) From 0996ac099cd8a64c52b54a2f78220a50c2359d70 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sat, 25 Sep 2021 23:38:33 +0000 Subject: [PATCH 87/88] Disabling Common Stage --- src/image_builder.py | 6 +++--- test/dlc_tests/sanity/test_safety_report_file.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/image_builder.py b/src/image_builder.py index f1c715b4a837..d1017b914f87 100644 --- a/src/image_builder.py +++ b/src/image_builder.py @@ -176,8 +176,8 @@ def image_builder(buildspec): # If for a pre_push stage image we create a common stage image, then we do not push the pre_push stage image # to the repository. Instead, we just push its common stage image to the repository. Therefore, # inside function get_common_stage_image_object we make pre_push_stage_image_object non pushable. - common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object, image_tag) - COMMON_STAGE_IMAGES.append(common_stage_image_object) + # common_stage_image_object = generate_common_stage_image_object(pre_push_stage_image_object, image_tag) + # COMMON_STAGE_IMAGES.append(common_stage_image_object) PRE_PUSH_STAGE_IMAGES.append(pre_push_stage_image_object) FORMATTER.separator() @@ -231,7 +231,7 @@ def process_images(pre_push_image_list, pre_push_image_type="Pre-push"): pushed and pushes them accordingly. Note that the common stage images should always be built after the pre-push images of a - particular kind. This is because the Common stage images use are built on respctive + particular kind. This is because the Common stage images use are built on respective Standard and Example images. :param pre_push_image_list: list[DockerImage], list of pre-push images diff --git a/test/dlc_tests/sanity/test_safety_report_file.py b/test/dlc_tests/sanity/test_safety_report_file.py index af433dd4344e..0b63cfa19029 100644 --- a/test/dlc_tests/sanity/test_safety_report_file.py +++ b/test/dlc_tests/sanity/test_safety_report_file.py @@ -57,6 +57,7 @@ def __post_init__(self): @pytest.mark.model("N/A") +@pytest.skip("Will be unskipped when Safety Scan Report Generation is enabled") def test_safety_file_exists_and_is_valid(image): """ Checks if the image has a safety report at the desired location and fails if any of the From e90439e4cdfd57e0c30cb4c4dc283ffa292bf131 Mon Sep 17 00:00:00 2001 From: Shantanu Tripathi Date: Sun, 26 Sep 2021 00:03:33 +0000 Subject: [PATCH 88/88] Fix --- test/dlc_tests/sanity/test_safety_report_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/dlc_tests/sanity/test_safety_report_file.py b/test/dlc_tests/sanity/test_safety_report_file.py index 0b63cfa19029..ed76ab10b25c 100644 --- a/test/dlc_tests/sanity/test_safety_report_file.py +++ b/test/dlc_tests/sanity/test_safety_report_file.py @@ -57,7 +57,7 @@ def __post_init__(self): @pytest.mark.model("N/A") -@pytest.skip("Will be unskipped when Safety Scan Report Generation is enabled") +@pytest.mark.skip(reason="Will be unskipped when Safety Scan Report Generation is enabled") def test_safety_file_exists_and_is_valid(image): """ Checks if the image has a safety report at the desired location and fails if any of the