From 3390626cc155732327c2239039c00ad0d53c013a Mon Sep 17 00:00:00 2001 From: yiyuanzzz Date: Thu, 18 Jul 2024 13:05:48 -0700 Subject: [PATCH 1/6] diable digest resolution for manifest v2 schema 1 --- agent/engine/docker_task_engine.go | 16 +++++++++++++ agent/engine/docker_task_engine_test.go | 31 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/agent/engine/docker_task_engine.go b/agent/engine/docker_task_engine.go index 13aaee0e316..322661bb0c7 100644 --- a/agent/engine/docker_task_engine.go +++ b/agent/engine/docker_task_engine.go @@ -1352,6 +1352,22 @@ func (engine *DockerTaskEngine) pullContainerManifest( }) return dockerapi.DockerContainerMetadata{Error: manifestPullErr} } + imageManifestMediatype := distInspect.Descriptor.MediaType + logger.Info("Fetched image manifest Mediatype for container from registry", logger.Fields{ + field.TaskARN: task.Arn, + field.ContainerName: container.Name, + field.ImageMediatype: imageManifestMediatype, + field.Image: container.Image, + }) + if strings.Contains(imageManifestMediatype, "application/vnd.docker.distribution.manifest.v1") { + logger.Info("skipping digest resolution for manifest v2 schema 1,", logger.Fields{ + field.TaskARN: task.Arn, + field.ContainerName: container.Name, + field.ImageMediatype: imageManifestMediatype, + field.Image: container.Image, + }) + return dockerapi.DockerContainerMetadata{} + } imageManifestDigest = distInspect.Descriptor.Digest logger.Info("Fetched image manifest digest for container from registry", logger.Fields{ field.TaskARN: task.Arn, diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index faea238fe0f..6bbf9bfbe47 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -4147,6 +4147,24 @@ func TestPullContainerManifest(t *testing.T) { name: "digest is not resolved if already available in image reference", image: "public.ecr.aws/library/alpine@" + testDigest.String(), }, + // New test case for isSchema1 function + { + name: "schema1 image - skip digest resolution", + image: "myimage", + imagePullBehavior: config.ImagePullAlwaysBehavior, + setDockerClientExpectations: func(c *gomock.Controller, d *mock_dockerapi.MockDockerClient) { + versioned := mock_dockerapi.NewMockDockerClient(c) + versioned.EXPECT(). + PullImageManifest(gomock.Any(), "myimage", nil). + Return( + registry.DistributionInspect{ + Descriptor: ocispec.Descriptor{MediaType: "application/vnd.docker.distribution.manifest.v1+json"}, + }, + nil) + d.EXPECT().WithVersion(dockerclient.Version_1_35).Return(versioned, nil) + }, + expectedResult: dockerapi.DockerContainerMetadata{}, + }, { name: "image pull not required - image inspect fails", image: "myimage", @@ -4229,7 +4247,12 @@ func TestPullContainerManifest(t *testing.T) { versioned.EXPECT(). PullImageManifest(gomock.Any(), "myimage", nil). Return( - registry.DistributionInspect{Descriptor: ocispec.Descriptor{Digest: testDigest}}, + registry.DistributionInspect{ + Descriptor: ocispec.Descriptor{ + MediaType: "application/vnd.docker.distribution.manifest.v2+json", + Digest: testDigest, + }, + }, nil) d.EXPECT().WithVersion(dockerclient.Version_1_35).Return(versioned, nil) }, @@ -4261,8 +4284,10 @@ func TestPullContainerManifest(t *testing.T) { PullImageManifest(gomock.Any(), "myimage", expectedRegistryAuthData). Return( registry.DistributionInspect{ - Descriptor: ocispec.Descriptor{Digest: digest.Digest(testDigest.String())}, - }, + Descriptor: ocispec.Descriptor{ + MediaType: "application/vnd.docker.distribution.manifest.v2+json", + Digest: testDigest, + }}, nil) d.EXPECT().WithVersion(dockerclient.Version_1_35).Return(versioned, nil) }, From 99e048968fb7341769e689cf2036f167096083f7 Mon Sep 17 00:00:00 2001 From: yiyuanzzz Date: Thu, 18 Jul 2024 13:48:09 -0700 Subject: [PATCH 2/6] fix --- ecs-agent/logger/field/constants.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs-agent/logger/field/constants.go b/ecs-agent/logger/field/constants.go index 9a93f4d8c25..811398e254f 100644 --- a/ecs-agent/logger/field/constants.go +++ b/ecs-agent/logger/field/constants.go @@ -49,6 +49,7 @@ const ( ImageLastUsedAt = "imageLastUsedAt" ImagePullSucceeded = "imagePullSucceeded" ImageDigest = "imageDigest" + ImageMediatype = "imageMediatype" ContainerName = "containerName" ContainerImage = "containerImage" ContainerExitCode = "containerExitCode" From 18098b3051adaad1ba4384a7c46a2f01c1890257 Mon Sep 17 00:00:00 2001 From: yiyuanzzz Date: Thu, 18 Jul 2024 13:57:09 -0700 Subject: [PATCH 3/6] go mod --- .../aws/amazon-ecs-agent/ecs-agent/logger/field/constants.go | 1 + 1 file changed, 1 insertion(+) diff --git a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/logger/field/constants.go b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/logger/field/constants.go index 9a93f4d8c25..811398e254f 100644 --- a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/logger/field/constants.go +++ b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/logger/field/constants.go @@ -49,6 +49,7 @@ const ( ImageLastUsedAt = "imageLastUsedAt" ImagePullSucceeded = "imagePullSucceeded" ImageDigest = "imageDigest" + ImageMediatype = "imageMediatype" ContainerName = "containerName" ContainerImage = "containerImage" ContainerExitCode = "containerExitCode" From 8117dc0bbd17971ab6a7a891b77a5b962fdf1e95 Mon Sep 17 00:00:00 2001 From: Steph Roberts <114956067+Ephylouise@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:06:54 -0400 Subject: [PATCH 4/6] Update CodeBuild CF stack template to add disabled encryption and remove webhook Booleans (#4243) * Remove webhook/triggers from CodeBuild CF stack template * Disable artifact encryption on CodeBuild devbuild stack template * Remove trailing space on EcryptionDisabled lines --- .../codebuild-devbuild-stack.yml | 66 +++---------------- 1 file changed, 9 insertions(+), 57 deletions(-) diff --git a/build-infrastructure/codebuild-devbuild-stack.yml b/build-infrastructure/codebuild-devbuild-stack.yml index ca9bc79cd70..ab0b26300c2 100644 --- a/build-infrastructure/codebuild-devbuild-stack.yml +++ b/build-infrastructure/codebuild-devbuild-stack.yml @@ -23,6 +23,7 @@ Resources: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -46,18 +47,12 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE UbuntuArmProject: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -81,18 +76,12 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE ArmProject: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -116,18 +105,12 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE AmdProject: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -151,13 +134,6 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE # Creates a CodeBuild project for Amazon Linux 2 ARM @@ -165,6 +141,7 @@ Resources: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -188,13 +165,6 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE # Creates a CodeBuild project for Amazon Linux 2 AMD @@ -202,6 +172,7 @@ Resources: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -225,13 +196,6 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE # Creates a CodeBuild project for Amazon Linux 2023 ARM @@ -239,6 +203,7 @@ Resources: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled: true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -262,13 +227,6 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE # Creates a CodeBuild project for Amazon Linux 2023 AMD @@ -276,6 +234,7 @@ Resources: Type: 'AWS::CodeBuild::Project' Properties: Artifacts: + EncryptionDisabled : true Location: !Ref BuildBucketName NamespaceType: NONE OverrideArtifactName: true @@ -299,13 +258,6 @@ Resources: Location: !Ref GithubFullRepoName Type: GITHUB TimeoutInMinutes: 60 - Triggers: - BuildType: BUILD - # Config list of developers allowlisted to create builds when creating PRs to GithubBranchName - # This allow list can be modified using aws-cli or aws-sdk - # CodeBuild also supports pattern matches using regex, but this is not useful for listing different Github IDs - # so they have to be listed separately - Webhook: true Visibility: PRIVATE # Defines the service roles for the CodeBuild projects @@ -652,4 +604,4 @@ Resources: - 's3:GetBucketAcl' - 's3:GetBucketLocation' PolicyName: !Sub '${AWS::StackName}-ServicePolicyAmzn2023Amd' - RoleName: !Sub '${AWS::StackName}-ServiceRoleAmzn2023Amd' \ No newline at end of file + RoleName: !Sub '${AWS::StackName}-ServiceRoleAmzn2023Amd' From 0ebbfbe74ccd60947b1372ae5fc899e00c5406ff Mon Sep 17 00:00:00 2001 From: yiyuanzzz Date: Thu, 18 Jul 2024 13:05:48 -0700 Subject: [PATCH 5/6] diable digest resolution for manifest v2 schema 1 --- agent/engine/docker_task_engine_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index 6bbf9bfbe47..7d49af06aef 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -4165,6 +4165,23 @@ func TestPullContainerManifest(t *testing.T) { }, expectedResult: dockerapi.DockerContainerMetadata{}, }, + { + name: "schema1 image - skip digest resolution", + image: "myimage", + imagePullBehavior: config.ImagePullAlwaysBehavior, + setDockerClientExpectations: func(c *gomock.Controller, d *mock_dockerapi.MockDockerClient) { + versioned := mock_dockerapi.NewMockDockerClient(c) + versioned.EXPECT(). + PullImageManifest(gomock.Any(), "myimage", nil). + Return( + registry.DistributionInspect{ + Descriptor: ocispec.Descriptor{MediaType: "application/vnd.docker.distribution.manifest.v1+json"}, + }, + nil) + d.EXPECT().WithVersion(dockerclient.Version_1_35).Return(versioned, nil) + }, + expectedResult: dockerapi.DockerContainerMetadata{}, + }, { name: "image pull not required - image inspect fails", image: "myimage", From 0deb5e1317c18c801b29c2adc02be5cae17e1a19 Mon Sep 17 00:00:00 2001 From: yiyuanzzz Date: Fri, 19 Jul 2024 11:23:15 -0700 Subject: [PATCH 6/6] fix --- agent/engine/docker_task_engine_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/engine/docker_task_engine_test.go b/agent/engine/docker_task_engine_test.go index 7d49af06aef..9545d8514bf 100644 --- a/agent/engine/docker_task_engine_test.go +++ b/agent/engine/docker_task_engine_test.go @@ -4147,7 +4147,6 @@ func TestPullContainerManifest(t *testing.T) { name: "digest is not resolved if already available in image reference", image: "public.ecr.aws/library/alpine@" + testDigest.String(), }, - // New test case for isSchema1 function { name: "schema1 image - skip digest resolution", image: "myimage",