diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json index c145d851c4c83..f4ff110239f7a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.assets.json @@ -1,7 +1,7 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { - "fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94": { + "8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a": { "source": { "path": "aws-cdk-codepipeline-ecs-deploy.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94.json", + "objectKey": "8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json index 67d54e84dc3e0..a2a555c5539b4 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/aws-cdk-codepipeline-ecs-deploy.template.json @@ -18,9 +18,6 @@ "VPCPublicSubnet1SubnetB4246D30": { "Type": "AWS::EC2::Subnet", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "AvailabilityZone": { "Fn::Select": [ 0, @@ -44,21 +41,24 @@ "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPublicSubnet1RouteTableFEE4B781": { "Type": "AWS::EC2::RouteTable", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "Tags": [ { "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPublicSubnet1RouteTableAssociation0B0896DC": { @@ -75,12 +75,12 @@ "VPCPublicSubnet1DefaultRoute91CEF279": { "Type": "AWS::EC2::Route", "Properties": { - "RouteTableId": { - "Ref": "VPCPublicSubnet1RouteTableFEE4B781" - }, "DestinationCidrBlock": "0.0.0.0/0", "GatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "RouteTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" } }, "DependsOn": [ @@ -102,15 +102,15 @@ "VPCPublicSubnet1NATGatewayE0556630": { "Type": "AWS::EC2::NatGateway", "Properties": { - "SubnetId": { - "Ref": "VPCPublicSubnet1SubnetB4246D30" - }, "AllocationId": { "Fn::GetAtt": [ "VPCPublicSubnet1EIP6AD938E8", "AllocationId" ] }, + "SubnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + }, "Tags": [ { "Key": "Name", @@ -126,9 +126,6 @@ "VPCPrivateSubnet1Subnet8BCA10E0": { "Type": "AWS::EC2::Subnet", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "AvailabilityZone": { "Fn::Select": [ 0, @@ -152,21 +149,24 @@ "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPrivateSubnet1RouteTableBE8A6027": { "Type": "AWS::EC2::RouteTable", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "Tags": [ { "Key": "Name", "Value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "VpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "VPCPrivateSubnet1RouteTableAssociation347902D1": { @@ -183,12 +183,12 @@ "VPCPrivateSubnet1DefaultRouteAE1D6490": { "Type": "AWS::EC2::Route", "Properties": { - "RouteTableId": { - "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" - }, "DestinationCidrBlock": "0.0.0.0/0", "NatGatewayId": { "Ref": "VPCPublicSubnet1NATGatewayE0556630" + }, + "RouteTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" } } }, @@ -206,11 +206,11 @@ "VPCVPCGW99B986DC": { "Type": "AWS::EC2::VPCGatewayAttachment", "Properties": { - "VpcId": { - "Ref": "VPCB9E5F0B4" - }, "InternetGatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "VpcId": { + "Ref": "VPCB9E5F0B4" } } }, @@ -492,6 +492,10 @@ "Artifacts": { "Type": "CODEPIPELINE" }, + "Cache": { + "Type": "NO_CACHE" + }, + "EncryptionKey": "alias/aws/s3", "Environment": { "ComputeType": "BUILD_GENERAL1_SMALL", "EnvironmentVariables": [ @@ -562,11 +566,7 @@ "Source": { "BuildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"pre_build\": {\n \"commands\": \"$(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)\"\n },\n \"build\": {\n \"commands\": \"docker build -t $REPOSITORY_URI:latest .\"\n },\n \"post_build\": {\n \"commands\": [\n \"docker push $REPOSITORY_URI:latest\",\n \"printf '[{ \\\"name\\\": \\\"Container\\\", \\\"imageUri\\\": \\\"%s\\\" }]' $REPOSITORY_URI:latest > imagedefinitions.json\"\n ]\n }\n },\n \"artifacts\": {\n \"files\": \"imagedefinitions.json\"\n }\n}", "Type": "CODEPIPELINE" - }, - "Cache": { - "Type": "NO_CACHE" - }, - "EncryptionKey": "alias/aws/s3" + } } }, "MyPipelineRoleC0D47CA4": { @@ -666,6 +666,12 @@ "MyPipelineAED38ECF": { "Type": "AWS::CodePipeline::Pipeline", "Properties": { + "ArtifactStore": { + "Location": { + "Ref": "MyBucketF68F3FF0" + }, + "Type": "S3" + }, "RoleArn": { "Fn::GetAtt": [ "MyPipelineRoleC0D47CA4", @@ -779,13 +785,7 @@ ], "Name": "Deploy" } - ], - "ArtifactStore": { - "Location": { - "Ref": "MyBucketF68F3FF0" - }, - "Type": "S3" - } + ] }, "DependsOn": [ "MyPipelineRoleDefaultPolicy34F09EFA", @@ -1010,6 +1010,7 @@ "ecs:DescribeTasks", "ecs:ListTasks", "ecs:RegisterTaskDefinition", + "ecs:TagResource", "ecs:UpdateService" ], "Effect": "Allow", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json index 376ce77f7d56b..b786b10b30c0e 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.pipeline-ecs-deploy": { "stacks": [ diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json index efd6c4aa1e491..846a7917d24c5 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-cdk-codepipeline-ecs-deploy.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-cdk-codepipeline-ecs-deploy.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/fadd6238a0ec388dc8d2cd3046a20a4218f1bde1c230b59050ee6df26f62dc94.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/8c9962322a7c4c8c90d88a3b9414e1c31d71f87e07f05b6d795e6d8355d5033a.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -171,6 +172,12 @@ "data": "EcsProject54EFDCA6" } ], + "/aws-cdk-codepipeline-ecs-deploy/MyPipeline": [ + { + "type": "aws:cdk:warning", + "data": "V1 pipeline type is implicitly selected when `pipelineType` is not set. If you want to use V2 type, set `PipelineType.V2`. [ack: @aws-cdk/aws-codepipeline:unspecifiedPipelineType]" + } + ], "/aws-cdk-codepipeline-ecs-deploy/MyPipeline/Role/Resource": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json index 9b1b21b788c29..bcb07e812f369 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-codepipeline-actions/test/integ.pipeline-ecs-deploy.js.snapshot/tree.json @@ -45,9 +45,6 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Subnet", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "availabilityZone": { "Fn::Select": [ 0, @@ -71,7 +68,10 @@ "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -93,15 +93,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::RouteTable", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "tags": [ { "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PublicSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -134,12 +134,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Route", "aws:cdk:cloudformation:props": { - "routeTableId": { - "Ref": "VPCPublicSubnet1RouteTableFEE4B781" - }, "destinationCidrBlock": "0.0.0.0/0", "gatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "routeTableId": { + "Ref": "VPCPublicSubnet1RouteTableFEE4B781" } } }, @@ -174,15 +174,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::NatGateway", "aws:cdk:cloudformation:props": { - "subnetId": { - "Ref": "VPCPublicSubnet1SubnetB4246D30" - }, "allocationId": { "Fn::GetAtt": [ "VPCPublicSubnet1EIP6AD938E8", "AllocationId" ] }, + "subnetId": { + "Ref": "VPCPublicSubnet1SubnetB4246D30" + }, "tags": [ { "key": "Name", @@ -212,9 +212,6 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Subnet", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "availabilityZone": { "Fn::Select": [ 0, @@ -238,7 +235,10 @@ "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -260,15 +260,15 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::RouteTable", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "tags": [ { "key": "Name", "value": "aws-cdk-codepipeline-ecs-deploy/VPC/PrivateSubnet1" } - ] + ], + "vpcId": { + "Ref": "VPCB9E5F0B4" + } } }, "constructInfo": { @@ -301,12 +301,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::Route", "aws:cdk:cloudformation:props": { - "routeTableId": { - "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" - }, "destinationCidrBlock": "0.0.0.0/0", "natGatewayId": { "Ref": "VPCPublicSubnet1NATGatewayE0556630" + }, + "routeTableId": { + "Ref": "VPCPrivateSubnet1RouteTableBE8A6027" } } }, @@ -346,11 +346,11 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::EC2::VPCGatewayAttachment", "aws:cdk:cloudformation:props": { - "vpcId": { - "Ref": "VPCB9E5F0B4" - }, "internetGatewayId": { "Ref": "VPCIGWB7E252D3" + }, + "vpcId": { + "Ref": "VPCB9E5F0B4" } } }, @@ -828,6 +828,10 @@ "artifacts": { "type": "CODEPIPELINE" }, + "cache": { + "type": "NO_CACHE" + }, + "encryptionKey": "alias/aws/s3", "environment": { "type": "LINUX_CONTAINER", "image": "aws/codebuild/docker:17.09.0", @@ -898,11 +902,7 @@ "source": { "type": "CODEPIPELINE", "buildSpec": "{\n \"version\": \"0.2\",\n \"phases\": {\n \"pre_build\": {\n \"commands\": \"$(aws ecr get-login --region $AWS_DEFAULT_REGION --no-include-email)\"\n },\n \"build\": {\n \"commands\": \"docker build -t $REPOSITORY_URI:latest .\"\n },\n \"post_build\": {\n \"commands\": [\n \"docker push $REPOSITORY_URI:latest\",\n \"printf '[{ \\\"name\\\": \\\"Container\\\", \\\"imageUri\\\": \\\"%s\\\" }]' $REPOSITORY_URI:latest > imagedefinitions.json\"\n ]\n }\n },\n \"artifacts\": {\n \"files\": \"imagedefinitions.json\"\n }\n}" - }, - "cache": { - "type": "NO_CACHE" - }, - "encryptionKey": "alias/aws/s3" + } } }, "constructInfo": { @@ -1064,6 +1064,12 @@ "attributes": { "aws:cdk:cloudformation:type": "AWS::CodePipeline::Pipeline", "aws:cdk:cloudformation:props": { + "artifactStore": { + "type": "S3", + "location": { + "Ref": "MyBucketF68F3FF0" + } + }, "roleArn": { "Fn::GetAtt": [ "MyPipelineRoleC0D47CA4", @@ -1177,13 +1183,7 @@ } ] } - ], - "artifactStore": { - "type": "S3", - "location": { - "Ref": "MyBucketF68F3FF0" - } - } + ] } }, "constructInfo": { @@ -1358,13 +1358,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } }, "Build": { @@ -1484,13 +1484,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } }, "Deploy": { @@ -1572,6 +1572,7 @@ "ecs:DescribeTasks", "ecs:ListTasks", "ecs:RegisterTaskDefinition", + "ecs:TagResource", "ecs:UpdateService" ], "Effect": "Allow", @@ -1651,13 +1652,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, @@ -1693,7 +1694,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts index 15cb5757e88b3..4bee0a054bcd8 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job-executable.ts @@ -169,14 +169,32 @@ interface PythonExecutableProps { /** * Additional Python files that AWS Glue adds to the Python path before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files and argument is not set * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; } +interface RayExecutableProps { + /** + * The Python version to use. + */ + readonly pythonVersion: PythonVersion; + + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files and argument is not set + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; +} + interface SharedJobExecutableProps { /** * Runtime. It is required for Ray jobs. @@ -199,10 +217,11 @@ interface SharedJobExecutableProps { /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-files`. * * @default [] - no extra files are copied to the working directory * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; } @@ -211,19 +230,21 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. * Only individual files are supported, directories are not supported. + * Equivalent to a job parameter `--extra-jars`. * * @default [] - no extra jars are added to the classpath * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default false - priority is not given to user-provided jars * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } @@ -234,8 +255,9 @@ interface SharedSparkJobExecutableProps extends SharedJobExecutableProps { export interface ScalaJobExecutableProps extends SharedSparkJobExecutableProps { /** * The fully qualified Scala class name that serves as the entry point for the job. + * Equivalent to a job parameter `--class`. * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className: string; } @@ -253,7 +275,7 @@ export interface PythonShellExecutableProps extends SharedJobExecutableProps, Py /** * Props for creating a Python Ray job executable */ -export interface PythonRayExecutableProps extends SharedJobExecutableProps, PythonExecutableProps {} +export interface PythonRayExecutableProps extends SharedJobExecutableProps, RayExecutableProps {} /** * The executable properties related to the Glue job's GlueVersion, JobType and code @@ -377,6 +399,9 @@ export class JobExecutable { if (JobLanguage.PYTHON !== config.language && config.extraPythonFiles) { throw new Error('extraPythonFiles is not supported for languages other than JobLanguage.PYTHON'); } + if (config.extraPythonFiles && type === JobType.RAY.name) { + throw new Error('extraPythonFiles is not supported for Ray jobs'); + } if (config.pythonVersion === PythonVersion.THREE_NINE && type !== JobType.PYTHON_SHELL.name && type !== JobType.RAY.name) { throw new Error('Specified PythonVersion PythonVersion.THREE_NINE is only supported for JobType Python Shell and Ray'); } @@ -384,7 +409,7 @@ export class JobExecutable { throw new Error('Specified PythonVersion PythonVersion.THREE is not supported for Ray'); } if (config.runtime === undefined && type === JobType.RAY.name) { - throw new Error('Runtime is required for Ray jobs.'); + throw new Error('Runtime is required for Ray jobs'); } this.config = config; } @@ -410,8 +435,9 @@ export interface JobExecutableConfig { /** * The language of the job (Scala or Python). + * Equivalent to a job parameter `--job-language`. * - * @see `--job-language` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly language: JobLanguage; @@ -441,46 +467,61 @@ export interface JobExecutableConfig { /** * The Scala class that serves as the entry point for the job. This applies only if your the job langauage is Scala. + * Equivalent to a job parameter `--class`. * * @default - no scala className specified * - * @see `--class` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly className?: string; /** * Additional Java .jar files that AWS Glue adds to the Java classpath before executing your script. + * Equivalent to a job parameter `--extra-jars`. * * @default - no extra jars specified. * - * @see `--extra-jars` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJars?: Code[]; /** * Additional Python files that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--extra-py-files`. * * @default - no extra python files specified. * - * @see `--extra-py-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraPythonFiles?: Code[]; + /** + * Additional Python modules that AWS Glue adds to the Python path before executing your script. + * Equivalent to a job parameter `--s3-py-modules`. + * + * @default - no extra python files specified. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html + */ + readonly s3PythonModules?: Code[]; + /** * Additional files, such as configuration files that AWS Glue copies to the working directory of your script before executing it. + * Equivalent to a job parameter `--extra-files`. * * @default - no extra files specified. * - * @see `--extra-files` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraFiles?: Code[]; /** * Setting this value to true prioritizes the customer's extra JAR files in the classpath. + * Equivalent to a job parameter `--user-jars-first`. * * @default - extra jars are not prioritized. * - * @see `--user-jars-first` in https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly extraJarsFirst?: boolean; } diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts index 526cb774017c3..8420d380bb675 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/job.ts @@ -598,10 +598,11 @@ export interface JobProps { /** * Enables the collection of metrics for job profiling. + * Equivalent to a job parameter `--enable-metrics`. * * @default - no profiling metrics emitted. * - * @see `--enable-metrics` at https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html + * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html */ readonly enableProfilingMetrics? :boolean; @@ -801,6 +802,9 @@ export class Job extends JobBase { if (config.extraPythonFiles && config.extraPythonFiles.length > 0) { args['--extra-py-files'] = config.extraPythonFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } + if (config.s3PythonModules && config.s3PythonModules.length > 0) { + args['--s3-py-modules'] = config.s3PythonModules.map(code => this.codeS3ObjectUrl(code)).join(','); + } if (config.extraFiles && config.extraFiles.length > 0) { args['--extra-files'] = config.extraFiles.map(code => this.codeS3ObjectUrl(code)).join(','); } @@ -886,8 +890,8 @@ export class Job extends JobBase { } /** - * Create a CloudWatch Metric that's based on Glue Job events - * {@see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types} + * Create a CloudWatch Metric that's based on Glue Job events. + * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types * The metric has namespace = 'AWS/Events', metricName = 'TriggeredRules' and RuleName = rule.ruleName dimension. * * @param rule for use in setting RuleName dimension value diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip differ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json index b371535051dc3..4be1daf9e3601 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "files": { "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { "source": { @@ -14,7 +14,33 @@ } } }, - "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": { + "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4": { + "source": { + "path": "asset.1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7": { + "source": { + "path": "asset.9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15": { "source": { "path": "aws-glue-job.template.json", "packaging": "file" @@ -22,7 +48,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "objectKey": "4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json index e524ee21d34da..6ed2d571b6a3e 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json @@ -149,12 +149,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -187,6 +181,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -338,12 +338,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -352,6 +346,12 @@ "GlueVersion": "2.0", "Name": "StreamingJob2.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -507,12 +507,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -545,6 +539,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -696,12 +696,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -710,6 +704,12 @@ "GlueVersion": "3.0", "Name": "StreamingJob3.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -865,12 +865,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -903,6 +897,12 @@ "NotifyDelayAfter": 1 }, "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1054,12 +1054,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1068,6 +1062,12 @@ "GlueVersion": "4.0", "Name": "StreamingJob4.0", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1181,12 +1181,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1194,6 +1188,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob", + "Role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1306,12 +1306,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1319,6 +1313,12 @@ }, "GlueVersion": "1.0", "Name": "ShellJob39", + "Role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "Tags": { "key": "value" } @@ -1427,25 +1427,37 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] } }, - "Role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "GlueVersion": "4.0", "Name": "RayJob", "NumberOfWorkers": 2, + "Role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "Tags": { "key": "value" }, @@ -1559,12 +1571,6 @@ ] } }, - "Role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "DefaultArguments": { "--job-language": "python" }, @@ -1572,6 +1578,12 @@ "GlueVersion": "3.0", "Name": "EtlJobWithFLEX", "NumberOfWorkers": 10, + "Role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "WorkerType": "G.1X" } } diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out index f0b901e7c06e5..1f0068d32659a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"32.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json index 3e404e817257e..9618d2538fb71 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "testCases": { "integ.job": { "stacks": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json index f729d71dca775..e0a0106226e31 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "32.0.0", + "version": "36.0.0", "artifacts": { "aws-glue-job.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "aws-glue-job.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4595f761e767da2650d387dfba7358d632f1da57744033b6023677b8bc150f15.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json index 1aad2f99aa438..6d88a4cb8fd4a 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/tree.json @@ -20,8 +20,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -59,8 +59,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -165,20 +165,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -193,40 +193,40 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, - "Code161d2d609eea130c2e9c740956bb950c": { - "id": "Code161d2d609eea130c2e9c740956bb950c", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c", + "Code2fe0fc936e45d982e718ad516d9c48b5": { + "id": "Code2fe0fc936e45d982e718ad516d9c48b5", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5", "children": { "Stage": { "id": "Stage", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/Stage", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/Stage", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" } }, "AssetBucket": { "id": "AssetBucket", - "path": "aws-glue-job/EtlJob2.0/Code161d2d609eea130c2e9c740956bb950c/AssetBucket", + "path": "aws-glue-job/EtlJob2.0/Code2fe0fc936e45d982e718ad516d9c48b5/AssetBucket", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -251,12 +251,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob20ServiceRoleD520CA20", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -289,6 +283,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob20ServiceRoleD520CA20", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -297,8 +297,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -346,20 +346,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob2.0": { @@ -374,8 +374,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob2.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -413,8 +413,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -482,20 +482,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -520,12 +520,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob20ServiceRole491E0FFF", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -534,6 +528,12 @@ "glueVersion": "2.0", "name": "StreamingJob2.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob20ServiceRole491E0FFF", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -541,14 +541,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob3.0": { @@ -563,8 +563,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -602,8 +602,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -708,20 +708,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -736,14 +736,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -768,12 +768,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob30ServiceRole8E675579", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -806,6 +800,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob30ServiceRole8E675579", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -814,8 +814,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -863,20 +863,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob3.0": { @@ -891,8 +891,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob3.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -930,8 +930,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -999,20 +999,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1037,12 +1037,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob30ServiceRole443B2FDE", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1051,6 +1045,12 @@ "glueVersion": "3.0", "name": "StreamingJob3.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob30ServiceRole443B2FDE", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1058,14 +1058,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJob4.0": { @@ -1080,8 +1080,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1119,8 +1119,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1225,20 +1225,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "SparkUIBucket": { @@ -1253,14 +1253,14 @@ "aws:cdk:cloudformation:props": {} }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.CfnBucket", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_s3.Bucket", + "version": "0.0.0" } }, "Resource": { @@ -1285,12 +1285,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJob40ServiceRoleBDD9998A", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "--enable-continuous-cloudwatch-log": "true", @@ -1323,6 +1317,12 @@ "notifyDelayAfter": 1 }, "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJob40ServiceRoleBDD9998A", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1331,8 +1331,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } }, "SuccessMetricRule": { @@ -1380,20 +1380,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_events.Rule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "StreamingJob4.0": { @@ -1408,8 +1408,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/StreamingJob4.0/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1447,8 +1447,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1516,20 +1516,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1554,12 +1554,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "StreamingJob40ServiceRole034BDEBD", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1568,6 +1562,12 @@ "glueVersion": "4.0", "name": "StreamingJob4.0", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "StreamingJob40ServiceRole034BDEBD", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -1575,14 +1575,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob": { @@ -1597,8 +1597,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1636,8 +1636,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1705,20 +1705,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1743,12 +1743,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "ShellJobServiceRoleCF97BC4B", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1756,20 +1750,26 @@ }, "glueVersion": "1.0", "name": "ShellJob", + "role": { + "Fn::GetAtt": [ + "ShellJobServiceRoleCF97BC4B", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "ShellJob39": { @@ -1784,8 +1784,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/ShellJob39/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -1823,8 +1823,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -1892,20 +1892,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -1930,12 +1930,6 @@ }, "pythonVersion": "3.9" }, - "role": { - "Fn::GetAtt": [ - "ShellJob39ServiceRole2F6F3768", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", "arg1": "value1", @@ -1943,20 +1937,26 @@ }, "glueVersion": "1.0", "name": "ShellJob39", + "role": { + "Fn::GetAtt": [ + "ShellJob39ServiceRole2F6F3768", + "Arn" + ] + }, "tags": { "key": "value" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "RayJob": { @@ -1971,8 +1971,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/RayJob/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2010,8 +2010,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2079,20 +2079,72 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code0115baecff23f24504399384f987872e": { + "id": "Code0115baecff23f24504399384f987872e", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code0115baecff23f24504399384f987872e/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Code7e4c8ab0af31dc38a92c1c69bf696244": { + "id": "Code7e4c8ab0af31dc38a92c1c69bf696244", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-glue-job/RayJob/Code7e4c8ab0af31dc38a92c1c69bf696244/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" } }, "Resource": { @@ -2111,27 +2163,39 @@ { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + "/9120cd4aa376310a46a09d707c5e0c75c164b9014f1740e4c5a3637a34dfafe7.py" ] ] }, "pythonVersion": "3.9", "runtime": "Ray2.4" }, - "role": { - "Fn::GetAtt": [ - "RayJobServiceRole51433C3D", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python", + "--s3-py-modules": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/1c055910db2de9d0fcf53321842cab491bb6985900b80e05d634fceef328c5c4.zip" + ] + ] + }, "arg1": "value1", "arg2": "value2" }, "glueVersion": "4.0", "name": "RayJob", "numberOfWorkers": 2, + "role": { + "Fn::GetAtt": [ + "RayJobServiceRole51433C3D", + "Arn" + ] + }, "tags": { "key": "value" }, @@ -2139,14 +2203,14 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "EtlJobWithFLEX": { @@ -2161,8 +2225,8 @@ "id": "ImportServiceRole", "path": "aws-glue-job/EtlJobWithFLEX/ServiceRole/ImportServiceRole", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" } }, "Resource": { @@ -2200,8 +2264,8 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" } }, "DefaultPolicy": { @@ -2269,20 +2333,20 @@ } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" } }, "Resource": { @@ -2307,12 +2371,6 @@ }, "pythonVersion": "3" }, - "role": { - "Fn::GetAtt": [ - "EtlJobWithFLEXServiceRoleBA7C99A5", - "Arn" - ] - }, "defaultArguments": { "--job-language": "python" }, @@ -2320,40 +2378,46 @@ "glueVersion": "3.0", "name": "EtlJobWithFLEX", "numberOfWorkers": 10, + "role": { + "Fn::GetAtt": [ + "EtlJobWithFLEXServiceRoleBA7C99A5", + "Arn" + ] + }, "workerType": "G.1X" } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" } }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-glue-job/BootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" } }, "CheckBootstrapVersion": { "id": "CheckBootstrapVersion", "path": "aws-glue-job/CheckBootstrapVersion", "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" } }, "Tree": { @@ -2361,13 +2425,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.55" + "version": "10.3.0" } } }, "constructInfo": { - "fqn": "constructs.Construct", - "version": "10.2.55" + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts index c16b5f9691ae1..7b405d53ca490 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.job.ts @@ -22,6 +22,8 @@ const app = new cdk.App(); const stack = new cdk.Stack(app, 'aws-glue-job'); const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const scriptResolveOptions = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'resolve_options.py')); +const moduleUtils = glue.Code.fromAsset(path.join(__dirname, 'module', 'utils.zip')); [glue.GlueVersion.V2_0, glue.GlueVersion.V3_0, glue.GlueVersion.V4_0].forEach((glueVersion) => { const etlJob = new glue.Job(stack, 'EtlJob' + glueVersion.name, { @@ -113,7 +115,8 @@ new glue.Job(stack, 'RayJob', { glueVersion: glue.GlueVersion.V4_0, pythonVersion: glue.PythonVersion.THREE_NINE, runtime: glue.Runtime.RAY_TWO_FOUR, - script, + s3PythonModules: [moduleUtils], + script: scriptResolveOptions, }), workerType: glue.WorkerType.Z_2X, workerCount: 2, diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts index 28c6225c542b6..d00faa55091ba 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-executable.test.ts @@ -254,5 +254,29 @@ describe('JobExecutable', () => { script, })).toBeDefined(); }); + + test('with JobTypeof("glueray") and extraPythonFiles set should throw', () => { + expect(() => glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + runtime: glue.Runtime.RAY_TWO_FOUR, + extraPythonFiles: [script], + script, + })).toThrow(/extraPythonFiles is not supported for Ray jobs/); + }); + + test('with JobType ray and s3PythonModules should succeed', () => { + expect(glue.JobExecutable.of({ + type: glue.JobType.of('glueray'), + glueVersion: glue.GlueVersion.V4_0, + language: glue.JobLanguage.PYTHON, + pythonVersion: glue.PythonVersion.THREE_NINE, + s3PythonModules: [script], + runtime: glue.Runtime.RAY_TWO_FOUR, + script, + })).toBeDefined(); + }); }); }); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py new file mode 100644 index 0000000000000..4fd465ff27636 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/job-script/resolve_options.py @@ -0,0 +1,4 @@ +import utils + +args = utils.get_job_args(['JOB_NAME', 'JOB_RUN_ID'], ['--s3-py-modules']) +print(args) diff --git a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts index 748d89b5668a2..48e395a3fedc9 100644 --- a/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts +++ b/packages/@aws-cdk/aws-glue-alpha/test/job.test.ts @@ -871,7 +871,7 @@ describe('Job', () => { }), workerType: glue.WorkerType.Z_2X, workerCount: 2, - })).toThrow('Runtime is required for Ray jobs.'); + })).toThrow('Runtime is required for Ray jobs'); }); }); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip new file mode 100644 index 0000000000000..3056cd4db8f27 Binary files /dev/null and b/packages/@aws-cdk/aws-glue-alpha/test/module/utils.zip differ diff --git a/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts b/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts index 2218cc8ce289c..60455d6c18db3 100644 --- a/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts +++ b/packages/aws-cdk-lib/aws-codepipeline-actions/lib/ecs/deploy-action.ts @@ -80,7 +80,7 @@ export class EcsDeployAction extends Action { protected bound(_scope: Construct, _stage: codepipeline.IStage, options: codepipeline.ActionBindOptions): codepipeline.ActionConfig { // permissions based on CodePipeline documentation: - // https://docs.aws.amazon.com/codepipeline/latest/userguide/how-to-custom-role.html#how-to-update-role-new-services + // https://docs.aws.amazon.com/codepipeline/latest/userguide/security-iam.html#how-to-custom-role options.role.addToPolicy(new iam.PolicyStatement({ actions: [ 'ecs:DescribeServices', @@ -88,6 +88,7 @@ export class EcsDeployAction extends Action { 'ecs:DescribeTasks', 'ecs:ListTasks', 'ecs:RegisterTaskDefinition', + 'ecs:TagResource', 'ecs:UpdateService', ], resources: ['*'], diff --git a/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts b/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts index 43548757380ea..1f49011ca1bd5 100644 --- a/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts +++ b/packages/aws-cdk-lib/aws-lambda/lib/event-source-mapping.ts @@ -292,8 +292,15 @@ export interface IEventSourceMapping extends cdk.IResource { * Usually, you won't need to define the mapping yourself. This will usually be done by * event sources. For example, to add an SQS event source to a function: * - * import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; - * lambda.addEventSource(new SqsEventSource(sqs)); + * ```ts + * import * as sqs from 'aws-cdk-lib/aws-sqs'; + * import * as eventsources from 'aws-cdk-lib/aws-lambda-event-sources'; + * + * declare const handler: lambda.Function; + * declare const queue: sqs.Queue; + * + * handler.addEventSource(new eventsources.SqsEventSource(queue)); + * ``` * * The `SqsEventSource` class will automatically create the mapping, and will also * modify the Lambda's execution role so it can consume messages from the queue.