Merge branch 'master' into nija-at/buildup-donottest

aws · Jun 29, 2020 · c6a6cca · c6a6cca
2 parents 8589fb2 + 83fd2ae
commit c6a6cca
Show file tree

Hide file tree

Showing 84 changed files with 1,438 additions and 621 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,6 @@ cdk.out/
 
 # Yarn error log
 yarn-error.log
+
+# Parcel default cache directory
+.parcel-cache
diff --git a/package.json b/package.json
@@ -52,6 +52,8 @@
       "@aws-cdk/aws-codepipeline-actions/case/**",
       "@aws-cdk/aws-ecr-assets/minimatch",
       "@aws-cdk/aws-ecr-assets/minimatch/**",
+      "@aws-cdk/aws-eks/yaml",
+      "@aws-cdk/aws-eks/yaml/**",
       "@aws-cdk/aws-lambda-nodejs/parcel-bundler",
       "@aws-cdk/aws-lambda-nodejs/parcel-bundler/**",
       "@aws-cdk/cloud-assembly-schema/jsonschema",

diff --git a/packages/@aws-cdk/aws-cloudfront/package.json b/packages/@aws-cdk/aws-cloudfront/package.json
@@ -64,7 +64,7 @@
   "devDependencies": {
     "@aws-cdk/assert": "0.0.0",
     "@types/nodeunit": "^0.0.31",
-    "aws-sdk": "^2.705.0",
+    "aws-sdk": "^2.706.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "cfn2ts": "0.0.0",

diff --git a/packages/@aws-cdk/aws-cloudtrail/package.json b/packages/@aws-cdk/aws-cloudtrail/package.json
@@ -64,7 +64,7 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@aws-cdk/assert": "0.0.0",
-    "aws-sdk": "^2.705.0",
+    "aws-sdk": "^2.706.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "cfn2ts": "0.0.0",

diff --git a/packages/@aws-cdk/aws-codebuild/package.json b/packages/@aws-cdk/aws-codebuild/package.json
@@ -70,7 +70,7 @@
     "@aws-cdk/aws-sns": "0.0.0",
     "@aws-cdk/aws-sqs": "0.0.0",
     "@types/nodeunit": "^0.0.31",
-    "aws-sdk": "^2.705.0",
+    "aws-sdk": "^2.706.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "cfn2ts": "0.0.0",

diff --git a/packages/@aws-cdk/aws-codecommit/package.json b/packages/@aws-cdk/aws-codecommit/package.json
@@ -70,7 +70,7 @@
     "@aws-cdk/assert": "0.0.0",
     "@aws-cdk/aws-sns": "0.0.0",
     "@types/nodeunit": "^0.0.31",
-    "aws-sdk": "^2.705.0",
+    "aws-sdk": "^2.706.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "cfn2ts": "0.0.0",

diff --git a/packages/@aws-cdk/aws-dynamodb/package.json b/packages/@aws-cdk/aws-dynamodb/package.json
@@ -65,7 +65,7 @@
   "devDependencies": {
     "@aws-cdk/assert": "0.0.0",
     "@types/jest": "^26.0.3",
-    "aws-sdk": "^2.705.0",
+    "aws-sdk": "^2.706.0",
     "aws-sdk-mock": "^5.1.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",

diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.l3s.ts b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.l3s.ts
@@ -436,7 +436,7 @@ export = {
       Type: 'A',
       AliasTarget: {
         HostedZoneId: { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'CanonicalHostedZoneID'] },
-        DNSName: { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'DNSName'] },
+        DNSName: { 'Fn::Join': ['', [ 'dualstack.', { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'DNSName'] } ] ] },
       },
     }));
 
@@ -501,7 +501,7 @@ export = {
       Type: 'A',
       AliasTarget: {
         HostedZoneId: { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'CanonicalHostedZoneID'] },
-        DNSName: { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'DNSName'] },
+        DNSName: { 'Fn::Join': [ '', [ 'dualstack.', { 'Fn::GetAtt': ['ServiceLBE9A1ADBC', 'DNSName'] } ] ] },
       },
     }));
 

diff --git a/.../@aws-cdk/aws-ecs-patterns/test/fargate/integ.load-balanced-fargate-service.expected.json b/.../@aws-cdk/aws-ecs-patterns/test/fargate/integ.load-balanced-fargate-service.expected.json
@@ -618,9 +618,18 @@
         "Type": "A",
         "AliasTarget": {
           "DNSName": {
-            "Fn::GetAtt": [
-              "myServiceLB168895E1",
-              "DNSName"
+            "Fn::Join":
+            [
+              "",
+              [
+                "dualstack.",
+                {
+                  "Fn::GetAtt": [
+                    "myServiceLB168895E1",
+                    "DNSName"
+                  ]
+                }
+              ]
             ]
           },
           "HostedZoneId": {

diff --git a/packages/@aws-cdk/aws-eks/lib/addons/neuron-device-plugin.yaml b/packages/@aws-cdk/aws-eks/lib/addons/neuron-device-plugin.yaml
@@ -0,0 +1,66 @@
+# source: https://github.com/aws/aws-neuron-sdk/blob/master/docs/neuron-container-tools/k8s-neuron-device-plugin.yml
+# https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: neuron-device-plugin-daemonset
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      name:  neuron-device-plugin-ds
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        name: neuron-device-plugin-ds
+    spec:
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: aws.amazon.com/neuron
+        operator: Exists
+        effect: NoSchedule
+      # Mark this pod as a critical add-on; when enabled, the critical add-on
+      # scheduler reserves resources for critical add-on pods so that they can
+      # be rescheduled after a failure.
+      # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+      priorityClassName: "system-node-critical"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "beta.kubernetes.io/instance-type"
+                    operator: In
+                    values:
+                      - inf1.xlarge
+                      - inf1.2xlarge
+                      - inf1.6xlarge
+                      - inf1.4xlarge
+              - matchExpressions:
+                  - key: "node.kubernetes.io/instance-type"
+                    operator: In
+                    values:
+                      - inf1.xlarge
+                      - inf1.2xlarge
+                      - inf1.6xlarge
+                      - inf1.24xlarge
+      containers:
+        - image: 790709498068.dkr.ecr.us-west-2.amazonaws.com/neuron-device-plugin:1.0.9043.0
+          imagePullPolicy: Always
+          name: k8s-neuron-device-plugin-ctr
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: device-plugin
+              mountPath: /var/lib/kubelet/device-plugins
+      volumes:
+        - name: device-plugin
+          hostPath:
+            path: /var/lib/kubelet/device-plugins
diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts
@@ -3,6 +3,9 @@ import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
 import * as ssm from '@aws-cdk/aws-ssm';
 import { CfnOutput, Construct, IResource, Resource, Stack, Tag, Token } from '@aws-cdk/core';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as YAML from 'yaml';
 import { AwsAuth } from './aws-auth';
 import { clusterArnComponents, ClusterResource } from './cluster-resource';
 import { CfnCluster, CfnClusterProps } from './eks.generated';
@@ -385,6 +388,8 @@ export class Cluster extends Resource implements ICluster {
 
   private _spotInterruptHandler?: HelmChart;
 
+  private _neuronDevicePlugin?: KubernetesResource;
+
   private readonly version: string | undefined;
 
   /**
@@ -537,6 +542,10 @@ export class Cluster extends Resource implements ICluster {
       machineImageType: options.machineImageType,
     });
 
+    if (nodeTypeForInstanceType(options.instanceType) === NodeType.INFERENTIA) {
+      this.addNeuronDevicePlugin();
+    }
+
     return asg;
   }
 
@@ -834,6 +843,20 @@ export class Cluster extends Resource implements ICluster {
     return this._spotInterruptHandler;
   }
 
+  /**
+   * Installs the Neuron device plugin on the cluster if it's not
+   * already added.
+   */
+  private addNeuronDevicePlugin() {
+    if (!this._neuronDevicePlugin) {
+      const fileContents = fs.readFileSync(path.join(__dirname, 'addons/neuron-device-plugin.yaml'), 'utf8');
+      const sanitized = YAML.parse(fileContents);
+      this._neuronDevicePlugin = this.addResource('NeuronDevicePlugin', sanitized);
+    }
+
+    return this._neuronDevicePlugin;
+  }
+
   /**
    * Opportunistically tag subnets with the required tags.
    *
@@ -1112,6 +1135,7 @@ export class EksOptimizedImage implements ec2.IMachineImage {
     this.amiParameterName = `/aws/service/eks/optimized-ami/${this.kubernetesVersion}/`
       + ( this.nodeType === NodeType.STANDARD ? 'amazon-linux-2/' : '' )
       + ( this.nodeType === NodeType.GPU ? 'amazon-linux-2-gpu/' : '' )
+      + (this.nodeType === NodeType.INFERENTIA ? 'amazon-linux-2-gpu/' : '')
       + 'recommended/image_id';
   }
 
@@ -1176,6 +1200,11 @@ export enum NodeType {
    * GPU instances
    */
   GPU = 'GPU',
+
+  /**
+   * Inferentia instances
+   */
+  INFERENTIA = 'INFERENTIA',
 }
 
 /**
@@ -1222,7 +1251,10 @@ export enum MachineImageType {
 }
 
 const GPU_INSTANCETYPES = ['p2', 'p3', 'g4'];
+const INFERENTIA_INSTANCETYPES = ['inf1'];
 
 function nodeTypeForInstanceType(instanceType: ec2.InstanceType) {
-  return GPU_INSTANCETYPES.includes(instanceType.toString().substring(0, 2)) ? NodeType.GPU : NodeType.STANDARD;
+  return GPU_INSTANCETYPES.includes(instanceType.toString().substring(0, 2)) ? NodeType.GPU :
+    INFERENTIA_INSTANCETYPES.includes(instanceType.toString().substring(0, 4)) ? NodeType.INFERENTIA :
+      NodeType.STANDARD;
 }
diff --git a/packages/@aws-cdk/aws-eks/lib/helm-chart.ts b/packages/@aws-cdk/aws-eks/lib/helm-chart.ts
@@ -100,7 +100,7 @@ export class HelmChart extends Construct {
         Chart: props.chart,
         Version: props.version,
         Wait: props.wait ?? false,
-        Timeout: timeout,
+        Timeout: timeout ? `${timeout.toString()}s` : undefined, // Helm v3 expects duration instead of integer
         Values: (props.values ? stack.toJsonString(props.values) : undefined),
         Namespace: props.namespace ?? 'default',
         Repository: props.repository,

diff --git a/packages/@aws-cdk/aws-eks/package.json b/packages/@aws-cdk/aws-eks/package.json
@@ -64,7 +64,8 @@
   "devDependencies": {
     "@aws-cdk/assert": "0.0.0",
     "@types/nodeunit": "^0.0.31",
-    "aws-sdk": "^2.705.0",
+    "@types/yaml": "1.2.0",
+    "aws-sdk": "^2.706.0",
     "cdk-build-tools": "0.0.0",
     "cdk-integ-tools": "0.0.0",
     "cfn2ts": "0.0.0",
@@ -80,8 +81,12 @@
     "@aws-cdk/aws-ssm": "0.0.0",
     "@aws-cdk/core": "0.0.0",
     "@aws-cdk/custom-resources": "0.0.0",
-    "constructs": "^3.0.2"
+    "constructs": "^3.0.2",
+    "yaml": "1.10.0"
   },
+  "bundledDependencies": [
+    "yaml"
+  ],
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-autoscaling": "0.0.0",