From cef0ab02148e5ac17c3e4e691ff90f3229f4ea26 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Thu, 25 Jun 2020 15:29:07 -0400
Subject: [PATCH 01/35] Attempt kops run.

---
 kops-test-config.yaml            | 42 ++++++++++++++++++++++++++++++++
 scripts/run-integration-tests.sh | 11 +++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 kops-test-config.yaml

diff --git a/kops-test-config.yaml b/kops-test-config.yaml
new file mode 100644
index 0000000000..0d68172d38
--- /dev/null
+++ b/kops-test-config.yaml
@@ -0,0 +1,42 @@
+cluster:
+  kops:
+    stateFile: s3://k8s-kops-csi-e2e
+    zones: us-west-2a,us-west-2b,us-west-2c
+    nodeCount: 3
+    nodeSize: c5.large
+    kubernetesVersion: 1.17.0
+
+build: |
+  eval $(aws ecr get-login --region us-west-2 --no-include-email)
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-efs-csi-driver
+  docker build -t $IMAGE_NAME:$IMAGE_TAG .
+  docker push $IMAGE_NAME:$IMAGE_TAG
+
+install: |
+  echo "Deploying driver"
+  #install helm
+  OS_ARCH=$(go env GOOS)-amd64
+  helm_name=helm-v3.1.1-linux-amd64.tar.gz
+  wget https://get.helm.sh/$helm_name
+  tar xvzf $helm_name
+  mv $OS_ARCH/helm /usr/local/bin/helm
+  
+  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+  IMAGE_TAG={{TEST_ID}}
+  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-efs-csi-driver
+
+  helm install aws-efs-csi-driver \
+      --set image.repository=$IMAGE_NAME \
+      --set image.tag=$IMAGE_TAG \
+      ./helm
+
+uninstall: |
+  echo "Removing driver"
+  helm uninstall aws-efs-csi-driver
+
+test: |
+  export KUBECONFIG=$HOME/.kube/config
+  cluster_name=test-cluster-{{TEST_ID}}.k8s.local
+  go test ./test/e2e/ -v -timeout=0 -kubeconfig=$HOME/.kube/config --cluster-name=$cluster_name --region=us-west-2 --report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]"
\ No newline at end of file
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 5bc65c8424..90a13c2826 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -186,6 +186,17 @@ echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
 echo "*******************************************************************************"
 echo "Running integration tests on current image:"
 echo ""
+
+ls $TESTER_PATH
+START=$SECONDS
+go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
+TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
+KOPS_TEST_DURATION=$((SECONDS - START))
+echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
+
+#TESTCONFIG=./kops-test-config.yaml ${TESTER_PATH}/e2e/tester/cmd/k8s-e2e-tester@master
+
+
 START=$SECONDS
 pushd ./test/integration
 GO111MODULE=on go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \

From 9e1539e743c0a957c61491d2fa1395242dcd37b6 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 26 Jun 2020 02:14:30 -0400
Subject: [PATCH 02/35] Try new kops run.

---
 scripts/run-integration-tests.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 90a13c2826..a352cc7f24 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -190,7 +190,9 @@ echo ""
 ls $TESTER_PATH
 START=$SECONDS
 go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
+TESTCONFIG=./kops-test-config.yaml 
+./go/bin/k8s-e2e-tester
+#${GOPATH}/bin/k8s-e2e-tester
 KOPS_TEST_DURATION=$((SECONDS - START))
 echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
 

From d53c8888819d64c7ee11dd86905360aa3cec47a4 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 26 Jun 2020 11:23:33 -0400
Subject: [PATCH 03/35] Revert "Try new kops run."

This reverts commit bc9b22ab6739cece2b71f1e11beeb01deecefb3d.
---
 scripts/run-integration-tests.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index a352cc7f24..90a13c2826 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -190,9 +190,7 @@ echo ""
 ls $TESTER_PATH
 START=$SECONDS
 go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-TESTCONFIG=./kops-test-config.yaml 
-./go/bin/k8s-e2e-tester
-#${GOPATH}/bin/k8s-e2e-tester
+TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
 KOPS_TEST_DURATION=$((SECONDS - START))
 echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
 

From efcb97ada0b27672e1f629eae0015119c39ddfa8 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 26 Jun 2020 11:53:13 -0400
Subject: [PATCH 04/35] Make changes to kops config yaml.

---
 kops-test-config.yaml            | 8 +++-----
 scripts/run-integration-tests.sh | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/kops-test-config.yaml b/kops-test-config.yaml
index 0d68172d38..0cbe7e5ce0 100644
--- a/kops-test-config.yaml
+++ b/kops-test-config.yaml
@@ -9,9 +9,8 @@ cluster:
 build: |
   eval $(aws ecr get-login --region us-west-2 --no-include-email)
   AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-  IMAGE_TAG={{TEST_ID}}
-  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-efs-csi-driver
-  docker build -t $IMAGE_NAME:$IMAGE_TAG .
+  IMAGE_TAG={{TEST_IMAGE_VERSION}}
+  docker build $(DOCKER_BUILD_FLAGS) -f scripts/dockerfiles/Dockerfile.release -t $IMAGE_NAME:$IMAGE_TAG .
   docker push $IMAGE_NAME:$IMAGE_TAG
 
 install: |
@@ -24,8 +23,7 @@ install: |
   mv $OS_ARCH/helm /usr/local/bin/helm
   
   AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-  IMAGE_TAG={{TEST_ID}}
-  IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-efs-csi-driver
+  IMAGE_TAG={{TEST_IMAGE_VERSION}}
 
   helm install aws-efs-csi-driver \
       --set image.repository=$IMAGE_NAME \
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 90a13c2826..2ec16daa82 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -187,7 +187,7 @@ echo "**************************************************************************
 echo "Running integration tests on current image:"
 echo ""
 
-ls $TESTER_PATH
+ls
 START=$SECONDS
 go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
 TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester

From f720dc5ac8608167a485c5a19f2ff8c8e1b5b25d Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 26 Jun 2020 16:03:57 -0400
Subject: [PATCH 05/35] Kops test attempt.

---
 scripts/run-integration-tests.sh | 38 +++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 2ec16daa82..d729be89f5 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -19,6 +19,7 @@ ARCH=$(go env GOARCH)
 : "${DEPROVISION:=true}"
 : "${BUILD:=true}"
 : "${RUN_CONFORMANCE:=false}"
+: "${RUN_KOPS_TEST:=false}"
 
 __cluster_created=0
 __cluster_deprovisioned=0
@@ -138,13 +139,23 @@ mkdir -p "$REPORT_DIR"
 mkdir -p "$TEST_CLUSTER_DIR"
 mkdir -p "$TEST_CONFIG_DIR"
 
-if [[ "$PROVISION" == true ]]; then
-    START=$SECONDS
+START=$SECONDS
+if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
-    UP_CLUSTER_DURATION=$((SECONDS - START))
-    echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds."
     __cluster_created=1
+else
+    aws s3api create-bucket --bucket kops-cni-test --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    export NAME=kops-cni-test.k8s.local
+    export KOPS_STATE_STORE=s3://kops-cni-test
+    kops create cluster \
+    --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
+    --networking amazon-vpc-routed-eni \
+    --node-count 2 \
+    ${NAME}
+    kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.6.3/config/v1.6/cni-metrics-helper.yaml
 fi
+UP_CLUSTER_DURATION=$((SECONDS - START))
+echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds."
 
 echo "Using $BASE_CONFIG_PATH as a template"
 cp "$BASE_CONFIG_PATH" "$TEST_CONFIG_PATH"
@@ -187,12 +198,12 @@ echo "**************************************************************************
 echo "Running integration tests on current image:"
 echo ""
 
-ls
-START=$SECONDS
-go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
-KOPS_TEST_DURATION=$((SECONDS - START))
-echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
+# ls
+# START=$SECONDS
+# go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
+# TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
+# KOPS_TEST_DURATION=$((SECONDS - START))
+# echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
 
 #TESTCONFIG=./kops-test-config.yaml ${TESTER_PATH}/e2e/tester/cmd/k8s-e2e-tester@master
 
@@ -225,7 +236,12 @@ fi
 if [[ "$DEPROVISION" == true ]]; then
     START=$SECONDS
 
-    down-test-cluster
+    if [[ "$RUN_KOPS_TEST" == false ]]; then
+        down-test-cluster
+    else
+        kops delete cluster --name $NAME --yes
+        aws s3 rb s3://kops-cni-test --region us-west-2
+    fi
 
     DOWN_DURATION=$((SECONDS - START))
     echo "TIMELINE: Down processes took $DOWN_DURATION seconds."

From 480f6bc1d60c77400fdefffb89b1477e28da7743 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 26 Jun 2020 17:41:08 -0400
Subject: [PATCH 06/35] Mark cluster created, try kops again.

---
 scripts/run-integration-tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index d729be89f5..674de8cfbb 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -153,6 +153,7 @@ else
     --node-count 2 \
     ${NAME}
     kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.6.3/config/v1.6/cni-metrics-helper.yaml
+    __cluster_created=1
 fi
 UP_CLUSTER_DURATION=$((SECONDS - START))
 echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds."

From 11a64b6ef5c25904b4d1dad02182b9bc66a589ef Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sat, 27 Jun 2020 17:00:15 -0400
Subject: [PATCH 07/35] Retry kops change.

---
 scripts/run-integration-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 674de8cfbb..251a432e13 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -144,7 +144,7 @@ if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
     __cluster_created=1
 else
-    aws s3api create-bucket --bucket kops-cni-test --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
     export NAME=kops-cni-test.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test
     kops create cluster \

From b6305d9d99f064240df9494cbec3af1788d25ff9 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 12:30:44 -0400
Subject: [PATCH 08/35] Install kops.

---
 scripts/run-integration-tests.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 251a432e13..59992c1d63 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -144,7 +144,10 @@ if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
     __cluster_created=1
 else
-    aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    aws s3api create-bucket --bucket kops-cni-test-temp2 --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
+    chmod +x kops-linux-amd64
+    sudo mv kops-linux-amd64 /usr/local/bin/kops
     export NAME=kops-cni-test.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test
     kops create cluster \

From 6848aad3f8ecffb985fc26c95a82570ea1696c16 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 12:54:42 -0400
Subject: [PATCH 09/35] Fix incorrect bucket change.

---
 scripts/run-integration-tests.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 59992c1d63..622c9203b3 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -144,12 +144,12 @@ if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
     __cluster_created=1
 else
-    aws s3api create-bucket --bucket kops-cni-test-temp2 --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    aws s3api create-bucket --bucket kops-cni-test-temp3 --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
     curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
     chmod +x kops-linux-amd64
     sudo mv kops-linux-amd64 /usr/local/bin/kops
-    export NAME=kops-cni-test.k8s.local
-    export KOPS_STATE_STORE=s3://kops-cni-test
+    export NAME=kops-cni-test
+    export KOPS_STATE_STORE=s3://kops-cni-test-temp3
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \

From b59210a35ad66fcf2fae32dcdc8d248518db820c Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 13:19:50 -0400
Subject: [PATCH 10/35] Full DNS name.

---
 scripts/run-integration-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 622c9203b3..a55c12575f 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -148,7 +148,7 @@ else
     curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
     chmod +x kops-linux-amd64
     sudo mv kops-linux-amd64 /usr/local/bin/kops
-    export NAME=kops-cni-test
+    export NAME=kops-cni-test.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp3
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \

From f73447079768b54d13a5a478f6e91012cf5cea8e Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 13:39:16 -0400
Subject: [PATCH 11/35] Correct bucket.

---
 scripts/run-integration-tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index a55c12575f..9a3604408f 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -144,12 +144,12 @@ if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
     __cluster_created=1
 else
-    aws s3api create-bucket --bucket kops-cni-test-temp3 --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
     curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
     chmod +x kops-linux-amd64
     sudo mv kops-linux-amd64 /usr/local/bin/kops
     export NAME=kops-cni-test.k8s.local
-    export KOPS_STATE_STORE=s3://kops-cni-test-temp3
+    export KOPS_STATE_STORE=s3://kops-cni-test-temp
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \

From c7f7d4abe45dff2547901ef58277b63e6aadb6e6 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 14:52:06 -0400
Subject: [PATCH 12/35] Create secret for kops.

---
 scripts/run-integration-tests.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 9a3604408f..d38b5cf0f0 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -150,6 +150,7 @@ else
     sudo mv kops-linux-amd64 /usr/local/bin/kops
     export NAME=kops-cni-test.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp
+    kops create secret --name kops-cni-test.k8s.local sshpublickey admin -i ~/.ssh/id_rsa.pub
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \

From 2242dc3843b90eaf3c8a7b02b7901d93c0ceb261 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 15:58:31 -0400
Subject: [PATCH 13/35] Poll for updated cluster.

---
 scripts/run-integration-tests.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index d38b5cf0f0..6387e121ef 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -156,6 +156,13 @@ else
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
     ${NAME}
+    kops update cluster --name kops-cni-test.k8s.local --yes
+    sleep 40
+    while [[ ! $(kops validate cluster | grep "is ready") ]]
+    do
+        sleep 5
+        echo "Waiting for cluster validation"
+    done
     kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.6.3/config/v1.6/cni-metrics-helper.yaml
     __cluster_created=1
 fi

From 340c4e8e7c3ec09fd5c24116474995b6c43daee6 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 16:07:18 -0400
Subject: [PATCH 14/35] Edit cluster name.

---
 scripts/run-integration-tests.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 6387e121ef..df306ebc90 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -148,15 +148,15 @@ else
     curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
     chmod +x kops-linux-amd64
     sudo mv kops-linux-amd64 /usr/local/bin/kops
-    export NAME=kops-cni-test.k8s.local
+    CLUSTER_NAME=kops-cni-test-cluster.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp
-    kops create secret --name kops-cni-test.k8s.local sshpublickey admin -i ~/.ssh/id_rsa.pub
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
-    ${NAME}
-    kops update cluster --name kops-cni-test.k8s.local --yes
+    ${CLUSTER_NAME}
+    kops create secret --name ${CLUSTER_NAME} sshpublickey admin -i ~/.ssh/id_rsa.pub
+    kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40
     while [[ ! $(kops validate cluster | grep "is ready") ]]
     do
@@ -251,7 +251,7 @@ if [[ "$DEPROVISION" == true ]]; then
     if [[ "$RUN_KOPS_TEST" == false ]]; then
         down-test-cluster
     else
-        kops delete cluster --name $NAME --yes
+        kops delete cluster --name ${CLUSTER_NAME} --yes
         aws s3 rb s3://kops-cni-test --region us-west-2
     fi
 

From 3027e409a12ac7510260ef9b088eb0546c5cf574 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 16:25:58 -0400
Subject: [PATCH 15/35] Move secret creations.

---
 scripts/run-integration-tests.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index df306ebc90..5108a0b31e 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -150,12 +150,12 @@ else
     sudo mv kops-linux-amd64 /usr/local/bin/kops
     CLUSTER_NAME=kops-cni-test-cluster.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp
+    kops create secret --name ${CLUSTER_NAME} sshpublickey admin -i ~/.ssh/id_rsa.pub
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
     ${CLUSTER_NAME}
-    kops create secret --name ${CLUSTER_NAME} sshpublickey admin -i ~/.ssh/id_rsa.pub
     kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40
     while [[ ! $(kops validate cluster | grep "is ready") ]]
@@ -252,7 +252,8 @@ if [[ "$DEPROVISION" == true ]]; then
         down-test-cluster
     else
         kops delete cluster --name ${CLUSTER_NAME} --yes
-        aws s3 rb s3://kops-cni-test --region us-west-2
+        aws s3 rm ${KOPS_STATE_STORE} --recursive
+        aws s3 rb ${KOPS_STATE_STORE} --region us-west-2
     fi
 
     DOWN_DURATION=$((SECONDS - START))

From 32862a4228432a6f21b158897bedd4d38bdd2aed Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 19:00:28 -0400
Subject: [PATCH 16/35] SSH key change

---
 scripts/run-integration-tests.sh | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 5108a0b31e..b2edeb67c6 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -150,11 +150,21 @@ else
     sudo mv kops-linux-amd64 /usr/local/bin/kops
     CLUSTER_NAME=kops-cni-test-cluster.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp
-    kops create secret --name ${CLUSTER_NAME} sshpublickey admin -i ~/.ssh/id_rsa.pub
+
+    SSH_KEYS=~/.ssh/devopsinuse
+    if [ ! -f "$SSH_KEYS" ]
+    then
+        echo -e "\nCreating SSH keys ..."
+        ssh-keygen -t rsa -N '' -f ~/.ssh/devopsinuse
+    else
+        echo -e "\nSSH keys are already in place!"
+    fi
+
     kops create cluster \
     --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
+    --ssh-public-key=~/.ssh/devopsinuse.pub \
     ${CLUSTER_NAME}
     kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40

From 6bceefa9c246a3531ee4b2e12b18f0ae116c10ed Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Sun, 28 Jun 2020 20:59:51 -0400
Subject: [PATCH 17/35] Attempt to run integration tests.

---
 scripts/run-integration-tests.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index b2edeb67c6..b538d46d0a 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -148,7 +148,7 @@ else
     curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
     chmod +x kops-linux-amd64
     sudo mv kops-linux-amd64 /usr/local/bin/kops
-    CLUSTER_NAME=kops-cni-test-cluster.k8s.local
+    CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local
     export KOPS_STATE_STORE=s3://kops-cni-test-temp
 
     SSH_KEYS=~/.ssh/devopsinuse
@@ -189,7 +189,15 @@ sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init
 sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PATH"
 
 export KUBECONFIG=$KUBECONFIG_PATH
-ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
+if [[ $RUN_KOPS_TEST != true ]]; then
+    ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
+else
+    pushd ./test/integration
+    go test ./test/e2e/ -v -timeout=0 -kubeconfig=$KUBECONFIG --cluster-name=$CLUSTER_NAME --region=us-west-2 -ginkgo.focus="\[cni-integration]" -ginkgo.skip="\[Disruptive\]"
+    TEST_PASS=$?
+    echo "Kops-provisioned cluster ran successfully!"
+    exit 0
+fi
 
 echo "*******************************************************************************"
 echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE"

From 6476d677182c23f6a2c9aee1eb3dd5d0c1b67d71 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 12:04:27 -0400
Subject: [PATCH 18/35] Attempt normal integration tests.

---
 scripts/run-integration-tests.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index b538d46d0a..00548cb7b4 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -49,6 +49,7 @@ CLUSTER_NAME=cni-test-$CLUSTER_ID
 TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME
 : "${CLUSTER_CONFIG:=${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml}"
 : "${KUBECONFIG_PATH:=${TEST_CLUSTER_DIR}/kubeconfig}"
+: "${ADDONS_CNI_IMAGE:=""}"
 
 # shared binaries
 : "${TESTER_DIR:=/tmp/aws-k8s-tester}"
@@ -191,12 +192,6 @@ sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PA
 export KUBECONFIG=$KUBECONFIG_PATH
 if [[ $RUN_KOPS_TEST != true ]]; then
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
-else
-    pushd ./test/integration
-    go test ./test/e2e/ -v -timeout=0 -kubeconfig=$KUBECONFIG --cluster-name=$CLUSTER_NAME --region=us-west-2 -ginkgo.focus="\[cni-integration]" -ginkgo.skip="\[Disruptive\]"
-    TEST_PASS=$?
-    echo "Kops-provisioned cluster ran successfully!"
-    exit 0
 fi
 
 echo "*******************************************************************************"

From 82e544477a3bce2f59c263ece4b1932be6915f7e Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 12:41:31 -0400
Subject: [PATCH 19/35] Try conformance tests.

---
 scripts/lib/cluster.sh           |  6 ++++++
 scripts/run-integration-tests.sh | 17 ++++++++++++++++-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
index c8458d6d30..2f6ac3f4ff 100644
--- a/scripts/lib/cluster.sh
+++ b/scripts/lib/cluster.sh
@@ -40,3 +40,9 @@ function up-test-cluster() {
         echo "ok."
     fi
 }
+
+function down-kops-cluster {
+    kops delete cluster --name ${CLUSTER_NAME} --yes
+    aws s3 rm ${KOPS_STATE_STORE} --recursive
+    aws s3 rb ${KOPS_STATE_STORE} --region us-west-2
+}
\ No newline at end of file
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 00548cb7b4..aaa32e9d5f 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -27,12 +27,15 @@ __cluster_deprovisioned=0
 on_error() {
     # Make sure we destroy any cluster that was created if we hit run into an
     # error when attempting to run tests against the cluster
-    if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true ]]; then
+    if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true && $RUN_KOPS_TEST == false ]]; then
         # prevent double-deprovisioning with ctrl-c during deprovisioning...
         __cluster_deprovisioned=1
         echo "Cluster was provisioned already. Deprovisioning it..."
         down-test-cluster
     fi
+    if [[ $RUN_KOPS_TEST == true ]]; then
+        down-kops-cluster
+    fi
     exit 1
 }
 
@@ -166,6 +169,7 @@ else
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
+    --kubernetes-version $K8S_VERSION
     ${CLUSTER_NAME}
     kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40
@@ -192,6 +196,17 @@ sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PA
 export KUBECONFIG=$KUBECONFIG_PATH
 if [[ $RUN_KOPS_TEST != true ]]; then
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
+else
+  go install github.com/onsi/ginkgo/ginkgo
+  wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
+  ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
+   --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+
+  /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+    --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"
+  echo "Kops conformance tests ran successfully!"
+  down-kops-cluster
+  exit 0
 fi
 
 echo "*******************************************************************************"

From 42c766551bd0a2ed6a7a69ea7a8ff7ad7b471714 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 12:54:11 -0400
Subject: [PATCH 20/35] Fix syntax error.

---
 scripts/run-integration-tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index aaa32e9d5f..dfc4465c1d 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -34,6 +34,7 @@ on_error() {
         down-test-cluster
     fi
     if [[ $RUN_KOPS_TEST == true ]]; then
+        echo "Cluster was provisioned already. Deprovisioning it..."
         down-kops-cluster
     fi
     exit 1
@@ -169,7 +170,7 @@ else
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
-    --kubernetes-version $K8S_VERSION
+    --kubernetes-version ${K8S_VERSION}
     ${CLUSTER_NAME}
     kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40

From 3298c4c7cf40d75070b60625484f0829d9285760 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 13:08:19 -0400
Subject: [PATCH 21/35] Missing backslash.

---
 scripts/run-integration-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index dfc4465c1d..07b7631af0 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -170,7 +170,7 @@ else
     --networking amazon-vpc-routed-eni \
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
-    --kubernetes-version ${K8S_VERSION}
+    --kubernetes-version ${K8S_VERSION} \
     ${CLUSTER_NAME}
     kops update cluster --name ${CLUSTER_NAME} --yes
     sleep 40

From 96dd409a02065a23ad6dd89732b7d9cb14fa50f0 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 15:23:07 -0400
Subject: [PATCH 22/35] More applying changes.

---
 .circleci/config.yml             |  5 -----
 scripts/run-integration-tests.sh | 31 ++++++++++++++++++-------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 313f810f6b..a6c0fa4cdf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -88,11 +88,6 @@ workflows:
     jobs:
       - build_x86_64
       - build_aarch64
-      - hold:
-          type: approval
-          requires:
-            - build_x86_64
-            - build_aarch64
       - integration_test:
           requires:
             - hold
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 07b7631af0..ead3eb0bf9 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -171,9 +171,15 @@ else
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
     --kubernetes-version ${K8S_VERSION} \
-    ${CLUSTER_NAME}
+    # ${CLUSTER_NAME}
+    # kops create cluster \
+    # --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
+    # --networking amazon-vpc-routed-eni \
+    # --node-count 2 \
+    # --kubernetes-version ${K8S_VERSION} \
+    # ${CLUSTER_NAME}
     kops update cluster --name ${CLUSTER_NAME} --yes
-    sleep 40
+    sleep 100
     while [[ ! $(kops validate cluster | grep "is ready") ]]
     do
         sleep 5
@@ -198,16 +204,17 @@ export KUBECONFIG=$KUBECONFIG_PATH
 if [[ $RUN_KOPS_TEST != true ]]; then
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
 else
-  go install github.com/onsi/ginkgo/ginkgo
-  wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
-  ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
-   --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+    $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
+    sleep 110
 
-  /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+    go install github.com/onsi/ginkgo/ginkgo
+    wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
+
+    /tmp/e2e.test --ginkgo.focus="Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
     --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"
-  echo "Kops conformance tests ran successfully!"
-  down-kops-cluster
-  exit 0
+    echo "Kops conformance tests ran successfully!"
+    down-kops-cluster
+    exit 0
 fi
 
 echo "*******************************************************************************"
@@ -280,9 +287,7 @@ if [[ "$DEPROVISION" == true ]]; then
     if [[ "$RUN_KOPS_TEST" == false ]]; then
         down-test-cluster
     else
-        kops delete cluster --name ${CLUSTER_NAME} --yes
-        aws s3 rm ${KOPS_STATE_STORE} --recursive
-        aws s3 rb ${KOPS_STATE_STORE} --region us-west-2
+        down-kops-cluster
     fi
 
     DOWN_DURATION=$((SECONDS - START))

From 2c18742f35b062c0aca5f42e941d2a30e6bae908 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 15:24:11 -0400
Subject: [PATCH 23/35] Fix circleci template.

---
 .circleci/config.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a6c0fa4cdf..f65acedfe0 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -90,7 +90,8 @@ workflows:
       - build_aarch64
       - integration_test:
           requires:
-            - hold
+            - build_x86_64
+            - build_aarch64
 
   # runs integration test when new git tag is pushed.
   run-test-on-tags:

From f504d5ed23bcfa793af113c9330123d295eb496c Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 16:40:09 -0400
Subject: [PATCH 24/35] Accidental comment.

---
 scripts/run-integration-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index ead3eb0bf9..4aa008c51d 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -171,7 +171,7 @@ else
     --node-count 2 \
     --ssh-public-key=~/.ssh/devopsinuse.pub \
     --kubernetes-version ${K8S_VERSION} \
-    # ${CLUSTER_NAME}
+    ${CLUSTER_NAME}
     # kops create cluster \
     # --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
     # --networking amazon-vpc-routed-eni \

From cfec829f0f65980937b5d25b6537f2419839680b Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 29 Jun 2020 16:56:07 -0400
Subject: [PATCH 25/35] Change kubectl command.

---
 scripts/run-integration-tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 4aa008c51d..96542251b3 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -204,7 +204,7 @@ export KUBECONFIG=$KUBECONFIG_PATH
 if [[ $RUN_KOPS_TEST != true ]]; then
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
 else
-    $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
+    kubectl apply -f "$TEST_CONFIG_PATH"
     sleep 110
 
     go install github.com/onsi/ginkgo/ginkgo
@@ -233,7 +233,7 @@ echo "**************************************************************************
 echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION"
 echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION"
 START=$SECONDS
-$KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
+kubectl apply -f "$TEST_CONFIG_PATH"
 
 # Delay based on 3 nodes, 30s grace period per CNI pod
 echo "TODO: Poll and wait for updates to complete instead!"

From 015bba805c64576254b7db764062f0476eb42ada Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Tue, 30 Jun 2020 12:54:08 -0400
Subject: [PATCH 26/35] Redirect kubeconfig.

---
 scripts/run-integration-tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 96542251b3..b11c1b3ef9 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -200,10 +200,11 @@ sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PA
 sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init,$INIT_IMAGE_NAME," "$TEST_CONFIG_PATH"
 sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PATH"
 
-export KUBECONFIG=$KUBECONFIG_PATH
 if [[ $RUN_KOPS_TEST != true ]]; then
+    export KUBECONFIG=$KUBECONFIG_PATH
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
 else
+    export KUBECONFIG=~/.kube/config
     kubectl apply -f "$TEST_CONFIG_PATH"
     sleep 110
 

From a38db089c01baab828fb0a5514a5c6a2d58e4e06 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Tue, 30 Jun 2020 13:54:30 -0400
Subject: [PATCH 27/35] Attempt kops run in parallel.

---
 scripts/run-integration-tests.sh | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index b11c1b3ef9..7eaed3a393 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -204,6 +204,8 @@ if [[ $RUN_KOPS_TEST != true ]]; then
     export KUBECONFIG=$KUBECONFIG_PATH
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
 else
+    START=$SECONDS
+
     export KUBECONFIG=~/.kube/config
     kubectl apply -f "$TEST_CONFIG_PATH"
     sleep 110
@@ -211,10 +213,20 @@ else
     go install github.com/onsi/ginkgo/ginkgo
     wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
 
-    /tmp/e2e.test --ginkgo.focus="Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+    ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
+    --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+
+    /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
     --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"
     echo "Kops conformance tests ran successfully!"
+
+    KOPS_TEST_DURATION=$((SECONDS - START))
+    echo "TIMELINE: KOPS tests took $KOPS_TEST_DURATION seconds."
+
+    START=$SECONDS
     down-kops-cluster
+    DOWN_KOPS_DURATION=$((SECONDS - START))
+    echo "TIMELINE: Down KOPS cluster took $DOWN_KOPS_DURATION seconds."
     exit 0
 fi
 
@@ -247,15 +259,6 @@ echo "**************************************************************************
 echo "Running integration tests on current image:"
 echo ""
 
-# ls
-# START=$SECONDS
-# go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-# TESTCONFIG=./kops-test-config.yaml ${GOPATH}/bin/k8s-e2e-tester
-# KOPS_TEST_DURATION=$((SECONDS - START))
-# echo "TIMELINE: Current image integration tests took $KOPS_TEST_DURATION seconds."
-
-#TESTCONFIG=./kops-test-config.yaml ${TESTER_PATH}/e2e/tester/cmd/k8s-e2e-tester@master
-
 
 START=$SECONDS
 pushd ./test/integration

From 249f22ef1829fd414405a61352bd7651a4467d58 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Wed, 1 Jul 2020 13:01:24 -0400
Subject: [PATCH 28/35] Clean up kops odds and ends.

---
 .circleci/config.yml             |  5 ++--
 kops-test-config.yaml            | 40 ------------------------------
 scripts/lib/cluster.sh           | 34 ++++++++++++++++++++++++++
 scripts/run-integration-tests.sh | 42 ++------------------------------
 4 files changed, 39 insertions(+), 82 deletions(-)
 delete mode 100644 kops-test-config.yaml

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f65acedfe0..e857167aaf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -88,10 +88,11 @@ workflows:
     jobs:
       - build_x86_64
       - build_aarch64
+      - hold:
+          type: approval
       - integration_test:
           requires:
-            - build_x86_64
-            - build_aarch64
+            - hold
 
   # runs integration test when new git tag is pushed.
   run-test-on-tags:
diff --git a/kops-test-config.yaml b/kops-test-config.yaml
deleted file mode 100644
index 0cbe7e5ce0..0000000000
--- a/kops-test-config.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-cluster:
-  kops:
-    stateFile: s3://k8s-kops-csi-e2e
-    zones: us-west-2a,us-west-2b,us-west-2c
-    nodeCount: 3
-    nodeSize: c5.large
-    kubernetesVersion: 1.17.0
-
-build: |
-  eval $(aws ecr get-login --region us-west-2 --no-include-email)
-  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-  IMAGE_TAG={{TEST_IMAGE_VERSION}}
-  docker build $(DOCKER_BUILD_FLAGS) -f scripts/dockerfiles/Dockerfile.release -t $IMAGE_NAME:$IMAGE_TAG .
-  docker push $IMAGE_NAME:$IMAGE_TAG
-
-install: |
-  echo "Deploying driver"
-  #install helm
-  OS_ARCH=$(go env GOOS)-amd64
-  helm_name=helm-v3.1.1-linux-amd64.tar.gz
-  wget https://get.helm.sh/$helm_name
-  tar xvzf $helm_name
-  mv $OS_ARCH/helm /usr/local/bin/helm
-  
-  AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-  IMAGE_TAG={{TEST_IMAGE_VERSION}}
-
-  helm install aws-efs-csi-driver \
-      --set image.repository=$IMAGE_NAME \
-      --set image.tag=$IMAGE_TAG \
-      ./helm
-
-uninstall: |
-  echo "Removing driver"
-  helm uninstall aws-efs-csi-driver
-
-test: |
-  export KUBECONFIG=$HOME/.kube/config
-  cluster_name=test-cluster-{{TEST_ID}}.k8s.local
-  go test ./test/e2e/ -v -timeout=0 -kubeconfig=$HOME/.kube/config --cluster-name=$cluster_name --region=us-west-2 --report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]"
\ No newline at end of file
diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
index 2f6ac3f4ff..6665f3fc61 100644
--- a/scripts/lib/cluster.sh
+++ b/scripts/lib/cluster.sh
@@ -41,6 +41,40 @@ function up-test-cluster() {
     fi
 }
 
+function up-kops-cluster {
+    aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
+    curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
+    chmod +x kops-linux-amd64
+    sudo mv kops-linux-amd64 /usr/local/bin/kops
+    CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local
+    export KOPS_STATE_STORE=s3://kops-cni-test-temp
+
+    SSH_KEYS=~/.ssh/devopsinuse
+    if [ ! -f "$SSH_KEYS" ]
+    then
+        echo -e "\nCreating SSH keys ..."
+        ssh-keygen -t rsa -N '' -f ~/.ssh/devopsinuse
+    else
+        echo -e "\nSSH keys are already in place!"
+    fi
+
+    kops create cluster \
+    --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
+    --networking amazon-vpc-routed-eni \
+    --node-count 2 \
+    --ssh-public-key=~/.ssh/devopsinuse.pub \
+    --kubernetes-version ${K8S_VERSION} \
+    ${CLUSTER_NAME}
+    kops update cluster --name ${CLUSTER_NAME} --yes
+    sleep 100
+    while [[ ! $(kops validate cluster | grep "is ready") ]]
+    do
+        sleep 5
+        echo "Waiting for cluster validation"
+    done
+    kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.6.3/config/v1.6/cni-metrics-helper.yaml
+}
+
 function down-kops-cluster {
     kops delete cluster --name ${CLUSTER_NAME} --yes
     aws s3 rm ${KOPS_STATE_STORE} --recursive
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 7eaed3a393..3f47b231fe 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -149,43 +149,7 @@ if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
     __cluster_created=1
 else
-    aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION
-    curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64
-    chmod +x kops-linux-amd64
-    sudo mv kops-linux-amd64 /usr/local/bin/kops
-    CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local
-    export KOPS_STATE_STORE=s3://kops-cni-test-temp
-
-    SSH_KEYS=~/.ssh/devopsinuse
-    if [ ! -f "$SSH_KEYS" ]
-    then
-        echo -e "\nCreating SSH keys ..."
-        ssh-keygen -t rsa -N '' -f ~/.ssh/devopsinuse
-    else
-        echo -e "\nSSH keys are already in place!"
-    fi
-
-    kops create cluster \
-    --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
-    --networking amazon-vpc-routed-eni \
-    --node-count 2 \
-    --ssh-public-key=~/.ssh/devopsinuse.pub \
-    --kubernetes-version ${K8S_VERSION} \
-    ${CLUSTER_NAME}
-    # kops create cluster \
-    # --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \
-    # --networking amazon-vpc-routed-eni \
-    # --node-count 2 \
-    # --kubernetes-version ${K8S_VERSION} \
-    # ${CLUSTER_NAME}
-    kops update cluster --name ${CLUSTER_NAME} --yes
-    sleep 100
-    while [[ ! $(kops validate cluster | grep "is ready") ]]
-    do
-        sleep 5
-        echo "Waiting for cluster validation"
-    done
-    kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.6.3/config/v1.6/cni-metrics-helper.yaml
+    up-kops-cluster
     __cluster_created=1
 fi
 UP_CLUSTER_DURATION=$((SECONDS - START))
@@ -246,7 +210,7 @@ echo "**************************************************************************
 echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION"
 echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION"
 START=$SECONDS
-kubectl apply -f "$TEST_CONFIG_PATH"
+$KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
 
 # Delay based on 3 nodes, 30s grace period per CNI pod
 echo "TODO: Poll and wait for updates to complete instead!"
@@ -258,8 +222,6 @@ echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
 echo "*******************************************************************************"
 echo "Running integration tests on current image:"
 echo ""
-
-
 START=$SECONDS
 pushd ./test/integration
 GO111MODULE=on go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \

From ba806faf6d65892c2d1dbcfa122e1528408b0865 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 3 Jul 2020 14:32:57 -0400
Subject: [PATCH 29/35] Poll for updated daemonset.

---
 scripts/run-integration-tests.sh | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 3f47b231fe..3a703aeedc 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -172,7 +172,13 @@ else
 
     export KUBECONFIG=~/.kube/config
     kubectl apply -f "$TEST_CONFIG_PATH"
-    sleep 110
+    sleep 30
+    while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
+    do
+        sleep 5
+        echo "Waiting for daemonset update"
+    done
+    echo "Updated!"
 
     go install github.com/onsi/ginkgo/ginkgo
     wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
@@ -211,11 +217,14 @@ echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION"
 echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION"
 START=$SECONDS
 $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
+sleep 30
+while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
+do
+    sleep 5
+    echo "Waiting for daemonset update"
+done
+echo "Updated!"
 
-# Delay based on 3 nodes, 30s grace period per CNI pod
-echo "TODO: Poll and wait for updates to complete instead!"
-echo "Sleeping for 110s"
-sleep 110
 CNI_IMAGE_UPDATE_DURATION=$((SECONDS - START))
 echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
 

From a69f3eeb68b8c348926f7930058b14c9d6278fac Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 3 Jul 2020 14:50:23 -0400
Subject: [PATCH 30/35] CircleCI step fix.

---
 .circleci/config.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e857167aaf..313f810f6b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -90,6 +90,9 @@ workflows:
       - build_aarch64
       - hold:
           type: approval
+          requires:
+            - build_x86_64
+            - build_aarch64
       - integration_test:
           requires:
             - hold

From 1506500e1cec26c921e488301bb94588783fa54c Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Fri, 3 Jul 2020 14:54:23 -0400
Subject: [PATCH 31/35] Reduce daemonset update sleep.

---
 scripts/run-integration-tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 3a703aeedc..3757a78bcd 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -172,7 +172,7 @@ else
 
     export KUBECONFIG=~/.kube/config
     kubectl apply -f "$TEST_CONFIG_PATH"
-    sleep 30
+    sleep 5
     while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
     do
         sleep 5
@@ -217,7 +217,7 @@ echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION"
 echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION"
 START=$SECONDS
 $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
-sleep 30
+sleep 5
 while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
 do
     sleep 5

From 962a6dfc34d97231568234321284daa9b7e2d21c Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Mon, 6 Jul 2020 14:17:39 -0400
Subject: [PATCH 32/35] Kubectl change.

---
 scripts/run-integration-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 3757a78bcd..833ea9fd48 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -173,7 +173,7 @@ else
     export KUBECONFIG=~/.kube/config
     kubectl apply -f "$TEST_CONFIG_PATH"
     sleep 5
-    while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
+    while [[ $(kubectl describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
     do
         sleep 5
         echo "Waiting for daemonset update"

From be8d4649e3e2bea86aaef13d566a5c60bf2fa4a7 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Tue, 7 Jul 2020 13:26:28 -0400
Subject: [PATCH 33/35] Extract kops conformance, improve organization.

---
 scripts/lib/integration.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 scripts/lib/integration.sh

diff --git a/scripts/lib/integration.sh b/scripts/lib/integration.sh
new file mode 100644
index 0000000000..9bc1111a95
--- /dev/null
+++ b/scripts/lib/integration.sh
@@ -0,0 +1,32 @@
+function run_kops_conformance() {
+    START=$SECONDS
+
+    export KUBECONFIG=~/.kube/config
+    kubectl apply -f "$TEST_CONFIG_PATH"
+    sleep 5
+    while [[ $(kubectl describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
+    do
+        sleep 5
+        echo "Waiting for daemonset update"
+    done
+    echo "Updated!"
+
+    go install github.com/onsi/ginkgo/ginkgo
+    wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
+
+    ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
+    --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
+
+    /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
+    --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"
+    echo "Kops conformance tests ran successfully!"
+
+    KOPS_TEST_DURATION=$((SECONDS - START))
+    echo "TIMELINE: KOPS tests took $KOPS_TEST_DURATION seconds."
+
+    START=$SECONDS
+    down-kops-cluster
+    DOWN_KOPS_DURATION=$((SECONDS - START))
+    echo "TIMELINE: Down KOPS cluster took $DOWN_KOPS_DURATION seconds."
+    exit 0
+}
\ No newline at end of file

From b6e347d9aaa9179cdc39b3233a0a8cce96b5f7ad Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Tue, 7 Jul 2020 13:26:39 -0400
Subject: [PATCH 34/35] Organization improved.

---
 scripts/run-integration-tests.sh | 64 ++++++++++----------------------
 1 file changed, 20 insertions(+), 44 deletions(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index 833ea9fd48..c9feebabb8 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -8,6 +8,7 @@ DIR=$(cd "$(dirname "$0")"; pwd)
 source "$DIR"/lib/common.sh
 source "$DIR"/lib/aws.sh
 source "$DIR"/lib/cluster.sh
+source "$DIR"/lib/integration.sh
 
 # Variables used in /lib/aws.sh
 OS=$(go env GOOS)
@@ -27,16 +28,20 @@ __cluster_deprovisioned=0
 on_error() {
     # Make sure we destroy any cluster that was created if we hit run into an
     # error when attempting to run tests against the cluster
-    if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true && $RUN_KOPS_TEST == false ]]; then
-        # prevent double-deprovisioning with ctrl-c during deprovisioning...
-        __cluster_deprovisioned=1
-        echo "Cluster was provisioned already. Deprovisioning it..."
-        down-test-cluster
-    fi
-    if [[ $RUN_KOPS_TEST == true ]]; then
-        echo "Cluster was provisioned already. Deprovisioning it..."
-        down-kops-cluster
+    if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true ]]; then
+        if [[ $RUN_KOPS_TEST == true ]]; then
+            __cluster_deprovisioned=1
+            echo "Cluster was provisioned already. Deprovisioning it..."
+            down-kops-cluster
+        fi
+        else
+            # prevent double-deprovisioning with ctrl-c during deprovisioning...
+            __cluster_deprovisioned=1
+            echo "Cluster was provisioned already. Deprovisioning it..."
+            down-test-cluster
+        fi
     fi
+    
     exit 1
 }
 
@@ -147,11 +152,11 @@ mkdir -p "$TEST_CONFIG_DIR"
 START=$SECONDS
 if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == false ]]; then
     up-test-cluster
-    __cluster_created=1
 else
     up-kops-cluster
-    __cluster_created=1
 fi
+__cluster_created=1
+
 UP_CLUSTER_DURATION=$((SECONDS - START))
 echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds."
 
@@ -168,36 +173,7 @@ if [[ $RUN_KOPS_TEST != true ]]; then
     export KUBECONFIG=$KUBECONFIG_PATH
     ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')
 else
-    START=$SECONDS
-
-    export KUBECONFIG=~/.kube/config
-    kubectl apply -f "$TEST_CONFIG_PATH"
-    sleep 5
-    while [[ $(kubectl describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]]
-    do
-        sleep 5
-        echo "Waiting for daemonset update"
-    done
-    echo "Updated!"
-
-    go install github.com/onsi/ginkgo/ginkgo
-    wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp  kubernetes/platforms/linux/amd64/e2e.test
-
-    ginkgo -p --focus="Conformance"  --failFast --flakeAttempts 2 \
-    --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG
-
-    /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \
-    --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]"
-    echo "Kops conformance tests ran successfully!"
-
-    KOPS_TEST_DURATION=$((SECONDS - START))
-    echo "TIMELINE: KOPS tests took $KOPS_TEST_DURATION seconds."
-
-    START=$SECONDS
-    down-kops-cluster
-    DOWN_KOPS_DURATION=$((SECONDS - START))
-    echo "TIMELINE: Down KOPS cluster took $DOWN_KOPS_DURATION seconds."
-    exit 0
+    run_kops_conformance
 fi
 
 echo "*******************************************************************************"
@@ -259,10 +235,10 @@ fi
 if [[ "$DEPROVISION" == true ]]; then
     START=$SECONDS
 
-    if [[ "$RUN_KOPS_TEST" == false ]]; then
-        down-test-cluster
-    else
+    if [[ "$RUN_KOPS_TEST" == true ]]; then
         down-kops-cluster
+    else
+        down-test-cluster
     fi
 
     DOWN_DURATION=$((SECONDS - START))

From 6263a8569582182f962483a0039507c62a8033c1 Mon Sep 17 00:00:00 2001
From: Ben Napolitan <bnapolitan@outlook.com>
Date: Tue, 7 Jul 2020 15:21:25 -0400
Subject: [PATCH 35/35] Fix syntax error.

---
 scripts/run-integration-tests.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
index c9feebabb8..c39379e628 100755
--- a/scripts/run-integration-tests.sh
+++ b/scripts/run-integration-tests.sh
@@ -33,7 +33,6 @@ on_error() {
             __cluster_deprovisioned=1
             echo "Cluster was provisioned already. Deprovisioning it..."
             down-kops-cluster
-        fi
         else
             # prevent double-deprovisioning with ctrl-c during deprovisioning...
             __cluster_deprovisioned=1