From 053bdd0c82a62a117cd890c14162604846d8f166 Mon Sep 17 00:00:00 2001
From: mccheah <mcheah@palantir.com>
Date: Tue, 23 May 2017 12:57:45 -0700
Subject: [PATCH] Replace submission v1 with submission v2. (#286)

* Replace submission v1 with submission v2.

* Address documentation changes.

* Fix documentation
---
 conf/kubernetes-resource-staging-server.yaml  |  63 ++
 .../org/apache/spark/deploy/SparkSubmit.scala |   2 +-
 docs/running-on-kubernetes.md                 | 416 ++++++----
 ....kubernetes.submit.v1.DriverServiceManager |   2 -
 .../deploy/kubernetes/CompressionUtils.scala  |  74 +-
 .../SparkPodInitContainerBootstrap.scala      |   2 +-
 .../spark/deploy/kubernetes/config.scala      |  84 +-
 .../kubernetes/submit/{v2 => }/Client.scala   |   5 +-
 .../ContainerLocalizedFilesResolver.scala     |   2 +-
 .../ContainerNameEqualityPredicate.scala      |   2 +-
 ...riverInitContainerComponentsProvider.scala |   4 +-
 ...riverPodKubernetesCredentialsMounter.scala |   2 +-
 ...KubernetesCredentialsMounterProvider.scala |   3 +-
 .../ExecutorInitContainerConfiguration.scala  |   2 +-
 .../submit/{v2 => }/InitContainerUtil.scala   |   2 +-
 ...opertiesConfigMapFromScalaMapBuilder.scala |   2 +-
 .../SparkInitContainerConfigMapBuilder.scala  |   3 +-
 .../SubmissionKubernetesClientProvider.scala  |   2 +-
 ...dDependencyInitContainerConfigPlugin.scala |   2 +-
 .../SubmittedDependencySecretBuilder.scala    |   2 +-
 .../SubmittedDependencyUploaderImpl.scala     |   5 +-
 .../submit/{v2 => }/SubmittedResources.scala  |   2 +-
 .../deploy/kubernetes/submit/v1/Client.scala  | 743 ------------------
 .../submit/v1/DriverServiceManager.scala      |  99 ---
 ...DriverSubmitSslConfigurationProvider.scala | 354 ---------
 ...rnalSuppliedUrisDriverServiceManager.scala | 105 ---
 .../submit/v1/KubernetesResourceCleaner.scala |  53 --
 .../v1/NodePortUrisDriverServiceManager.scala |  70 --
 ...esSparkRestApi.scala => FileFetcher.scala} |  24 +-
 ...SparkDependencyDownloadInitContainer.scala |  50 +-
 .../{v1 => }/PemsToKeyStoreConverter.scala    |   3 +-
 .../{v2 => }/ResourceStagingServer.scala      |   2 +-
 ...ourceStagingServerSslOptionsProvider.scala |   3 +-
 .../{v2 => }/ResourceStagingService.scala     |   4 +-
 .../{v2 => }/ResourceStagingServiceImpl.scala |  11 +-
 .../ResourceStagingServiceRetrofit.scala      |   4 +-
 .../{v2 => }/RetrofitClientFactory.scala      |   2 +-
 .../{v2 => }/SparkConfPropertiesParser.scala  |   4 +-
 .../rest/kubernetes/v1/HttpClientUtil.scala   | 131 ---
 .../v1/KubernetesRestProtocolMessages.scala   |  75 --
 .../v1/KubernetesSparkRestServer.scala        | 483 ------------
 .../v1/MultiServerFeignTarget.scala           |  89 ---
 .../spark/deploy/kubernetes/SSLUtils.scala    |   2 +-
 .../submit/{v2 => }/ClientV2Suite.scala       |   3 +-
 ...ContainerLocalizedFilesResolverSuite.scala |   2 +-
 ...PodKubernetesCredentialsMounterSuite.scala |   4 +-
 ...cutorInitContainerConfigurationSuite.scala |   2 +-
 .../submit/{v2 => }/SSLFilePairs.scala        |   2 +-
 ...rkInitContainerConfigMapBuilderSuite.scala |   2 +-
 ...ndencyInitContainerConfigPluginSuite.scala |   6 +-
 ...ubmittedDependencySecretBuilderSuite.scala |   4 +-
 .../SubmittedDependencyUploaderSuite.scala    |   4 +-
 ...DependencyDownloadInitContainerSuite.scala |   4 +-
 ...StagingServerSslOptionsProviderSuite.scala |   2 +-
 .../{v2 => }/ResourceStagingServerSuite.scala |   2 +-
 .../ResourceStagingServiceImplSuite.scala     |   2 +-
 .../src/main/docker/driver-v2/Dockerfile      |  43 -
 .../src/main/docker/driver/Dockerfile         |  18 +-
 .../Dockerfile                                |   2 +-
 .../docker/resource-staging-server/Dockerfile |   2 +-
 .../integrationtest/KubernetesSuite.scala     | 248 +++++-
 .../KubernetesTestComponents.scala            |  29 +-
 .../integrationtest/KubernetesV1Suite.scala   | 339 --------
 .../integrationtest/KubernetesV2Suite.scala   | 265 -------
 .../ResourceStagingServerLauncher.scala       |   2 +-
 .../docker/SparkDockerImageBuilder.scala      |  10 +-
 66 files changed, 668 insertions(+), 3323 deletions(-)
 create mode 100644 conf/kubernetes-resource-staging-server.yaml
 delete mode 100644 resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.deploy.kubernetes.submit.v1.DriverServiceManager
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/Client.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ContainerLocalizedFilesResolver.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ContainerNameEqualityPredicate.scala (95%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/DriverInitContainerComponentsProvider.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/DriverPodKubernetesCredentialsMounter.scala (99%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/DriverPodKubernetesCredentialsMounterProvider.scala (92%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ExecutorInitContainerConfiguration.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/InitContainerUtil.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/PropertiesConfigMapFromScalaMapBuilder.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SparkInitContainerConfigMapBuilder.scala (95%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmissionKubernetesClientProvider.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencyInitContainerConfigPlugin.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencySecretBuilder.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencyUploaderImpl.scala (95%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedResources.scala (96%)
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/Client.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverServiceManager.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverSubmitSslConfigurationProvider.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/ExternalSuppliedUrisDriverServiceManager.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/KubernetesResourceCleaner.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/NodePortUrisDriverServiceManager.scala
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v1/KubernetesSparkRestApi.scala => FileFetcher.scala} (56%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/KubernetesSparkDependencyDownloadInitContainer.scala (95%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v1 => }/PemsToKeyStoreConverter.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServer.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServerSslOptionsProvider.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingService.scala (97%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServiceImpl.scala (91%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServiceRetrofit.scala (93%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/RetrofitClientFactory.scala (98%)
 rename resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/SparkConfPropertiesParser.scala (94%)
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/HttpClientUtil.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesRestProtocolMessages.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestServer.scala
 delete mode 100644 resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/MultiServerFeignTarget.scala
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ClientV2Suite.scala (99%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ContainerLocalizedFilesResolverSuite.scala (98%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/DriverPodKubernetesCredentialsMounterSuite.scala (99%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/ExecutorInitContainerConfigurationSuite.scala (97%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SSLFilePairs.scala (94%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SparkInitContainerConfigMapBuilderSuite.scala (98%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencyInitContainerConfigPluginSuite.scala (96%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencySecretBuilderSuite.scala (97%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/{v2 => }/SubmittedDependencyUploaderSuite.scala (97%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/KubernetesSparkDependencyDownloadInitContainerSuite.scala (98%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServerSslOptionsProviderSuite.scala (99%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServerSuite.scala (99%)
 rename resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/{v2 => }/ResourceStagingServiceImplSuite.scala (98%)
 delete mode 100644 resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-v2/Dockerfile
 rename resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/{driver-init => init-container}/Dockerfile (95%)
 delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV1Suite.scala
 delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV2Suite.scala

diff --git a/conf/kubernetes-resource-staging-server.yaml b/conf/kubernetes-resource-staging-server.yaml
new file mode 100644
index 0000000000000..de0da3edcb901
--- /dev/null
+++ b/conf/kubernetes-resource-staging-server.yaml
@@ -0,0 +1,63 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: spark-resource-staging-server
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        resource-staging-server-instance: default
+    spec:
+      volumes:
+        - name: resource-staging-server-properties
+          configMap:
+            name: spark-resource-staging-server-config
+      containers:
+        - name: spark-resource-staging-server
+          image: kubespark/spark-resource-staging-server:v2.1.0-kubernetes-0.1.0-alpha.3
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 100m
+              memory: 256Mi
+          volumeMounts:
+            - name: resource-staging-server-properties
+              mountPath: '/etc/spark-resource-staging-server'
+          args:
+            - '/etc/spark-resource-staging-server/resource-staging-server.properties'
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: spark-resource-staging-server-config
+data:
+  resource-staging-server.properties: |
+    spark.kubernetes.resourceStagingServer.port=10000
+    spark.ssl.kubernetes.resourceStagingServer.enabled=false
+# Other possible properties are listed below, primarily for setting up TLS. The paths given by KeyStore, password, and PEM files here should correspond to
+# files that are securely mounted into the resource staging server container, via e.g. secret volumes.
+#   spark.ssl.kubernetes.resourceStagingServer.keyStore=/mnt/secrets/resource-staging-server/keyStore.jks
+#   spark.ssl.kubernetes.resourceStagingServer.keyStorePassword=changeit
+#   spark.ssl.kubernetes.resourceStagingServer.keyPassword=changeit
+#   spark.ssl.kubernetes.resourceStagingServer.keyStorePasswordFile=/mnt/secrets/resource-staging-server/keystore-password.txt
+#   spark.ssl.kubernetes.resourceStagingServer.keyPasswordFile=/mnt/secrets/resource-staging-server/keystore-key-password.txt
+#   spark.ssl.kubernetes.resourceStagingServer.keyPem=/mnt/secrets/resource-staging-server/key.pem
+#   spark.ssl.kubernetes.resourceStagingServer.serverCertPem=/mnt/secrets/resource-staging-server/cert.pem
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: spark-resource-staging-service
+spec:
+  type: NodePort
+  selector:
+    resource-staging-server-instance: default
+  ports:
+    - protocol: TCP
+      port: 10000
+      targetPort: 10000
+      nodePort: 31000
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 2a505bb4ca254..05188fc1f6b4a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -655,7 +655,7 @@ object SparkSubmit extends CommandLineUtils {
     }
 
     if (isKubernetesCluster) {
-      childMainClass = "org.apache.spark.deploy.kubernetes.submit.v1.Client"
+      childMainClass = "org.apache.spark.deploy.kubernetes.submit.Client"
       childArgs += args.primaryResource
       childArgs += args.mainClass
       childArgs ++= args.childArgs
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 5b7bb6cc612c5..98393cbbbba2d 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -3,15 +3,25 @@ layout: global
 title: Running Spark on Kubernetes
 ---
 
-Support for running on [Kubernetes](https://kubernetes.io/docs/whatisk8s/) is available in experimental status. The feature set is
-currently limited and not well-tested. This should not be used in production environments.
+Support for running on [Kubernetes](https://kubernetes.io/docs/whatisk8s/) is available in experimental status. The
+feature set is currently limited and not well-tested. This should not be used in production environments.
 
 ## Prerequisites
 
-* You must have a running Kubernetes cluster with access configured to it using [kubectl](https://kubernetes.io/docs/user-guide/prereqs/). If you do not already have a working Kubernetes cluster, you may setup a test cluster on your local machine using [minikube](https://kubernetes.io/docs/getting-started-guides/minikube/).
-  * We recommend that minikube be updated to the most recent version (0.18.0 at the time of this documentation), as some earlier versions may not start up the kubernetes cluster with all the necessary components.
-* You must have appropriate permissions to create and list [pods](https://kubernetes.io/docs/user-guide/pods/), [nodes](https://kubernetes.io/docs/admin/node/) and [services](https://kubernetes.io/docs/user-guide/services/) in your cluster. You can verify that you can list these resources by running `kubectl get nodes`, `kubectl get pods` and `kubectl get svc` which should give you a list of nodes, pods and services (if any) respectively.
-* You must have a spark distribution with Kubernetes support. This may be obtained from the [release tarball](https://github.com/apache-spark-on-k8s/spark/releases) or by [building Spark with Kubernetes support](../resource-managers/kubernetes/README.md#building-spark-with-kubernetes-support).
+* You must have a running Kubernetes cluster with access configured to it
+using [kubectl](https://kubernetes.io/docs/user-guide/prereqs/). If you do not already have a working Kubernetes
+cluster, you may setup a test cluster on your local machine using
+[minikube](https://kubernetes.io/docs/getting-started-guides/minikube/).
+  * We recommend that minikube be updated to the most recent version (0.19.0 at the time of this documentation), as some
+  earlier versions may not start up the kubernetes cluster with all the necessary components.
+* You must have appropriate permissions to create and list [pods](https://kubernetes.io/docs/user-guide/pods/),
+[ConfigMaps](https://kubernetes.io/docs/tasks/configure-pod-container/configmap/) and
+[secrets](https://kubernetes.io/docs/concepts/configuration/secret/) in your cluster. You can verify that
+you can list these resources by running `kubectl get pods` `kubectl get configmap`, and `kubectl get secrets` which
+should give you a list of pods and configmaps (if any) respectively.
+* You must have a spark distribution with Kubernetes support. This may be obtained from the
+[release tarball](https://github.com/apache-spark-on-k8s/spark/releases) or by
+[building Spark with Kubernetes support](../resource-managers/kubernetes/README.md#building-spark-with-kubernetes-support).
 
 ## Driver & Executor Images
 
@@ -19,7 +29,8 @@ Kubernetes requires users to supply images that can be deployed into containers
 be run in a container runtime environment that Kubernetes supports. Docker is a container runtime environment that is
 frequently used with Kubernetes, so Spark provides some support for working with Docker to get started quickly.
 
-If you wish to use pre-built docker images, you may use the images published in [kubespark](https://hub.docker.com/u/kubespark/). The images are as follows:
+If you wish to use pre-built docker images, you may use the images published in
+[kubespark](https://hub.docker.com/u/kubespark/). The images are as follows:
 
 <table class="table">
 <tr><th>Component</th><th>Image</th></tr>
@@ -31,20 +42,27 @@ If you wish to use pre-built docker images, you may use the images published in
   <td>Spark Executor Image</td>
   <td><code>kubespark/spark-executor:v2.1.0-kubernetes-0.1.0-alpha.2</code></td>
 </tr>
+<tr>
+  <td>Spark Initialization Image</td>
+  <td><code>kubespark/spark-init:v2.1.0-kubernetes-0.1.0-alpha.2</code></td>
+</tr>
 </table>
 
-You may also build these docker images from sources, or customize them as required. Spark distributions include the Docker files for the driver and the executor at
-`dockerfiles/driver/Dockerfile` and `dockerfiles/executor/Dockerfile`, respectively. Use these Docker files to build the
-Docker images, and then tag them with the registry that the images should be sent to. Finally, push the images to the
-registry.
+You may also build these docker images from sources, or customize them as required. Spark distributions include the
+Docker files for the driver, executor, and init-container at `dockerfiles/driver/Dockerfile`,
+`dockerfiles/executor/Dockerfile`, and `dockerfiles/init-container/Dockerfile` respectively. Use these Docker files to
+build the Docker images, and then tag them with the registry that the images should be sent to. Finally, push the images
+to the registry.
 
 For example, if the registry host is `registry-host` and the registry is listening on port 5000:
 
     cd $SPARK_HOME
     docker build -t registry-host:5000/spark-driver:latest -f dockerfiles/driver/Dockerfile .
     docker build -t registry-host:5000/spark-executor:latest -f dockerfiles/executor/Dockerfile .
+    docker build -t registry-host:5000/spark-init:latest -f dockerfiles/init-container/Dockerfile .
     docker push registry-host:5000/spark-driver:latest
     docker push registry-host:5000/spark-executor:latest
+    docker push registry-host:5000/spark-init:latest
 
 ## Submitting Applications to Kubernetes
 
@@ -60,7 +78,8 @@ are set up as described above:
       --conf spark.app.name=spark-pi \
       --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.1.0-alpha.2 \
       --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.1.0-alpha.2 \
-      examples/jars/spark_examples_2.11-2.2.0.jar
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 The Spark master, specified either via passing the `--master` command line argument to `spark-submit` or by setting
 `spark.master` in the application's configuration, must be a URL with the format `k8s://<api_server_url>`. Prefixing the
@@ -80,13 +99,53 @@ In the above example, the specific Kubernetes cluster can be used with spark sub
 Note that applications can currently only be executed in cluster mode, where the driver and its executors are running on
 the cluster.
 
-### Specifying input files
+Finally, notice that in the above example we specify a jar with a specific URI with a scheme of `local://`. This URI is
+the location of the example jar that is already in the Docker image. Using dependencies that are on your machine's local
+disk is discussed below.
+
+## Dependency Management
+
+Application dependencies that are being submitted from your machine need to be sent to a **resource staging server**
+that the driver and executor can then communicate with to retrieve those dependencies. A YAML file denoting a minimal
+set of Kubernetes resources that runs this service is located in the file `conf/kubernetes-resource-staging-server.yaml`.
+This YAML file configures a Deployment with one pod running the resource staging server configured with a ConfigMap,
+and exposes the server through a Service with a fixed NodePort. Deploying a resource staging server with the included
+YAML file requires you to have permissions to create Deployments, Services, and ConfigMaps.
+
+To run the resource staging server with default configurations, the Kubernetes resources can be created:
+
+    kubectl create -f conf/kubernetes-resource-staging-server.yaml
+
+and then you can compute the value of Pi as follows:
+
+    bin/spark-submit \
+      --deploy-mode cluster \
+      --class org.apache.spark.examples.SparkPi \
+      --master k8s://<k8s-apiserver-host>:<k8s-apiserver-port> \
+      --kubernetes-namespace default \
+      --conf spark.executor.instances=5 \
+      --conf spark.app.name=spark-pi \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.resourceStagingServer.uri=http://<address-of-any-cluster-node>:31000 \
+      examples/jars/spark_examples_2.11-2.2.0.jar
+
+The Docker image for the resource staging server may also be built from source, in a similar manner to the driver
+and executor images. The Dockerfile is provided in `dockerfiles/resource-staging-server/Dockerfile`.
+
+The provided YAML file specifically sets the NodePort to 31000 on the service's specification. If port 31000 is not
+available on any of the nodes of your cluster, you should remove the NodePort field from the service's specification
+and allow the Kubernetes cluster to determine the NodePort itself. Be sure to provide the correct port in the resource
+staging server URI when submitting your application, in accordance to the NodePort chosen by the Kubernetes cluster.
+
+### Dependency Management Without The Resource Staging Server
 
-Spark supports specifying JAR paths that are either on the submitting host's disk, or are located on the disk of the
-driver and executors. Refer to the [application submission](submitting-applications.html#advanced-dependency-management)
-section for details. Note that files specified with the `local://` scheme should be added to the container image of both
-the driver and the executors. Files without a scheme or with the scheme `file://` are treated as being on the disk of
-the submitting machine, and are uploaded to the driver running in Kubernetes before launching the application.
+Note that this resource staging server is only required for submitting local dependencies. If your application's
+dependencies are all hosted in remote locations like HDFS or http servers, they may be referred to by their appropriate
+remote URIs. Also, application dependencies can be pre-mounted into custom-built Docker images. Those dependencies
+can be added to the classpath by referencing them with `local://` URIs and/or setting the `SPARK_EXTRA_CLASSPATH`
+environment variable in your Dockerfiles.
 
 ### Accessing Kubernetes Clusters
 
@@ -111,70 +170,127 @@ If our local proxy were listening on port 8001, we would have our submission loo
       --conf spark.app.name=spark-pi \
       --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.1.0-alpha.2 \
       --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.1.0-alpha.2 \
-      examples/jars/spark_examples_2.11-2.2.0.jar
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar
 
 Communication between Spark and Kubernetes clusters is performed using the fabric8 kubernetes-client library.
 The above mechanism using `kubectl proxy` can be used when we have authentication providers that the fabric8
 kubernetes-client library does not support. Authentication using X509 Client Certs and OAuth tokens
 is currently supported.
 
+## Dynamic Executor Scaling
+
+Spark on Kubernetes supports Dynamic Allocation with cluster mode. This mode requires running
+an external shuffle service. This is typically a [daemonset](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/)
+with a provisioned [hostpath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volume.
+This shuffle service may be shared by executors belonging to different SparkJobs. Using Spark with dynamic allocation
+on Kubernetes assumes that a cluster administrator has set up one or more shuffle-service daemonsets in the cluster.
+
+A sample configuration file is provided in `conf/kubernetes-shuffle-service.yaml` which can be customized as needed
+for a particular cluster. It is important to note that `spec.template.metadata.labels` are setup appropriately for the shuffle
+service because there may be multiple shuffle service instances running in a cluster. The labels give Spark applications
+a way to target a particular shuffle service.
+
+For example, if the shuffle service we want to use is in the default namespace, and
+has pods with labels `app=spark-shuffle-service` and `spark-version=2.1.0`, we can
+use those tags to target that particular shuffle service at job launch time. In order to run a job with dynamic allocation enabled,
+the command may then look like the following:
+
+    bin/spark-submit \
+      --deploy-mode cluster \
+      --class org.apache.spark.examples.GroupByTest \
+      --master k8s://<k8s-master>:<port> \
+      --kubernetes-namespace default \
+      --conf spark.app.name=group-by-test \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:latest \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:latest \
+      --conf spark.dynamicAllocation.enabled=true \
+      --conf spark.shuffle.service.enabled=true \
+      --conf spark.kubernetes.shuffle.namespace=default \
+      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.1.0" \
+      local:///opt/spark/examples/jars/spark_examples_2.11-2.2.0.jar 10 400000 2
+
 ## Advanced
- 
-### Setting Up TLS For Submitting the Driver
-
-When submitting to Kubernetes, a pod is started for the driver, and the pod starts an HTTP server. This HTTP server
-receives the driver's configuration, including uploaded driver jars, from the client before starting the application.
-Spark supports using TLS to encrypt the traffic in this bootstrapping process. It is recommended to configure this
-whenever possible. 
-
-See the [security page](security.html) and [configuration](configuration.html) sections for more information on
-configuring TLS; use the prefix `spark.ssl.kubernetes.driversubmitserver` in configuring the TLS-related fields in the context
-of submitting to Kubernetes. For example, to set the trustStore used when the local machine communicates with the driver
-pod in starting the application, set `spark.ssl.kubernetes.driversubmitserver.trustStore`.
-
-One note about the keyStore is that it can be specified as either a file on the client machine or a file in the
-container image's disk. Thus `spark.ssl.kubernetes.driversubmitserver.keyStore` can be a URI with a scheme of either `file:`
-or `local:`. A scheme of `file:` corresponds to the keyStore being located on the client machine; it is mounted onto
-the driver container as a [secret volume](https://kubernetes.io/docs/user-guide/secrets/). When the URI has the scheme
-`local:`, the file is assumed to already be on the container's disk at the appropriate path.
-
-Finally, the submission server and client can be configured to use PEM files instead of Java keyStores. When using
-this mode, set `spark.ssl.kubernetes.driversubmitserver.keyPem` and
-`spark.ssl.kubernetes.driversubmitserver.serverCertPem` to configure the key and certificate files on the driver
-submission server. These files can be uploaded from the submitter's machine if they have no scheme or a scheme of
-`file:`, or they can be located on the container's disk if they have the scheme `local:`. The client's certificate
-file should be provided via setting `spark.ssl.kubernetes.driversubmitserver.clientCertPem`, and this file must be
-located on the submitting machine's local disk.
-
-### Submission of Local Files through Ingress/External controller
-
-Kubernetes pods run with their own IP address space. If Spark is run in cluster mode, the driver pod may not be
-accessible to the submitter. However, the submitter needs to send local dependencies from its local disk to the driver
-pod.
-
-By default, Spark will place a [Service](https://kubernetes.io/docs/user-guide/services/#type-nodeport) with a NodePort
-that is opened on every node. The submission client will then contact the driver at one of the node's
-addresses with the appropriate service port.
-
-There may be cases where the nodes cannot be reached by the submission client. For example, the cluster may
-only be reachable through an external load balancer. The user may provide their own external URI for Spark driver
-services. To use a your own external URI instead of a node's IP and node port, first set
-`spark.kubernetes.driver.serviceManagerType` to `ExternalAnnotation`. A service will be created with the annotation
-`spark-job.alpha.apache.org/provideExternalUri`, and this service routes to the driver pod. You will need to run a
-separate process that watches the API server for services that are created with this annotation in the application's
-namespace (set by `spark.kubernetes.namespace`). The process should determine a URI that routes to this service
-(potentially configuring infrastructure to handle the URI behind the scenes), and patch the service to include an
-annotation `spark-job.alpha.apache.org/resolvedExternalUri`, which has its value as the external URI that your process
-has provided (e.g. `https://example.com:8080/my-job`).
-
-Note that the URI provided in the annotation needs to route traffic to the appropriate destination on the pod, which has
-a empty path portion of the URI. This means the external URI provider will likely need to rewrite the path from the
-external URI to the destination on the pod, e.g. https://example.com:8080/spark-app-1/submit will need to route traffic
-to https://<pod_ip>:<service_port>/. Note that the paths of these two URLs are different.
-
-If the above is confusing, keep in mind that this functionality is only necessary if the submitter cannot reach any of
-the nodes at the driver's node port. It is recommended to use the default configuration with the node port service
-whenever possible.
+
+### Securing the Resource Staging Server with TLS
+
+The default configuration of the resource staging server is not secured with TLS. It is highly recommended to configure
+this to protect the secrets and jars/files being submitted through the staging server.
+
+The YAML file in `conf/kubernetes-resource-staging-server.yaml` includes a ConfigMap resource that holds the resource
+staging server's configuration. The properties can be adjusted here to make the resource staging server listen over TLS.
+Refer to the [security](security.html) page for the available settings related to TLS. The namespace for the
+resource staging server is `kubernetes.resourceStagingServer`, so for example the path to the server's keyStore would
+be set by `spark.ssl.kubernetes.resourceStagingServer.keyStore`.
+
+In addition to the settings specified by the previously linked security page, the resource staging server supports the
+following additional configurations:
+
+<table class="table">
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.keyPem</code></td>
+  <td>(none)</td>
+  <td>
+    Private key file encoded in PEM format that the resource staging server uses to secure connections over TLS. If this
+    is specified, the associated public key file must be specified in
+    <code>spark.ssl.kubernetes.resourceStagingServer.serverCertPem</code>. PEM files and a keyStore file (set by
+    <code>spark.ssl.kubernetes.resourceStagingServer.keyStore</code>) cannot both be specified at the same time.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.serverCertPem</code></td>
+  <td>(none)</td>
+  <td>
+    Certificate file encoded in PEM format that the resource staging server uses to secure connections over TLS. If this
+    is specified, the associated private key file must be specified in
+    <code>spark.ssl.kubernetes.resourceStagingServer.keyPem</code>. PEM files and a keyStore file (set by
+    <code>spark.ssl.kubernetes.resourceStagingServer.keyStore</code>) cannot both be specified at the same time.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.keyStorePasswordFile</code></td>
+  <td>(none)</td>
+  <td>
+    Provides the KeyStore password through a file in the container instead of a static value. This is useful if the
+    keyStore's password is to be mounted into the container with a secret.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.keyPasswordFile</code></td>
+  <td>(none)</td>
+  <td>
+    Provides the keyStore's key password using a file in the container instead of a static value. This is useful if the
+    keyStore's key password is to be mounted into the container with a secret.
+  </td>
+</tr>
+</table>
+
+Note that while the properties can be set in the ConfigMap, you will still need to consider the means of mounting the
+appropriate secret files into the resource staging server's container. A common mechanism that is used for this is
+to use [Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret/) that are mounted as secret
+volumes. Refer to the appropriate Kubernetes documentation for guidance and adjust the resource staging server's
+specification in the provided YAML file accordingly.
+
+Finally, when you submit your application, you must specify either a trustStore or a PEM-encoded certificate file to
+communicate with the resource staging server over TLS. The trustStore can be set with
+`spark.ssl.kubernetes.resourceStagingServer.trustStore`, or a certificate file can be set with
+`spark.ssl.kubernetes.resourceStagingServer.clientCertPem`. For example, our SparkPi example now looks like this:
+
+    bin/spark-submit \
+      --deploy-mode cluster \
+      --class org.apache.spark.examples.SparkPi \
+      --master k8s://https://<k8s-apiserver-host>:<k8s-apiserver-port> \
+      --kubernetes-namespace default \
+      --conf spark.executor.instances=5 \
+      --conf spark.app.name=spark-pi \
+      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.initcontainer.docker.image=kubespark/spark-init:v2.1.0-kubernetes-0.1.0-alpha.2 \
+      --conf spark.kubernetes.resourceStagingServer.uri=https://<address-of-any-cluster-node>:31000 \
+      --conf spark.ssl.kubernetes.resourceStagingServer.enabled=true \
+      --conf spark.ssl.kubernetes.resourceStagingServer.clientCertPem=/home/myuser/cert.pem \
+      examples/jars/spark_examples_2.11-2.2.0.jar
 
 ### Spark Properties
 
@@ -208,6 +324,16 @@ from the other deployment modes. See the [configuration page](configuration.html
     <a href="https://docs.docker.com/engine/reference/commandline/tag/">Docker tag</a> format.
   </td>
 </tr>
+<tr>
+  <td><code>spark.kubernetes.initcontainer.docker.image</code></td>
+  <td><code>spark-init:2.2.0</code></td>
+  <td>
+    Docker image to use for the init-container that is run before the driver and executor containers. Specify this using
+    the standard <a href="https://docs.docker.com/engine/reference/commandline/tag/">Docker tag</a> format. The
+    init-container is responsible for fetching application dependencies from both remote locations like HDFS or S3,
+    and from the resource staging server, if applicable.
+  </td>
+</tr>
 <tr>
   <td><code>spark.kubernetes.shuffle.namespace</code></td>
   <td><code>default</code></td>
@@ -218,7 +344,7 @@ from the other deployment modes. See the [configuration page](configuration.html
 </tr>
 <tr>
   <td><code>spark.kubernetes.shuffle.labels</code></td>
-  <td><code>(none)</code></td>
+  <td>(none)</td>
   <td>
     Labels that will be used to look up shuffle service pods. This should be a comma-separated list of label key-value pairs,
     where each label is in the format <code>key=value</code>. The labels chosen must be such that
@@ -334,123 +460,113 @@ from the other deployment modes. See the [configuration page](configuration.html
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.submissionServerMemory</code></td>
-  <td>256m</td>
+  <td><code>spark.kubernetes.driver.labels</code></td>
+  <td>(none)</td>
   <td>
-    The amount of memory to allocate for the driver submission server.
+    Custom labels that will be added to the driver pod. This should be a comma-separated list of label key-value pairs,
+    where each label is in the format <code>key=value</code>. Note that Spark also adds its own labels to the driver pod
+    for bookkeeping purposes.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.memoryOverhead</code></td>
-  <td>(driverMemory + driverSubmissionServerMemory) * 0.10, with minimum of 384</td>
+  <td><code>spark.kubernetes.driver.annotations</code></td>
+  <td>(none)</td>
   <td>
-    The amount of off-heap memory (in megabytes) to be allocated for the driver and the driver submission server. This
-    is memory that accounts for things like VM overheads, interned strings, other native overheads, etc. This tends to
-    grow with the driver size (typically 6-10%).
+    Custom annotations that will be added to the driver pod. This should be a comma-separated list of label key-value
+    pairs, where each annotation is in the format <code>key=value</code>.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.labels</code></td>
+  <td><code>spark.kubernetes.driver.pod.name</code></td>
   <td>(none)</td>
   <td>
-    Custom labels that will be added to the driver pod. This should be a comma-separated list of label key-value pairs,
-    where each label is in the format <code>key=value</code>. Note that Spark also adds its own labels to the driver pod
-    for bookkeeping purposes.
+    Name of the driver pod. If not set, the driver pod name is set to "spark.app.name" suffixed by the current timestamp
+    to avoid name conflicts.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.annotations</code></td>
+  <td><code>spark.kubernetes.submission.waitAppCompletion</code></td>
+  <td><code>true</code></td>
+  <td>
+    In cluster mode, whether to wait for the application to finish before exiting the launcher process.  When changed to
+    false, the launcher has a "fire-and-forget" behavior when launching the Spark job.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.resourceStagingServer.port</code></td>
+  <td><code>10000</code></td>
+  <td>
+    Port for the resource staging server to listen on when it is deployed.
+  </td>
+</tr>
+<tr>
+  <td><code>spark.kubernetes.resourceStagingServer.uri</code></td>
   <td>(none)</td>
   <td>
-    Custom annotations that will be added to the driver pod. This should be a comma-separated list of label key-value
-    pairs, where each annotation is in the format <code>key=value</code>.
+    URI of the resource staging server that Spark should use to distribute the application's local dependencies. Note
+    that by default, this URI must be reachable by both the submitting machine and the pods running in the cluster. If
+    one URI is not simultaneously reachable both by the submitter and the driver/executor pods, configure the pods to
+    access the staging server at a different URI by setting
+    <code>spark.kubernetes.resourceStagingServer.internal.uri</code> as discussed below.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driverSubmissionTimeout</code></td>
-  <td>60s</td>
+  <td><code>spark.kubernetes.resourceStagingServer.internal.uri</code></td>
+  <td>Value of <code>spark.kubernetes.resourceStagingServer.uri</code></td>
   <td>
-    Time to wait for the driver pod to start running before aborting its execution.
+    URI of the resource staging server to communicate with when init-containers bootstrap the driver and executor pods
+    with submitted local dependencies. Note that this URI must by the pods running in the cluster. This is useful to
+    set if the resource staging server has a separate "internal" URI that must be accessed by components running in the
+    cluster.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.service.exposeUiPort</code></td>
-  <td><code>false</code></td>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.internal.trustStore</code></td>
+  <td>Value of <code>spark.ssl.kubernetes.resourceStagingServer.trustStore</code></td>
   <td>
-    Whether to expose the driver Web UI port as a service NodePort. Turned off by default because NodePort is a limited
-    resource.
+    Location of the trustStore file to use when communicating with the resource staging server over TLS, as
+    init-containers bootstrap the driver and executor pods with submitted local dependencies. This can be a URI with a
+    scheme of <code>local://</code>, which denotes that the file is pre-mounted on the pod's disk. A uri without a
+    scheme or a scheme of <code>file://</code> will result in this file being mounted from the submitting machine's
+    disk as a secret into the init-containers.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.pod.name</code></td>
-  <td><code>(none)</code></td>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.internal.trustStorePassword</code></td>
+  <td>Value of <code><code>spark.ssl.kubernetes.resourceStagingServer.trustStorePassword</code></td>
   <td>
-    Name of the driver pod. If not set, the driver pod name is set to "spark.app.name" suffixed by the current timestamp to avoid name conflicts.
+    Password of the trustStore file that is used when communicating with the resource staging server over TLS, as
+    init-containers bootstrap the driver and executor pods with submitted local dependencies.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.submission.waitAppCompletion</code></td>
-  <td><code>true</code></td>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.internal.trustStoreType</code></td>
+  <td>Value of <code><code>spark.ssl.kubernetes.resourceStagingServer.trustStoreType</code></td>
   <td>
-    In cluster mode, whether to wait for the application to finish before exiting the launcher process.  When changed to
-    false, the launcher has a "fire-and-forget" behavior when launching the Spark job.
+    Type of the trustStore file that is used when communicating with the resource staging server over TLS, when
+    init-containers bootstrap the driver and executor pods with submitted local dependencies.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.report.interval</code></td>
-  <td><code>1s</code></td>
+  <td><code>spark.ssl.kubernetes.resourceStagingServer.internal.clientCertPem</code></td>
+  <td>Value of <code>spark.ssl.kubernetes.resourceStagingServer.clientCertPem</code></td>
   <td>
-    Interval between reports of the current Spark job status in cluster mode.
+    Location of the certificate file to use when communicating with the resource staging server over TLS, as
+    init-containers bootstrap the driver and executor pods with submitted local dependencies. This can be a URI with a
+    scheme of <code>local://</code>, which denotes that the file is pre-mounted on the pod's disk. A uri without a
+    scheme or a scheme of <code>file://</code> will result in this file being mounted from the submitting machine's
+    disk as a secret into the init-containers.
   </td>
 </tr>
 <tr>
-  <td><code>spark.kubernetes.driver.serviceManagerType</code></td>
-  <td><code>NodePort</code></td>
+  <td><code>spark.kubernetes.report.interval</code></td>
+  <td><code>1s</code></td>
   <td>
-    A tag indicating which class to use for creating the Kubernetes service and determining its URI for the submission
-    client. Valid values are currently <code>NodePort</code> and <code>ExternalAnnotation</code>. By default, a service
-    is created with the <code>NodePort</code> type, and the driver will be contacted at one of the nodes at the port
-    that the nodes expose for the service. If the nodes cannot be contacted from the submitter's machine, consider
-    setting this to <code>ExternalAnnotation</code> as described in "Determining the Driver Base URI" above. One may
-    also include a custom implementation of <code>org.apache.spark.deploy.rest.kubernetes.DriverServiceManager</code> on
-    the submitter's classpath - spark-submit service loads an instance of that class. To use the custom
-    implementation, set this value to the custom implementation's return value of 
-    <code>DriverServiceManager#getServiceManagerType()</code>. This method should only be done as a last resort.
+    Interval between reports of the current Spark job status in cluster mode.
   </td>
 </tr>
 </table>
 
-## Dynamic Executor Scaling
-
-Spark on Kubernetes supports Dynamic Allocation with cluster mode. This mode requires running
-an external shuffle service. This is typically a [daemonset](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/)
-with a provisioned [hostpath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) volume.
-This shuffle service may be shared by executors belonging to different SparkJobs. Using Spark with dynamic allocation
-on Kubernetes assumes that a cluster administrator has set up one or more shuffle-service daemonsets in the cluster.
-
-A sample configuration file is provided in `conf/kubernetes-shuffle-service.yaml` which can be customized as needed
-for a particular cluster. It is important to note that `spec.template.metadata.labels` are setup appropriately for the shuffle
-service because there may be multiple shuffle service instances running in a cluster. The labels give us a way to target a particular
-shuffle service.
-
-For example, if the shuffle service we want to use is in the default namespace, and
-has pods with labels `app=spark-shuffle-service` and `spark-version=2.1.0`, we can
-use those tags to target that particular shuffle service at job launch time. In order to run a job with dynamic allocation enabled,
-the command may then look like the following:
-
-    bin/spark-submit \
-      --deploy-mode cluster \
-      --class org.apache.spark.examples.GroupByTest \
-      --master k8s://<k8s-master>:<port> \
-      --kubernetes-namespace default \
-      --conf spark.app.name=group-by-test \
-      --conf spark.kubernetes.driver.docker.image=kubespark/spark-driver:latest \
-      --conf spark.kubernetes.executor.docker.image=kubespark/spark-executor:latest \
-      --conf spark.dynamicAllocation.enabled=true \
-      --conf spark.shuffle.service.enabled=true \
-      --conf spark.kubernetes.shuffle.namespace=default \
-      --conf spark.kubernetes.shuffle.labels="app=spark-shuffle-service,spark-version=2.1.0" \
-      examples/jars/spark_examples_2.11-2.2.0.jar 10 400000 2
 
 ## Current Limitations
 
diff --git a/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.deploy.kubernetes.submit.v1.DriverServiceManager b/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.deploy.kubernetes.submit.v1.DriverServiceManager
deleted file mode 100644
index 2ed0387c51bc6..0000000000000
--- a/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.deploy.kubernetes.submit.v1.DriverServiceManager
+++ /dev/null
@@ -1,2 +0,0 @@
-org.apache.spark.deploy.kubernetes.submit.v1.ExternalSuppliedUrisDriverServiceManager
-org.apache.spark.deploy.kubernetes.submit.v1.NodePortUrisDriverServiceManager
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/CompressionUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/CompressionUtils.scala
index 03991ba26a6f7..a6f0ca502f6f0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/CompressionUtils.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/CompressionUtils.scala
@@ -16,19 +16,17 @@
  */
 package org.apache.spark.deploy.kubernetes
 
-import java.io.{ByteArrayInputStream, File, FileInputStream, FileOutputStream, InputStream, OutputStream}
+import java.io.{File, FileInputStream, FileOutputStream, InputStream, OutputStream}
 import java.util.zip.{GZIPInputStream, GZIPOutputStream}
 
 import com.google.common.io.Files
-import org.apache.commons.codec.binary.Base64
 import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveInputStream, TarArchiveOutputStream}
 import org.apache.commons.compress.utils.CharsetNames
 import org.apache.commons.io.IOUtils
 import scala.collection.mutable
 
-import org.apache.spark.deploy.rest.kubernetes.v1.TarGzippedData
 import org.apache.spark.internal.Logging
-import org.apache.spark.util.{ByteBufferOutputStream, Utils}
+import org.apache.spark.util.Utils
 
 private[spark] object CompressionUtils extends Logging {
   // Defaults from TarArchiveOutputStream
@@ -36,30 +34,6 @@ private[spark] object CompressionUtils extends Logging {
   private val RECORD_SIZE = 512
   private val ENCODING = CharsetNames.UTF_8
 
-  /**
-   * Compresses all of the given paths into a gzipped-tar archive, returning the compressed data in
-   * memory as an instance of {@link TarGzippedData}. The files are taken without consideration to
-   * their original folder structure, and are added to the tar archive in a flat hierarchy.
-   * Directories are not allowed, and duplicate file names are de-duplicated by appending a numeric
-   * suffix to the file name, before the file extension. For example, if paths a/b.txt and b/b.txt
-   * were provided, then the files added to the tar archive would be b.txt and b-1.txt.
-   * @param paths A list of file paths to be archived
-   * @return An in-memory representation of the compressed data.
-   */
-  def createTarGzip(paths: Iterable[String]): TarGzippedData = {
-    val compressedBytesStream = Utils.tryWithResource(new ByteBufferOutputStream()) { raw =>
-      writeTarGzipToStream(raw, paths)
-      raw
-    }
-    val compressedAsBase64 = Base64.encodeBase64String(compressedBytesStream.toByteBuffer.array)
-    TarGzippedData(
-      dataBase64 = compressedAsBase64,
-      blockSize = BLOCK_SIZE,
-      recordSize = RECORD_SIZE,
-      encoding = ENCODING
-    )
-  }
-
   def writeTarGzipToStream(outputStream: OutputStream, paths: Iterable[String]): Unit = {
     Utils.tryWithResource(new GZIPOutputStream(outputStream)) { gzipping =>
       Utils.tryWithResource(new TarArchiveOutputStream(
@@ -98,50 +72,14 @@ private[spark] object CompressionUtils extends Logging {
     }
   }
 
-  /**
-   * Decompresses the provided tar archive to a directory.
-   * @param compressedData In-memory representation of the compressed data, ideally created via
-   *                       {@link createTarGzip}.
-   * @param rootOutputDir  Directory to write the output files to. All files from the tarball
-   *                       are written here in a flat hierarchy.
-   * @return List of file paths for each file that was unpacked from the archive.
-   */
-  def unpackAndWriteCompressedFiles(
-      compressedData: TarGzippedData,
-      rootOutputDir: File): Seq[String] = {
-    val compressedBytes = Base64.decodeBase64(compressedData.dataBase64)
-    if (!rootOutputDir.exists) {
-      if (!rootOutputDir.mkdirs) {
-        throw new IllegalStateException(s"Failed to create output directory for unpacking" +
-          s" files at ${rootOutputDir.getAbsolutePath}")
-      }
-    } else if (rootOutputDir.isFile) {
-      throw new IllegalArgumentException(s"Root dir for writing decompressed files: " +
-         s"${rootOutputDir.getAbsolutePath} exists and is not a directory.")
-    }
-    Utils.tryWithResource(new ByteArrayInputStream(compressedBytes)) { compressedBytesStream =>
-      unpackTarStreamToDirectory(
-        compressedBytesStream,
-        rootOutputDir,
-        compressedData.blockSize,
-        compressedData.recordSize,
-        compressedData.encoding)
-    }
-  }
-
-  def unpackTarStreamToDirectory(
-      inputStream: InputStream,
-      outputDir: File,
-      blockSize: Int = BLOCK_SIZE,
-      recordSize: Int = RECORD_SIZE,
-      encoding: String = ENCODING): Seq[String] = {
+  def unpackTarStreamToDirectory(inputStream: InputStream, outputDir: File): Seq[String] = {
     val paths = mutable.Buffer.empty[String]
     Utils.tryWithResource(new GZIPInputStream(inputStream)) { gzipped =>
       Utils.tryWithResource(new TarArchiveInputStream(
           gzipped,
-          blockSize,
-          recordSize,
-          encoding)) { tarInputStream =>
+          BLOCK_SIZE,
+          RECORD_SIZE,
+          ENCODING)) { tarInputStream =>
         var nextTarEntry = tarInputStream.getNextTarEntry
         while (nextTarEntry != null) {
           val outputFile = new File(outputDir, nextTarEntry.getName)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/SparkPodInitContainerBootstrap.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/SparkPodInitContainerBootstrap.scala
index 227420db4636d..0d4e82566643d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/SparkPodInitContainerBootstrap.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/SparkPodInitContainerBootstrap.scala
@@ -19,7 +19,7 @@ package org.apache.spark.deploy.kubernetes
 import io.fabric8.kubernetes.api.model.{ContainerBuilder, EmptyDirVolumeSource, PodBuilder, VolumeMount, VolumeMountBuilder}
 
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.v2.{ContainerNameEqualityPredicate, InitContainerUtil}
+import org.apache.spark.deploy.kubernetes.submit.{ContainerNameEqualityPredicate, InitContainerUtil}
 
 private[spark] trait SparkPodInitContainerBootstrap {
   /**
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
index 759a7df505829..bcb9a96cae960 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/config.scala
@@ -20,7 +20,6 @@ import java.util.concurrent.TimeUnit
 
 import org.apache.spark.{SPARK_VERSION => sparkVersion}
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.v1.NodePortUrisDriverServiceManager
 import org.apache.spark.internal.Logging
 import org.apache.spark.internal.config.ConfigBuilder
 import org.apache.spark.network.util.ByteUnit
@@ -212,77 +211,6 @@ package object config extends Logging {
       .stringConf
       .createOptional
 
-  private[spark] val KUBERNETES_DRIVER_SUBMIT_TIMEOUT =
-    ConfigBuilder("spark.kubernetes.driverSubmissionTimeout")
-      .doc("Time to wait for the driver process to start running before aborting its execution.")
-      .timeConf(TimeUnit.SECONDS)
-      .createWithDefault(60L)
-
-  private[spark] val KUBERNETES_DRIVER_SUBMIT_SSL_KEYSTORE =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.keyStore")
-      .doc("KeyStore file for the driver submission server listening on SSL. Can be pre-mounted" +
-        " on the driver container or uploaded from the submitting client.")
-      .stringConf
-      .createOptional
-
-  private[spark] val KUBERNETES_DRIVER_SUBMIT_SSL_TRUSTSTORE =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.trustStore")
-      .doc("TrustStore containing certificates for communicating to the driver submission server" +
-        " over SSL.")
-      .stringConf
-      .createOptional
-
-  private[spark] val DRIVER_SUBMIT_SSL_ENABLED =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.enabled")
-      .doc("Whether or not to use SSL when sending the application dependencies to the driver pod.")
-      .booleanConf
-      .createWithDefault(false)
-
-  private[spark] val DRIVER_SUBMIT_SSL_KEY_PEM =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.keyPem")
-      .doc("Key PEM file that the driver submission server will use when setting up TLS" +
-        " connections. Can be pre-mounted on the driver pod's disk or uploaded from the" +
-        " submitting client's machine.")
-      .stringConf
-      .createOptional
-
-  private[spark] val DRIVER_SUBMIT_SSL_SERVER_CERT_PEM =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.serverCertPem")
-      .doc("Certificate PEM file that is associated with the key PEM file" +
-        " the submission server uses to set up TLS connections. Can be pre-mounted" +
-        " on the driver pod's disk or uploaded from the submitting client's machine.")
-      .stringConf
-      .createOptional
-
-  private[spark] val DRIVER_SUBMIT_SSL_CLIENT_CERT_PEM =
-    ConfigBuilder("spark.ssl.kubernetes.driversubmitserver.clientCertPem")
-      .doc("Certificate pem file that the submission client uses to connect to the submission" +
-        " server over TLS. This should often be the same as the server certificate, but can be" +
-        " different if the submission client will contact the driver through a proxy instead of" +
-        " the driver service directly.")
-      .stringConf
-      .createOptional
-
-  private[spark] val KUBERNETES_DRIVER_SERVICE_NAME =
-    ConfigBuilder("spark.kubernetes.driver.service.name")
-        .doc("Kubernetes service that exposes the driver pod for external access.")
-        .internal()
-        .stringConf
-        .createOptional
-
-  private[spark] val KUBERNETES_DRIVER_SUBMIT_SERVER_MEMORY =
-    ConfigBuilder("spark.kubernetes.driver.submissionServerMemory")
-      .doc("The amount of memory to allocate for the driver submission server.")
-      .bytesConf(ByteUnit.MiB)
-      .createWithDefaultString("256m")
-
-  private[spark] val EXPOSE_KUBERNETES_DRIVER_SERVICE_UI_PORT =
-    ConfigBuilder("spark.kubernetes.driver.service.exposeUiPort")
-      .doc("Whether to expose the driver Web UI port as a service NodePort. Turned off by default" +
-        " because NodePort is a limited resource. Use alternatives if possible.")
-      .booleanConf
-      .createWithDefault(false)
-
   private[spark] val KUBERNETES_DRIVER_POD_NAME =
     ConfigBuilder("spark.kubernetes.driver.pod.name")
       .doc("Name of the driver pod.")
@@ -327,13 +255,6 @@ package object config extends Logging {
       .longConf
       .createWithDefault(1)
 
-  private[spark] val DRIVER_SERVICE_MANAGER_TYPE =
-    ConfigBuilder("spark.kubernetes.driver.serviceManagerType")
-      .doc("A tag indicating which class to use for creating the Kubernetes service and" +
-        " determining its URI for the submission client.")
-      .stringConf
-      .createWithDefault(NodePortUrisDriverServiceManager.TYPE)
-
   private[spark] val WAIT_FOR_APP_COMPLETION =
     ConfigBuilder("spark.kubernetes.submission.waitAppCompletion")
       .doc("In cluster mode, whether to wait for the application to finish before exiting the" +
@@ -347,8 +268,7 @@ package object config extends Logging {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
-  // Spark dependency server for submission v2
-
+  // Spark resource staging server.
   private[spark] val RESOURCE_STAGING_SERVER_PORT =
     ConfigBuilder("spark.kubernetes.resourceStagingServer.port")
       .doc("Port for the Kubernetes resource staging server to listen on.")
@@ -451,7 +371,7 @@ package object config extends Logging {
       .stringConf
       .createOptional
 
-  // Driver and Init-Container parameters for submission v2
+  // Driver and Init-Container parameters
   private[spark] val RESOURCE_STAGING_SERVER_URI =
     ConfigBuilder("spark.kubernetes.resourceStagingServer.uri")
       .doc("Base URI for the Spark resource staging server.")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/Client.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/Client.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
index e4ca5c1458abe..bfb0bc3ffb0f3 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/Client.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/Client.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 import java.util.Collections
@@ -25,8 +25,7 @@ import scala.collection.JavaConverters._
 import org.apache.spark.{SparkConf, SparkException}
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.{LoggingPodStatusWatcher, LoggingPodStatusWatcherImpl}
-import org.apache.spark.deploy.rest.kubernetes.v2.ResourceStagingServerSslOptionsProviderImpl
+import org.apache.spark.deploy.rest.kubernetes.ResourceStagingServerSslOptionsProviderImpl
 import org.apache.spark.internal.Logging
 import org.apache.spark.launcher.SparkLauncher
 import org.apache.spark.util.Utils
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolver.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolver.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolver.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolver.scala
index 5505d87fa8072..c635484c4c124 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolver.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolver.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerNameEqualityPredicate.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerNameEqualityPredicate.scala
similarity index 95%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerNameEqualityPredicate.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerNameEqualityPredicate.scala
index 5101e1506e4d5..434919208ba2e 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerNameEqualityPredicate.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ContainerNameEqualityPredicate.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.lang.Boolean
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverInitContainerComponentsProvider.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverInitContainerComponentsProvider.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverInitContainerComponentsProvider.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverInitContainerComponentsProvider.scala
index 0a5e6cd216011..7fbb0c9274bf5 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverInitContainerComponentsProvider.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverInitContainerComponentsProvider.scala
@@ -14,13 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.{SparkConf, SSLOptions}
 import org.apache.spark.deploy.kubernetes.{InitContainerResourceStagingServerSecretPluginImpl, OptionRequirements, SparkPodInitContainerBootstrap, SparkPodInitContainerBootstrapImpl}
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.rest.kubernetes.v2.RetrofitClientFactoryImpl
+import org.apache.spark.deploy.rest.kubernetes.RetrofitClientFactoryImpl
 import org.apache.spark.util.Utils
 
 /**
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounter.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounter.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounter.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounter.scala
index 9759669335774..ded0237732ce0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounter.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounter.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import io.fabric8.kubernetes.api.model.{PodBuilder, Secret, SecretBuilder}
 import scala.collection.JavaConverters._
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterProvider.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterProvider.scala
similarity index 92%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterProvider.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterProvider.scala
index e981c54d23a9d..3f0e7d97275a5 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterProvider.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterProvider.scala
@@ -14,11 +14,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.submit.DriverPodKubernetesCredentialsProvider
 
 private[spark] trait DriverPodKubernetesCredentialsMounterProvider {
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfiguration.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfiguration.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfiguration.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfiguration.scala
index adfdc060f0d0f..2292365995d1f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfiguration.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfiguration.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.kubernetes.config._
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/InitContainerUtil.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/InitContainerUtil.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/InitContainerUtil.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/InitContainerUtil.scala
index 0526ca53baaab..9b7faaa78a9aa 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/InitContainerUtil.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/InitContainerUtil.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/PropertiesConfigMapFromScalaMapBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/PropertiesConfigMapFromScalaMapBuilder.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/PropertiesConfigMapFromScalaMapBuilder.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/PropertiesConfigMapFromScalaMapBuilder.scala
index cb9194552d2b6..8103272c27518 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/PropertiesConfigMapFromScalaMapBuilder.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/PropertiesConfigMapFromScalaMapBuilder.scala
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.StringWriter
 import java.util.Properties
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilder.scala
similarity index 95%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilder.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilder.scala
index 362fbbdf517dc..4062a3113eddf 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilder.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilder.scala
@@ -14,12 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import io.fabric8.kubernetes.api.model.ConfigMap
 
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.submit.KubernetesFileUtils
 
 private[spark] trait SparkInitContainerConfigMapBuilder {
   /**
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmissionKubernetesClientProvider.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmissionKubernetesClientProvider.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmissionKubernetesClientProvider.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmissionKubernetesClientProvider.scala
index af3de6ce85026..17b61d4a6ace0 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmissionKubernetesClientProvider.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmissionKubernetesClientProvider.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient, KubernetesClient}
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPlugin.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPlugin.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPlugin.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPlugin.scala
index 1b086e60d3d0d..06d3648efb89f 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPlugin.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPlugin.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.SparkException
 import org.apache.spark.deploy.kubernetes.config._
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilder.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilder.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilder.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilder.scala
index 1a33757e45aa0..7850853df97e6 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilder.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilder.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderImpl.scala
similarity index 95%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderImpl.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderImpl.scala
index 5f98facfb691f..9d0d863d174bc 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderImpl.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.{File, FileOutputStream}
 import javax.ws.rs.core.MediaType
@@ -26,8 +26,7 @@ import retrofit2.Call
 
 import org.apache.spark.{SparkException, SSLOptions}
 import org.apache.spark.deploy.kubernetes.{CompressionUtils, KubernetesCredentials}
-import org.apache.spark.deploy.kubernetes.submit.KubernetesFileUtils
-import org.apache.spark.deploy.rest.kubernetes.v2.{ResourceStagingServiceRetrofit, RetrofitClientFactory}
+import org.apache.spark.deploy.rest.kubernetes.{ResourceStagingServiceRetrofit, RetrofitClientFactory}
 import org.apache.spark.util.Utils
 
 private[spark] trait SubmittedDependencyUploader {
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedResources.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedResources.scala
similarity index 96%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedResources.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedResources.scala
index f4e5e991180ce..225972c1057f2 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedResources.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedResources.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 case class SubmittedResourceIdAndSecret(resourceId: String, resourceSecret: String)
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/Client.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/Client.scala
deleted file mode 100644
index 32fc434cb693a..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/Client.scala
+++ /dev/null
@@ -1,743 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import java.io.File
-import java.security.SecureRandom
-import java.util.ServiceLoader
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-
-import com.google.common.io.Files
-import com.google.common.util.concurrent.SettableFuture
-import io.fabric8.kubernetes.api.model._
-import io.fabric8.kubernetes.client.{ConfigBuilder => K8SConfigBuilder, DefaultKubernetesClient, KubernetesClient, KubernetesClientException, Watcher}
-import io.fabric8.kubernetes.client.Watcher.Action
-import org.apache.commons.codec.binary.Base64
-import scala.collection.JavaConverters._
-
-import org.apache.spark.{SparkConf, SparkException}
-import org.apache.spark.deploy.kubernetes.{CompressionUtils, KubernetesCredentials}
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.{DriverPodKubernetesCredentialsProvider, KubernetesFileUtils, LoggingPodStatusWatcherImpl}
-import org.apache.spark.deploy.rest.kubernetes.v1.{AppResource, ContainerAppResource, HttpClientUtil, KubernetesCreateSubmissionRequest, KubernetesSparkRestApi, RemoteAppResource, UploadedAppResource}
-import org.apache.spark.internal.Logging
-import org.apache.spark.util.{ShutdownHookManager, Utils}
-
-private[spark] class Client(
-    sparkConf: SparkConf,
-    mainClass: String,
-    mainAppResource: String,
-    appArgs: Array[String]) extends Logging {
-  import Client._
-
-  private val namespace = sparkConf.get(KUBERNETES_NAMESPACE)
-  private val master = resolveK8sMaster(sparkConf.get("spark.master"))
-
-  private val launchTime = System.currentTimeMillis
-  private val appName = sparkConf.getOption("spark.app.name")
-    .getOrElse("spark")
-  private val kubernetesAppId = s"$appName-$launchTime".toLowerCase.replaceAll("\\.", "-")
-  private val kubernetesDriverPodName = sparkConf.get(KUBERNETES_DRIVER_POD_NAME)
-    .getOrElse(kubernetesAppId)
-  private val secretName = s"$SUBMISSION_APP_SECRET_PREFIX-$kubernetesAppId"
-  private val secretDirectory = s"$DRIVER_CONTAINER_SUBMISSION_SECRETS_BASE_DIR/$kubernetesAppId"
-  private val driverDockerImage = sparkConf.get(DRIVER_DOCKER_IMAGE)
-  private val uiPort = sparkConf.getInt("spark.ui.port", DEFAULT_UI_PORT)
-  private val driverSubmitTimeoutSecs = sparkConf.get(KUBERNETES_DRIVER_SUBMIT_TIMEOUT)
-  private val driverServiceManagerType = sparkConf.get(DRIVER_SERVICE_MANAGER_TYPE)
-  private val sparkFiles = sparkConf.getOption("spark.files")
-    .map(_.split(","))
-    .getOrElse(Array.empty[String])
-  private val sparkJars = sparkConf.getOption("spark.jars")
-    .map(_.split(","))
-    .getOrElse(Array.empty[String])
-
-  // CPU settings
-  private val driverCpuCores = sparkConf.getOption("spark.driver.cores").getOrElse("1")
-
-  // Memory settings
-  private val driverMemoryMb = sparkConf.get(org.apache.spark.internal.config.DRIVER_MEMORY)
-  private val driverSubmitServerMemoryMb = sparkConf.get(KUBERNETES_DRIVER_SUBMIT_SERVER_MEMORY)
-  private val driverSubmitServerMemoryString = sparkConf.get(
-    KUBERNETES_DRIVER_SUBMIT_SERVER_MEMORY.key,
-    KUBERNETES_DRIVER_SUBMIT_SERVER_MEMORY.defaultValueString)
-  private val driverContainerMemoryMb = driverMemoryMb + driverSubmitServerMemoryMb
-  private val memoryOverheadMb = sparkConf
-    .get(KUBERNETES_DRIVER_MEMORY_OVERHEAD)
-    .getOrElse(math.max((MEMORY_OVERHEAD_FACTOR * driverContainerMemoryMb).toInt,
-      MEMORY_OVERHEAD_MIN))
-  private val driverContainerMemoryWithOverhead = driverContainerMemoryMb + memoryOverheadMb
-
-  private val waitForAppCompletion = sparkConf.get(WAIT_FOR_APP_COMPLETION)
-  private val loggingInterval = Some(sparkConf.get(REPORT_INTERVAL))
-    .filter( _ => waitForAppCompletion)
-
-  private val secretBase64String = {
-    val secretBytes = new Array[Byte](128)
-    SECURE_RANDOM.nextBytes(secretBytes)
-    Base64.encodeBase64String(secretBytes)
-  }
-
-  private val serviceAccount = sparkConf.get(KUBERNETES_SERVICE_ACCOUNT_NAME)
-  private val customLabels = sparkConf.get(KUBERNETES_DRIVER_LABELS)
-  private val customAnnotations = sparkConf.get(KUBERNETES_DRIVER_ANNOTATIONS)
-
-  private val kubernetesResourceCleaner = new KubernetesResourceCleaner
-
-  def run(): Unit = {
-    logInfo(s"Starting application $kubernetesAppId in Kubernetes...")
-    val submitterLocalFiles = KubernetesFileUtils.getOnlySubmitterLocalFiles(sparkFiles)
-    val submitterLocalJars = KubernetesFileUtils.getOnlySubmitterLocalFiles(sparkJars)
-    (submitterLocalFiles ++ submitterLocalJars).foreach { file =>
-      if (!new File(Utils.resolveURI(file).getPath).isFile) {
-        throw new SparkException(s"File $file does not exist or is a directory.")
-      }
-    }
-    if (KubernetesFileUtils.isUriLocalFile(mainAppResource) &&
-        !new File(Utils.resolveURI(mainAppResource).getPath).isFile) {
-      throw new SparkException(s"Main app resource file $mainAppResource is not a file or" +
-        s" is a directory.")
-    }
-    val driverServiceManager = getDriverServiceManager
-    val parsedCustomLabels = parseKeyValuePairs(customLabels, KUBERNETES_DRIVER_LABELS.key,
-      "labels")
-    parsedCustomLabels.keys.foreach { key =>
-      require(key != SPARK_APP_ID_LABEL, "Label with key" +
-        s" $SPARK_APP_ID_LABEL cannot be used in" +
-        " spark.kubernetes.driver.labels, as it is reserved for Spark's" +
-        " internal configuration.")
-    }
-    val parsedCustomAnnotations = parseKeyValuePairs(
-      customAnnotations,
-      KUBERNETES_DRIVER_ANNOTATIONS.key,
-      "annotations")
-    val driverPodKubernetesCredentials = new DriverPodKubernetesCredentialsProvider(sparkConf).get()
-    var k8ConfBuilder = new K8SConfigBuilder()
-      .withApiVersion("v1")
-      .withMasterUrl(master)
-      .withNamespace(namespace)
-    sparkConf.get(KUBERNETES_SUBMIT_CA_CERT_FILE).foreach {
-      f => k8ConfBuilder = k8ConfBuilder.withCaCertFile(f)
-    }
-    sparkConf.get(KUBERNETES_SUBMIT_CLIENT_KEY_FILE).foreach {
-      f => k8ConfBuilder = k8ConfBuilder.withClientKeyFile(f)
-    }
-    sparkConf.get(KUBERNETES_SUBMIT_CLIENT_CERT_FILE).foreach {
-      f => k8ConfBuilder = k8ConfBuilder.withClientCertFile(f)
-    }
-    sparkConf.get(KUBERNETES_SUBMIT_OAUTH_TOKEN).foreach { token =>
-      k8ConfBuilder = k8ConfBuilder.withOauthToken(token)
-    }
-
-    val k8ClientConfig = k8ConfBuilder.build
-    Utils.tryWithResource(new DefaultKubernetesClient(k8ClientConfig)) { kubernetesClient =>
-      driverServiceManager.start(kubernetesClient, kubernetesAppId, sparkConf)
-      // start outer watch for status logging of driver pod
-      // only enable interval logging if in waitForAppCompletion mode
-      val loggingWatch = new LoggingPodStatusWatcherImpl(
-        kubernetesAppId, loggingInterval)
-      Utils.tryWithResource(kubernetesClient
-          .pods()
-          .withName(kubernetesDriverPodName)
-          .watch(loggingWatch)) { _ =>
-        loggingWatch.start()
-        val resourceCleanShutdownHook = ShutdownHookManager.addShutdownHook(() =>
-          kubernetesResourceCleaner.deleteAllRegisteredResourcesFromKubernetes(kubernetesClient))
-        val cleanupServiceManagerHook = ShutdownHookManager.addShutdownHook(
-          ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY)(
-          () => driverServiceManager.stop())
-        // Place the error hook at a higher priority in order for the error hook to run before
-        // the stop hook.
-        val serviceManagerErrorHook = ShutdownHookManager.addShutdownHook(
-          ShutdownHookManager.DEFAULT_SHUTDOWN_PRIORITY + 1)(() =>
-          driverServiceManager.handleSubmissionError(
-            new SparkException("Submission shutting down early...")))
-        try {
-          val sslConfigurationProvider = new DriverSubmitSslConfigurationProvider(
-            sparkConf, kubernetesAppId, kubernetesClient, kubernetesResourceCleaner)
-          val submitServerSecret = kubernetesClient.secrets().createNew()
-            .withNewMetadata()
-            .withName(secretName)
-            .endMetadata()
-            .withData(Map((SUBMISSION_APP_SECRET_NAME, secretBase64String)).asJava)
-            .withType("Opaque")
-            .done()
-          kubernetesResourceCleaner.registerOrUpdateResource(submitServerSecret)
-          val sslConfiguration = sslConfigurationProvider.getSslConfiguration()
-          val (driverPod, driverService) = launchDriverKubernetesComponents(
-            kubernetesClient,
-            driverServiceManager,
-            parsedCustomLabels,
-            parsedCustomAnnotations,
-            submitServerSecret,
-            sslConfiguration)
-          configureOwnerReferences(
-            kubernetesClient,
-            submitServerSecret,
-            sslConfiguration.sslSecret,
-            driverPod,
-            driverService)
-          submitApplicationToDriverServer(
-            kubernetesClient,
-            driverServiceManager,
-            sslConfiguration,
-            driverService,
-            submitterLocalFiles,
-            submitterLocalJars,
-            driverPodKubernetesCredentials)
-          // Now that the application has started, persist the components that were created beyond
-          // the shutdown hook. We still want to purge the one-time secrets, so do not unregister
-          // those.
-          kubernetesResourceCleaner.unregisterResource(driverPod)
-          kubernetesResourceCleaner.unregisterResource(driverService)
-        } catch {
-          case e: Throwable =>
-            driverServiceManager.handleSubmissionError(e)
-            throw e
-        } finally {
-          Utils.tryLogNonFatalError {
-            kubernetesResourceCleaner.deleteAllRegisteredResourcesFromKubernetes(kubernetesClient)
-          }
-          Utils.tryLogNonFatalError {
-            driverServiceManager.stop()
-          }
-          // Remove the shutdown hooks that would be redundant
-          Utils.tryLogNonFatalError {
-            ShutdownHookManager.removeShutdownHook(resourceCleanShutdownHook)
-          }
-          Utils.tryLogNonFatalError {
-            ShutdownHookManager.removeShutdownHook(cleanupServiceManagerHook)
-          }
-          Utils.tryLogNonFatalError {
-            ShutdownHookManager.removeShutdownHook(serviceManagerErrorHook)
-          }
-        }
-        // wait if configured to do so
-        if (waitForAppCompletion) {
-          logInfo(s"Waiting for application $kubernetesAppId to finish...")
-          loggingWatch.awaitCompletion()
-          logInfo(s"Application $kubernetesAppId finished.")
-        } else {
-          logInfo(s"Application $kubernetesAppId successfully launched.")
-        }
-      }
-    }
-  }
-
-  private def submitApplicationToDriverServer(
-      kubernetesClient: KubernetesClient,
-      driverServiceManager: DriverServiceManager,
-      sslConfiguration: DriverSubmitSslConfiguration,
-      driverService: Service,
-      submitterLocalFiles: Iterable[String],
-      submitterLocalJars: Iterable[String],
-      driverPodKubernetesCredentials: KubernetesCredentials): Unit = {
-    sparkConf.getOption("spark.app.id").foreach { id =>
-      logWarning(s"Warning: Provided app id in spark.app.id as $id will be" +
-        s" overridden as $kubernetesAppId")
-    }
-    sparkConf.setIfMissing(KUBERNETES_DRIVER_POD_NAME, kubernetesDriverPodName)
-    sparkConf.set(KUBERNETES_DRIVER_SERVICE_NAME, driverService.getMetadata.getName)
-    sparkConf.set("spark.app.id", kubernetesAppId)
-    sparkConf.setIfMissing("spark.app.name", appName)
-    sparkConf.setIfMissing("spark.driver.port", DEFAULT_DRIVER_PORT.toString)
-    sparkConf.setIfMissing("spark.driver.blockManager.port", DEFAULT_BLOCKMANAGER_PORT.toString)
-    sparkConf.setIfMissing("spark.blockManager.port", DEFAULT_BLOCKMANAGER_PORT.toString)
-    sparkConf.get(KUBERNETES_SUBMIT_OAUTH_TOKEN).foreach { _ =>
-      sparkConf.set(KUBERNETES_SUBMIT_OAUTH_TOKEN, "<present_but_redacted>")
-    }
-    sparkConf.get(KUBERNETES_DRIVER_OAUTH_TOKEN).foreach { _ =>
-      sparkConf.set(KUBERNETES_DRIVER_OAUTH_TOKEN, "<present_but_redacted>")
-    }
-    val driverSubmitter = buildDriverSubmissionClient(
-      kubernetesClient,
-      driverServiceManager,
-      driverService,
-      sslConfiguration)
-    // Sanity check to see if the driver submitter is even reachable.
-    driverSubmitter.ping()
-    logInfo(s"Submitting local resources to driver pod for application " +
-      s"$kubernetesAppId ...")
-    val submitRequest = buildSubmissionRequest(
-      submitterLocalFiles,
-      submitterLocalJars,
-      driverPodKubernetesCredentials)
-    driverSubmitter.submitApplication(submitRequest)
-    logInfo("Successfully submitted local resources and driver configuration to" +
-      " driver pod.")
-    // After submitting, adjust the service to only expose the Spark UI
-    val uiServiceType = if (sparkConf.get(EXPOSE_KUBERNETES_DRIVER_SERVICE_UI_PORT)) "NodePort"
-      else "ClusterIP"
-    val uiServicePort = new ServicePortBuilder()
-      .withName(UI_PORT_NAME)
-      .withPort(uiPort)
-      .withNewTargetPort(uiPort)
-      .build()
-    val resolvedService = kubernetesClient.services().withName(kubernetesAppId).edit()
-      .editSpec()
-        .withType(uiServiceType)
-        .withPorts(uiServicePort)
-        .endSpec()
-      .done()
-    kubernetesResourceCleaner.registerOrUpdateResource(resolvedService)
-    logInfo("Finished submitting application to Kubernetes.")
-  }
-
-  private def launchDriverKubernetesComponents(
-      kubernetesClient: KubernetesClient,
-      driverServiceManager: DriverServiceManager,
-      customLabels: Map[String, String],
-      customAnnotations: Map[String, String],
-      submitServerSecret: Secret,
-      sslConfiguration: DriverSubmitSslConfiguration): (Pod, Service) = {
-    val driverKubernetesSelectors = (Map(
-      SPARK_DRIVER_LABEL -> kubernetesAppId,
-      SPARK_APP_ID_LABEL -> kubernetesAppId,
-      SPARK_APP_NAME_LABEL -> appName)
-      ++ customLabels)
-    val endpointsReadyFuture = SettableFuture.create[Endpoints]
-    val endpointsReadyWatcher = new DriverEndpointsReadyWatcher(endpointsReadyFuture)
-    val serviceReadyFuture = SettableFuture.create[Service]
-    val serviceReadyWatcher = new DriverServiceReadyWatcher(serviceReadyFuture)
-    val podReadyFuture = SettableFuture.create[Pod]
-    val podWatcher = new DriverPodReadyWatcher(podReadyFuture)
-    Utils.tryWithResource(kubernetesClient
-        .pods()
-        .withName(kubernetesDriverPodName)
-        .watch(podWatcher)) { _ =>
-      Utils.tryWithResource(kubernetesClient
-          .services()
-          .withName(kubernetesAppId)
-          .watch(serviceReadyWatcher)) { _ =>
-        Utils.tryWithResource(kubernetesClient
-            .endpoints()
-            .withName(kubernetesAppId)
-            .watch(endpointsReadyWatcher)) { _ =>
-          val serviceTemplate = createDriverServiceTemplate(driverKubernetesSelectors)
-          val driverService = kubernetesClient.services().create(
-            driverServiceManager.customizeDriverService(serviceTemplate).build())
-          kubernetesResourceCleaner.registerOrUpdateResource(driverService)
-          val driverPod = createDriverPod(
-            kubernetesClient,
-            driverKubernetesSelectors,
-            customAnnotations,
-            submitServerSecret,
-            sslConfiguration)
-          waitForReadyKubernetesComponents(kubernetesClient, endpointsReadyFuture,
-            serviceReadyFuture, podReadyFuture)
-          (driverPod, driverService)
-        }
-      }
-    }
-  }
-
-  /**
-   * Sets the owner reference for all the kubernetes components to link to the driver pod.
-   *
-   * @return The driver service after it has been adjusted to reflect the new owner
-   * reference.
-   */
-  private def configureOwnerReferences(
-      kubernetesClient: KubernetesClient,
-      submitServerSecret: Secret,
-      sslSecret: Option[Secret],
-      driverPod: Pod,
-      driverService: Service): Service = {
-    val driverPodOwnerRef = new OwnerReferenceBuilder()
-      .withName(driverPod.getMetadata.getName)
-      .withUid(driverPod.getMetadata.getUid)
-      .withApiVersion(driverPod.getApiVersion)
-      .withKind(driverPod.getKind)
-      .withController(true)
-      .build()
-    sslSecret.foreach(secret => {
-      val updatedSecret = kubernetesClient.secrets().withName(secret.getMetadata.getName).edit()
-        .editMetadata()
-        .addToOwnerReferences(driverPodOwnerRef)
-        .endMetadata()
-        .done()
-      kubernetesResourceCleaner.registerOrUpdateResource(updatedSecret)
-    })
-    val updatedSubmitServerSecret = kubernetesClient
-      .secrets()
-      .withName(submitServerSecret.getMetadata.getName)
-      .edit()
-        .editMetadata()
-          .addToOwnerReferences(driverPodOwnerRef)
-          .endMetadata()
-        .done()
-    kubernetesResourceCleaner.registerOrUpdateResource(updatedSubmitServerSecret)
-    val updatedService = kubernetesClient
-      .services()
-      .withName(driverService.getMetadata.getName)
-      .edit()
-        .editMetadata()
-          .addToOwnerReferences(driverPodOwnerRef)
-          .endMetadata()
-        .done()
-    kubernetesResourceCleaner.registerOrUpdateResource(updatedService)
-    updatedService
-  }
-
-  private def waitForReadyKubernetesComponents(
-      kubernetesClient: KubernetesClient,
-      endpointsReadyFuture: SettableFuture[Endpoints],
-      serviceReadyFuture: SettableFuture[Service],
-      podReadyFuture: SettableFuture[Pod]) = {
-    try {
-      podReadyFuture.get(driverSubmitTimeoutSecs, TimeUnit.SECONDS)
-      logInfo("Driver pod successfully created in Kubernetes cluster.")
-    } catch {
-      case e: Throwable =>
-        val finalErrorMessage: String = buildSubmitFailedErrorMessage(kubernetesClient, e)
-        logError(finalErrorMessage, e)
-        throw new SparkException(finalErrorMessage, e)
-    }
-    try {
-      serviceReadyFuture.get(driverSubmitTimeoutSecs, TimeUnit.SECONDS)
-      logInfo("Driver service created successfully in Kubernetes.")
-    } catch {
-      case e: Throwable =>
-        throw new SparkException(s"The driver service was not ready" +
-          s" in $driverSubmitTimeoutSecs seconds.", e)
-    }
-    try {
-      endpointsReadyFuture.get(driverSubmitTimeoutSecs, TimeUnit.SECONDS)
-      logInfo("Driver endpoints ready to receive application submission")
-    } catch {
-      case e: Throwable =>
-        throw new SparkException(s"The driver service endpoint was not ready" +
-          s" in $driverSubmitTimeoutSecs seconds.", e)
-    }
-  }
-
-  private def createDriverPod(
-      kubernetesClient: KubernetesClient,
-      driverKubernetesSelectors: Map[String, String],
-      customAnnotations: Map[String, String],
-      submitServerSecret: Secret,
-      sslConfiguration: DriverSubmitSslConfiguration): Pod = {
-    val containerPorts = buildContainerPorts()
-    val probePingHttpGet = new HTTPGetActionBuilder()
-      .withScheme(if (sslConfiguration.enabled) "HTTPS" else "HTTP")
-      .withPath("/v1/submissions/ping")
-      .withNewPort(SUBMISSION_SERVER_PORT_NAME)
-      .build()
-    val driverCpuQuantity = new QuantityBuilder(false)
-      .withAmount(driverCpuCores)
-      .build()
-    val driverMemoryQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverContainerMemoryMb}M")
-      .build()
-    val driverMemoryLimitQuantity = new QuantityBuilder(false)
-      .withAmount(s"${driverContainerMemoryWithOverhead}M")
-      .build()
-    val driverPod = kubernetesClient.pods().createNew()
-      .withNewMetadata()
-        .withName(kubernetesDriverPodName)
-        .withLabels(driverKubernetesSelectors.asJava)
-        .withAnnotations(customAnnotations.asJava)
-        .endMetadata()
-      .withNewSpec()
-        .withRestartPolicy("Never")
-        .addNewVolume()
-          .withName(SUBMISSION_APP_SECRET_VOLUME_NAME)
-          .withNewSecret()
-            .withSecretName(submitServerSecret.getMetadata.getName)
-            .endSecret()
-          .endVolume()
-        .addToVolumes(sslConfiguration.sslPodVolume.toSeq: _*)
-        .withServiceAccount(serviceAccount.getOrElse("default"))
-        .addNewContainer()
-          .withName(DRIVER_CONTAINER_NAME)
-          .withImage(driverDockerImage)
-          .withImagePullPolicy("IfNotPresent")
-          .addNewVolumeMount()
-            .withName(SUBMISSION_APP_SECRET_VOLUME_NAME)
-            .withMountPath(secretDirectory)
-            .withReadOnly(true)
-            .endVolumeMount()
-          .addToVolumeMounts(sslConfiguration.sslPodVolumeMount.toSeq: _*)
-          .addNewEnv()
-            .withName(ENV_SUBMISSION_SECRET_LOCATION)
-            .withValue(s"$secretDirectory/$SUBMISSION_APP_SECRET_NAME")
-            .endEnv()
-          .addNewEnv()
-            .withName(ENV_SUBMISSION_SERVER_PORT)
-            .withValue(SUBMISSION_SERVER_PORT.toString)
-            .endEnv()
-          // Note that SPARK_DRIVER_MEMORY only affects the REST server via spark-class.
-          .addNewEnv()
-            .withName(ENV_DRIVER_MEMORY)
-            .withValue(driverSubmitServerMemoryString)
-            .endEnv()
-          .addToEnv(sslConfiguration.sslPodEnvVars: _*)
-          .withNewResources()
-            .addToRequests("cpu", driverCpuQuantity)
-            .addToLimits("cpu", driverCpuQuantity)
-            .addToRequests("memory", driverMemoryQuantity)
-            .addToLimits("memory", driverMemoryLimitQuantity)
-            .endResources()
-          .withPorts(containerPorts.asJava)
-          .withNewReadinessProbe().withHttpGet(probePingHttpGet).endReadinessProbe()
-          .endContainer()
-        .endSpec()
-      .done()
-    kubernetesResourceCleaner.registerOrUpdateResource(driverPod)
-    driverPod
-  }
-
-   private def createDriverServiceTemplate(driverKubernetesSelectors: Map[String, String])
-      : ServiceBuilder = {
-    val driverSubmissionServicePort = new ServicePortBuilder()
-      .withName(SUBMISSION_SERVER_PORT_NAME)
-      .withPort(SUBMISSION_SERVER_PORT)
-      .withNewTargetPort(SUBMISSION_SERVER_PORT)
-      .build()
-    new ServiceBuilder()
-      .withNewMetadata()
-        .withName(kubernetesAppId)
-        .withLabels(driverKubernetesSelectors.asJava)
-        .endMetadata()
-      .withNewSpec()
-        .withSelector(driverKubernetesSelectors.asJava)
-        .withPorts(driverSubmissionServicePort)
-        .endSpec()
-  }
-
-  private class DriverPodReadyWatcher(resolvedDriverPod: SettableFuture[Pod]) extends Watcher[Pod] {
-    override def eventReceived(action: Action, pod: Pod): Unit = {
-      if ((action == Action.ADDED || action == Action.MODIFIED)
-          && pod.getStatus.getPhase == "Running"
-          && !resolvedDriverPod.isDone) {
-        pod.getStatus
-          .getContainerStatuses
-          .asScala
-          .find(status =>
-            status.getName == DRIVER_CONTAINER_NAME && status.getReady)
-          .foreach { _ => resolvedDriverPod.set(pod) }
-      }
-    }
-
-    override def onClose(cause: KubernetesClientException): Unit = {
-      logDebug("Driver pod readiness watch closed.", cause)
-    }
-  }
-
-  private class DriverEndpointsReadyWatcher(resolvedDriverEndpoints: SettableFuture[Endpoints])
-      extends Watcher[Endpoints] {
-    override def eventReceived(action: Action, endpoints: Endpoints): Unit = {
-      if ((action == Action.ADDED || action == Action.MODIFIED)
-          && (endpoints != null)
-          && (endpoints.getSubsets != null)
-          && endpoints.getSubsets.asScala.nonEmpty
-          && endpoints.getSubsets.asScala.exists(_.getAddresses.asScala.nonEmpty)
-          && !resolvedDriverEndpoints.isDone) {
-        resolvedDriverEndpoints.set(endpoints)
-      }
-    }
-
-    override def onClose(cause: KubernetesClientException): Unit = {
-      logDebug("Driver endpoints readiness watch closed.", cause)
-    }
-  }
-
-  private class DriverServiceReadyWatcher(resolvedDriverService: SettableFuture[Service])
-      extends Watcher[Service] {
-    override def eventReceived(action: Action, service: Service): Unit = {
-      if ((action == Action.ADDED || action == Action.MODIFIED)
-          && !resolvedDriverService.isDone) {
-        resolvedDriverService.set(service)
-      }
-    }
-
-    override def onClose(cause: KubernetesClientException): Unit = {
-      logDebug("Driver service readiness watch closed.", cause)
-    }
-  }
-
-  private def buildSubmitFailedErrorMessage(
-      kubernetesClient: KubernetesClient,
-      e: Throwable): String = {
-    val driverPod = try {
-      kubernetesClient.pods().withName(kubernetesDriverPodName).get()
-    } catch {
-      case throwable: Throwable =>
-        logError(s"Timed out while waiting $driverSubmitTimeoutSecs seconds for the" +
-          " driver pod to start, but an error occurred while fetching the driver" +
-          " pod's details.", throwable)
-        throw new SparkException(s"Timed out while waiting $driverSubmitTimeoutSecs" +
-          " seconds for the driver pod to start. Unfortunately, in attempting to fetch" +
-          " the latest state of the pod, another error was thrown. Check the logs for" +
-          " the error that was thrown in looking up the driver pod.", e)
-    }
-    val topLevelMessage = s"The driver pod with name ${driverPod.getMetadata.getName}" +
-      s" in namespace ${driverPod.getMetadata.getNamespace} was not ready in" +
-      s" $driverSubmitTimeoutSecs seconds."
-    val podStatusPhase = if (driverPod.getStatus.getPhase != null) {
-      s"Latest phase from the pod is: ${driverPod.getStatus.getPhase}"
-    } else {
-      "The pod had no final phase."
-    }
-    val podStatusMessage = if (driverPod.getStatus.getMessage != null) {
-      s"Latest message from the pod is: ${driverPod.getStatus.getMessage}"
-    } else {
-      "The pod had no final message."
-    }
-    val failedDriverContainerStatusString = driverPod.getStatus
-      .getContainerStatuses
-      .asScala
-      .find(_.getName == DRIVER_CONTAINER_NAME)
-      .map(status => {
-        val lastState = status.getState
-        if (lastState.getRunning != null) {
-          "Driver container last state: Running\n" +
-            s"Driver container started at: ${lastState.getRunning.getStartedAt}"
-        } else if (lastState.getWaiting != null) {
-          "Driver container last state: Waiting\n" +
-            s"Driver container wait reason: ${lastState.getWaiting.getReason}\n" +
-            s"Driver container message: ${lastState.getWaiting.getMessage}\n"
-        } else if (lastState.getTerminated != null) {
-          "Driver container last state: Terminated\n" +
-            s"Driver container started at: ${lastState.getTerminated.getStartedAt}\n" +
-            s"Driver container finished at: ${lastState.getTerminated.getFinishedAt}\n" +
-            s"Driver container exit reason: ${lastState.getTerminated.getReason}\n" +
-            s"Driver container exit code: ${lastState.getTerminated.getExitCode}\n" +
-            s"Driver container message: ${lastState.getTerminated.getMessage}"
-        } else {
-          "Driver container last state: Unknown"
-        }
-      }).getOrElse("The driver container wasn't found in the pod; expected to find" +
-      s" container with name $DRIVER_CONTAINER_NAME")
-    s"$topLevelMessage\n" +
-      s"$podStatusPhase\n" +
-      s"$podStatusMessage\n\n$failedDriverContainerStatusString"
-  }
-
-  private def buildContainerPorts(): Seq[ContainerPort] = {
-    Seq((DRIVER_PORT_NAME, sparkConf.getInt("spark.driver.port", DEFAULT_DRIVER_PORT)),
-      (BLOCK_MANAGER_PORT_NAME,
-        sparkConf.getInt("spark.blockManager.port", DEFAULT_BLOCKMANAGER_PORT)),
-      (SUBMISSION_SERVER_PORT_NAME, SUBMISSION_SERVER_PORT),
-      (UI_PORT_NAME, uiPort)).map(port => new ContainerPortBuilder()
-        .withName(port._1)
-        .withContainerPort(port._2)
-        .build())
-  }
-
-  private def buildSubmissionRequest(
-      submitterLocalFiles: Iterable[String],
-      submitterLocalJars: Iterable[String],
-      driverPodKubernetesCredentials: KubernetesCredentials): KubernetesCreateSubmissionRequest = {
-    val mainResourceUri = Utils.resolveURI(mainAppResource)
-    val resolvedAppResource: AppResource = Option(mainResourceUri.getScheme)
-        .getOrElse("file") match {
-      case "file" =>
-        val appFile = new File(mainResourceUri.getPath)
-        val fileBytes = Files.toByteArray(appFile)
-        val fileBase64 = Base64.encodeBase64String(fileBytes)
-        UploadedAppResource(resourceBase64Contents = fileBase64, name = appFile.getName)
-      case "local" => ContainerAppResource(mainAppResource)
-      case other => RemoteAppResource(other)
-    }
-    val uploadFilesBase64Contents = CompressionUtils.createTarGzip(submitterLocalFiles.map(
-      Utils.resolveURI(_).getPath))
-    val uploadJarsBase64Contents = CompressionUtils.createTarGzip(submitterLocalJars.map(
-      Utils.resolveURI(_).getPath))
-    KubernetesCreateSubmissionRequest(
-      appResource = resolvedAppResource,
-      mainClass = mainClass,
-      appArgs = appArgs,
-      secret = secretBase64String,
-      sparkProperties = sparkConf.getAll.toMap,
-      uploadedJarsBase64Contents = uploadJarsBase64Contents,
-      uploadedFilesBase64Contents = uploadFilesBase64Contents,
-      driverPodKubernetesCredentials = driverPodKubernetesCredentials)
-  }
-
-  private def buildDriverSubmissionClient(
-      kubernetesClient: KubernetesClient,
-      driverServiceManager: DriverServiceManager,
-      service: Service,
-      sslConfiguration: DriverSubmitSslConfiguration): KubernetesSparkRestApi = {
-    val serviceUris = driverServiceManager.getDriverServiceSubmissionServerUris(service)
-    require(serviceUris.nonEmpty, "No uris found to contact the driver!")
-    HttpClientUtil.createClient[KubernetesSparkRestApi](
-      uris = serviceUris,
-      maxRetriesPerServer = 10,
-      sslSocketFactory = sslConfiguration
-        .driverSubmitClientSslContext
-        .getSocketFactory,
-      trustContext = sslConfiguration
-        .driverSubmitClientTrustManager
-        .orNull,
-      connectTimeoutMillis = 5000)
-  }
-
-  private def parseKeyValuePairs(
-      maybeKeyValues: Option[String],
-      configKey: String,
-      keyValueType: String): Map[String, String] = {
-    maybeKeyValues.map(keyValues => {
-      keyValues.split(",").map(_.trim).filterNot(_.isEmpty).map(keyValue => {
-        keyValue.split("=", 2).toSeq match {
-          case Seq(k, v) =>
-            (k, v)
-          case _ =>
-            throw new SparkException(s"Custom $keyValueType set by $configKey must be a" +
-              s" comma-separated list of key-value pairs, with format <key>=<value>." +
-              s" Got value: $keyValue. All values: $keyValues")
-        }
-      }).toMap
-    }).getOrElse(Map.empty[String, String])
-  }
-
-  private def getDriverServiceManager: DriverServiceManager = {
-    val driverServiceManagerLoader = ServiceLoader.load(classOf[DriverServiceManager])
-    val matchingServiceManagers = driverServiceManagerLoader
-      .iterator()
-      .asScala
-      .filter(_.getServiceManagerType == driverServiceManagerType)
-      .toList
-    require(matchingServiceManagers.nonEmpty,
-      s"No driver service manager found matching type $driverServiceManagerType")
-    require(matchingServiceManagers.size == 1, "Multiple service managers found" +
-      s" matching type $driverServiceManagerType, got: " +
-      matchingServiceManagers.map(_.getClass).toList.mkString(","))
-    matchingServiceManagers.head
-  }
-}
-
-private[spark] object Client extends Logging {
-
-  private[spark] val SECURE_RANDOM = new SecureRandom()
-
-  def main(args: Array[String]): Unit = {
-    require(args.length >= 2, s"Too few arguments. Usage: ${getClass.getName} <mainAppResource>" +
-      s" <mainClass> [<application arguments>]")
-    val mainAppResource = args(0)
-    val mainClass = args(1)
-    val appArgs = args.drop(2)
-    val sparkConf = new SparkConf(true)
-    new Client(
-      mainAppResource = mainAppResource,
-      mainClass = mainClass,
-      sparkConf = sparkConf,
-      appArgs = appArgs).run()
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverServiceManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverServiceManager.scala
deleted file mode 100644
index c7d394fcf00ad..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverServiceManager.scala
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import io.fabric8.kubernetes.api.model.{Service, ServiceBuilder}
-import io.fabric8.kubernetes.client.KubernetesClient
-
-import org.apache.spark.SparkConf
-
-/**
- * Implementations of this interface are responsible for exposing the driver pod by:
- * - Creating a Kubernetes Service that is backed by the driver pod, and
- * - Providing one or more URIs that the service can be reached at from the submission client.
- *
- * In general, one should not need to implement custom variants of this interface. Consider
- * if the built-in service managers, NodePort and ExternalAnnotation, suit your needs first.
- *
- * This API is in an alpha state and may break without notice.
- */
-trait DriverServiceManager {
-
-  protected var kubernetesClient: KubernetesClient = _
-  protected var serviceName: String = _
-  protected var sparkConf: SparkConf = _
-
-  /**
-   * The tag that identifies this service manager type. This service manager will be loaded
-   * only if the Spark configuration spark.kubernetes.driver.serviceManagerType matches this
-   * value.
-   */
-  def getServiceManagerType: String
-
-  final def start(
-      kubernetesClient: KubernetesClient,
-      serviceName: String,
-      sparkConf: SparkConf): Unit = {
-    this.kubernetesClient = kubernetesClient
-    this.serviceName = serviceName
-    this.sparkConf = sparkConf
-    onStart(kubernetesClient, serviceName, sparkConf)
-  }
-
-  /**
-   * Guaranteed to be called before {@link createDriverService} or
-   * {@link getDriverServiceSubmissionServerUris} is called.
-   */
-  protected def onStart(
-      kubernetesClient: KubernetesClient,
-      serviceName: String,
-      sparkConf: SparkConf): Unit = {}
-
-  /**
-   * Customize the driver service that overlays on the driver pod.
-   *
-   * Implementations are expected to take the service template and adjust it
-   * according to the particular needs of how the Service will be accessed by
-   * URIs provided in {@link getDriverServiceSubmissionServerUris}.
-   *
-   * @param driverServiceTemplate Base settings for the driver service.
-   * @return The same ServiceBuilder object with any required customizations.
-   */
-  def customizeDriverService(driverServiceTemplate: ServiceBuilder): ServiceBuilder
-
-  /**
-   * Return the set of URIs that can be used to reach the submission server that
-   * is running on the driver pod.
-   */
-  def getDriverServiceSubmissionServerUris(driverService: Service): Set[String]
-
-  /**
-   * Called when the Spark application failed to start. Allows the service
-   * manager to clean up any state it may have created that should not be persisted
-   * in the case of an unsuccessful launch. Note that stop() is still called
-   * regardless if this method is called.
-   */
-  def handleSubmissionError(cause: Throwable): Unit = {}
-
-  final def stop(): Unit = onStop()
-
-  /**
-   * Perform any cleanup of this service manager.
-   * the super implementation.
-   */
-  protected def onStop(): Unit = {}
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverSubmitSslConfigurationProvider.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverSubmitSslConfigurationProvider.scala
deleted file mode 100644
index 174e9c57a65ca..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/DriverSubmitSslConfigurationProvider.scala
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import java.io.{File, FileInputStream}
-import java.security.{KeyStore, SecureRandom}
-import javax.net.ssl.{SSLContext, TrustManagerFactory, X509TrustManager}
-
-import com.google.common.base.Charsets
-import com.google.common.io.{BaseEncoding, Files}
-import io.fabric8.kubernetes.api.model.{EnvVar, EnvVarBuilder, Secret, Volume, VolumeBuilder, VolumeMount, VolumeMountBuilder}
-import io.fabric8.kubernetes.client.KubernetesClient
-import scala.collection.JavaConverters._
-
-import org.apache.spark.{SecurityManager => SparkSecurityManager, SparkConf, SparkException, SSLOptions}
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.KubernetesFileUtils
-import org.apache.spark.deploy.rest.kubernetes.v1.PemsToKeyStoreConverter
-import org.apache.spark.util.Utils
-
-/**
- * Raw SSL configuration as the user specified in SparkConf for setting up the driver
- * submission server.
- */
-private case class DriverSubmitSslConfigurationParameters(
-    storeBasedSslOptions: SSLOptions,
-    isKeyStoreLocalFile: Boolean,
-    driverSubmitServerKeyPem: Option[File],
-    isDriverSubmitKeyPemLocalFile: Boolean,
-    driverSubmitServerCertPem: Option[File],
-    isDriverSubmitServerCertPemLocalFile: Boolean,
-    submissionClientCertPem: Option[File])
-
-/**
- * Resolved from translating options provided in
- * {@link DriverSubmitSslConfigurationParameters} into Kubernetes volumes, environment variables
- * for the driver pod, Kubernetes secrets, client-side trust managers, and the client-side SSL
- * context. This is used for setting up the SSL connection for the submission server where the
- * application local dependencies and configuration is provided from.
- */
-private[spark] case class DriverSubmitSslConfiguration(
-    enabled: Boolean,
-    sslPodEnvVars: Array[EnvVar],
-    sslPodVolume: Option[Volume],
-    sslPodVolumeMount: Option[VolumeMount],
-    sslSecret: Option[Secret],
-    driverSubmitClientTrustManager: Option[X509TrustManager],
-    driverSubmitClientSslContext: SSLContext)
-
-/**
- * Provides the SSL configuration for bootstrapping the driver pod to listen for the driver
- * submission over SSL, and then supply the client-side configuration for establishing the
- * SSL connection. This is done in two phases: first, interpreting the raw configuration
- * values from the SparkConf object; then second, converting the configuration parameters
- * into the appropriate Kubernetes constructs, namely the volume and volume mount to add to the
- * driver pod, and the secret to create at the API server; and finally, constructing the
- * client-side trust manager and SSL context for sending the local dependencies.
- */
-private[spark] class DriverSubmitSslConfigurationProvider(
-    sparkConf: SparkConf,
-    kubernetesAppId: String,
-    kubernetesClient: KubernetesClient,
-    kubernetesResourceCleaner: KubernetesResourceCleaner) {
-  private val SECURE_RANDOM = new SecureRandom()
-  private val sslSecretsName = s"$SUBMISSION_SSL_SECRETS_PREFIX-$kubernetesAppId"
-  private val sslSecretsDirectory = DRIVER_CONTAINER_SUBMISSION_SECRETS_BASE_DIR +
-    s"/$kubernetesAppId-ssl"
-
-  def getSslConfiguration(): DriverSubmitSslConfiguration = {
-    val sslConfigurationParameters = parseSslConfigurationParameters()
-    if (sslConfigurationParameters.storeBasedSslOptions.enabled) {
-      val storeBasedSslOptions = sslConfigurationParameters.storeBasedSslOptions
-      val keyStoreSecret = resolveFileToSecretMapping(
-          sslConfigurationParameters.isKeyStoreLocalFile,
-          SUBMISSION_SSL_KEYSTORE_SECRET_NAME,
-          storeBasedSslOptions.keyStore,
-          "KeyStore")
-      val keyStorePathEnv = resolveFilePathEnv(
-          sslConfigurationParameters.isKeyStoreLocalFile,
-          ENV_SUBMISSION_KEYSTORE_FILE,
-          SUBMISSION_SSL_KEYSTORE_SECRET_NAME,
-          storeBasedSslOptions.keyStore)
-      val storePasswordSecret = storeBasedSslOptions.keyStorePassword.map(password => {
-        val passwordBase64 = BaseEncoding.base64().encode(password.getBytes(Charsets.UTF_8))
-        (SUBMISSION_SSL_KEYSTORE_PASSWORD_SECRET_NAME, passwordBase64)
-      }).toMap
-      val storePasswordLocationEnv = storeBasedSslOptions.keyStorePassword.map(_ => {
-        new EnvVarBuilder()
-          .withName(ENV_SUBMISSION_KEYSTORE_PASSWORD_FILE)
-          .withValue(s"$sslSecretsDirectory/$SUBMISSION_SSL_KEYSTORE_PASSWORD_SECRET_NAME")
-          .build()
-      })
-      val storeKeyPasswordSecret = storeBasedSslOptions.keyPassword.map(password => {
-        val passwordBase64 = BaseEncoding.base64().encode(password.getBytes(Charsets.UTF_8))
-        (SUBMISSION_SSL_KEY_PASSWORD_SECRET_NAME, passwordBase64)
-      }).toMap
-      val storeKeyPasswordEnv = storeBasedSslOptions.keyPassword.map(_ => {
-        new EnvVarBuilder()
-          .withName(ENV_SUBMISSION_KEYSTORE_KEY_PASSWORD_FILE)
-          .withValue(s"$sslSecretsDirectory/$SUBMISSION_SSL_KEY_PASSWORD_SECRET_NAME")
-          .build()
-      })
-      val storeTypeEnv = storeBasedSslOptions.keyStoreType.map(storeType => {
-        new EnvVarBuilder()
-          .withName(ENV_SUBMISSION_KEYSTORE_TYPE)
-          .withValue(storeType)
-          .build()
-      })
-      val keyPemSecret = resolveFileToSecretMapping(
-        sslConfigurationParameters.isDriverSubmitKeyPemLocalFile,
-        secretName = SUBMISSION_SSL_KEY_PEM_SECRET_NAME,
-        secretType = "Key pem",
-        secretFile = sslConfigurationParameters.driverSubmitServerKeyPem)
-      val keyPemLocationEnv = resolveFilePathEnv(
-        sslConfigurationParameters.isDriverSubmitKeyPemLocalFile,
-        envName = ENV_SUBMISSION_KEY_PEM_FILE,
-        secretName = SUBMISSION_SSL_KEY_PEM_SECRET_NAME,
-        maybeFile = sslConfigurationParameters.driverSubmitServerKeyPem)
-      val certPemSecret = resolveFileToSecretMapping(
-        sslConfigurationParameters.isDriverSubmitServerCertPemLocalFile,
-        secretName = SUBMISSION_SSL_CERT_PEM_SECRET_NAME,
-        secretType = "Cert pem",
-        secretFile = sslConfigurationParameters.driverSubmitServerCertPem)
-      val certPemLocationEnv = resolveFilePathEnv(
-        sslConfigurationParameters.isDriverSubmitServerCertPemLocalFile,
-        envName = ENV_SUBMISSION_CERT_PEM_FILE,
-        secretName = SUBMISSION_SSL_CERT_PEM_SECRET_NAME,
-        maybeFile = sslConfigurationParameters.driverSubmitServerCertPem)
-      val useSslEnv = new EnvVarBuilder()
-        .withName(ENV_SUBMISSION_USE_SSL)
-        .withValue("true")
-        .build()
-      val sslVolume = new VolumeBuilder()
-        .withName(SUBMISSION_SSL_SECRETS_VOLUME_NAME)
-        .withNewSecret()
-        .withSecretName(sslSecretsName)
-        .endSecret()
-        .build()
-      val sslVolumeMount = new VolumeMountBuilder()
-        .withName(SUBMISSION_SSL_SECRETS_VOLUME_NAME)
-        .withReadOnly(true)
-        .withMountPath(sslSecretsDirectory)
-        .build()
-      val allSecrets = keyStoreSecret ++
-        storePasswordSecret ++
-        storeKeyPasswordSecret ++
-        keyPemSecret ++
-        certPemSecret
-      val sslSecret = kubernetesClient.secrets().createNew()
-        .withNewMetadata()
-        .withName(sslSecretsName)
-        .endMetadata()
-        .withData(allSecrets.asJava)
-        .withType("Opaque")
-        .done()
-      kubernetesResourceCleaner.registerOrUpdateResource(sslSecret)
-      val allSslEnvs = keyStorePathEnv ++
-        storePasswordLocationEnv ++
-        storeKeyPasswordEnv ++
-        storeTypeEnv ++
-        keyPemLocationEnv ++
-        Array(useSslEnv) ++
-        certPemLocationEnv
-      val (driverSubmitClientTrustManager, driverSubmitClientSslContext) =
-        buildSslConnectionConfiguration(sslConfigurationParameters)
-      DriverSubmitSslConfiguration(
-        true,
-        allSslEnvs.toArray,
-        Some(sslVolume),
-        Some(sslVolumeMount),
-        Some(sslSecret),
-        driverSubmitClientTrustManager,
-        driverSubmitClientSslContext)
-    } else {
-      DriverSubmitSslConfiguration(
-        false,
-        Array[EnvVar](),
-        None,
-        None,
-        None,
-        None,
-        SSLContext.getDefault)
-    }
-  }
-
-  private def resolveFilePathEnv(
-      isLocal: Boolean,
-      envName: String,
-      secretName: String,
-      maybeFile: Option[File]): Option[EnvVar] = {
-    maybeFile.map(file => {
-      val pemPath = if (isLocal) {
-        s"$sslSecretsDirectory/$secretName"
-      } else {
-        file.getAbsolutePath
-      }
-      new EnvVarBuilder()
-        .withName(envName)
-        .withValue(pemPath)
-        .build()
-    })
-  }
-
-  private def resolveFileToSecretMapping(
-      isLocal: Boolean,
-      secretName: String,
-      secretFile: Option[File],
-      secretType: String): Map[String, String] = {
-    secretFile.filter(_ => isLocal).map(file => {
-      if (!file.isFile) {
-        throw new SparkException(s"$secretType specified at ${file.getAbsolutePath} is not" +
-          s" a file or does not exist.")
-      }
-      val keyStoreBytes = Files.toByteArray(file)
-      (secretName, BaseEncoding.base64().encode(keyStoreBytes))
-    }).toMap
-  }
-
-  private def parseSslConfigurationParameters(): DriverSubmitSslConfigurationParameters = {
-    val maybeKeyStore = sparkConf.get(KUBERNETES_DRIVER_SUBMIT_SSL_KEYSTORE)
-    val maybeTrustStore = sparkConf.get(KUBERNETES_DRIVER_SUBMIT_SSL_TRUSTSTORE)
-    val maybeKeyPem = sparkConf.get(DRIVER_SUBMIT_SSL_KEY_PEM)
-    val maybeDriverSubmitServerCertPem = sparkConf.get(DRIVER_SUBMIT_SSL_SERVER_CERT_PEM)
-    val maybeDriverSubmitClientCertPem = sparkConf.get(DRIVER_SUBMIT_SSL_CLIENT_CERT_PEM)
-    validatePemsDoNotConflictWithStores(
-      maybeKeyStore,
-      maybeTrustStore,
-      maybeKeyPem,
-      maybeDriverSubmitServerCertPem,
-      maybeDriverSubmitClientCertPem)
-    val resolvedSparkConf = sparkConf.clone()
-    val (isLocalKeyStore, resolvedKeyStore) = resolveLocalFile(maybeKeyStore, "keyStore")
-    resolvedKeyStore.foreach {
-      resolvedSparkConf.set(KUBERNETES_DRIVER_SUBMIT_SSL_KEYSTORE, _)
-    }
-    val (isLocalDriverSubmitServerCertPem, resolvedDriverSubmitServerCertPem) =
-      resolveLocalFile(maybeDriverSubmitServerCertPem, "server cert PEM")
-    val (isLocalKeyPem, resolvedKeyPem) = resolveLocalFile(maybeKeyPem, "key PEM")
-    maybeTrustStore.foreach { trustStore =>
-      require(KubernetesFileUtils.isUriLocalFile(trustStore), s"Invalid trustStore URI" +
-        s" $trustStore; trustStore URI for submit server must have no scheme, or scheme file://")
-      resolvedSparkConf.set(KUBERNETES_DRIVER_SUBMIT_SSL_TRUSTSTORE,
-        Utils.resolveURI(trustStore).getPath)
-    }
-    val driverSubmitClientCertPem = maybeDriverSubmitClientCertPem.map { driverSubmitClientCert =>
-      require(KubernetesFileUtils.isUriLocalFile(driverSubmitClientCert),
-        "Invalid client certificate PEM URI $driverSubmitClientCert: client certificate URI must" +
-          " have no scheme, or scheme file://")
-      Utils.resolveURI(driverSubmitClientCert).getPath
-    }
-    val securityManager = new SparkSecurityManager(resolvedSparkConf)
-    val storeBasedSslOptions = securityManager.getSSLOptions(DRIVER_SUBMIT_SSL_NAMESPACE)
-    DriverSubmitSslConfigurationParameters(
-      storeBasedSslOptions,
-      isLocalKeyStore,
-      resolvedKeyPem.map(new File(_)),
-      isLocalKeyPem,
-      resolvedDriverSubmitServerCertPem.map(new File(_)),
-      isLocalDriverSubmitServerCertPem,
-      driverSubmitClientCertPem.map(new File(_)))
-  }
-
-  private def resolveLocalFile(file: Option[String],
-      fileType: String): (Boolean, Option[String]) = {
-    file.map { f =>
-      require(isValidSslFileScheme(f), s"Invalid $fileType URI $f, $fileType URI" +
-        s" for submit server must have scheme file:// or local:// (no scheme defaults to file://")
-      val isLocal = KubernetesFileUtils.isUriLocalFile(f)
-      (isLocal, Option.apply(Utils.resolveURI(f).getPath))
-    }.getOrElse(false, None)
-  }
-
-  private def validatePemsDoNotConflictWithStores(
-      maybeKeyStore: Option[String],
-      maybeTrustStore: Option[String],
-      maybeKeyPem: Option[String],
-      maybeDriverSubmitServerCertPem: Option[String],
-      maybeSubmitClientCertPem: Option[String]) = {
-    maybeKeyPem.orElse(maybeDriverSubmitServerCertPem).foreach { _ =>
-      require(maybeKeyStore.isEmpty,
-        "Cannot specify server PEM files and key store files; must specify only one or the other.")
-    }
-    maybeKeyPem.foreach { _ =>
-      require(maybeDriverSubmitServerCertPem.isDefined,
-        "When specifying the key PEM file, the server certificate PEM file must also be provided.")
-    }
-    maybeDriverSubmitServerCertPem.foreach { _ =>
-      require(maybeKeyPem.isDefined,
-        "When specifying the server certificate PEM file, the key PEM file must also be provided.")
-    }
-    maybeTrustStore.foreach { _ =>
-      require(maybeSubmitClientCertPem.isEmpty,
-        "Cannot specify client cert file and truststore file; must specify only one or the other.")
-    }
-  }
-
-  private def isValidSslFileScheme(rawUri: String): Boolean = {
-    val resolvedScheme = Option.apply(Utils.resolveURI(rawUri).getScheme).getOrElse("file")
-    resolvedScheme == "file" || resolvedScheme == "local"
-  }
-
-  private def buildSslConnectionConfiguration(
-      sslConfigurationParameters: DriverSubmitSslConfigurationParameters)
-      : (Option[X509TrustManager], SSLContext) = {
-    val maybeTrustStore = sslConfigurationParameters.submissionClientCertPem.map { certPem =>
-      PemsToKeyStoreConverter.convertCertPemToTrustStore(
-        certPem,
-        sslConfigurationParameters.storeBasedSslOptions.trustStoreType)
-    }.orElse(sslConfigurationParameters.storeBasedSslOptions.trustStore.map { trustStoreFile =>
-      if (!trustStoreFile.isFile) {
-        throw new SparkException(s"TrustStore file at ${trustStoreFile.getAbsolutePath}" +
-          s" does not exist or is not a file.")
-      }
-      val trustStore = KeyStore.getInstance(
-        sslConfigurationParameters
-          .storeBasedSslOptions
-          .trustStoreType
-          .getOrElse(KeyStore.getDefaultType))
-      Utils.tryWithResource(new FileInputStream(trustStoreFile)) { trustStoreStream =>
-        val trustStorePassword = sslConfigurationParameters
-          .storeBasedSslOptions
-          .trustStorePassword
-          .map(_.toCharArray)
-          .orNull
-        trustStore.load(trustStoreStream, trustStorePassword)
-      }
-      trustStore
-    })
-    maybeTrustStore.map { trustStore =>
-      val trustManagerFactory = TrustManagerFactory.getInstance(
-        TrustManagerFactory.getDefaultAlgorithm)
-      trustManagerFactory.init(trustStore)
-      val trustManagers = trustManagerFactory.getTrustManagers
-      val sslContext = SSLContext.getInstance("TLSv1.2")
-      sslContext.init(null, trustManagers, SECURE_RANDOM)
-      (Option.apply(trustManagers(0).asInstanceOf[X509TrustManager]), sslContext)
-    }.getOrElse((Option.empty[X509TrustManager], SSLContext.getDefault))
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/ExternalSuppliedUrisDriverServiceManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/ExternalSuppliedUrisDriverServiceManager.scala
deleted file mode 100644
index 4c784aeb5692f..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/ExternalSuppliedUrisDriverServiceManager.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import java.util.concurrent.TimeUnit
-
-import com.google.common.util.concurrent.SettableFuture
-import io.fabric8.kubernetes.api.model.{Service, ServiceBuilder}
-import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientException, Watch, Watcher}
-import io.fabric8.kubernetes.client.Watcher.Action
-import scala.collection.JavaConverters._
-
-import org.apache.spark.SparkConf
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.internal.Logging
-import org.apache.spark.util.Utils
-
-/**
- * Creates the service with an annotation that is expected to be detected by another process
- * which the user provides and is not built in this project. When the external process detects
- * the creation of the service with the appropriate annotation, it is expected to populate the
- * value of a second annotation that is the URI of the driver submission server.
- */
-private[spark] class ExternalSuppliedUrisDriverServiceManager
-  extends DriverServiceManager with Logging {
-
-  private val externalUriFuture = SettableFuture.create[String]
-  private var externalUriSetWatch: Option[Watch] = None
-
-  override def onStart(
-      kubernetesClient: KubernetesClient,
-      serviceName: String,
-      sparkConf: SparkConf): Unit = {
-    externalUriSetWatch = Some(kubernetesClient
-      .services()
-      .withName(serviceName)
-      .watch(new ExternalUriSetWatcher(externalUriFuture)))
-  }
-
-  override def getServiceManagerType: String = ExternalSuppliedUrisDriverServiceManager.TYPE
-
-  override def customizeDriverService(driverServiceTemplate: ServiceBuilder): ServiceBuilder = {
-    require(serviceName != null, "Service name was null; was start() called?")
-    driverServiceTemplate
-      .editMetadata()
-      .addToAnnotations(ANNOTATION_PROVIDE_EXTERNAL_URI, "true")
-      .endMetadata()
-      .editSpec()
-      .withType("ClusterIP")
-      .endSpec()
-  }
-
-  override def getDriverServiceSubmissionServerUris(driverService: Service): Set[String] = {
-    val timeoutSeconds = sparkConf.get(KUBERNETES_DRIVER_SUBMIT_TIMEOUT)
-    require(externalUriSetWatch.isDefined, "The watch that listens for the provision of" +
-      " the external URI was not started; was start() called?")
-    Set(externalUriFuture.get(timeoutSeconds, TimeUnit.SECONDS))
-  }
-
-  override def onStop(): Unit = {
-    Utils.tryLogNonFatalError {
-      externalUriSetWatch.foreach(_.close())
-      externalUriSetWatch = None
-    }
-  }
-}
-
-private[spark] object ExternalSuppliedUrisDriverServiceManager {
-  val TYPE = "ExternalAnnotation"
-}
-
-private[spark] class ExternalUriSetWatcher(externalUriFuture: SettableFuture[String])
-  extends Watcher[Service] with Logging {
-
-  override def eventReceived(action: Action, service: Service): Unit = {
-    if (action == Action.MODIFIED && !externalUriFuture.isDone) {
-      service
-        .getMetadata
-        .getAnnotations
-        .asScala
-        .get(ANNOTATION_RESOLVED_EXTERNAL_URI)
-        .foreach(externalUriFuture.set)
-    }
-  }
-
-  override def onClose(cause: KubernetesClientException): Unit = {
-    logDebug("External URI set watcher closed.", cause)
-  }
-}
-
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/KubernetesResourceCleaner.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/KubernetesResourceCleaner.scala
deleted file mode 100644
index 266ec652ed8ae..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/KubernetesResourceCleaner.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import io.fabric8.kubernetes.api.model.HasMetadata
-import io.fabric8.kubernetes.client.KubernetesClient
-import scala.collection.mutable
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.util.Utils
-
-private[spark] class KubernetesResourceCleaner extends Logging {
-
-  private val resources = mutable.HashMap.empty[(String, String), HasMetadata]
-
-  // Synchronized because deleteAllRegisteredResourcesFromKubernetes may be called from a
-  // shutdown hook
-  def registerOrUpdateResource(resource: HasMetadata): Unit = synchronized {
-    resources.put((resource.getMetadata.getName, resource.getKind), resource)
-  }
-
-  def unregisterResource(resource: HasMetadata): Unit = synchronized {
-    resources.remove((resource.getMetadata.getName, resource.getKind))
-  }
-
-  def deleteAllRegisteredResourcesFromKubernetes(kubernetesClient: KubernetesClient): Unit = {
-    synchronized {
-      val resourceCount = resources.size
-      logInfo(s"Deleting ${resourceCount} registered Kubernetes resources...")
-      resources.values.foreach { resource =>
-        Utils.tryLogNonFatalError {
-          kubernetesClient.resource(resource).delete()
-        }
-      }
-      resources.clear()
-      logInfo(s"Deleted ${resourceCount} registered Kubernetes resources.")
-    }
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/NodePortUrisDriverServiceManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/NodePortUrisDriverServiceManager.scala
deleted file mode 100644
index 965d71917403e..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/submit/v1/NodePortUrisDriverServiceManager.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.submit.v1
-
-import io.fabric8.kubernetes.api.model.{Service, ServiceBuilder}
-import scala.collection.JavaConverters._
-
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.internal.Logging
-
-/**
- * Creates the service with an open NodePort. The URI to reach the submission server is thus
- * at the address of any of the nodes through the service's node port.
- */
-private[spark] class NodePortUrisDriverServiceManager extends DriverServiceManager with Logging {
-
-  override def getServiceManagerType: String = NodePortUrisDriverServiceManager.TYPE
-
-  override def customizeDriverService(driverServiceTemplate: ServiceBuilder): ServiceBuilder = {
-    driverServiceTemplate.editSpec().withType("NodePort").endSpec()
-  }
-
-  override def getDriverServiceSubmissionServerUris(driverService: Service): Set[String] = {
-    val urlScheme = if (sparkConf.get(DRIVER_SUBMIT_SSL_ENABLED)) {
-      "https"
-    } else {
-      logWarning("Submitting application details, application secret, Kubernetes credentials," +
-        " and local jars to the cluster over an insecure connection. You should configure SSL" +
-        " to secure this step.")
-      "http"
-    }
-    val servicePort = driverService.getSpec.getPorts.asScala
-      .filter(_.getName == SUBMISSION_SERVER_PORT_NAME)
-      .head.getNodePort
-    val nodeUrls = kubernetesClient.nodes.list.getItems.asScala
-      .filterNot(node => node.getSpec.getUnschedulable != null &&
-        node.getSpec.getUnschedulable)
-      .flatMap(_.getStatus.getAddresses.asScala)
-      // The list contains hostnames, internal and external IP addresses.
-      // (https://kubernetes.io/docs/admin/node/#addresses)
-      // we want only external IP addresses and legacyHostIP addresses in our list
-      // legacyHostIPs are deprecated and will be removed in the future.
-      // (https://github.com/kubernetes/kubernetes/issues/9267)
-      .filter(address => address.getType == "ExternalIP" || address.getType == "LegacyHostIP")
-      .map(address => {
-        s"$urlScheme://${address.getAddress}:$servicePort"
-      }).toSet
-    require(nodeUrls.nonEmpty, "No nodes found to contact the driver!")
-    nodeUrls
-  }
-}
-
-private[spark] object NodePortUrisDriverServiceManager {
-  val TYPE = "NodePort"
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestApi.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/FileFetcher.scala
similarity index 56%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestApi.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/FileFetcher.scala
index 270e7ea0e77bf..d050e0a41a15a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestApi.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/FileFetcher.scala
@@ -14,25 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v1
+package org.apache.spark.deploy.rest.kubernetes
 
-import javax.ws.rs.{Consumes, GET, Path, POST, Produces}
-import javax.ws.rs.core.MediaType
+import java.io.File
 
-import org.apache.spark.deploy.rest.CreateSubmissionResponse
-
-@Path("/v1/submissions/")
-trait KubernetesSparkRestApi {
-
-  @POST
-  @Consumes(Array(MediaType.APPLICATION_JSON))
-  @Produces(Array(MediaType.APPLICATION_JSON))
-  @Path("/create")
-  def submitApplication(request: KubernetesCreateSubmissionRequest): CreateSubmissionResponse
-
-  @GET
-  @Consumes(Array(MediaType.APPLICATION_JSON))
-  @Produces(Array(MediaType.APPLICATION_JSON))
-  @Path("/ping")
-  def ping(): PingResponse
+// Extracted for testing so that unit tests don't have to depend on Utils.fetchFile
+private[spark] trait FileFetcher {
+  def fetchFile(uri: String, targetDir: File): Unit
 }
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainer.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainer.scala
similarity index 95%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainer.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainer.scala
index 7f21087159145..9bdc224f10c90 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainer.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainer.scala
@@ -14,8 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.File
 import java.util.concurrent.TimeUnit
@@ -30,8 +29,8 @@ import scala.concurrent.duration.Duration
 
 import org.apache.spark.{SecurityManager => SparkSecurityManager, SparkConf, SSLOptions}
 import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.deploy.kubernetes.{CompressionUtils, KubernetesCredentials}
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.CompressionUtils
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.{ThreadUtils, Utils}
 
@@ -63,26 +62,6 @@ private class DownloadTarGzCallback(downloadDir: File) extends WaitableCallback[
     }
   }
 }
-
-// Extracted for testing so that unit tests don't have to depend on Utils.fetchFile
-private[v2] trait FileFetcher {
-  def fetchFile(uri: String, targetDir: File): Unit
-}
-
-private class FileFetcherImpl(sparkConf: SparkConf, securityManager: SparkSecurityManager)
-    extends FileFetcher {
-  def fetchFile(uri: String, targetDir: File): Unit = {
-    Utils.fetchFile(
-      url = uri,
-      targetDir = targetDir,
-      conf = sparkConf,
-      securityMgr = securityManager,
-      hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf),
-      timestamp = System.currentTimeMillis(),
-      useCache = false)
-  }
-}
-
 /**
  * Process that fetches files from a resource staging server and/or arbitrary remote locations.
  *
@@ -97,6 +76,7 @@ private[spark] class KubernetesSparkDependencyDownloadInitContainer(
     fileFetcher: FileFetcher,
     resourceStagingServerSslOptions: SSLOptions) extends Logging {
 
+
   private implicit val downloadExecutor = ExecutionContext.fromExecutorService(
     ThreadUtils.newDaemonCachedThreadPool("download-executor"))
   private val maybeResourceStagingServerUri = sparkConf.get(RESOURCE_STAGING_SERVER_URI)
@@ -184,8 +164,7 @@ private[spark] class KubernetesSparkDependencyDownloadInitContainer(
         val resourceSecret = Files.toString(resourceSecretLocation, Charsets.UTF_8)
         val downloadResourceCallback = new DownloadTarGzCallback(resourceDownloadDir)
         logInfo(downloadStartMessage)
-        service.downloadResources(resourceId, resourceSecret)
-          .enqueue(downloadResourceCallback)
+        service.downloadResources(resourceId, resourceSecret).enqueue(downloadResourceCallback)
         downloadResourceCallback.waitForCompletion(downloadTimeoutMinutes, TimeUnit.MINUTES)
         logInfo(downloadFinishedMessage)
       }
@@ -211,6 +190,27 @@ private[spark] class KubernetesSparkDependencyDownloadInitContainer(
   }
 }
 
+private class FileFetcherImpl(sparkConf: SparkConf, securityManager: SparkSecurityManager)
+    extends FileFetcher {
+  def fetchFile(uri: String, targetDir: File): Unit = {
+    Utils.fetchFile(
+      url = uri,
+      targetDir = targetDir,
+      conf = sparkConf,
+      securityMgr = securityManager,
+      hadoopConf = SparkHadoopUtil.get.newConfiguration(sparkConf),
+      timestamp = System.currentTimeMillis(),
+      useCache = false)
+  }
+}
+
+private case class StagedResources(
+    resourceSecret: String,
+    podLabels: Map[String, String],
+    podNamespace: String,
+    resourcesFile: File,
+    kubernetesCredentials: KubernetesCredentials)
+
 object KubernetesSparkDependencyDownloadInitContainer extends Logging {
   def main(args: Array[String]): Unit = {
     logInfo("Starting init-container to download Spark application dependencies.")
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/PemsToKeyStoreConverter.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/PemsToKeyStoreConverter.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/PemsToKeyStoreConverter.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/PemsToKeyStoreConverter.scala
index 178956a136d1c..17f90118e150d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/PemsToKeyStoreConverter.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/PemsToKeyStoreConverter.scala
@@ -14,10 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v1
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{File, FileInputStream, FileOutputStream, InputStreamReader}
-import java.nio.file.Paths
 import java.security.{KeyStore, PrivateKey}
 import java.security.cert.Certificate
 import java.util.UUID
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServer.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServer.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServer.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServer.scala
index 4ecb6369ff3b0..34594ba518b62 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServer.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServer.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.File
 
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProvider.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProvider.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProvider.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProvider.scala
index 0dd0b08433def..cb1e65421c013 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProvider.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProvider.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.File
 import java.security.SecureRandom
@@ -26,7 +26,6 @@ import org.apache.commons.lang3.RandomStringUtils
 import org.apache.spark.{SecurityManager, SparkConf, SparkException, SSLOptions}
 import org.apache.spark.deploy.kubernetes.OptionRequirements
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.rest.kubernetes.v1.PemsToKeyStoreConverter
 import org.apache.spark.internal.Logging
 
 private[spark] trait ResourceStagingServerSslOptionsProvider {
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingService.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingService.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingService.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingService.scala
index 5dbe55b72bd8b..525711e78c01c 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingService.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingService.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.InputStream
 import javax.ws.rs.{Consumes, GET, HeaderParam, Path, PathParam, POST, Produces}
@@ -23,7 +23,7 @@ import javax.ws.rs.core.{MediaType, StreamingOutput}
 import org.glassfish.jersey.media.multipart.FormDataParam
 
 import org.apache.spark.deploy.kubernetes.KubernetesCredentials
-import org.apache.spark.deploy.kubernetes.submit.v2.SubmittedResourceIdAndSecret
+import org.apache.spark.deploy.kubernetes.submit.SubmittedResourceIdAndSecret
 
 /**
  * Service that receives application data that can be retrieved later on. This is primarily used
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImpl.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImpl.scala
similarity index 91%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImpl.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImpl.scala
index 34c3192ae6780..abe956da9914d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImpl.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImpl.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{File, FileOutputStream, InputStream, OutputStream}
 import java.security.SecureRandom
@@ -27,7 +27,7 @@ import scala.collection.concurrent.TrieMap
 
 import org.apache.spark.SparkException
 import org.apache.spark.deploy.kubernetes.KubernetesCredentials
-import org.apache.spark.deploy.kubernetes.submit.v2.SubmittedResourceIdAndSecret
+import org.apache.spark.deploy.kubernetes.submit.SubmittedResourceIdAndSecret
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.Utils
 
@@ -92,10 +92,3 @@ private[spark] class ResourceStagingServiceImpl(dependenciesRootDir: File)
 
   override def ping(): String = "pong"
 }
-
-private case class StagedResources(
-  resourceSecret: String,
-  podLabels: Map[String, String],
-  podNamespace: String,
-  resourcesFile: File,
-  kubernetesCredentials: KubernetesCredentials)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceRetrofit.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceRetrofit.scala
similarity index 93%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceRetrofit.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceRetrofit.scala
index e0079a372f0d9..3c2fe8ebbc3c8 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceRetrofit.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceRetrofit.scala
@@ -14,13 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import okhttp3.{RequestBody, ResponseBody}
 import retrofit2.Call
 import retrofit2.http.{Multipart, Path, Streaming}
 
-import org.apache.spark.deploy.kubernetes.submit.v2.SubmittedResourceIdAndSecret
+import org.apache.spark.deploy.kubernetes.submit.SubmittedResourceIdAndSecret
 
 /**
  * Retrofit-compatible variant of {@link ResourceStagingService}. For documentation on
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/RetrofitClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/RetrofitClientFactory.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/RetrofitClientFactory.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/RetrofitClientFactory.scala
index f906423524944..a374982444f79 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/RetrofitClientFactory.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/RetrofitClientFactory.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.FileInputStream
 import java.security.{KeyStore, SecureRandom}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/SparkConfPropertiesParser.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/SparkConfPropertiesParser.scala
similarity index 94%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/SparkConfPropertiesParser.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/SparkConfPropertiesParser.scala
index cf9decab127c5..9e2b8a780df29 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v2/SparkConfPropertiesParser.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/SparkConfPropertiesParser.scala
@@ -14,13 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{File, FileInputStream}
 import java.util.Properties
 
 import com.google.common.collect.Maps
-import scala.collection.JavaConverters.mapAsScalaMapConverter
+import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkConf
 import org.apache.spark.internal.config.{ConfigReader, SparkConfigProvider}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/HttpClientUtil.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/HttpClientUtil.scala
deleted file mode 100644
index ea1abed72c07f..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/HttpClientUtil.scala
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.rest.kubernetes.v1
-
-import java.io.IOException
-import java.net.{InetSocketAddress, ProxySelector, SocketAddress, URI}
-import java.util.Collections
-import javax.net.ssl.{SSLContext, SSLSocketFactory, X509TrustManager}
-
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper}
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import feign.{Client, Feign, Request, Response}
-import feign.Request.Options
-import feign.jackson.{JacksonDecoder, JacksonEncoder}
-import feign.jaxrs.JAXRSContract
-import io.fabric8.kubernetes.client.Config
-import okhttp3.OkHttpClient
-import scala.reflect.ClassTag
-
-import org.apache.spark.SparkException
-import org.apache.spark.internal.Logging
-import org.apache.spark.status.api.v1.JacksonMessageWriter
-
-private[spark] object HttpClientUtil extends Logging {
-
-  def createClient[T: ClassTag](
-      uris: Set[String],
-      maxRetriesPerServer: Int = 1,
-      sslSocketFactory: SSLSocketFactory = SSLContext.getDefault.getSocketFactory,
-      trustContext: X509TrustManager = null,
-      readTimeoutMillis: Int = 20000,
-      connectTimeoutMillis: Int = 20000): T = {
-    var httpClientBuilder = new OkHttpClient.Builder()
-    Option.apply(trustContext).foreach(context => {
-      httpClientBuilder = httpClientBuilder.sslSocketFactory(sslSocketFactory, context)
-    })
-    val uriObjects = uris.map(URI.create)
-    val httpUris = uriObjects.filter(uri => uri.getScheme == "http")
-    val httpsUris = uriObjects.filter(uri => uri.getScheme == "https")
-    val maybeAllProxy = Option.apply(System.getProperty(Config.KUBERNETES_ALL_PROXY))
-    val maybeHttpProxy = Option.apply(System.getProperty(Config.KUBERNETES_HTTP_PROXY))
-      .orElse(maybeAllProxy)
-      .map(uriStringToProxy)
-    val maybeHttpsProxy = Option.apply(System.getProperty(Config.KUBERNETES_HTTPS_PROXY))
-      .orElse(maybeAllProxy)
-      .map(uriStringToProxy)
-    val maybeNoProxy = Option.apply(System.getProperty(Config.KUBERNETES_NO_PROXY))
-      .map(_.split(","))
-      .toSeq
-      .flatten
-    val proxySelector = new ProxySelector {
-      override def select(uri: URI): java.util.List[java.net.Proxy] = {
-        val directProxy = java.net.Proxy.NO_PROXY
-        val resolvedProxy = maybeNoProxy.find( _ == uri.getHost)
-          .map( _ => directProxy)
-          .orElse(uri.getScheme match {
-            case "http" =>
-              logDebug(s"Looking up http proxies to route $uri")
-              maybeHttpProxy.filter { _ =>
-                matchingUriExists(uri, httpUris)
-              }
-            case "https" =>
-              logDebug(s"Looking up https proxies to route $uri")
-              maybeHttpsProxy.filter { _ =>
-                matchingUriExists(uri, httpsUris)
-              }
-            case _ => None
-        }).getOrElse(directProxy)
-        logDebug(s"Routing $uri through ${resolvedProxy.address()} with proxy" +
-          s" type ${resolvedProxy.`type`()}")
-        Collections.singletonList(resolvedProxy)
-      }
-
-      override def connectFailed(uri: URI, sa: SocketAddress, ioe: IOException) = {
-        throw new SparkException(s"Failed to connect to proxy through uri $uri," +
-          s" socket address: $sa", ioe)
-      }
-    }
-    httpClientBuilder = httpClientBuilder.proxySelector(proxySelector)
-    val objectMapper = new ObjectMapper()
-      .registerModule(new DefaultScalaModule)
-      .setDateFormat(JacksonMessageWriter.makeISODateFormat)
-    objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-    val target = new MultiServerFeignTarget[T](uris.toSeq, maxRetriesPerServer)
-    val baseHttpClient = new feign.okhttp.OkHttpClient(httpClientBuilder.build())
-    val resetTargetHttpClient = new Client {
-      override def execute(request: Request, options: Options): Response = {
-        val response = baseHttpClient.execute(request, options)
-        if (response.status() / 100 == 2) {
-          target.reset()
-        }
-        response
-      }
-    }
-    Feign.builder()
-      .client(resetTargetHttpClient)
-      .contract(new JAXRSContract)
-      .encoder(new JacksonEncoder(objectMapper))
-      .decoder(new JacksonDecoder(objectMapper))
-      .options(new Options(connectTimeoutMillis, readTimeoutMillis))
-      .retryer(target)
-      .target(target)
-  }
-
-  private def matchingUriExists(uri: URI, httpUris: Set[URI]): Boolean = {
-    httpUris.exists(httpUri => {
-      httpUri.getScheme == uri.getScheme && httpUri.getHost == uri.getHost &&
-      httpUri.getPort == uri.getPort
-    })
-  }
-
-  private def uriStringToProxy(uriString: String): java.net.Proxy = {
-    val uriObject = URI.create(uriString)
-    new java.net.Proxy(java.net.Proxy.Type.HTTP,
-      new InetSocketAddress(uriObject.getHost, uriObject.getPort))
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesRestProtocolMessages.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesRestProtocolMessages.scala
deleted file mode 100644
index bdd4a85da8f85..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesRestProtocolMessages.scala
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.rest.kubernetes.v1
-
-import com.fasterxml.jackson.annotation.{JsonIgnore, JsonSubTypes, JsonTypeInfo}
-
-import org.apache.spark.SPARK_VERSION
-import org.apache.spark.deploy.kubernetes.KubernetesCredentials
-import org.apache.spark.deploy.rest.{SubmitRestProtocolRequest, SubmitRestProtocolResponse}
-import org.apache.spark.util.Utils
-
-case class KubernetesCreateSubmissionRequest(
-    appResource: AppResource,
-    mainClass: String,
-    appArgs: Array[String],
-    sparkProperties: Map[String, String],
-    secret: String,
-    driverPodKubernetesCredentials: KubernetesCredentials,
-    uploadedJarsBase64Contents: TarGzippedData,
-    uploadedFilesBase64Contents: TarGzippedData) extends SubmitRestProtocolRequest {
-  @JsonIgnore
-  override val messageType: String = s"kubernetes.v1.${Utils.getFormattedClassName(this)}"
-  override val action = messageType
-  message = "create"
-  clientSparkVersion = SPARK_VERSION
-}
-
-case class TarGzippedData(
-  dataBase64: String,
-  blockSize: Int = 10240,
-  recordSize: Int = 512,
-  encoding: String
-)
-
-@JsonTypeInfo(
-  use = JsonTypeInfo.Id.NAME,
-  include = JsonTypeInfo.As.PROPERTY,
-  property = "type")
-@JsonSubTypes(value = Array(
-  new JsonSubTypes.Type(value = classOf[UploadedAppResource], name = "UploadedAppResource"),
-  new JsonSubTypes.Type(value = classOf[ContainerAppResource], name = "ContainerLocalAppResource"),
-  new JsonSubTypes.Type(value = classOf[RemoteAppResource], name = "RemoteAppResource")))
-abstract class AppResource
-
-case class UploadedAppResource(
-  resourceBase64Contents: String,
-  name: String = "spark-app-resource") extends AppResource
-
-case class ContainerAppResource(resourcePath: String) extends AppResource
-
-case class RemoteAppResource(resource: String) extends AppResource
-
-class PingResponse extends SubmitRestProtocolResponse {
-  val text = "pong"
-  message = "pong"
-  serverSparkVersion = SPARK_VERSION
-  @JsonIgnore
-  override val messageType: String = s"kubernetes.v1.${Utils.getFormattedClassName(this)}"
-  override val action: String = messageType
-}
-
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestServer.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestServer.scala
deleted file mode 100644
index 5cd24a8f9b75e..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/KubernetesSparkRestServer.scala
+++ /dev/null
@@ -1,483 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.rest.kubernetes.v1
-
-import java.io.{File, FileOutputStream, StringReader}
-import java.net.URI
-import java.nio.file.Paths
-import java.security.SecureRandom
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.atomic.AtomicInteger
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
-
-import com.google.common.base.Charsets
-import com.google.common.io.{BaseEncoding, ByteStreams, Files}
-import org.apache.commons.codec.binary.Base64
-import org.apache.commons.lang3.RandomStringUtils
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.{SecurityManager, SPARK_VERSION => sparkVersion, SparkConf, SparkException, SSLOptions}
-import org.apache.spark.deploy.SparkHadoopUtil
-import org.apache.spark.deploy.kubernetes.{CompressionUtils, KubernetesCredentials}
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.submit.KubernetesFileUtils
-import org.apache.spark.deploy.rest._
-import org.apache.spark.internal.config.OptionalConfigEntry
-import org.apache.spark.util.{ShutdownHookManager, ThreadUtils, Utils}
-
-private case class KubernetesSparkRestServerArguments(
-    host: Option[String] = None,
-    port: Option[Int] = None,
-    useSsl: Boolean = false,
-    secretFile: Option[String] = None,
-    keyStoreFile: Option[String] = None,
-    keyStorePasswordFile: Option[String] = None,
-    keyStoreType: Option[String] = None,
-    keyPasswordFile: Option[String] = None,
-    keyPemFile: Option[String] = None,
-    certPemFile: Option[String] = None) {
-  def validate(): KubernetesSparkRestServerArguments = {
-    require(host.isDefined, "Hostname not set via --hostname.")
-    require(port.isDefined, "Port not set via --port")
-    require(secretFile.isDefined, "Secret file not set via --secret-file")
-    this
-  }
-}
-
-private object KubernetesSparkRestServerArguments {
-  def fromArgsArray(inputArgs: Array[String]): KubernetesSparkRestServerArguments = {
-    var args = inputArgs.toList
-    var resolvedArguments = KubernetesSparkRestServerArguments()
-    while (args.nonEmpty) {
-      resolvedArguments = args match {
-        case "--hostname" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(host = Some(value))
-        case "--port" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(port = Some(value.toInt))
-        case "--secret-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(secretFile = Some(value))
-        case "--use-ssl" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(useSsl = value.toBoolean)
-        case "--keystore-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(keyStoreFile = Some(value))
-        case "--keystore-password-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(keyStorePasswordFile = Some(value))
-        case "--keystore-type" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(keyStoreType = Some(value))
-        case "--keystore-key-password-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(keyPasswordFile = Some(value))
-        case "--key-pem-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(keyPemFile = Some(value))
-        case "--cert-pem-file" :: value :: tail =>
-          args = tail
-          resolvedArguments.copy(certPemFile = Some(value))
-        // TODO polish usage message
-        case Nil => resolvedArguments
-        case unknown => throw new IllegalStateException(s"Unknown argument(s) found: $unknown")
-      }
-    }
-    resolvedArguments.validate()
-  }
-}
-
-/**
- * Runs in the driver pod and receives a request to run an application. Note that
- * unlike the submission rest server in standalone mode, this server is expected
- * to be used to run one application only, and then shut down once that application
- * is complete.
- */
-private[spark] class KubernetesSparkRestServer(
-    host: String,
-    port: Int,
-    conf: SparkConf,
-    expectedApplicationSecret: Array[Byte],
-    shutdownLock: CountDownLatch,
-    exitCode: AtomicInteger,
-    sslOptions: SSLOptions = new SSLOptions)
-  extends RestSubmissionServer(host, port, conf, sslOptions) {
-
-  private val SERVLET_LOCK = new Object
-  private val javaExecutable = s"${System.getenv("JAVA_HOME")}/bin/java"
-  private val sparkHome = System.getenv("SPARK_HOME")
-  private val securityManager = new SecurityManager(conf)
-  override protected lazy val contextToServlet = Map[String, RestServlet](
-    s"$baseContext/create/*" -> submitRequestServlet,
-    s"$baseContext/ping/*" -> pingServlet)
-
-  private val pingServlet = new PingServlet
-  override protected val submitRequestServlet: SubmitRequestServlet
-    = new KubernetesSubmitRequestServlet
-  // TODO
-  override protected val statusRequestServlet: StatusRequestServlet = null
-  override protected val killRequestServlet: KillRequestServlet = null
-
-  private class PingServlet extends RestServlet {
-    protected override def doGet(
-      request: HttpServletRequest,
-      response: HttpServletResponse): Unit = {
-      sendResponse(new PingResponse, response)
-    }
-  }
-
-  private class KubernetesSubmitRequestServlet extends SubmitRequestServlet {
-
-    private val waitForProcessCompleteExecutor = ThreadUtils
-        .newDaemonSingleThreadExecutor("wait-for-spark-app-complete")
-    private var startedApplication = false
-
-    // TODO validating the secret should be done as part of a header of the request.
-    // Instead here we have to specify the secret in the body.
-    override protected def handleSubmit(
-        requestMessageJson: String,
-        requestMessage: SubmitRestProtocolMessage,
-        responseServlet: HttpServletResponse): SubmitRestProtocolResponse = {
-      SERVLET_LOCK.synchronized {
-        if (startedApplication) {
-          throw new IllegalStateException("Application has already been submitted.")
-        } else {
-          requestMessage match {
-            case KubernetesCreateSubmissionRequest(
-                appResource,
-                mainClass,
-                appArgs,
-                sparkProperties,
-                secret,
-                driverPodKubernetesCredentials,
-                uploadedJars,
-                uploadedFiles) =>
-              val decodedSecret = Base64.decodeBase64(secret)
-              if (!expectedApplicationSecret.sameElements(decodedSecret)) {
-                responseServlet.setStatus(HttpServletResponse.SC_UNAUTHORIZED)
-                handleError("Unauthorized to submit application.")
-              } else {
-                val tempDir = Utils.createTempDir()
-                val resolvedAppResource = resolveAppResource(appResource, tempDir)
-                val writtenJars = writeUploadedJars(uploadedJars, tempDir)
-                val writtenFiles = writeUploadedFiles(uploadedFiles)
-                val resolvedSparkProperties = new mutable.HashMap[String, String]
-                resolvedSparkProperties ++= sparkProperties
-                val originalJars = sparkProperties.get("spark.jars")
-                  .map(_.split(","))
-                  .getOrElse(Array.empty)
-
-                // The driver at this point has handed us the value of spark.jars verbatim as
-                // specified in spark-submit. At this point, remove all jars that were local
-                // to the submitting user's disk, and replace them with the paths that were
-                // written to disk above.
-                val onlyContainerLocalOrRemoteJars = KubernetesFileUtils
-                  .getNonSubmitterLocalFiles(originalJars)
-                val resolvedJars = (writtenJars ++
-                  onlyContainerLocalOrRemoteJars ++
-                  Array(resolvedAppResource.sparkJarPath)).toSet
-                if (resolvedJars.nonEmpty) {
-                  resolvedSparkProperties("spark.jars") = resolvedJars.mkString(",")
-                } else {
-                  resolvedSparkProperties.remove("spark.jars")
-                }
-
-                // Determining the driver classpath is similar. It's the combination of:
-                // - Jars written from uploads
-                // - Jars in (spark.jars + mainAppResource) that has a "local" prefix
-                // - spark.driver.extraClasspath
-                // - Spark core jars from the installation
-                val sparkCoreJars = new File(sparkHome, "jars").listFiles().map(_.getAbsolutePath)
-                val driverExtraClasspath = sparkProperties
-                  .get("spark.driver.extraClassPath")
-                  .map(_.split(","))
-                  .getOrElse(Array.empty[String])
-                val onlyContainerLocalJars = KubernetesFileUtils
-                  .getOnlyContainerLocalFiles(originalJars)
-                val driverClasspath = driverExtraClasspath ++
-                  Seq(resolvedAppResource.localPath) ++
-                  writtenJars ++
-                  onlyContainerLocalJars ++
-                  sparkCoreJars
-
-                // Resolve spark.files similarly to spark.jars.
-                val originalFiles = sparkProperties.get("spark.files")
-                  .map(_.split(","))
-                  .getOrElse(Array.empty[String])
-                val onlyContainerLocalOrRemoteFiles = KubernetesFileUtils
-                  .getNonSubmitterLocalFiles(originalFiles)
-                val resolvedFiles = writtenFiles ++ onlyContainerLocalOrRemoteFiles
-                if (resolvedFiles.nonEmpty) {
-                  resolvedSparkProperties("spark.files") = resolvedFiles.mkString(",")
-                } else {
-                  resolvedSparkProperties.remove("spark.files")
-                }
-                resolvedSparkProperties ++= writeKubernetesCredentials(
-                  driverPodKubernetesCredentials, tempDir)
-
-                val command = new ArrayBuffer[String]
-                command += javaExecutable
-                command += "-cp"
-                command += s"${driverClasspath.mkString(":")}"
-                for (prop <- resolvedSparkProperties) {
-                  command += s"-D${prop._1}=${prop._2}"
-                }
-                val driverMemory = resolvedSparkProperties.getOrElse("spark.driver.memory", "1g")
-                command += s"-Xms$driverMemory"
-                command += s"-Xmx$driverMemory"
-                val extraJavaOpts = resolvedSparkProperties.get("spark.driver.extraJavaOptions")
-                  .map(Utils.splitCommandString)
-                  .getOrElse(Seq.empty)
-                command ++= extraJavaOpts
-                command += mainClass
-                command ++= appArgs
-                val pb = new ProcessBuilder(command: _*).inheritIO()
-                val process = pb.start()
-                ShutdownHookManager.addShutdownHook(() => {
-                  logInfo("Received stop command, shutting down the running Spark application...")
-                  process.destroy()
-                  shutdownLock.countDown()
-                })
-                waitForProcessCompleteExecutor.submit(new Runnable {
-                  override def run(): Unit = {
-                    // set the REST service's exit code to the exit code of the driver subprocess
-                    exitCode.set(process.waitFor)
-                    SERVLET_LOCK.synchronized {
-                      logInfo("Spark application complete. Shutting down submission server...")
-                      KubernetesSparkRestServer.this.stop
-                      shutdownLock.countDown()
-                    }
-                  }
-                })
-                startedApplication = true
-                val response = new CreateSubmissionResponse
-                response.success = true
-                response.submissionId = null
-                response.message = "success"
-                response.serverSparkVersion = sparkVersion
-                response
-              }
-            case unexpected =>
-              responseServlet.setStatus(HttpServletResponse.SC_BAD_REQUEST)
-              handleError(s"Received message of unexpected type ${unexpected.messageType}.")
-          }
-        }
-      }
-    }
-
-    private def writeUploadedJars(jars: TarGzippedData, rootTempDir: File):
-        Seq[String] = {
-      val resolvedDirectory = new File(rootTempDir, "jars")
-      if (!resolvedDirectory.mkdir()) {
-        throw new IllegalStateException(s"Failed to create jars dir at " +
-          resolvedDirectory.getAbsolutePath)
-      }
-      CompressionUtils.unpackAndWriteCompressedFiles(jars, resolvedDirectory)
-    }
-
-    private def writeUploadedFiles(files: TarGzippedData): Seq[String] = {
-      val workingDir = Paths.get("").toFile.getAbsoluteFile
-      CompressionUtils.unpackAndWriteCompressedFiles(files, workingDir)
-    }
-
-    private def writeKubernetesCredentials(
-        kubernetesCredentials: KubernetesCredentials,
-        rootTempDir: File): Map[String, String] = {
-      val resolvedDirectory = new File(rootTempDir, "kubernetes-credentials")
-      if (!resolvedDirectory.mkdir()) {
-        throw new IllegalStateException(s"Failed to create credentials dir at "
-          + resolvedDirectory.getAbsolutePath)
-      }
-      val oauthTokenFile = writeRawStringCredentialAndGetConf("oauth-token.txt", resolvedDirectory,
-        KUBERNETES_DRIVER_MOUNTED_OAUTH_TOKEN,
-        kubernetesCredentials.oauthTokenBase64.map { base64 =>
-          new String(BaseEncoding.base64().decode(base64), Charsets.UTF_8)
-        })
-      val caCertFile = writeBase64CredentialAndGetConf("ca.crt", resolvedDirectory,
-        KUBERNETES_DRIVER_MOUNTED_CA_CERT_FILE, kubernetesCredentials.caCertDataBase64)
-      val clientKeyFile = writeBase64CredentialAndGetConf("key.key", resolvedDirectory,
-        KUBERNETES_DRIVER_MOUNTED_CLIENT_KEY_FILE, kubernetesCredentials.clientKeyDataBase64)
-      val clientCertFile = writeBase64CredentialAndGetConf("cert.crt", resolvedDirectory,
-        KUBERNETES_DRIVER_MOUNTED_CLIENT_CERT_FILE, kubernetesCredentials.clientCertDataBase64)
-      (oauthTokenFile ++ caCertFile ++ clientKeyFile ++ clientCertFile).toMap
-    }
-
-    private def writeRawStringCredentialAndGetConf(
-        fileName: String,
-        dir: File,
-        conf: OptionalConfigEntry[String],
-        credential: Option[String]): Option[(String, String)] = {
-      credential.map { cred =>
-        val credentialFile = new File(dir, fileName)
-        Files.write(cred, credentialFile, Charsets.UTF_8)
-        (conf.key, credentialFile.getAbsolutePath)
-      }
-    }
-
-    private def writeBase64CredentialAndGetConf(
-        fileName: String,
-        dir: File,
-        conf: OptionalConfigEntry[String],
-        credential: Option[String]): Option[(String, String)] = {
-      credential.map { cred =>
-        val credentialFile = new File(dir, fileName)
-        Files.write(BaseEncoding.base64().decode(cred), credentialFile)
-        (conf.key, credentialFile.getAbsolutePath)
-      }
-    }
-
-    /**
-     * Retrieve the path on the driver container where the main app resource is, and what value it
-     * ought to have in the spark.jars property. The two may be different because for non-local
-     * dependencies, we have to fetch the resource (if it is not "local") but still want to use
-     * the full URI in spark.jars.
-     */
-    private def resolveAppResource(appResource: AppResource, tempDir: File):
-        ResolvedAppResource = {
-      appResource match {
-        case UploadedAppResource(resourceContentsBase64, resourceName) =>
-          val resourceFile = new File(tempDir, resourceName)
-          val resourceFilePath = resourceFile.getAbsolutePath
-          if (resourceFile.createNewFile()) {
-            Utils.tryWithResource(new StringReader(resourceContentsBase64)) { reader =>
-              Utils.tryWithResource(new FileOutputStream(resourceFile)) { os =>
-                Utils.tryWithResource(BaseEncoding.base64().decodingStream(reader)) {
-                    decodingStream =>
-                  ByteStreams.copy(decodingStream, os)
-                }
-              }
-            }
-            ResolvedAppResource(resourceFile.getAbsolutePath, resourceFile.getAbsolutePath)
-          } else {
-            throw new IllegalStateException(s"Failed to write main app resource file" +
-              s" to $resourceFilePath")
-          }
-        case ContainerAppResource(resource) =>
-          ResolvedAppResource(Utils.resolveURI(resource).getPath, resource)
-        case RemoteAppResource(resource) =>
-          Utils.fetchFile(resource, tempDir, conf,
-            securityManager, SparkHadoopUtil.get.newConfiguration(conf),
-            System.currentTimeMillis(), useCache = false)
-          val fileName = Utils.decodeFileNameInURI(URI.create(resource))
-          val downloadedFile = new File(tempDir, fileName)
-          val downloadedFilePath = downloadedFile.getAbsolutePath
-          if (!downloadedFile.isFile) {
-            throw new IllegalStateException(s"Main app resource is not a file or" +
-              s" does not exist at $downloadedFilePath")
-          }
-          ResolvedAppResource(downloadedFilePath, resource)
-      }
-    }
-  }
-
-  private case class ResolvedAppResource(localPath: String, sparkJarPath: String)
-}
-
-private[spark] object KubernetesSparkRestServer {
-  private val barrier = new CountDownLatch(1)
-  private val SECURE_RANDOM = new SecureRandom()
-
-  def main(args: Array[String]): Unit = {
-    val parsedArguments = KubernetesSparkRestServerArguments.fromArgsArray(args)
-    val secretFile = new File(parsedArguments.secretFile.get)
-    require(secretFile.isFile, "Secret file specified by --secret-file is not a file, or" +
-      " does not exist.")
-    val sslOptions = if (parsedArguments.useSsl) {
-      validateSslOptions(parsedArguments)
-      val keyPassword = parsedArguments
-        .keyPasswordFile
-        .map(new File(_))
-        .map(Files.toString(_, Charsets.UTF_8))
-        // If key password isn't set but we're using PEM files, generate a password
-        .orElse(parsedArguments.keyPemFile.map(_ => randomPassword()))
-      val keyStorePassword = parsedArguments
-        .keyStorePasswordFile
-        .map(new File(_))
-        .map(Files.toString(_, Charsets.UTF_8))
-        // If keystore password isn't set but we're using PEM files, generate a password
-        .orElse(parsedArguments.keyPemFile.map(_ => randomPassword()))
-      val resolvedKeyStore = parsedArguments.keyStoreFile.map(new File(_)).orElse(
-        for {
-          keyPemFile <- parsedArguments.keyPemFile
-          certPemFile <- parsedArguments.certPemFile
-          resolvedKeyStorePassword <- keyStorePassword
-          resolvedKeyPassword <- keyPassword
-        } yield {
-          PemsToKeyStoreConverter.convertPemsToTempKeyStoreFile(
-            new File(keyPemFile),
-            new File(certPemFile),
-            "provided-key",
-            resolvedKeyStorePassword,
-            resolvedKeyPassword,
-            parsedArguments.keyStoreType)
-        })
-      new SSLOptions(
-        enabled = true,
-        keyStore = resolvedKeyStore,
-        keyStoreType = parsedArguments.keyStoreType,
-        keyStorePassword = keyStorePassword,
-        keyPassword = keyPassword)
-    } else {
-      new SSLOptions
-    }
-    val secretBytes = Files.toByteArray(secretFile)
-    val sparkConf = new SparkConf(true)
-    val exitCode = new AtomicInteger(0)
-    val server = new KubernetesSparkRestServer(
-      parsedArguments.host.get,
-      parsedArguments.port.get,
-      sparkConf,
-      secretBytes,
-      barrier,
-      exitCode,
-      sslOptions)
-    server.start()
-    ShutdownHookManager.addShutdownHook(() => {
-      try {
-        server.stop()
-      } finally {
-        barrier.countDown()
-      }
-    })
-    barrier.await()
-    System.exit(exitCode.get())
-  }
-
-  private def validateSslOptions(parsedArguments: KubernetesSparkRestServerArguments): Unit = {
-    parsedArguments.keyStoreFile.foreach { _ =>
-      require(parsedArguments.keyPemFile.orElse(parsedArguments.certPemFile).isEmpty,
-        "Cannot provide both key/cert PEM files and a keyStore file; select one or the other" +
-          " for configuring SSL.")
-    }
-    parsedArguments.keyPemFile.foreach { _ =>
-      require(parsedArguments.certPemFile.isDefined,
-        "When providing the key PEM file, the certificate PEM file must also be provided.")
-    }
-    parsedArguments.certPemFile.foreach { _ =>
-      require(parsedArguments.keyPemFile.isDefined,
-        "When providing the certificate PEM file, the key PEM file must also be provided.")
-    }
-  }
-
-  private def randomPassword(): String = {
-    RandomStringUtils.random(1024, 0, Integer.MAX_VALUE, false, false, null, SECURE_RANDOM)
-  }
-}
-
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/MultiServerFeignTarget.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/MultiServerFeignTarget.scala
deleted file mode 100644
index 56ff82ea2fc33..0000000000000
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/rest/kubernetes/v1/MultiServerFeignTarget.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.rest.kubernetes.v1
-
-import feign.{Request, RequestTemplate, RetryableException, Retryer, Target}
-import scala.reflect.ClassTag
-import scala.util.Random
-
-import org.apache.spark.internal.Logging
-
-private[kubernetes] class MultiServerFeignTarget[T : ClassTag](
-    private val servers: Seq[String],
-    private val maxRetriesPerServer: Int = 1,
-    private val delayBetweenRetriesMillis: Int = 1000) extends Target[T] with Retryer with Logging {
-  require(servers.nonEmpty, "Must provide at least one server URI.")
-
-  private val threadLocalShuffledServers = new ThreadLocal[Seq[String]] {
-    override def initialValue(): Seq[String] = Random.shuffle(servers)
-  }
-  private val threadLocalCurrentAttempt = new ThreadLocal[Int] {
-    override def initialValue(): Int = 0
-  }
-
-  override def `type`(): Class[T] = {
-    implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]]
-  }
-
-  /**
-   * Cloning the target is done on every request, for use on the current
-   * thread - thus it's important that clone returns a "fresh" target.
-   */
-  override def clone(): Retryer = {
-    reset()
-    this
-  }
-
-  override def name(): String = {
-    s"${getClass.getSimpleName} with servers [${servers.mkString(",")}]"
-  }
-
-  override def apply(requestTemplate: RequestTemplate): Request = {
-    if (!requestTemplate.url().startsWith("http")) {
-      requestTemplate.insert(0, url())
-    }
-    requestTemplate.request()
-  }
-
-  override def url(): String = threadLocalShuffledServers.get.head
-
-  override def continueOrPropagate(e: RetryableException): Unit = {
-    threadLocalCurrentAttempt.set(threadLocalCurrentAttempt.get + 1)
-    val currentAttempt = threadLocalCurrentAttempt.get
-    if (threadLocalCurrentAttempt.get < maxRetriesPerServer) {
-      logWarning(s"Attempt $currentAttempt of $maxRetriesPerServer failed for" +
-        s" server ${url()}. Retrying request...", e)
-      Thread.sleep(delayBetweenRetriesMillis)
-    } else {
-      val previousUrl = url()
-      threadLocalShuffledServers.set(threadLocalShuffledServers.get.drop(1))
-      if (threadLocalShuffledServers.get.isEmpty) {
-        logError(s"Failed request to all servers $maxRetriesPerServer times.", e)
-        throw e
-      } else {
-        logWarning(s"Failed request to $previousUrl $maxRetriesPerServer times." +
-          s" Trying to access ${url()} instead.", e)
-        threadLocalCurrentAttempt.set(0)
-      }
-    }
-  }
-
-  def reset(): Unit = {
-    threadLocalShuffledServers.set(Random.shuffle(servers))
-    threadLocalCurrentAttempt.set(0)
-  }
-}
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/SSLUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/SSLUtils.scala
index 886484ffb4692..8de0f56f007dc 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/SSLUtils.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/SSLUtils.scala
@@ -30,7 +30,7 @@ import org.bouncycastle.cert.jcajce.{JcaX509CertificateConverter, JcaX509v3Certi
 import org.bouncycastle.openssl.jcajce.JcaPEMWriter
 import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder
 
-import org.apache.spark.deploy.kubernetes.submit.v2.{KeyAndCertPem, KeyStoreAndTrustStore}
+import org.apache.spark.deploy.kubernetes.submit.{KeyAndCertPem, KeyStoreAndTrustStore}
 import org.apache.spark.util.Utils
 
 private[spark] object SSLUtils {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ClientV2Suite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientV2Suite.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ClientV2Suite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientV2Suite.scala
index 9ad46e52747fd..d4d3882bb8bab 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ClientV2Suite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ClientV2Suite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 
@@ -35,7 +35,6 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.kubernetes.SparkPodInitContainerBootstrap
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.submit.LoggingPodStatusWatcher
 
 class ClientV2Suite extends SparkFunSuite with BeforeAndAfter {
   private val JARS_RESOURCE = SubmittedResourceIdAndSecret("jarsId", "jarsSecret")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolverSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolverSuite.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolverSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolverSuite.scala
index 6804f0010b6a5..ca5cd1fff9b74 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ContainerLocalizedFilesResolverSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ContainerLocalizedFilesResolverSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.SparkFunSuite
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterSuite.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterSuite.scala
index d4413076fb092..c1005a176408c 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/DriverPodKubernetesCredentialsMounterSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/DriverPodKubernetesCredentialsMounterSuite.scala
@@ -14,16 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import io.fabric8.kubernetes.api.model.{PodBuilder, SecretBuilder}
 import org.scalatest.prop.TableDrivenPropertyChecks
 import scala.collection.JavaConverters._
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.deploy.kubernetes.KubernetesCredentials
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.KubernetesCredentials
 
 class DriverPodKubernetesCredentialsMounterSuite
     extends SparkFunSuite with TableDrivenPropertyChecks {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfigurationSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfigurationSuite.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfigurationSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfigurationSuite.scala
index 62bfd127d17e2..ead1d49b8a37c 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/ExecutorInitContainerConfigurationSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/ExecutorInitContainerConfigurationSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import org.apache.spark.{SparkConf, SparkFunSuite}
 import org.apache.spark.deploy.kubernetes.config._
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SSLFilePairs.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SSLFilePairs.scala
similarity index 94%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SSLFilePairs.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SSLFilePairs.scala
index 3d3ff7ad7011a..5240128743b76 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SSLFilePairs.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SSLFilePairs.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilderSuite.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilderSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilderSuite.scala
index 7c6fbf5ce6da2..f1e1ff7013496 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SparkInitContainerConfigMapBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SparkInitContainerConfigMapBuilderSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.StringReader
 import java.util.Properties
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPluginSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPluginSuite.scala
similarity index 96%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPluginSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPluginSuite.scala
index 09b41dc1bcaaf..8431b77c9e85f 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyInitContainerConfigPluginSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyInitContainerConfigPluginSuite.scala
@@ -14,11 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
-import java.io.File
-
-import org.apache.spark.{SparkFunSuite, SSLOptions}
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.deploy.kubernetes.config._
 
 class SubmittedDependencyInitContainerConfigPluginSuite extends SparkFunSuite {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilderSuite.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilderSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilderSuite.scala
index 358edbecf8708..83fd568e7a3aa 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencySecretBuilderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencySecretBuilderSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.File
 
@@ -24,7 +24,7 @@ import io.fabric8.kubernetes.api.model.Secret
 import scala.collection.JavaConverters._
 import scala.collection.Map
 
-import org.apache.spark.{SparkFunSuite, SSLOptions}
+import org.apache.spark.SparkFunSuite
 import org.apache.spark.util.Utils
 
 class SubmittedDependencySecretBuilderSuite extends SparkFunSuite {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderSuite.scala
similarity index 97%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderSuite.scala
index 7b259aa2c3a0c..8693ff4e15372 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/v2/SubmittedDependencyUploaderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/kubernetes/submit/SubmittedDependencyUploaderSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.kubernetes.submit.v2
+package org.apache.spark.deploy.kubernetes.submit
 
 import java.io.{ByteArrayInputStream, ByteArrayOutputStream, File}
 import java.util.UUID
@@ -35,7 +35,7 @@ import retrofit2.{Call, Response}
 
 import org.apache.spark.{SparkFunSuite, SSLOptions}
 import org.apache.spark.deploy.kubernetes.CompressionUtils
-import org.apache.spark.deploy.rest.kubernetes.v2.{ResourceStagingServiceRetrofit, RetrofitClientFactory}
+import org.apache.spark.deploy.rest.kubernetes.{ResourceStagingServiceRetrofit, RetrofitClientFactory}
 import org.apache.spark.util.Utils
 
 private[spark] class SubmittedDependencyUploaderSuite extends SparkFunSuite with BeforeAndAfter {
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainerSuite.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainerSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainerSuite.scala
index c551fbc01d060..f2fdf026390cd 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/KubernetesSparkDependencyDownloadInitContainerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/KubernetesSparkDependencyDownloadInitContainerSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{ByteArrayOutputStream, File}
 import java.util.UUID
@@ -32,7 +32,7 @@ import org.scalatest.BeforeAndAfter
 import org.scalatest.mock.MockitoSugar._
 import retrofit2.{Call, Callback, Response}
 
-import org.apache.spark.{SecurityManager => SparkSecurityManager, SparkConf, SparkFunSuite, SSLOptions}
+import org.apache.spark.{SparkConf, SparkFunSuite, SSLOptions}
 import org.apache.spark.deploy.kubernetes.CompressionUtils
 import org.apache.spark.deploy.kubernetes.config._
 import org.apache.spark.util.Utils
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProviderSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProviderSuite.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProviderSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProviderSuite.scala
index c33d8beb2c397..3bb318d713a54 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSslOptionsProviderSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSslOptionsProviderSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{File, FileInputStream, StringWriter}
 import java.security.KeyStore
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSuite.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSuite.scala
index 4ffb0d4dfa887..0604e0d6494ae 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServerSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.net.ServerSocket
 import javax.ws.rs.core.MediaType
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImplSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImplSuite.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImplSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImplSuite.scala
index 9677d12681a16..53396a3f27a1a 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/v2/ResourceStagingServiceImplSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/rest/kubernetes/ResourceStagingServiceImplSuite.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.deploy.rest.kubernetes.v2
+package org.apache.spark.deploy.rest.kubernetes
 
 import java.io.{ByteArrayInputStream, File}
 import java.nio.file.Paths
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-v2/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-v2/Dockerfile
deleted file mode 100644
index 40f9459dc06dc..0000000000000
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-v2/Dockerfile
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM openjdk:8-alpine
-
-# If this docker file is being used in the context of building your images from a Spark distribution, the docker build
-# command should be invoked from the top level directory of the Spark distribution. E.g.:
-# docker build -t spark-driver:latest -f dockerfiles/driver/Dockerfile .
-
-RUN apk upgrade --update
-RUN apk add --update bash
-RUN mkdir -p /opt/spark
-RUN touch /opt/spark/RELEASE
-
-ADD jars /opt/spark/jars
-ADD examples /opt/spark/examples
-ADD bin /opt/spark/bin
-ADD sbin /opt/spark/sbin
-ADD conf /opt/spark/conf
-
-ENV SPARK_HOME /opt/spark
-
-WORKDIR /opt/spark
-
-CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
-    if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
-    exec ${JAVA_HOME}/bin/java $SPARK_DRIVER_JAVA_OPTS -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
index 8ab7a58704505..40f9459dc06dc 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver/Dockerfile
@@ -36,16 +36,8 @@ ENV SPARK_HOME /opt/spark
 
 WORKDIR /opt/spark
 
-CMD SSL_ARGS="" && \
-    if ! [ -z ${SPARK_SUBMISSION_USE_SSL+x} ]; then SSL_ARGS="$SSL_ARGS --use-ssl $SPARK_SUBMISSION_USE_SSL"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_KEYSTORE_FILE+x} ]; then SSL_ARGS="$SSL_ARGS --keystore-file $SPARK_SUBMISSION_KEYSTORE_FILE"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_KEYSTORE_TYPE+x} ]; then SSL_ARGS="$SSL_ARGS --keystore-type $SPARK_SUBMISSION_KEYSTORE_TYPE"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_KEYSTORE_PASSWORD_FILE+x} ]; then SSL_ARGS="$SSL_ARGS --keystore-password-file $SPARK_SUBMISSION_KEYSTORE_PASSWORD_FILE"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_KEYSTORE_KEY_PASSWORD_FILE+x} ]; then SSL_ARGS="$SSL_ARGS --keystore-key-password-file $SPARK_SUBMISSION_KEYSTORE_KEY_PASSWORD_FILE"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_KEY_PEM_FILE+x} ]; then SSL_ARGS="$SSL_ARGS --key-pem-file $SPARK_SUBMISSION_KEY_PEM_FILE"; fi && \
-    if ! [ -z ${SPARK_SUBMISSION_CERT_PEM_FILE+x} ]; then SSL_ARGS="$SSL_ARGS --cert-pem-file $SPARK_SUBMISSION_CERT_PEM_FILE"; fi && \
-    exec bin/spark-class org.apache.spark.deploy.rest.kubernetes.v1.KubernetesSparkRestServer \
-      --hostname $HOSTNAME \
-      --port $SPARK_SUBMISSION_SERVER_PORT \
-      --secret-file $SPARK_SUBMISSION_SECRET_LOCATION \
-      ${SSL_ARGS}
+CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \
+    if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \
+    if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
+    if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH"; fi && \
+    exec ${JAVA_HOME}/bin/java $SPARK_DRIVER_JAVA_OPTS -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-init/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
similarity index 95%
rename from resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-init/Dockerfile
rename to resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
index 59029a6c08b4a..bb249a4ea86b6 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-init/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/init-container/Dockerfile
@@ -35,4 +35,4 @@ ENV SPARK_HOME /opt/spark
 
 WORKDIR /opt/spark
 
-ENTRYPOINT [ "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.v2.KubernetesSparkDependencyDownloadInitContainer" ]
+ENTRYPOINT [ "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.KubernetesSparkDependencyDownloadInitContainer" ]
diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
index 15e1ce75815df..125749c71c79a 100644
--- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
+++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/resource-staging-server/Dockerfile
@@ -35,4 +35,4 @@ ENV SPARK_HOME /opt/spark
 
 WORKDIR /opt/spark
 
-ENTRYPOINT [ "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.v2.ResourceStagingServer" ]
+ENTRYPOINT [ "bin/spark-class", "org.apache.spark.deploy.rest.kubernetes.ResourceStagingServer" ]
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
index 56fcf692b8ff7..d23bfcdbc5251 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesSuite.scala
@@ -17,31 +17,257 @@
 package org.apache.spark.deploy.kubernetes.integrationtest
 
 import java.nio.file.Paths
+import java.util.UUID
 
 import com.google.common.base.Charsets
 import com.google.common.io.Files
-import org.scalatest.Suite
-import org.scalatest.concurrent.PatienceConfiguration
+import io.fabric8.kubernetes.client.internal.readiness.Readiness
+import org.scalatest.BeforeAndAfter
+import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
 import org.scalatest.time.{Minutes, Seconds, Span}
+import scala.collection.JavaConverters._
 
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.deploy.kubernetes.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory}
+import org.apache.spark.{SparkConf, SparkFunSuite, SSLOptions}
+import org.apache.spark.deploy.kubernetes.SSLUtils
+import org.apache.spark.deploy.kubernetes.config._
+import org.apache.spark.deploy.kubernetes.integrationtest.backend.IntegrationTestBackendFactory
+import org.apache.spark.deploy.kubernetes.integrationtest.backend.minikube.Minikube
+import org.apache.spark.deploy.kubernetes.integrationtest.constants.MINIKUBE_TEST_BACKEND
+import org.apache.spark.deploy.kubernetes.submit.{Client, KeyAndCertPem}
+import org.apache.spark.launcher.SparkLauncher
 
-private[spark] class KubernetesSuite extends SparkFunSuite {
-  private val testBackend: IntegrationTestBackend = IntegrationTestBackendFactory.getTestBackend()
+private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter {
+  import KubernetesSuite._
+  private val testBackend = IntegrationTestBackendFactory.getTestBackend()
+
+  private val APP_LOCATOR_LABEL = UUID.randomUUID().toString.replaceAll("-", "")
+  private var kubernetesTestComponents: KubernetesTestComponents = _
+  private var sparkConf: SparkConf = _
+  private var resourceStagingServerLauncher: ResourceStagingServerLauncher = _
+  private var staticAssetServerLauncher: StaticAssetServerLauncher = _
 
   override def beforeAll(): Unit = {
     testBackend.initialize()
+    kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient)
+    resourceStagingServerLauncher = new ResourceStagingServerLauncher(
+      kubernetesTestComponents.kubernetesClient.inNamespace(kubernetesTestComponents.namespace))
+    staticAssetServerLauncher = new StaticAssetServerLauncher(
+      kubernetesTestComponents.kubernetesClient.inNamespace(kubernetesTestComponents.namespace))
   }
 
   override def afterAll(): Unit = {
     testBackend.cleanUp()
   }
 
-  override def nestedSuites: scala.collection.immutable.IndexedSeq[Suite] = {
-      Vector(
-        new KubernetesV1Suite(testBackend),
-        new KubernetesV2Suite(testBackend))
+  before {
+    sparkConf = kubernetesTestComponents.newSparkConf()
+      .set(INIT_CONTAINER_DOCKER_IMAGE, s"spark-init:latest")
+      .set(DRIVER_DOCKER_IMAGE, s"spark-driver:latest")
+      .set(KUBERNETES_DRIVER_LABELS, s"spark-app-locator=$APP_LOCATOR_LABEL")
+    kubernetesTestComponents.createNamespace()
+  }
+
+  after {
+    kubernetesTestComponents.deleteNamespace()
+  }
+
+  test("Simple submission test with the resource staging server.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+
+    launchStagingServer(SSLOptions(), None)
+    runSparkPiAndVerifyCompletion(SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
+  }
+
+  test("Enable SSL on the resource staging server") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+
+    val keyStoreAndTrustStore = SSLUtils.generateKeyStoreTrustStorePair(
+      ipAddress = Minikube.getMinikubeIp,
+      keyStorePassword = "keyStore",
+      keyPassword = "key",
+      trustStorePassword = "trustStore")
+    sparkConf.set(RESOURCE_STAGING_SERVER_SSL_ENABLED, true)
+      .set("spark.ssl.kubernetes.resourceStagingServer.keyStore",
+          keyStoreAndTrustStore.keyStore.getAbsolutePath)
+      .set("spark.ssl.kubernetes.resourceStagingServer.trustStore",
+          keyStoreAndTrustStore.trustStore.getAbsolutePath)
+      .set("spark.ssl.kubernetes.resourceStagingServer.keyStorePassword", "keyStore")
+      .set("spark.ssl.kubernetes.resourceStagingServer.keyPassword", "key")
+      .set("spark.ssl.kubernetes.resourceStagingServer.trustStorePassword", "trustStore")
+    launchStagingServer(SSLOptions(
+      enabled = true,
+      keyStore = Some(keyStoreAndTrustStore.keyStore),
+      trustStore = Some(keyStoreAndTrustStore.trustStore),
+      keyStorePassword = Some("keyStore"),
+      keyPassword = Some("key"),
+      trustStorePassword = Some("trustStore")),
+      None)
+    runSparkPiAndVerifyCompletion(SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
+  }
+
+  test("Use container-local resources without the resource staging server") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+
+    sparkConf.setJars(Seq(CONTAINER_LOCAL_HELPER_JAR_PATH))
+    runSparkPiAndVerifyCompletion(CONTAINER_LOCAL_MAIN_APP_RESOURCE)
+  }
+
+  test("Dynamic executor scaling basic test") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+
+    launchStagingServer(SSLOptions(), None)
+    createShuffleServiceDaemonSet()
+
+    sparkConf.setJars(Seq(CONTAINER_LOCAL_HELPER_JAR_PATH))
+    sparkConf.set("spark.dynamicAllocation.enabled", "true")
+    sparkConf.set("spark.shuffle.service.enabled", "true")
+    sparkConf.set("spark.kubernetes.shuffle.labels", "app=spark-shuffle-service")
+    sparkConf.set("spark.kubernetes.shuffle.namespace", kubernetesTestComponents.namespace)
+    sparkConf.set("spark.app.name", "group-by-test")
+    runSparkGroupByTestAndVerifyCompletion(SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
+  }
+
+  test("Use remote resources without the resource staging server.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    val assetServerUri = staticAssetServerLauncher.launchStaticAssetServer()
+    sparkConf.setJars(Seq(
+      s"$assetServerUri/${EXAMPLES_JAR_FILE.getName}",
+      s"$assetServerUri/${HELPER_JAR_FILE.getName}"
+    ))
+    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
+  }
+
+  test("Mix remote resources with submitted ones.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    launchStagingServer(SSLOptions(), None)
+    val assetServerUri = staticAssetServerLauncher.launchStaticAssetServer()
+    sparkConf.setJars(Seq(
+      SUBMITTER_LOCAL_MAIN_APP_RESOURCE, s"$assetServerUri/${HELPER_JAR_FILE.getName}"
+    ))
+    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
+  }
+
+  test("Use key and certificate PEM files for TLS.") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    val keyAndCertificate = SSLUtils.generateKeyCertPemPair(Minikube.getMinikubeIp)
+    launchStagingServer(
+        SSLOptions(enabled = true),
+        Some(keyAndCertificate))
+    sparkConf.set(RESOURCE_STAGING_SERVER_SSL_ENABLED, true)
+        .set(
+            RESOURCE_STAGING_SERVER_CLIENT_CERT_PEM.key, keyAndCertificate.certPem.getAbsolutePath)
+    runSparkPiAndVerifyCompletion(SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
+  }
+
+  test("Use client key and client cert file when requesting executors") {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+    sparkConf.setJars(Seq(
+        CONTAINER_LOCAL_MAIN_APP_RESOURCE,
+        CONTAINER_LOCAL_HELPER_JAR_PATH))
+    sparkConf.set(KUBERNETES_DRIVER_CLIENT_KEY_FILE,
+        kubernetesTestComponents.clientConfig.getClientKeyFile)
+    sparkConf.set(KUBERNETES_DRIVER_CLIENT_CERT_FILE,
+        kubernetesTestComponents.clientConfig.getClientCertFile)
+    sparkConf.set(KUBERNETES_DRIVER_CA_CERT_FILE,
+        kubernetesTestComponents.clientConfig.getCaCertFile)
+    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
+  }
+
+  private def launchStagingServer(
+      resourceStagingServerSslOptions: SSLOptions, keyAndCertPem: Option[KeyAndCertPem]): Unit = {
+    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
+
+    val resourceStagingServerPort = resourceStagingServerLauncher.launchStagingServer(
+      resourceStagingServerSslOptions, keyAndCertPem)
+    val resourceStagingServerUriScheme = if (resourceStagingServerSslOptions.enabled) {
+      "https"
+    } else {
+      "http"
+    }
+    sparkConf.set(RESOURCE_STAGING_SERVER_URI,
+      s"$resourceStagingServerUriScheme://" +
+        s"${Minikube.getMinikubeIp}:$resourceStagingServerPort")
+  }
+
+  private def runSparkPiAndVerifyCompletion(appResource: String): Unit = {
+    Client.run(sparkConf, appResource, SPARK_PI_MAIN_CLASS, Array.empty[String])
+    val driverPod = kubernetesTestComponents.kubernetesClient
+      .pods()
+      .withLabel("spark-app-locator", APP_LOCATOR_LABEL)
+      .list()
+      .getItems
+      .get(0)
+    Eventually.eventually(TIMEOUT, INTERVAL) {
+      assert(kubernetesTestComponents.kubernetesClient
+        .pods()
+        .withName(driverPod.getMetadata.getName)
+        .getLog
+        .contains("Pi is roughly 3"), "The application did not compute the value of pi.")
+    }
+  }
+
+  private def runSparkGroupByTestAndVerifyCompletion(appResource: String): Unit = {
+    Client.run(
+      sparkConf = sparkConf,
+      appArgs = Array.empty[String],
+      mainClass = GROUP_BY_MAIN_CLASS,
+      mainAppResource = appResource)
+    val driverPod = kubernetesTestComponents.kubernetesClient
+      .pods()
+      .withLabel("spark-app-locator", APP_LOCATOR_LABEL)
+      .list()
+      .getItems
+      .get(0)
+    Eventually.eventually(TIMEOUT, INTERVAL) {
+      assert(kubernetesTestComponents.kubernetesClient
+        .pods()
+        .withName(driverPod.getMetadata.getName)
+        .getLog
+        .contains("The Result is"), "The application did not complete.")
+    }
+  }
+
+  private def createShuffleServiceDaemonSet(): Unit = {
+    val ds = kubernetesTestComponents.kubernetesClient.extensions().daemonSets()
+      .createNew()
+        .withNewMetadata()
+        .withName("shuffle")
+      .endMetadata()
+      .withNewSpec()
+        .withNewTemplate()
+          .withNewMetadata()
+            .withLabels(Map("app" -> "spark-shuffle-service").asJava)
+          .endMetadata()
+          .withNewSpec()
+            .addNewVolume()
+              .withName("shuffle-dir")
+              .withNewHostPath()
+                .withPath("/tmp")
+              .endHostPath()
+            .endVolume()
+            .addNewContainer()
+              .withName("shuffle")
+              .withImage("spark-shuffle:latest")
+              .withImagePullPolicy("IfNotPresent")
+              .addNewVolumeMount()
+                .withName("shuffle-dir")
+                .withMountPath("/tmp")
+              .endVolumeMount()
+            .endContainer()
+          .endSpec()
+        .endTemplate()
+      .endSpec()
+      .done()
+
+    // wait for daemonset to become available.
+    Eventually.eventually(TIMEOUT, INTERVAL) {
+      val pods = kubernetesTestComponents.kubernetesClient.pods()
+        .withLabel("app", "spark-shuffle-service").list().getItems
+
+      if (pods.size() == 0 || !Readiness.isReady(pods.get(0))) {
+        throw ShuffleNotReadyException
+      }
+    }
   }
 }
 
@@ -70,5 +296,5 @@ private[spark] object KubernetesSuite {
   val GROUP_BY_MAIN_CLASS = "org.apache.spark.deploy.kubernetes" +
     ".integrationtest.jobs.GroupByTest"
 
-  case class ShuffleNotReadyException() extends Exception
+  case object ShuffleNotReadyException extends Exception
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesTestComponents.scala
index 677c0db606a47..9ae0d9ade7dc2 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesTestComponents.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesTestComponents.scala
@@ -17,18 +17,13 @@
 package org.apache.spark.deploy.kubernetes.integrationtest
 
 import java.util.UUID
-import javax.net.ssl.X509TrustManager
-
-import scala.collection.JavaConverters._
-import scala.reflect.ClassTag
 
 import io.fabric8.kubernetes.client.DefaultKubernetesClient
-import io.fabric8.kubernetes.client.internal.SSLUtils
 import org.scalatest.concurrent.Eventually
+import scala.collection.JavaConverters._
 
 import org.apache.spark.SparkConf
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.rest.kubernetes.v1.HttpClientUtil
 
 private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) {
 
@@ -73,26 +68,4 @@ private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesCl
       .set("spark.testing", "false")
       .set(WAIT_FOR_APP_COMPLETION, false)
   }
-
-  def getService[T: ClassTag](
-    serviceName: String,
-    namespace: String,
-    servicePortName: String,
-    servicePath: String = ""): T = synchronized {
-    val kubernetesMaster = s"${defaultClient.getMasterUrl}"
-
-    val url = s"${
-      Array[String](
-        s"${kubernetesClient.getMasterUrl}",
-        "api", "v1", "proxy",
-        "namespaces", namespace,
-        "services", serviceName).mkString("/")
-    }" +
-      s":$servicePortName$servicePath"
-    val userHome = System.getProperty("user.home")
-    val kubernetesConf = kubernetesClient.getConfiguration
-    val sslContext = SSLUtils.sslContext(kubernetesConf)
-    val trustManager = SSLUtils.trustManagers(kubernetesConf)(0).asInstanceOf[X509TrustManager]
-    HttpClientUtil.createClient[T](Set(url), 5, sslContext.getSocketFactory, trustManager)
-  }
 }
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV1Suite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV1Suite.scala
deleted file mode 100644
index 559cb281c7c62..0000000000000
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV1Suite.scala
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.integrationtest
-
-import java.util.concurrent.TimeUnit
-
-import scala.collection.JavaConverters._
-
-import com.google.common.collect.ImmutableList
-import com.google.common.util.concurrent.SettableFuture
-import io.fabric8.kubernetes.api.model.Pod
-import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher}
-import io.fabric8.kubernetes.client.Watcher.Action
-import org.scalatest.{BeforeAndAfter, DoNotDiscover}
-import org.scalatest.concurrent.Eventually
-
-import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
-import org.apache.spark.deploy.kubernetes.SSLUtils
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.constants._
-import org.apache.spark.deploy.kubernetes.integrationtest.backend.IntegrationTestBackend
-import org.apache.spark.deploy.kubernetes.integrationtest.backend.minikube.Minikube
-import org.apache.spark.deploy.kubernetes.integrationtest.constants.MINIKUBE_TEST_BACKEND
-import org.apache.spark.deploy.kubernetes.integrationtest.restapis.SparkRestApiV1
-import org.apache.spark.deploy.kubernetes.submit.v1.{Client, ExternalSuppliedUrisDriverServiceManager}
-import org.apache.spark.status.api.v1.{ApplicationStatus, StageStatus}
-import org.apache.spark.util.Utils
-
-@DoNotDiscover
-private[spark] class KubernetesV1Suite(testBackend: IntegrationTestBackend)
-  extends SparkFunSuite with BeforeAndAfter {
-
-  private var kubernetesTestComponents: KubernetesTestComponents = _
-  private var sparkConf: SparkConf = _
-
-  override def beforeAll(): Unit = {
-    kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient)
-    kubernetesTestComponents.createNamespace()
-  }
-
-  override def afterAll(): Unit = {
-    kubernetesTestComponents.deleteNamespace()
-  }
-
-  before {
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      val podsList = kubernetesTestComponents.kubernetesClient.pods().list()
-      assert(podsList == null
-        || podsList.getItems == null
-        || podsList.getItems.isEmpty
-      )
-      val servicesList = kubernetesTestComponents.kubernetesClient.services().list()
-      assert(servicesList == null
-        || servicesList.getItems == null
-        || servicesList.getItems.isEmpty)
-    }
-    sparkConf = kubernetesTestComponents.newSparkConf()
-  }
-
-  after {
-    val pods = kubernetesTestComponents.kubernetesClient.pods().list().getItems.asScala
-    pods.par.foreach(pod => {
-      kubernetesTestComponents.kubernetesClient.pods()
-        .withName(pod.getMetadata.getName)
-        .withGracePeriod(60)
-        .delete
-    })
-  }
-
-  private def getSparkMetricsService(sparkBaseAppName: String): SparkRestApiV1 = {
-    val serviceName = kubernetesTestComponents.kubernetesClient.services()
-      .withLabel("spark-app-name", sparkBaseAppName)
-      .list()
-      .getItems
-      .get(0)
-      .getMetadata
-      .getName
-    kubernetesTestComponents.getService[SparkRestApiV1](serviceName,
-      kubernetesTestComponents.namespace, "spark-ui-port")
-  }
-
-  private def expectationsForStaticAllocation(sparkMetricsService: SparkRestApiV1): Unit = {
-    val apps = Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      val result = sparkMetricsService
-        .getApplications(ImmutableList.of(ApplicationStatus.RUNNING, ApplicationStatus.COMPLETED))
-      assert(result.size == 1
-        && !result.head.id.equalsIgnoreCase("appid")
-        && !result.head.id.equalsIgnoreCase("{appId}"))
-      result
-    }
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      val result = sparkMetricsService.getExecutors(apps.head.id)
-      assert(result.size == 2)
-      assert(result.count(exec => exec.id != "driver") == 1)
-      result
-    }
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      val result = sparkMetricsService.getStages(
-        apps.head.id, Seq(StageStatus.COMPLETE).asJava)
-      assert(result.size == 1)
-      result
-    }
-  }
-
-  test("Run a simple example") {
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-    val sparkMetricsService = getSparkMetricsService("spark-pi")
-    expectationsForStaticAllocation(sparkMetricsService)
-  }
-
-  test("Run with the examples jar on the docker image") {
-    sparkConf.setJars(Seq(KubernetesSuite.CONTAINER_LOCAL_HELPER_JAR_PATH))
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.CONTAINER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-    val sparkMetricsService = getSparkMetricsService("spark-pi")
-    expectationsForStaticAllocation(sparkMetricsService)
-  }
-
-  test("Run with custom labels and annotations") {
-    sparkConf.set(KUBERNETES_DRIVER_LABELS, "label1=label1value,label2=label2value")
-    sparkConf.set(KUBERNETES_DRIVER_ANNOTATIONS, "annotation1=annotation1value," +
-        "annotation2=annotation2value")
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-    val driverPodMetadata = kubernetesTestComponents.kubernetesClient
-      .pods
-      .withLabel("spark-app-name", "spark-pi")
-      .list()
-      .getItems
-      .get(0)
-      .getMetadata
-    val driverPodLabels = driverPodMetadata.getLabels
-    // We can't match all of the selectors directly since one of the selectors is based on the
-    // launch time.
-    assert(driverPodLabels.size === 5, "Unexpected number of pod labels.")
-    assert(driverPodLabels.get("spark-app-name") === "spark-pi", "Unexpected value for" +
-      " spark-app-name label.")
-    assert(driverPodLabels.get("spark-app-id").startsWith("spark-pi"), "Unexpected value for" +
-      " spark-app-id label (should be prefixed with the app name).")
-    assert(driverPodLabels.get("label1") === "label1value", "Unexpected value for label1")
-    assert(driverPodLabels.get("label2") === "label2value", "Unexpected value for label2")
-    val driverPodAnnotations = driverPodMetadata.getAnnotations
-    assert(driverPodAnnotations.size === 2, "Unexpected number of pod annotations.")
-    assert(driverPodAnnotations.get("annotation1") === "annotation1value",
-      "Unexpected value for annotation1")
-    assert(driverPodAnnotations.get("annotation2") === "annotation2value",
-      "Unexpected value for annotation2")
-  }
-
-  test("Run with driver pod name") {
-    sparkConf.set(KUBERNETES_DRIVER_POD_NAME, "spark-pi")
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-    val driverPodMetadata = kubernetesTestComponents.kubernetesClient
-      .pods()
-      .withName("spark-pi")
-      .get()
-      .getMetadata()
-    val driverName = driverPodMetadata.getName
-    assert(driverName === "spark-pi", "Unexpected driver pod name.")
-  }
-
-  test("Enable SSL on the driver submit server") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    val keyStoreAndTrustStore = SSLUtils.generateKeyStoreTrustStorePair(
-      Minikube.getMinikubeIp,
-      "changeit",
-      "changeit",
-      "changeit")
-    sparkConf.set(KUBERNETES_DRIVER_SUBMIT_SSL_KEYSTORE,
-        s"file://${keyStoreAndTrustStore.keyStore.getAbsolutePath}")
-    sparkConf.set("spark.ssl.kubernetes.driversubmitserver.keyStorePassword", "changeit")
-    sparkConf.set("spark.ssl.kubernetes.driversubmitserver.keyPassword", "changeit")
-    sparkConf.set(KUBERNETES_DRIVER_SUBMIT_SSL_TRUSTSTORE,
-        s"file://${keyStoreAndTrustStore.trustStore.getAbsolutePath}")
-    sparkConf.set("spark.ssl.kubernetes.driversubmitserver.trustStorePassword", "changeit")
-    sparkConf.set(DRIVER_SUBMIT_SSL_ENABLED, true)
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-  }
-
-  test("Enable SSL on the driver submit server using PEM files") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    val keyAndCertPem = SSLUtils.generateKeyCertPemPair(Minikube.getMinikubeIp)
-    sparkConf.set(DRIVER_SUBMIT_SSL_KEY_PEM, s"file://${keyAndCertPem.keyPem.getAbsolutePath}")
-    sparkConf.set(
-        DRIVER_SUBMIT_SSL_CLIENT_CERT_PEM, s"file://${keyAndCertPem.certPem.getAbsolutePath}")
-    sparkConf.set(
-        DRIVER_SUBMIT_SSL_SERVER_CERT_PEM, s"file://${keyAndCertPem.certPem.getAbsolutePath}")
-    sparkConf.set(DRIVER_SUBMIT_SSL_ENABLED, true)
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-  }
-
-  test("Added files should exist on the driver.") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    sparkConf.set("spark.files", KubernetesSuite.TEST_EXISTENCE_FILE.getAbsolutePath)
-    sparkConf.setAppName("spark-file-existence-test")
-    val podCompletedFuture = SettableFuture.create[Boolean]
-    val watch = new Watcher[Pod] {
-      override def eventReceived(action: Action, pod: Pod): Unit = {
-        val containerStatuses = pod.getStatus.getContainerStatuses.asScala
-        val allSuccessful = containerStatuses.nonEmpty && containerStatuses
-          .forall(status => {
-            status.getState.getTerminated != null && status.getState.getTerminated.getExitCode == 0
-        })
-        if (allSuccessful) {
-          podCompletedFuture.set(true)
-        } else {
-          val failedContainers = containerStatuses.filter(container => {
-            container.getState.getTerminated != null &&
-              container.getState.getTerminated.getExitCode != 0
-          })
-          if (failedContainers.nonEmpty) {
-            podCompletedFuture.setException(new SparkException(
-              "One or more containers in the driver failed with a nonzero exit code."))
-          }
-        }
-      }
-
-      override def onClose(e: KubernetesClientException): Unit = {
-        logWarning("Watch closed", e)
-      }
-    }
-    Utils.tryWithResource(kubernetesTestComponents.kubernetesClient
-        .pods
-        .withLabel("spark-app-name", "spark-file-existence-test")
-        .watch(watch)) { _ =>
-      new Client(
-        sparkConf = sparkConf,
-        mainClass = KubernetesSuite.FILE_EXISTENCE_MAIN_CLASS,
-        mainAppResource = KubernetesSuite.CONTAINER_LOCAL_MAIN_APP_RESOURCE,
-        appArgs = Array(KubernetesSuite.TEST_EXISTENCE_FILE.getName,
-          KubernetesSuite.TEST_EXISTENCE_FILE_CONTENTS)).run()
-      assert(podCompletedFuture.get(60, TimeUnit.SECONDS), "Failed to run driver pod")
-      val driverPod = kubernetesTestComponents.kubernetesClient
-        .pods
-        .withLabel("spark-app-name", "spark-file-existence-test")
-        .list()
-        .getItems
-        .get(0)
-      val podLog = kubernetesTestComponents.kubernetesClient
-        .pods
-        .withName(driverPod.getMetadata.getName)
-        .getLog
-      assert(podLog.contains(s"File found at" +
-        s" /opt/spark/${KubernetesSuite.TEST_EXISTENCE_FILE.getName} with correct contents."),
-        "Job did not find the file as expected.")
-    }
-  }
-
-  test("Use external URI provider") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    val externalUriProviderWatch =
-      new ExternalUriProviderWatch(kubernetesTestComponents.kubernetesClient)
-    Utils.tryWithResource(kubernetesTestComponents.kubernetesClient.services()
-        .withLabel("spark-app-name", "spark-pi")
-        .watch(externalUriProviderWatch)) { _ =>
-      sparkConf.set(DRIVER_SERVICE_MANAGER_TYPE, ExternalSuppliedUrisDriverServiceManager.TYPE)
-      new Client(
-        sparkConf = sparkConf,
-        mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-        mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-        appArgs = Array.empty[String]).run()
-      val sparkMetricsService = getSparkMetricsService("spark-pi")
-      expectationsForStaticAllocation(sparkMetricsService)
-      assert(externalUriProviderWatch.annotationSet.get)
-      val driverService = kubernetesTestComponents.kubernetesClient
-        .services()
-        .withLabel("spark-app-name", "spark-pi")
-        .list()
-        .getItems
-        .asScala(0)
-      assert(driverService.getMetadata.getAnnotations.containsKey(ANNOTATION_PROVIDE_EXTERNAL_URI),
-          "External URI request annotation was not set on the driver service.")
-      // Unfortunately we can't check the correctness of the actual value of the URI, as it depends
-      // on the driver submission port set on the driver service but we remove that port from the
-      // service once the submission is complete.
-      assert(driverService.getMetadata.getAnnotations.containsKey(ANNOTATION_RESOLVED_EXTERNAL_URI),
-        "Resolved URI annotation not set on driver service.")
-    }
-  }
-
-  test("Mount the Kubernetes credentials onto the driver pod") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    sparkConf.set(KUBERNETES_DRIVER_CA_CERT_FILE,
-      kubernetesTestComponents.clientConfig.getCaCertFile)
-    sparkConf.set(KUBERNETES_DRIVER_CLIENT_KEY_FILE,
-      kubernetesTestComponents.clientConfig.getClientKeyFile)
-    sparkConf.set(KUBERNETES_DRIVER_CLIENT_CERT_FILE,
-      kubernetesTestComponents.clientConfig.getClientCertFile)
-    new Client(
-      sparkConf = sparkConf,
-      mainClass = KubernetesSuite.SPARK_PI_MAIN_CLASS,
-      mainAppResource = KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      appArgs = Array.empty[String]).run()
-    val sparkMetricsService = getSparkMetricsService("spark-pi")
-    expectationsForStaticAllocation(sparkMetricsService)
-  }
-
-}
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV2Suite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV2Suite.scala
deleted file mode 100644
index e9900b90cb588..0000000000000
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/KubernetesV2Suite.scala
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.deploy.kubernetes.integrationtest
-
-import java.util.UUID
-
-import io.fabric8.kubernetes.client.internal.readiness.Readiness
-import org.scalatest.{BeforeAndAfter, DoNotDiscover}
-import org.scalatest.concurrent.Eventually
-import scala.collection.JavaConverters._
-
-import org.apache.spark.{SparkConf, SparkFunSuite, SSLOptions}
-import org.apache.spark.deploy.kubernetes.SSLUtils
-import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.integrationtest.backend.IntegrationTestBackend
-import org.apache.spark.deploy.kubernetes.integrationtest.backend.minikube.Minikube
-import org.apache.spark.deploy.kubernetes.integrationtest.constants.MINIKUBE_TEST_BACKEND
-import org.apache.spark.deploy.kubernetes.submit.v2.{Client, KeyAndCertPem}
-import org.apache.spark.launcher.SparkLauncher
-
-@DoNotDiscover
-private[spark] class KubernetesV2Suite(testBackend: IntegrationTestBackend)
-  extends SparkFunSuite with BeforeAndAfter {
-
-  private val APP_LOCATOR_LABEL = UUID.randomUUID().toString.replaceAll("-", "")
-  private var kubernetesTestComponents: KubernetesTestComponents = _
-  private var sparkConf: SparkConf = _
-  private var resourceStagingServerLauncher: ResourceStagingServerLauncher = _
-  private var staticAssetServerLauncher: StaticAssetServerLauncher = _
-
-  override def beforeAll(): Unit = {
-    kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient)
-    resourceStagingServerLauncher = new ResourceStagingServerLauncher(
-      kubernetesTestComponents.kubernetesClient.inNamespace(kubernetesTestComponents.namespace))
-    staticAssetServerLauncher = new StaticAssetServerLauncher(
-      kubernetesTestComponents.kubernetesClient.inNamespace(kubernetesTestComponents.namespace))
-  }
-
-  before {
-    sparkConf = kubernetesTestComponents.newSparkConf()
-      .set(INIT_CONTAINER_DOCKER_IMAGE, s"spark-driver-init:latest")
-      .set(DRIVER_DOCKER_IMAGE, s"spark-driver-v2:latest")
-      .set(KUBERNETES_DRIVER_LABELS, s"spark-app-locator=$APP_LOCATOR_LABEL")
-    kubernetesTestComponents.createNamespace()
-  }
-
-  after {
-    kubernetesTestComponents.deleteNamespace()
-  }
-
-  test("Use submission v2.") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    launchStagingServer(SSLOptions(), None)
-    runSparkPiAndVerifyCompletion(KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
-  }
-
-  test("Enable SSL on the submission server") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    val keyStoreAndTrustStore = SSLUtils.generateKeyStoreTrustStorePair(
-      ipAddress = Minikube.getMinikubeIp,
-      keyStorePassword = "keyStore",
-      keyPassword = "key",
-      trustStorePassword = "trustStore")
-    sparkConf.set(RESOURCE_STAGING_SERVER_SSL_ENABLED, true)
-      .set("spark.ssl.kubernetes.resourceStagingServer.keyStore",
-          keyStoreAndTrustStore.keyStore.getAbsolutePath)
-      .set("spark.ssl.kubernetes.resourceStagingServer.trustStore",
-          keyStoreAndTrustStore.trustStore.getAbsolutePath)
-      .set("spark.ssl.kubernetes.resourceStagingServer.keyStorePassword", "keyStore")
-      .set("spark.ssl.kubernetes.resourceStagingServer.keyPassword", "key")
-      .set("spark.ssl.kubernetes.resourceStagingServer.trustStorePassword", "trustStore")
-    launchStagingServer(SSLOptions(
-      enabled = true,
-      keyStore = Some(keyStoreAndTrustStore.keyStore),
-      trustStore = Some(keyStoreAndTrustStore.trustStore),
-      keyStorePassword = Some("keyStore"),
-      keyPassword = Some("key"),
-      trustStorePassword = Some("trustStore")),
-      None)
-    runSparkPiAndVerifyCompletion(KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
-  }
-
-  test("Use container-local resources without the resource staging server") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    sparkConf.setJars(Seq(
-      KubernetesSuite.CONTAINER_LOCAL_HELPER_JAR_PATH))
-    runSparkPiAndVerifyCompletion(KubernetesSuite.CONTAINER_LOCAL_MAIN_APP_RESOURCE)
-  }
-
-  test("Dynamic executor scaling basic test") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    launchStagingServer(SSLOptions(), None)
-    createShuffleServiceDaemonSet()
-
-    sparkConf.setJars(Seq(KubernetesSuite.CONTAINER_LOCAL_HELPER_JAR_PATH))
-    sparkConf.set("spark.dynamicAllocation.enabled", "true")
-    sparkConf.set("spark.shuffle.service.enabled", "true")
-    sparkConf.set("spark.kubernetes.shuffle.labels", "app=spark-shuffle-service")
-    sparkConf.set("spark.kubernetes.shuffle.namespace", kubernetesTestComponents.namespace)
-    sparkConf.set("spark.app.name", "group-by-test")
-    runSparkGroupByTestAndVerifyCompletion(KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
-  }
-
-  test("Use remote resources without the resource staging server.") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-    val assetServerUri = staticAssetServerLauncher.launchStaticAssetServer()
-    sparkConf.setJars(Seq(
-      s"$assetServerUri/${KubernetesSuite.EXAMPLES_JAR_FILE.getName}",
-      s"$assetServerUri/${KubernetesSuite.HELPER_JAR_FILE.getName}"
-    ))
-    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
-  }
-
-  test("Mix remote resources with submitted ones.") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-    launchStagingServer(SSLOptions(), None)
-    val assetServerUri = staticAssetServerLauncher.launchStaticAssetServer()
-    sparkConf.setJars(Seq(
-      KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE,
-      s"$assetServerUri/${KubernetesSuite.HELPER_JAR_FILE.getName}"
-    ))
-    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
-  }
-
-  test("Use key and certificate PEM files for TLS.") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-    val keyAndCertificate = SSLUtils.generateKeyCertPemPair(Minikube.getMinikubeIp)
-    launchStagingServer(
-        SSLOptions(enabled = true),
-        Some(keyAndCertificate))
-    sparkConf.set(RESOURCE_STAGING_SERVER_SSL_ENABLED, true)
-        .set(
-            RESOURCE_STAGING_SERVER_CLIENT_CERT_PEM.key, keyAndCertificate.certPem.getAbsolutePath)
-    runSparkPiAndVerifyCompletion(KubernetesSuite.SUBMITTER_LOCAL_MAIN_APP_RESOURCE)
-  }
-
-  test("Use client key and client cert file when requesting executors") {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-    sparkConf.setJars(Seq(
-        KubernetesSuite.CONTAINER_LOCAL_MAIN_APP_RESOURCE,
-        KubernetesSuite.CONTAINER_LOCAL_HELPER_JAR_PATH))
-    sparkConf.set(KUBERNETES_DRIVER_CLIENT_KEY_FILE,
-        kubernetesTestComponents.clientConfig.getClientKeyFile)
-    sparkConf.set(KUBERNETES_DRIVER_CLIENT_CERT_FILE,
-        kubernetesTestComponents.clientConfig.getClientCertFile)
-    sparkConf.set(KUBERNETES_DRIVER_CA_CERT_FILE,
-        kubernetesTestComponents.clientConfig.getCaCertFile)
-    runSparkPiAndVerifyCompletion(SparkLauncher.NO_RESOURCE)
-  }
-
-  private def launchStagingServer(
-      resourceStagingServerSslOptions: SSLOptions, keyAndCertPem: Option[KeyAndCertPem]): Unit = {
-    assume(testBackend.name == MINIKUBE_TEST_BACKEND)
-
-    val resourceStagingServerPort = resourceStagingServerLauncher.launchStagingServer(
-      resourceStagingServerSslOptions, keyAndCertPem)
-    val resourceStagingServerUriScheme = if (resourceStagingServerSslOptions.enabled) {
-      "https"
-    } else {
-      "http"
-    }
-    sparkConf.set(RESOURCE_STAGING_SERVER_URI,
-      s"$resourceStagingServerUriScheme://" +
-        s"${Minikube.getMinikubeIp}:$resourceStagingServerPort")
-  }
-
-  private def runSparkPiAndVerifyCompletion(appResource: String): Unit = {
-    Client.run(sparkConf, appResource, KubernetesSuite.SPARK_PI_MAIN_CLASS, Array.empty[String])
-    val driverPod = kubernetesTestComponents.kubernetesClient
-      .pods()
-      .withLabel("spark-app-locator", APP_LOCATOR_LABEL)
-      .list()
-      .getItems
-      .get(0)
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      assert(kubernetesTestComponents.kubernetesClient
-        .pods()
-        .withName(driverPod.getMetadata.getName)
-        .getLog
-        .contains("Pi is roughly 3"), "The application did not compute the value of pi.")
-    }
-  }
-
-  private def runSparkGroupByTestAndVerifyCompletion(appResource: String): Unit = {
-    Client.run(
-      sparkConf = sparkConf,
-      appArgs = Array.empty[String],
-      mainClass = KubernetesSuite.GROUP_BY_MAIN_CLASS,
-      mainAppResource = appResource)
-    val driverPod = kubernetesTestComponents.kubernetesClient
-      .pods()
-      .withLabel("spark-app-locator", APP_LOCATOR_LABEL)
-      .list()
-      .getItems
-      .get(0)
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      assert(kubernetesTestComponents.kubernetesClient
-        .pods()
-        .withName(driverPod.getMetadata.getName)
-        .getLog
-        .contains("The Result is"), "The application did not complete.")
-    }
-  }
-
-  private def createShuffleServiceDaemonSet(): Unit = {
-    val ds = kubernetesTestComponents.kubernetesClient.extensions().daemonSets()
-      .createNew()
-        .withNewMetadata()
-        .withName("shuffle")
-      .endMetadata()
-      .withNewSpec()
-        .withNewTemplate()
-          .withNewMetadata()
-            .withLabels(Map("app" -> "spark-shuffle-service").asJava)
-          .endMetadata()
-          .withNewSpec()
-            .addNewVolume()
-              .withName("shuffle-dir")
-              .withNewHostPath()
-                .withPath("/tmp")
-              .endHostPath()
-            .endVolume()
-            .addNewContainer()
-              .withName("shuffle")
-              .withImage("spark-shuffle:latest")
-              .withImagePullPolicy("IfNotPresent")
-              .addNewVolumeMount()
-                .withName("shuffle-dir")
-                .withMountPath("/tmp")
-              .endVolumeMount()
-            .endContainer()
-          .endSpec()
-        .endTemplate()
-      .endSpec()
-      .done()
-
-    // wait for daemonset to become available.
-    Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) {
-      val pods = kubernetesTestComponents.kubernetesClient.pods()
-        .withLabel("app", "spark-shuffle-service").list().getItems()
-
-      if (pods.size() == 0 || Readiness.isReady(pods.get(0))) {
-        throw KubernetesSuite.ShuffleNotReadyException()
-      }
-    }
-  }
-}
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/ResourceStagingServerLauncher.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/ResourceStagingServerLauncher.scala
index 1ba54c131c196..e5e1b1f085f9f 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/ResourceStagingServerLauncher.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/ResourceStagingServerLauncher.scala
@@ -26,7 +26,7 @@ import scala.collection.JavaConverters._
 
 import org.apache.spark.SSLOptions
 import org.apache.spark.deploy.kubernetes.config._
-import org.apache.spark.deploy.kubernetes.submit.v2.{ContainerNameEqualityPredicate, KeyAndCertPem}
+import org.apache.spark.deploy.kubernetes.submit.{ContainerNameEqualityPredicate, KeyAndCertPem}
 import org.apache.spark.util.Utils
 
 /**
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/docker/SparkDockerImageBuilder.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/docker/SparkDockerImageBuilder.scala
index 0692cf55db848..3ff72829f88a7 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/docker/SparkDockerImageBuilder.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/kubernetes/integrationtest/docker/SparkDockerImageBuilder.scala
@@ -28,11 +28,10 @@ private[spark] class SparkDockerImageBuilder(private val dockerEnv: Map[String,
 
   private val DOCKER_BUILD_PATH = Paths.get("target", "docker")
   // Dockerfile paths must be relative to the build path.
-  private val DRIVER_V1_DOCKER_FILE = "dockerfiles/driver/Dockerfile"
-  private val DRIVER_V2_DOCKER_FILE = "dockerfiles/driver-v2/Dockerfile"
+  private val DRIVER_DOCKER_FILE = "dockerfiles/driver/Dockerfile"
   private val EXECUTOR_DOCKER_FILE = "dockerfiles/executor/Dockerfile"
   private val SHUFFLE_SERVICE_DOCKER_FILE = "dockerfiles/shuffle-service/Dockerfile"
-  private val DRIVER_INIT_DOCKER_FILE = "dockerfiles/driver-init/Dockerfile"
+  private val INIT_CONTAINER_DOCKER_FILE = "dockerfiles/init-container/Dockerfile"
   private val STAGING_SERVER_DOCKER_FILE = "dockerfiles/resource-staging-server/Dockerfile"
   private val STATIC_ASSET_SERVER_DOCKER_FILE =
     "dockerfiles/integration-test-asset-server/Dockerfile"
@@ -61,12 +60,11 @@ private[spark] class SparkDockerImageBuilder(private val dockerEnv: Map[String,
 
   def buildSparkDockerImages(): Unit = {
     Eventually.eventually(TIMEOUT, INTERVAL) { dockerClient.ping() }
-    buildImage("spark-driver", DRIVER_V1_DOCKER_FILE)
+    buildImage("spark-driver", DRIVER_DOCKER_FILE)
     buildImage("spark-executor", EXECUTOR_DOCKER_FILE)
     buildImage("spark-shuffle", SHUFFLE_SERVICE_DOCKER_FILE)
-    buildImage("spark-driver-v2", DRIVER_V2_DOCKER_FILE)
     buildImage("spark-resource-staging-server", STAGING_SERVER_DOCKER_FILE)
-    buildImage("spark-driver-init", DRIVER_INIT_DOCKER_FILE)
+    buildImage("spark-init", INIT_CONTAINER_DOCKER_FILE)
     buildImage("spark-integration-test-asset-server", STATIC_ASSET_SERVER_DOCKER_FILE)
   }