From b0478c093c48fe608b8011fdaf76648f9f67ecda Mon Sep 17 00:00:00 2001
From: Patrick Wendell <pwendell@gmail.com>
Date: Sun, 12 Oct 2014 23:27:07 -0700
Subject: [PATCH] Changes to simplify the build of SPARK-2706

This simply uses two profiles and defines the source
location based on the profiles. It also removes a bunch
of other excludes. Finally, this doesn't use dynamic
profile activiation which could interfere with third-party
libraries that try to link against the spark-hive artifact.
---
 assembly/pom.xml       |   6 ++
 docs/_config.yml       |   2 -
 docs/building-spark.md |  14 ++-
 pom.xml                |  33 ++++----
 sql/hive/pom.xml       | 187 +++++++++++------------------------------
 5 files changed, 82 insertions(+), 160 deletions(-)

diff --git a/assembly/pom.xml b/assembly/pom.xml
index 81219a37a0e6b..ccdc150e143da 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -197,6 +197,12 @@
           <artifactId>spark-hive_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <!-- TODO: Move this to "hive" profile once 0.13 JDBC is supported -->
+      <id>hive-0.12.0</id>
+      <dependencies>
         <dependency>
           <groupId>org.apache.spark</groupId>
           <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
diff --git a/docs/_config.yml b/docs/_config.yml
index f4bf242ac191b..78c92281a49d5 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -1,7 +1,5 @@
 highlighter: pygments
 markdown: kramdown
-gems:
-  - jekyll-redirect-from
 
 # For some reason kramdown seems to behave differently on different
 # OS/packages wrt encoding. So we hard code this config.
diff --git a/docs/building-spark.md b/docs/building-spark.md
index b2940ee4029e8..2150f223f4915 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -97,12 +97,20 @@ mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -DskipTests clean package
 mvn -Pyarn-alpha -Phadoop-2.3 -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package
 {% endhighlight %}
 
+<!--- TODO: Update this when Hive 0.13 JDBC is added -->
+
 # Building With Hive and JDBC Support
 To enable Hive integration for Spark SQL along with its JDBC server and CLI,
-add the `-Phive` profile to your existing build options.
+add the `-Phive` profile to your existing build options. By default Spark
+will build with Hive 0.12.0 bindings. You can also build for Hive 0.13.1 using
+the `-Phive-0.13.1` profile. NOTE: currently the JDBC server is only
+supported for Hive 12.
 {% highlight bash %}
-# Apache Hadoop 2.4.X with Hive support
+# Apache Hadoop 2.4.X with Hive 12 support
 mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -DskipTests clean package
+
+# Apache Hadoop 2.4.X with Hive 13 support
+mvn -Pyarn -Phive-0.13.1 -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -DskipTests clean package
 {% endhighlight %}
 
 # Spark Tests in Maven
@@ -192,4 +200,4 @@ To run test suites of a specific sub project as follows:
 compiler. When run locally as a background process, it speeds up builds of Scala-based projects
 like Spark. Developers who regularly recompile Spark with Maven will be the most interested in
 Zinc. The project site gives instructions for building and running `zinc`; OS X users can
-install it using `brew install zinc`.
\ No newline at end of file
+install it using `brew install zinc`.
diff --git a/pom.xml b/pom.xml
index 8e5720aa8cc8c..31f456a03bb3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -127,6 +127,11 @@
     <hbase.version>0.94.6</hbase.version>
     <flume.version>1.4.0</flume.version>
     <zookeeper.version>3.4.5</zookeeper.version>
+    <!-- Version used in Maven Hive dependency -->
+    <hive.version>0.12.0-protobuf-2.5</hive.version>
+    <!-- Version used for internal directory structure -->
+    <hive.version.short>0.12.0</hive.version.short>
+    <derby.version>10.4.2.0</derby.version>
     <parquet.version>1.4.3</parquet.version>
     <jblas.version>1.2.3</jblas.version>
     <jetty.version>8.1.14.v20131031</jetty.version>
@@ -1260,34 +1265,28 @@
       </dependencies>
     </profile>
     <profile>
-      <id>hive-default</id>
-      <activation>
-        <property>
-          <name>!hive.version</name>
-        </property>
-      </activation>
-      <properties>
-        <hive.version>0.12.0</hive.version>
-        <derby.version>10.4.2.0</derby.version>
-      </properties>
-    </profile>
-    <profile>
-      <id>hive</id>
+      <id>hive-0.12.0</id>
       <activation>
         <activeByDefault>false</activeByDefault>
       </activation>
+      <!-- TODO: Move this to "hive" profile once 0.13 JDBC is supported -->
       <modules>
         <module>sql/hive-thriftserver</module>
       </modules>
+      <properties>
+        <hive.version>0.12.0-protobuf-2.5</hive.version>
+        <hive.version.short>0.12.0-protobuf-2.5</hive.version.short>
+        <derby.version>10.4.2.0</derby.version>
+      </properties>
     </profile>
     <profile>
-      <id>hive-versions</id>
+      <id>hive-0.13.1</id>
       <activation>
-        <property>
-          <name>hive.version</name>
-        </property>
+        <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
+        <hive.version>0.13.1</hive.version>
+        <hive.version.short>0.13.1</hive.version.short>
         <derby.version>10.10.1.1</derby.version>
       </properties>
     </profile>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index d788f0f2ac72f..cdeba8accb70d 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -36,6 +36,11 @@
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>com.twitter</groupId>
+      <artifactId>parquet-hive-bundle</artifactId>
+      <version>1.5.0</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -46,15 +51,46 @@
       <artifactId>spark-sql_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.spark-project.hive</groupId>
+      <artifactId>hive-metastore</artifactId>
+      <version>${hive.version}</version>
+    </dependency>
     <dependency>
       <groupId>commons-httpclient</groupId>
       <artifactId>commons-httpclient</artifactId>
       <version>3.1</version>
     </dependency>
+    <dependency>
+      <groupId>org.spark-project.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>${hive.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-mapper-asl</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.spark-project.hive</groupId>
+      <artifactId>hive-serde</artifactId>
+      <version>${hive.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <!-- hive-serde already depends on avro, but this brings in customized config of avro deps from parent -->
     <dependency>
       <groupId>org.apache.avro</groupId>
@@ -80,76 +116,7 @@
       <scope>test</scope>
     </dependency>
   </dependencies>
-
   <profiles>
-    <profile>
-      <id>hive-default</id>
-      <activation>
-        <property>
-          <name>!hive.version</name>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>com.twitter</groupId>
-          <artifactId>parquet-hive-bundle</artifactId>
-          <version>1.5.0</version>
-        </dependency>
-        <dependency>
-          <groupId>org.spark-project.hive</groupId>
-          <artifactId>hive-metastore</artifactId>
-          <version>${hive.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.spark-project.hive</groupId>
-          <artifactId>hive-exec</artifactId>
-          <version>${hive.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-              <artifactId>commons-logging</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.spark-project.hive</groupId>
-          <artifactId>hive-serde</artifactId>
-          <version>${hive.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-              <artifactId>commons-logging</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-              <artifactId>commons-logging-api</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.codehaus.mojo</groupId>
-            <artifactId>build-helper-maven-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>add-default-sources</id>
-                <phase>generate-sources</phase>
-                <goals>
-                  <goal>add-source</goal>
-                </goals>
-                <configuration>
-                  <sources>
-                    <source>v${hive.version}/src/main/scala</source>
-                  </sources>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
     <profile>
       <id>hive</id>
       <build>
@@ -176,91 +143,35 @@
         </plugins>
       </build>
     </profile>
-    <profile>
-      <id>hive-versions</id>
-      <activation>
-        <property>
-          <name>hive.version</name>
-        </property>
-      </activation>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>hive-metastore</artifactId>
-          <version>${hive.version}</version>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>hive-exec</artifactId>
-          <version>${hive.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-                <artifactId>commons-logging</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hive</groupId>
-          <artifactId>hive-serde</artifactId>
-          <version>${hive.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-              <artifactId>commons-logging</artifactId>
-            </exclusion>
-            <exclusion>
-              <groupId>commons-logging</groupId>
-              <artifactId>commons-logging-api</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
-      <build>
-        <plugins>
+  </profiles>
+
+  <build>
+    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+    <plugins>
+      <plugin>
+        <groupId>org.scalatest</groupId>
+        <artifactId>scalatest-maven-plugin</artifactId>
+      </plugin>
+
           <plugin>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>build-helper-maven-plugin</artifactId>
             <executions>
               <execution>
-                <id>add-v13-sources</id>
+                <id>add-default-sources</id>
                 <phase>generate-sources</phase>
                 <goals>
                   <goal>add-source</goal>
                 </goals>
                 <configuration>
                   <sources>
-                    <source>v${hive.version}/src/main/scala</source>
-                  </sources>
-                </configuration>
-              </execution>
-              <execution>
-                <id>add-scala-test-sources</id>
-                <phase>generate-test-sources</phase>
-                <goals>
-                  <goal>add-test-source</goal>
-                </goals>
-                <configuration>
-                  <sources>
-                    <source>src/test/scala</source>
+                    <source>v${hive.version.short}/src/main/scala</source>
                   </sources>
                 </configuration>
               </execution>
             </executions>
           </plugin>
-        </plugins>
-      </build>
-    </profile>
-  </profiles>
-
-  <build>
-    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
-    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <plugins>
-      <plugin>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest-maven-plugin</artifactId>
-      </plugin>
 
       <!-- Deploy datanucleus jars to the spark/lib_managed/jars directory -->
       <plugin>