Bring back hive (#158)

circle.yml

@@ -37,13 +37,13 @@ dependencies:
         # Copy contents into current build directory
         rsync --info=stats2,misc1,flist0 -a build_classes/ .
       fi
-    - ./build/mvn -DskipTests -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr install
+    - ./build/mvn -DskipTests -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Phive -Psparkr install
       # Copy all of */target/scala_2.11/classes to build_classes/
     - >
       rsync --info=stats2,misc1,flist0 -a --delete-excluded --prune-empty-dirs --exclude build_classes/ --exclude 'target/streams' --exclude 'assembly/target' --exclude 'common/network-yarn/target' --exclude 'examples/target' --exclude '***/*.jar' --include 'target/***' --include '**/' --exclude '*' . build_classes/
     - |
       # Make sbt fetch all the external deps to ~/.ivy2 so it gets cached
-      ./build/sbt -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr externalDependencyClasspath
+      ./build/sbt -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Phive -Psparkr externalDependencyClasspath
   cache_directories:
     - "build_classes"
     - "build"

dev/deps/spark-deps-hadoop-palantir

@@ -1,9 +1,14 @@
+JavaEWAH-0.3.2.jar
 RoaringBitmap-0.5.11.jar
+ST4-4.0.4.jar
 activation-1.1.1.jar
 animal-sniffer-annotation-1.0.jar
+antlr-2.7.7.jar
+antlr-runtime-3.4.jar
 antlr4-runtime-4.5.3.jar
 aopalliance-1.0.jar
 aopalliance-repackaged-2.4.0-b34.jar
+apache-log4j-extras-1.2.17.jar
 apacheds-i18n-2.0.0-M15.jar
 apacheds-kerberos-codec-2.0.0-M15.jar
 api-asn1-api-1.0.0-M20.jar
@@ -16,8 +21,12 @@ avro-mapred-1.7.7-hadoop2.jar
 base64-2.3.8.jar
 bcpkix-jdk15on-1.52.jar
 bcprov-jdk15on-1.51.jar
+bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.11-0.12.jar
 breeze_2.11-0.12.jar
+calcite-avatica-1.2.0-incubating.jar
+calcite-core-1.2.0-incubating.jar
+calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.8.0.jar
 chill_2.11-0.8.0.jar
 commons-beanutils-1.7.0.jar
@@ -29,10 +38,13 @@ commons-compiler-3.0.0.jar
 commons-compress-1.4.1.jar
 commons-configuration-1.6.jar
 commons-crypto-1.0.0.jar
+commons-dbcp-1.4.jar
 commons-digester-1.8.jar
+commons-httpclient-3.1.jar
 commons-io-2.4.jar
 commons-lang-2.6.jar
 commons-lang3-3.5.jar
+commons-logging-1.1.3.jar
 commons-math3-3.4.1.jar
 commons-net-2.2.jar
 commons-pool-1.5.4.jar
@@ -41,6 +53,11 @@ core-1.1.2.jar
 curator-client-2.7.1.jar
 curator-framework-2.7.1.jar
 curator-recipes-2.7.1.jar
+datanucleus-api-jdo-3.2.6.jar
+datanucleus-core-3.2.10.jar
+datanucleus-rdbms-3.2.9.jar
+derby-10.12.1.1.jar
+eigenbase-properties-1.1.5.jar
 fastutil-6.5.7.jar
 feign-core-8.18.0.jar
 feign-jackson-8.18.0.jar
@@ -95,9 +112,11 @@ javax.inject-1.jar
 javax.inject-2.4.0-b34.jar
 javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
+javolution-5.5.1.jar
 jaxb-api-2.2.2.jar
 jcip-annotations-1.0.jar
 jcl-over-slf4j-1.7.16.jar
+jdo-api-3.0.1.jar
 jersey-client-2.22.2.jar
 jersey-common-2.22.2.jar
 jersey-container-servlet-2.22.2.jar
@@ -110,18 +129,23 @@ jetty-6.1.26.jar
 jetty-sslengine-6.1.26.jar
 jetty-util-6.1.26.jar
 jline-2.12.1.jar
+joda-time-2.9.3.jar
+jodd-core-3.5.2.jar
 json-smart-1.1.1.jar
 json4s-ast_2.11-3.2.11.jar
 json4s-core_2.11-3.2.11.jar
 json4s-jackson_2.11-3.2.11.jar
 jsp-api-2.1.jar
 jsr305-1.3.9.jar
+jta-1.1.jar
 jtransforms-2.4.0.jar
 jul-to-slf4j-1.7.16.jar
 kryo-shaded-3.0.3.jar
 kubernetes-client-2.2.1.jar
 kubernetes-model-1.0.67.jar
 leveldbjni-all-1.8.jar
+libfb303-0.9.3.jar
+libthrift-0.9.3.jar
 log4j-1.2.17.jar
 logging-interceptor-3.6.0.jar
 lz4-1.3.0.jar
@@ -148,6 +172,7 @@ parquet-common-1.9.1-palantir1.jar
 parquet-encoding-1.9.1-palantir1.jar
 parquet-format-2.3.1.jar
 parquet-hadoop-1.9.1-palantir1.jar
+parquet-hadoop-bundle-1.6.0.jar
 parquet-jackson-1.9.1-palantir1.jar
 pmml-model-1.2.15.jar
 pmml-schema-1.2.15.jar
@@ -164,11 +189,14 @@ shapeless_2.11-2.0.0.jar
 slf4j-api-1.7.16.jar
 slf4j-log4j12-1.7.16.jar
 snakeyaml-1.15.jar
+snappy-0.2.jar
 snappy-java-1.1.2.6.jar
 spire-macros_2.11-0.7.4.jar
 spire_2.11-0.7.4.jar
 stax-api-1.0-2.jar
+stax-api-1.0.1.jar
 stream-2.7.0.jar
+stringtemplate-3.2.1.jar
 univocity-parsers-2.2.1.jar
 validation-api-1.1.0.Final.jar
 xbean-asm5-shaded-4.4.jar

dev/publish.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 version=$(git describe --tags)
 
-PALANTIR_FLAGS=(-Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
+PALANTIR_FLAGS=(-Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
 
 publish_artifacts() {
   tmp_settings="tmp-settings.xml"
@@ -28,4 +28,4 @@ make_dist() {
 
 publish_artifacts
 make_dist hadoop-2.8.0-palantir3 "${PALANTIR_FLAGS[*]}" --clean
-make_dist without-hadoop "-Phadoop-provided -Pkubernetes -Pyarn -Psparkr" --clean
+make_dist without-hadoop "-Phadoop-provided -Pkubernetes -Phive -Pyarn -Psparkr" --clean

dev/test-dependencies.sh

@@ -29,7 +29,7 @@ export LC_ALL=C
 # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
 
 # NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Pkubernetes -Pyarn"
+HADOOP2_MODULE_PROFILES="-Pkubernetes -Pyarn -Phive"
 MVN="build/mvn"
 HADOOP_PROFILES=(
     hadoop-palantir

dists/hadoop-palantir/pom.xml

@@ -143,7 +143,11 @@
       <artifactId>guava</artifactId>
       <scope>${hadoop.deps.scope}</scope>
     </dependency>
-
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-hive_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-yarn_${scala.binary.version}</artifactId>