[NSE-326] update scala unit tests to spark-3.1.1 (#327)

* move back * update Unit Tests to Spark.3.1.1 * move back tests on columnar shuffle * enable scala unit test Signed-off-by: Yuan Zhou <yuan.zhou@intel.com> * add tests in travis Co-authored-by: Yuan Zhou <yuan.zhou@intel.com>
oap-project · May 25, 2021 · 8fca8a5 · 8fca8a5
1 parent fbc366f
commit 8fca8a5
Show file tree

Hide file tree

Showing 1,718 changed files with 297,065 additions and 12,245 deletions.
diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -60,7 +60,6 @@ jobs:
           ctest -R
 
   scala-unit-test:
-    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '@github-actions scala-unit-test') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

diff --git a/native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java b/native-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadTest.java
diff --git a/...-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java b/...-sql-engine/core/src/test/java/com/intel/oap/datasource/parquet/ParquetReadWriteTest.java
diff --git a/...ap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet b/...ap/datasource/parquet/part-00000-d648dd34-c9d2-4fe9-87f2-770ef3551442-c000.snappy.parquet
diff --git a/...ore/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider b/...ore/src/test/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider
@@ -0,0 +1 @@
+org.apache.spark.sql.execution.datasources.jdbc.connection.IntentionallyFaultyConnectionProvider
diff --git a/...core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/...core/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister
@@ -0,0 +1,15 @@
+org.apache.spark.sql.sources.FakeSourceOne
+org.apache.spark.sql.sources.FakeSourceTwo
+org.apache.spark.sql.sources.FakeSourceThree
+org.apache.spark.sql.sources.FakeSourceFour
+org.apache.fakesource.FakeExternalSourceOne
+org.apache.fakesource.FakeExternalSourceTwo
+org.apache.fakesource.FakeExternalSourceThree
+org.apache.spark.sql.streaming.sources.FakeReadMicroBatchOnly
+org.apache.spark.sql.streaming.sources.FakeReadContinuousOnly
+org.apache.spark.sql.streaming.sources.FakeReadBothModes
+org.apache.spark.sql.streaming.sources.FakeReadNeitherMode
+org.apache.spark.sql.streaming.sources.FakeWriteOnly
+org.apache.spark.sql.streaming.sources.FakeNoWrite
+org.apache.spark.sql.streaming.sources.FakeWriteSupportingExternalMetadata
+org.apache.spark.sql.streaming.sources.FakeWriteSupportProviderV1Fallback
diff --git a/native-sql-engine/core/src/test/resources/hive-site.xml b/native-sql-engine/core/src/test/resources/hive-site.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<configuration>
+  <property>
+    <name>hive.in.test</name>
+      <value>true</value>
+      <description>Internal marker for test.</description>
+  </property>
+  <property>
+    <name>hadoop.tmp.dir</name>
+    <value>/tmp/hive_one</value>
+    <description>default is /tmp/hadoop-${user.name} and will be overridden </description>
+  </property>
+</configuration>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		org.apache.spark.sql.execution.datasources.jdbc.connection.IntentionallyFaultyConnectionProvider