move unit tests into Scala

apache · May 20, 2015 · 6836a80 · 6836a80
1 parent b97af11
commit 6836a80
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 23 deletions.
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
@@ -512,28 +512,6 @@ def test_save_and_load(self):
 
         shutil.rmtree(tmpPath)
 
-    def test_json_with_map(self):
-        # regression test for SPARK-7565
-        tmpPath = tempfile.mkdtemp()
-        shutil.rmtree(tmpPath)
-        rdd = self.sc.parallelize(['{"obj": {"a": "hello"}}', '{"obj": {"b": "world"}}'], 1)
-        schema = StructType([StructField("obj", MapType(StringType(), StringType()), True)])
-        df = self.sqlCtx.jsonRDD(rdd, schema)
-        rs = [({'a': 'hello'},), ({'b': 'world'},)]
-        self.assertEqual(rs, list(map(tuple, df.collect())))
-        df.write.parquet(tmpPath)
-        df2 = self.sqlCtx.read.parquet(tmpPath)
-        self.assertEqual(rs, list(map(tuple, df2.collect())))
-
-        self.sqlCtx.setConf("spark.sql.json.useJacksonStreamingAPI", "false")
-        df3 = self.sqlCtx.jsonRDD(rdd, schema)
-        self.assertEqual(rs, list(map(tuple, df3.collect())))
-        rs = list(map(tuple, df.collect()))
-        df.write.parquet(tmpPath, 'overwrite')
-        df4 = self.sqlCtx.read.parquet(tmpPath)
-        self.assertEqual(rs, list(map(tuple, df4.collect())))
-        shutil.rmtree(tmpPath)
-
     def test_help_command(self):
         # Regression test for SPARK-5464
         rdd = self.sc.parallelize(['{"foo":"bar"}', '{"foo":"baz"}'])

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/json/JsonSuite.scala
@@ -25,7 +25,6 @@ import org.scalactic.Tolerance._
 
 import org.apache.spark.sql.TestData._
 import org.apache.spark.sql.catalyst.util.DateUtils
-import org.apache.spark.sql.functions._
 import org.apache.spark.sql.json.InferSchema.compatibleType
 import org.apache.spark.sql.sources.LogicalRelation
 import org.apache.spark.sql.test.TestSQLContext
@@ -1074,4 +1073,16 @@ class JsonSuite extends QueryTest {
     assert(StructType(Seq()) === emptySchema)
   }
 
+  test("SPARK-7565 MapType in JsonRDD") {
+    val schemaWithSimpleMap = StructType(
+      StructField("map", MapType(StringType, IntegerType, true), false) :: Nil)
+    val df = read.schema(schemaWithSimpleMap).json(mapType1)
+    val temp = Utils.createTempDir()
+    df.write.mode("overwrite").parquet(temp.getPath)
+
+    setConf("spark.sql.json.useJacksonStreamingAPI", "false")
+    val df2 = read.schema(schemaWithSimpleMap).json(mapType1)
+    df2.write.mode("overwrite").parquet(temp.getPath)
+  }
+
 }