Merge pull request #42 from eyala/remove_udafs

DATAFU-174 Remove deprecated SparkUDAFs
apache · Feb 20, 2024 · 526fbd6 · 526fbd6
2 parents 6c91749 + 5010f82
commit 526fbd6
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 466 deletions.
diff --git a/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala b/datafu-spark/src/main/scala/datafu/spark/SparkUDAFs.scala
diff --git a/datafu-spark/src/test/scala/datafu/spark/TestAggregators.scala b/datafu-spark/src/test/scala/datafu/spark/TestAggregators.scala
@@ -158,7 +158,7 @@ class TestAggregators extends FunSuite with DataFrameSuiteBase {
       sqlContext.createDataFrame(List(mapExp(Map("dsa" -> 1, "asd" -> 5)))),
       spark.table("mas_table2").groupBy().agg(mas2($"arr").as("map_col")))
 
-    val mas1 = new SparkUDAFs.MultiArraySet[String](maxKeys = 1)
+    val mas1 = udaf(new Aggregators.MultiArraySet[String](maxKeys = 1))
     assertDataFrameEquals(
       sqlContext.createDataFrame(List(mapExp(Map("asd" -> 5)))),
       spark.table("mas_table2").groupBy().agg(mas1($"arr").as("map_col")))
@@ -247,17 +247,17 @@ class TestAggregators extends FunSuite with DataFrameSuiteBase {
         Exp6(Option(2), Option(1))
       ))
 
-    assertDataFrameEquals(results3DF,
+    assertDataFrameNoOrderEquals(results3DF,
       inputDF
         .groupBy("col_grp")
         .agg(countDistinctUpTo3($"col_ord").as("col_ord")))
 
-    assertDataFrameEquals(results6DF,
+    assertDataFrameNoOrderEquals(results6DF,
       inputDF
         .groupBy("col_grp")
         .agg(countDistinctUpTo6($"col_ord").as("col_ord")))
 
-    assertDataFrameEquals(results2DF, inputDF
+    assertDataFrameNoOrderEquals(results2DF, inputDF
       .groupBy("col_ord")
       .agg(countDistinctUpTo2($"col_grp").as("col_grp")))
   }