Upgrade Spark dependency from 3.1.2 to 3.2.0

.circleci/config.yml

@@ -33,7 +33,7 @@ create_docs_env: &create_docs_env
       fi
 
 install_pyspark2: &install_pyspark2
-  run: # Evict PySpark 3.1.2 in favor of PySpark 2.4.5
+  run: # Evict PySpark 3.2.0 in favor of PySpark 2.4.5
     name: Install PySpark 2.4.5
     command: |
       export PATH=$HOME/conda/bin:$PATH
@@ -226,7 +226,7 @@ jobs:
           environment:
           command: |
             export PATH=$HOME/conda/envs/glow/bin:$PATH
-            export SPARK_VERSION="3.1.2"
+            export SPARK_VERSION="3.2.0"
             export SCALA_VERSION="2.12.8"
             sbt core/test exit
       - run:
@@ -235,7 +235,7 @@ jobs:
           environment:
           command: |
             export PATH=$HOME/conda/envs/glow/bin:$PATH
-            export SPARK_VERSION="3.1.2"
+            export SPARK_VERSION="3.2.0"
             export SCALA_VERSION="2.12.8"
             sbt docs/test exit
       - run:
@@ -244,15 +244,15 @@ jobs:
           environment:
           command: |
             export PATH=$HOME/conda/envs/glow/bin:$PATH
-            export SPARK_VERSION="3.1.2"
+            export SPARK_VERSION="3.2.0"
             export SCALA_VERSION="2.12.8"
             sbt python/test exit
       - run:
           name: Run Hail tests
           environment:
           command: |
             export PATH=$HOME/conda/envs/glow/bin:$HOME/conda/bin:$PATH
-            export SPARK_VERSION="3.1.2"
+            export SPARK_VERSION="3.2.0"
             export SCALA_VERSION="2.12.8"
             export HAIL_VERSION="0.2.74"
             sudo apt-get update

README.md

@@ -68,7 +68,7 @@ conda env update -f python/environment.yml
 
 Start an sbt shell using the `sbt` command.
 
-> **FYI**: The following SBT projects are built on Spark 3.1.2/Scala 2.12.8 by default. To change the Spark version and
+> **FYI**: The following SBT projects are built on Spark 3.2.0/Scala 2.12.8 by default. To change the Spark version and
 Scala version, set the environment variables `SPARK_VERSION` and `SCALA_VERSION`.
 
 To compile the main code:

build.sbt

@@ -10,10 +10,10 @@ import sbt.nio.Keys._
 lazy val scala212 = "2.12.8"
 lazy val scala211 = "2.11.12"
 
-lazy val spark3 = "3.1.2"
+lazy val spark3 = "3.2.0"
 lazy val spark2 = "2.4.5"
 
-lazy val hailOnSpark3 = "0.2.74"
+lazy val hailOnSpark3 = "0.2.78"
 lazy val hailOnSpark2 = "0.2.58"
 
 lazy val sparkVersion = settingKey[String]("sparkVersion")

core/src/main/scala/io/projectglow/sql/expressions/AggregateByIndex.scala

@@ -143,6 +143,8 @@ trait AggregateByIndex extends DeclarativeAggregate with HigherOrderFunction {
  */
 trait UnwrappedAggregateFunction extends AggregateFunction {
   def asWrapped: AggregateFunction
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+    super.legacyWithNewChildren(newChildren)
 }
 
 case class UnwrappedAggregateByIndex(

core/src/main/scala/io/projectglow/sql/expressions/glueExpressions.scala

@@ -33,8 +33,8 @@ import org.apache.spark.sql.types._
 case class ExpandStruct(struct: Expression) extends Expression with Unevaluable {
   override def children: Seq[Expression] = Seq(struct)
   override lazy val resolved: Boolean = false
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
   def expand(): Seq[NamedExpression] = {
     if (!struct.dataType.isInstanceOf[StructType]) {
       throw SQLUtils.newAnalysisException("Only structs can be expanded.")

core/src/main/scala/io/projectglow/sql/util/ExpectsGenotypeFields.scala

@@ -126,8 +126,8 @@ trait ExpectsGenotypeFields extends Expression {
 trait Rewrite extends Expression with Unevaluable {
   def rewrite: Expression
 
-  override def dataType: DataType = throw new UnresolvedException(this, "dataType")
-  override def nullable: Boolean = throw new UnresolvedException(this, "nullable")
+  override def dataType: DataType = throw new UnresolvedException("dataType")
+  override def nullable: Boolean = throw new UnresolvedException("nullable")
 }
 
 /**

core/src/main/shim/3.1/SparkShim.scala → core/src/main/shim/3.2/SparkShim.scala

@@ -19,7 +19,7 @@ package io.projectglow
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.ExpressionInfo
 
-// Spark 3.1 APIs that are not inter-version compatible
+// Spark 3.2 APIs that are not inter-version compatible
 object SparkShim extends SparkShimBase {
   // [SPARK-25393][SQL] Adding new function from_csv()
   // Refactors classes from [[org.apache.spark.sql.execution.datasources.csv]] to [[org.apache.spark.sql.catalyst.csv]]

docs/source/getting-started.rst

@@ -4,7 +4,7 @@ Getting Started
 Running Locally
 ---------------
 
-Glow requires Apache Spark 3.1.2.
+Glow requires Apache Spark 3.2.0.
 
 .. tabs::
 
@@ -14,7 +14,7 @@ Glow requires Apache Spark 3.1.2.
 
         .. code-block:: sh
 
-          pip install pyspark==3.1.2
+          pip install pyspark==3.2.0
 
         or `download a specific distribution <https://spark.apache.org/downloads.html>`_.
 

python/environment.yml

@@ -16,7 +16,7 @@ dependencies:
   - pandas=1.0.1
   - pip
   - pyarrow=1.0.1
-  - pyspark=3.1.2
+  - pyspark=3.2.0
   - pytest
   - pyyaml
   - scipy=1.4.1