WIP: Regexp Unicode support

integration_tests/run_pyspark_from_build.sh

@@ -14,6 +14,9 @@
 # limitations under the License.
 set -ex
 
+# set LANG so that regular expressions run on GPU
+export LANG=en_US.UTF-8
+
 SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 cd "$SCRIPTPATH"
 

sql-plugin/src/main/scala/com/nvidia/spark/rapids/Plugin.scala

@@ -136,7 +136,8 @@ object RapidsPluginUtils extends Logging {
           s"the RAPIDS Accelerator. Please disable the RAPIDS Accelerator or use a supported " +
           s"serializer ($JAVA_SERIALIZER_NAME, $KRYO_SERIALIZER_NAME).")
     }
-    // set driver timezone
+    // set driver language & timezone
+    conf.set(RapidsConf.DRIVER_LANG.key, sys.env.getOrElse("LANG", ""))
     conf.set(RapidsConf.DRIVER_TIMEZONE.key, ZoneId.systemDefault().normalized().toString)
   }
 
@@ -209,6 +210,15 @@ class RapidsExecutorPlugin extends ExecutorPlugin with Logging {
       // by setting this config spark.rapids.cudfVersionOverride=true
       checkCudfVersion(conf)
 
+      // Validate driver and executor LANG are the same
+      // Note that driverLang is never set to None and defaults to empty string when set
+      val driverLang = conf.driverLang.getOrElse("")
+      val executorLang = sys.env.getOrElse("LANG", "")
+      if (driverLang != executorLang) {
+        throw new RuntimeException("Driver and executor LANG mismatch. " +
+          s"Driver LANG is '${driverLang}' and executor LANG is '$executorLang'.")
+      }
+
       // Validate driver and executor time zone are same if the driver time zone is supported by
       // the plugin.
       val driverTimezone = conf.driverTimeZone match {

sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala

@@ -459,6 +459,13 @@ object RapidsConf {
     .bytesConf(ByteUnit.BYTE)
     .createWithDefault(Integer.MAX_VALUE)
 
+  val DRIVER_LANG = conf("spark.rapids.driver.user.lang")
+    .doc("This config is used to inform the executor plugin about the driver's language " +
+      "and is not intended to be set by the user.")
+    .internal()
+    .stringConf
+    .createOptional
+
   val DRIVER_TIMEZONE = conf("spark.rapids.driver.user.timezone")
     .doc("This config is used to inform the executor plugin about the driver's timezone " +
       "and is not intended to be set by the user.")
@@ -1891,6 +1898,8 @@ class RapidsConf(conf: Map[String, String]) extends Logging {
 
   lazy val getAlluxioPathsToReplace: Option[Seq[String]] = get(ALLUXIO_PATHS_REPLACE)
 
+  lazy val driverLang: Option[String] = get(DRIVER_LANG)
+
   lazy val driverTimeZone: Option[String] = get(DRIVER_TIMEZONE)
 
   lazy val isRangeWindowByteEnabled: Boolean = get(ENABLE_RANGE_WINDOW_BYTES)

sql-plugin/src/main/scala/org/apache/spark/sql/rapids/stringFunctions.scala

@@ -835,6 +835,14 @@ object GpuRegExpUtils {
       meta.willNotWorkOnGpu(s"regular expression support is disabled. " +
         s"Set ${RapidsConf.ENABLE_REGEXP}=true to enable it")
     }
+    sys.env.get("LANG") match {
+      case Some("en_US.UTF-8") =>
+        // supported
+      case _ => meta.willNotWorkOnGpu(
+        "regular expression support is disabled because environment variable 'LANG' " +
+          "is not set to 'en_US.UTF-8' and this would result in incorrect handling of " +
+          "Unicode data")
+    }
   }
 
 }

tests/src/test/scala/com/nvidia/spark/rapids/RegularExpressionTranspilerSuite.scala

@@ -570,6 +570,22 @@ class RegularExpressionTranspilerSuite extends FunSuite with Arm {
       RegexReplaceMode)
   }
 
+  test("AST fuzz test - regexp_find - full unicode input") {
+    assume(isUnicodeEnabled())
+    doAstFuzzTest(None, REGEXP_LIMITED_CHARS_REPLACE,
+      RegexFindMode)
+  }
+
+  test("AST fuzz test - regexp_replace - full unicode input") {
+    assume(isUnicodeEnabled())
+    doAstFuzzTest(None, REGEXP_LIMITED_CHARS_REPLACE,
+      RegexReplaceMode)
+  }
+
+  def isUnicodeEnabled(): Boolean = {
+    sys.env.getOrElse("LANG", "") == "en_US.UTF-8"
+  }
+
   test("AST fuzz test - regexp_find - anchor focused") {
     doAstFuzzTest(validDataChars = Some("\r\nabc"),
       validPatternChars = "^$\\AZz\r\n()[]-", mode = RegexFindMode)