From edc35a7a61580bba6a4e22cf652ccd5d4eef0298 Mon Sep 17 00:00:00 2001 From: songzhxlh-max <59271430+songzhxlh-max@users.noreply.github.com> Date: Tue, 24 May 2022 14:23:47 +0800 Subject: [PATCH] KE-36161 fix replace function (#471) * KE-36501 fix replace function when first argument length equals second argument length * KE-36499 fix replace function when second argument is "" pushdown error * KE-36494 fix replace function when second argument has special symbol "$", ")" * fix spark version * fix ut --- assembly/pom.xml | 2 +- common/kvstore/pom.xml | 2 +- common/network-common/pom.xml | 2 +- common/network-shuffle/pom.xml | 2 +- common/network-yarn/pom.xml | 2 +- common/sketch/pom.xml | 2 +- common/tags/pom.xml | 2 +- common/unsafe/pom.xml | 2 +- core/pom.xml | 2 +- examples/pom.xml | 2 +- external/avro/pom.xml | 2 +- external/docker-integration-tests/pom.xml | 2 +- external/kafka-0-10-assembly/pom.xml | 2 +- external/kafka-0-10-sql/pom.xml | 2 +- external/kafka-0-10-token-provider/pom.xml | 2 +- external/kafka-0-10/pom.xml | 2 +- external/kinesis-asl-assembly/pom.xml | 2 +- external/kinesis-asl/pom.xml | 2 +- external/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- hadoop-cloud/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib-local/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- resource-managers/kubernetes/core/pom.xml | 2 +- .../kubernetes/integration-tests/pom.xml | 2 +- resource-managers/mesos/pom.xml | 2 +- resource-managers/yarn/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- .../expressions/regexpExpressions.scala | 109 +++++++++++++++--- .../expressions/stringExpressions.scala | 11 +- .../expressions/RegexpExpressionsSuite.scala | 10 ++ .../expressions/StringExpressionsSuite.scala | 4 + sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tools/pom.xml | 2 +- 40 files changed, 148 insertions(+), 58 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 8cd9355bb6f89..c5821de0de7cb 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml index cfde991acc410..1fc2ee62fc942 100644 --- a/common/kvstore/pom.xml +++ b/common/kvstore/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml index c1e8566782d3c..2594ddbd2e758 100644 --- a/common/network-common/pom.xml +++ b/common/network-common/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml index 3058de2bb268c..6a9c4d51a8441 100644 --- a/common/network-shuffle/pom.xml +++ b/common/network-shuffle/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml index 48e9788b48abf..1507435c4f2e0 100644 --- a/common/network-yarn/pom.xml +++ b/common/network-yarn/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml index 786d0b3b1737a..39ea94d88d05d 100644 --- a/common/sketch/pom.xml +++ b/common/sketch/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/tags/pom.xml b/common/tags/pom.xml index 8d07319c9e6a3..872ebbb2e4309 100644 --- a/common/tags/pom.xml +++ b/common/tags/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml index 1cb8b8792da7d..ae97b5a16798b 100644 --- a/common/unsafe/pom.xml +++ b/common/unsafe/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/core/pom.xml b/core/pom.xml index aa5d1f13ac1d1..b9bfc9f7b1efe 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index d5a3526a42d4d..1243b1f1cfbc6 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/external/avro/pom.xml b/external/avro/pom.xml index 66f11fcfd08a9..cef0d3487ae35 100644 --- a/external/avro/pom.xml +++ b/external/avro/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 1b39e564e1fb2..1d41e00f57204 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml index e59da7a111367..e15b1cf3ea14b 100644 --- a/external/kafka-0-10-assembly/pom.xml +++ b/external/kafka-0-10-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml index 44fe161425f7c..d0017253e4d3a 100644 --- a/external/kafka-0-10-sql/pom.xml +++ b/external/kafka-0-10-sql/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kafka-0-10-token-provider/pom.xml b/external/kafka-0-10-token-provider/pom.xml index 5abeeb2d3e40b..749d65ac2e3e2 100644 --- a/external/kafka-0-10-token-provider/pom.xml +++ b/external/kafka-0-10-token-provider/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml index 4e8ddf06dcc7e..00a4ff61de435 100644 --- a/external/kafka-0-10/pom.xml +++ b/external/kafka-0-10/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml index 72a1ccbf7e76a..1144bf99d47e5 100644 --- a/external/kinesis-asl-assembly/pom.xml +++ b/external/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml index 490b7d17243a7..7c1dd9ce0f092 100644 --- a/external/kinesis-asl/pom.xml +++ b/external/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml index 99329f715b304..1ed1d67a3c68b 100644 --- a/external/spark-ganglia-lgpl/pom.xml +++ b/external/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index c12652607fbd3..e96ced2c2d44c 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 23f89c7e63d7e..a956f2b41d28d 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 1f0ec1aeeca46..2f10c88f86d98 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 76bc97c36b7af..d3f476b7a993d 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 2df3c611ffbcd..1154e14ed717a 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/pom.xml b/pom.xml index f59dc114e3cdd..46c1459157bec 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 0653fa038fbe3..c87e24520eb42 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 71c92b4a6df8d..6a03fe7e26988 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../../pom.xml diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index ce7507185ebf0..20a9e7ca81f65 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../../pom.xml diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml index ac88fbb69f7bd..14c3470fbb401 100644 --- a/resource-managers/mesos/pom.xml +++ b/resource-managers/mesos/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index c9204bf7733fe..8928a958d99ac 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 5dd72ac7cc6b2..f0edef3964931 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 57d7d76268795..d8833a1be7b58 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.catalyst.expressions import java.util.Locale -import java.util.regex.{Matcher, MatchResult, Pattern} +import java.util.regex.{Matcher, MatchResult, Pattern, PatternSyntaxException} import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer @@ -556,10 +556,27 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio @transient private lazy val result: StringBuffer = new StringBuffer override def nullSafeEval(s: Any, p: Any, r: Any, i: Any): Any = { + if (s.toString.indexOf('$') > -1 || p.toString.indexOf('$') > -1) { + if (!isRegex(p.toString)) { + val ss = UTF8String.fromString(s.toString.replace(p.toString, r.toString)) + return ss + } + } if (!p.equals(lastRegex)) { // regex value changed lastRegex = p.asInstanceOf[UTF8String].clone() - pattern = Pattern.compile(lastRegex.toString) + val lastRegexStr = lastRegex.toString + if (lastRegexStr.indexOf(')') > -1 && !isRegex(lastRegexStr)) { + val array = lastRegexStr.toCharArray + val buffer = new StringBuffer() + for (i <- 0 until array.length) { + if (array(i) == ')' && (i == 0 || array(i - 1) != '\\')) buffer.append("\\") + buffer.append(array(i)) + } + pattern = Pattern.compile(buffer.toString()) + } else { + pattern = Pattern.compile(lastRegexStr) + } } if (!r.equals(lastReplacementInUTF8)) { // replacement string changed @@ -568,7 +585,7 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio } val source = s.toString() val position = i.asInstanceOf[Int] - 1 - if (position < source.length) { + if (position <= source.length) { val m = pattern.matcher(source) m.region(position, source.length) result.delete(0, result.length()) @@ -608,24 +625,63 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio "" } + val specialSymbol = "$" nullSafeCodeGen(ctx, ev, (subject, regexp, rep, pos) => { s""" - if (!$regexp.equals($termLastRegex)) { - // regex value changed - $termLastRegex = $regexp.clone(); - $termPattern = $classNamePattern.compile($termLastRegex.toString()); - } - if (!$rep.equals($termLastReplacementInUTF8)) { - // replacement string changed - $termLastReplacementInUTF8 = $rep.clone(); - $termLastReplacement = $termLastReplacementInUTF8.toString(); + Boolean isRegex = true; + if ($rep.toString().indexOf('$specialSymbol') > -1 || $regexp.toString() + .indexOf('$specialSymbol') > -1) { + try { + java.util.regex.Pattern.compile($rep.toString()); + isRegex = true; + } catch (java.util.regex.PatternSyntaxException e) { + isRegex = false; + } + if (!isRegex) { + ${ev.value} = UTF8String.fromString( + $subject.toString().replace($regexp.toString(), $rep.toString())); + } } - String $source = $subject.toString(); - int $position = $pos - 1; - if ($position < $source.length()) { - $classNameStringBuffer $termResult = new $classNameStringBuffer(); - java.util.regex.Matcher $matcher = $termPattern.matcher($source); - $matcher.region($position, $source.length()); + if (isRegex) { + if (!$regexp.equals($termLastRegex)) { + // regex value changed + $termLastRegex = $regexp.clone(); + String termLastRegexStr = $termLastRegex.toString(); + if (termLastRegexStr.indexOf(')') > -1) { + try { + java.util.regex.Pattern.compile(termLastRegexStr); + isRegex = true; + } catch (java.util.regex.PatternSyntaxException e) { + isRegex = false; + } + if (!isRegex) { + char[] array = termLastRegexStr.toCharArray(); + StringBuffer buffer = new StringBuffer(); + for (int k = 0; k < array.length; k++) { + if (array[k] == ')' && (k == 0 || array[k - 1] != '\\\\')) { + buffer.append("\\\\"); + } + buffer.append(array[k]); + } + $termPattern = $classNamePattern.compile(buffer.toString()); + } else { + $termPattern = $classNamePattern.compile(termLastRegexStr); + } + } else { + $termPattern = $classNamePattern.compile(termLastRegexStr); + } + } + if (!$rep.equals($termLastReplacementInUTF8)) { + // replacement string changed + $termLastReplacementInUTF8 = $rep.clone(); + $termLastReplacement = $termLastReplacementInUTF8.toString(); + } + String $source = $subject.toString(); + int $position = $pos - 1; + if ($position <= $source.length()) { + $classNameStringBuffer $termResult = new $classNameStringBuffer(); + java.util.regex.Matcher $matcher = $termPattern.matcher($source); + $matcher.region($position, $source.length()); while ($matcher.find()) { $matcher.appendReplacement($termResult, $termLastReplacement); @@ -636,11 +692,26 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio } else { ${ev.value} = $subject; } - $setEvNotNull + } + $setEvNotNull """ }) } + def isRegex(input: String): Boolean = + { + var isRegex = false + try { + Pattern.compile(input) + isRegex = true + } catch { + case e: PatternSyntaxException => + isRegex = false + } + isRegex + } + + override def first: Expression = subject override def second: Expression = regexp override def third: Expression = rep diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 5956c3e882118..77df807523c80 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -526,13 +526,18 @@ case class StringReplace(srcExpr: Expression, searchExpr: Expression, replaceExp } override def nullSafeEval(srcEval: Any, searchEval: Any, replaceEval: Any): Any = { - srcEval.asInstanceOf[UTF8String].replace( - searchEval.asInstanceOf[UTF8String], replaceEval.asInstanceOf[UTF8String]) + val s = srcEval.asInstanceOf[UTF8String].clone().toString().replace( + searchEval.asInstanceOf[UTF8String].clone().toString() + , replaceEval.asInstanceOf[UTF8String].clone().toString()) + val ss = UTF8String.fromString(s) + ss } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { nullSafeCodeGen(ctx, ev, (src, search, replace) => { - s"""${ev.value} = $src.replace($search, $replace);""" + s"""${ev.value} = + | UTF8String.fromString($src.toString().replace($search.toString(), $replace.toString())); + | """.stripMargin }) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala index 019857580d077..590ef05d6c623 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjectio import org.apache.spark.sql.catalyst.optimizer.ConstantFolding import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StringType +import org.apache.spark.unsafe.types.UTF8String /** * Unit tests for regular expression (regexp) related SQL expressions. @@ -298,6 +299,15 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val p = 'p.string.at(1) val r = 'r.string.at(2) + val regReplaceStr = RegExpReplace(Literal("abc)"), Literal(")"), Literal("vv")) + assert(regReplaceStr.eval(null).equals(UTF8String.fromString("abcvv"))) + + val regReplaceStr2 = RegExpReplace(Literal("abc$"), Literal("$"), Literal("vv")) + assert(regReplaceStr2.eval(null).equals(UTF8String.fromString("abcvv"))) + + val regReplaceStr3 = RegExpReplace(Literal(""), Literal(""), Literal("vv")) + assert(regReplaceStr3.eval(null).equals(UTF8String.fromString("vv"))) + val expr = RegExpReplace(s, p, r) checkEvaluation(expr, "num-num", row1) checkEvaluation(expr, "###-###", row2) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index d25240fec13d1..d840b76344593 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { @@ -427,6 +428,9 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("replace") { + val stringReplaceExpression = StringReplace(Literal(""), Literal(""), Literal("vv")) + val stringReplaceResult = stringReplaceExpression.eval(null) + assert(stringReplaceResult.equals(UTF8String.fromString("vv"))) checkEvaluation( StringReplace(Literal("replace"), Literal("pl"), Literal("123")), "re123ace") checkEvaluation(StringReplace(Literal("replace"), Literal("pl"), Literal("")), "reace") diff --git a/sql/core/pom.xml b/sql/core/pom.xml index a5749f41466e5..61156964bc379 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 07752dc673572..fd940c42cfe33 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 9a35bb24f9b8d..5876b995eb971 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -22,7 +22,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index de2844b444160..d55fbe02766a8 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index d6657341d8105..4913df0a21e7a 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.12 - 3.2.0-kylin-4.x-r65 + 3.2.0-kylin-4.x-r66 ../pom.xml