From 473f0c3ee28f0a2056e4b16fa02b58793ba256dc Mon Sep 17 00:00:00 2001 From: currantw Date: Tue, 17 Dec 2024 09:19:41 -0800 Subject: [PATCH] Revert `cast` changes. Signed-off-by: currantw --- docs/ppl-lang/PPL-Example-Commands.md | 2 +- .../spark/ppl/FlintSparkPPLParseITSuite.scala | 2 +- .../PPLLogicalPlanParseTranslatorTestSuite.scala | 14 ++++---------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/docs/ppl-lang/PPL-Example-Commands.md b/docs/ppl-lang/PPL-Example-Commands.md index 0afee9255..0450d05a4 100644 --- a/docs/ppl-lang/PPL-Example-Commands.md +++ b/docs/ppl-lang/PPL-Example-Commands.md @@ -274,7 +274,7 @@ source = table | where ispresent(a) | - `source=accounts | parse email '.+@(?.+)' | stats count() by host` - `source=accounts | parse email '.+@(?.+)' | eval eval_result=1 | fields host, eval_result` - `source=accounts | parse email '.+@(?.+)' | where age > 45 | sort - age | fields age, email, host` -- `source=accounts | parse address '(?\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort streetNumber | fields streetNumber, street` +- `source=accounts | parse address '(?\d+) (?.+)' | where streetNumber > 500 | sort streetNumber | fields streetNumber, street` - Limitation: [see limitations](ppl-parse-command.md#limitations) #### **Grok** diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLParseITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLParseITSuite.scala index f091596d0..5d38ad32d 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLParseITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLParseITSuite.scala @@ -216,7 +216,7 @@ class FlintSparkPPLParseITSuite test("test parse email & host expressions including cast and sort commands") { val frame = sql(s""" - | source = $testTable| parse street_address '(?\\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort streetNumber | fields streetNumber, street + | source = $testTable| parse street_address '(?\\d+) (?.+)' | where streetNumber > 500 | sort streetNumber | fields streetNumber, street | """.stripMargin) // Retrieve the results val results: Array[Row] = frame.collect() diff --git a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanParseTranslatorTestSuite.scala b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanParseTranslatorTestSuite.scala index 61f02cae3..4cbdaa070 100644 --- a/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanParseTranslatorTestSuite.scala +++ b/ppl-spark-integration/src/test/scala/org/opensearch/flint/spark/ppl/PPLLogicalPlanParseTranslatorTestSuite.scala @@ -13,10 +13,9 @@ import org.scalatest.matchers.should.Matchers import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.ScalaReflection.universe.Star import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} -import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Cast, Coalesce, Descending, GreaterThan, Literal, NamedExpression, NullsFirst, NullsLast, RegExpExtract, SortOrder} +import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Coalesce, Descending, GreaterThan, Literal, NamedExpression, NullsFirst, NullsLast, RegExpExtract, SortOrder} import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, GlobalLimit, LocalLimit, Project, Sort} -import org.apache.spark.sql.types.{DataType, IntegerType} class PPLLogicalPlanParseTranslatorTestSuite extends SparkFunSuite @@ -121,26 +120,24 @@ class PPLLogicalPlanParseTranslatorTestSuite assert(compareByString(expectedPlan) === compareByString(logPlan)) } - test("test parse street number & address expressions including cast and sort commands") { + test("test parse email & host expressions including cast and sort commands") { val context = new CatalystPlanContext val logPlan = planTransformer.visit( plan( pplParser, - "source=t | parse address '(?\\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort streetNumber | fields streetNumber, street"), + "source=t | parse address '(?\\d+) (?.+)' | where streetNumber > 500 | sort streetNumber | fields streetNumber, street"), context) val addressAttribute = UnresolvedAttribute("address") val streetNumberAttribute = UnresolvedAttribute("streetNumber") val streetAttribute = UnresolvedAttribute("street") - val streetNumberExpression = Alias( RegExpExtract( addressAttribute, Literal("(?\\d+) (?.+)"), Literal("1")), "streetNumber")() - val streetExpression = Alias( RegExpExtract( addressAttribute, @@ -148,19 +145,16 @@ class PPLLogicalPlanParseTranslatorTestSuite Literal("2")), "street")() - val castExpression = Cast(streetNumberAttribute, IntegerType) - val expectedPlan = Project( Seq(streetNumberAttribute, streetAttribute), Sort( Seq(SortOrder(streetNumberAttribute, Ascending, NullsFirst, Seq.empty)), global = true, Filter( - GreaterThan(castExpression, Literal(500)), + GreaterThan(streetNumberAttribute, Literal(500)), Project( Seq(addressAttribute, streetNumberExpression, streetExpression, UnresolvedStar(None)), UnresolvedRelation(Seq("t")))))) - assert(compareByString(expectedPlan) === compareByString(logPlan)) }