diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 13f4e66842ef1..614fdd49d83df 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -499,7 +499,7 @@ jobs: - >- pyspark-sql, pyspark-resource, pyspark-testing - >- - pyspark-core, pyspark-errors, pyspark-streaming + pyspark-core, pyspark-errors, pyspark-streaming, pyspark-logger - >- pyspark-mllib, pyspark-ml, pyspark-ml-connect - >- @@ -743,7 +743,7 @@ jobs: python-version: '3.11' - name: Install dependencies for Python CodeGen check run: | - python3.11 -m pip install 'black==23.9.1' 'protobuf==5.29.1' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' + python3.11 -m pip install 'black==23.12.1' 'protobuf==5.29.1' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' python3.11 -m pip list - name: Python CodeGen check for branch-3.5 if: inputs.branch == 'branch-3.5' @@ -1094,6 +1094,7 @@ jobs: spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true - name: Run TPC-DS queries on collated data + if: inputs.branch != 'branch-3.5' run: | SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSCollationQueryTestSuite" - name: Upload test results to report @@ -1225,6 +1226,7 @@ jobs: - name: Start Minikube uses: medyagh/setup-minikube@v0.0.18 with: + kubernetes-version: "1.32.0" # Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for better resource statistic cpus: 2 memory: 6144m diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index 565bb8c7d6e68..ac139147beb91 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -122,6 +122,19 @@ jobs: - name: Image digest (PySpark with old dependencies) if: hashFiles('dev/spark-test-image/python-minimum/Dockerfile') != '' run: echo ${{ steps.docker_build_pyspark_python_minimum.outputs.digest }} + - name: Build and push (PySpark PS with old dependencies) + if: hashFiles('dev/spark-test-image/python-ps-minimum/Dockerfile') != '' + id: docker_build_pyspark_python_ps_minimum + uses: docker/build-push-action@v6 + with: + context: ./dev/spark-test-image/python-ps-minimum/ + push: true + tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-ps-minimum-cache:${{ github.ref_name }}-static + cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-ps-minimum-cache:${{ github.ref_name }} + cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-ps-minimum-cache:${{ github.ref_name }},mode=max + - name: Image digest (PySpark PS with old dependencies) + if: hashFiles('dev/spark-test-image/python-ps-minimum/Dockerfile') != '' + run: echo ${{ steps.docker_build_pyspark_python_ps_minimum.outputs.digest }} - name: Build and push (PySpark with PyPy 3.10) if: hashFiles('dev/spark-test-image/pypy-310/Dockerfile') != '' id: docker_build_pyspark_pypy_310 diff --git a/.github/workflows/build_python_ps_minimum.yml b/.github/workflows/build_python_ps_minimum.yml new file mode 100644 index 0000000000000..742d578e27418 --- /dev/null +++ b/.github/workflows/build_python_ps_minimum.yml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Build / Python-only (master, Python PS with old dependencies)" + +on: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + +jobs: + run-build: + permissions: + packages: write + name: Run + uses: ./.github/workflows/build_and_test.yml + if: github.repository == 'apache/spark' + with: + java: 17 + branch: master + hadoop: hadoop3 + envs: >- + { + "PYSPARK_IMAGE_TO_TEST": "python-ps-minimum", + "PYTHON_TO_TEST": "python3.9" + } + jobs: >- + { + "pyspark": "true", + "pyspark-pandas": "true" + } diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 637abb86b36cd..4bcc275064d3c 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -62,7 +62,7 @@ jobs: run: | pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow 'pandas==2.2.3' 'plotly>=4.8' 'docutils<0.18.0' \ - 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.9.1' \ + 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \ 'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpcio-status==1.67.0' 'protobuf==5.29.1' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' - name: Install Ruby for documentation generation diff --git a/LICENSE-binary b/LICENSE-binary index 40d28fbe71e6b..5cf099cb4d3c4 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -286,6 +286,10 @@ io.netty:netty-transport-classes-kqueue io.netty:netty-transport-native-epoll io.netty:netty-transport-native-kqueue io.netty:netty-transport-native-unix-common +io.vertx:vertx-auth-common +io.vertx:vertx-core +io.vertx:vertx-web-client +io.vertx:vertx-web-common jakarta.inject:jakarta.inject-api jakarta.validation:jakarta.validation-api javax.jdo:jdo-api diff --git a/common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java b/common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java index 90dddc2cb08c1..3482c6addfee3 100644 --- a/common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java +++ b/common/utils/src/main/java/org/apache/spark/network/util/JavaUtils.java @@ -22,6 +22,7 @@ import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.*; import java.util.concurrent.TimeUnit; @@ -125,10 +126,11 @@ public static void deleteRecursively(File file, FilenameFilter filter) throws IO private static void deleteRecursivelyUsingJavaIO( File file, FilenameFilter filter) throws IOException { - if (!file.exists()) return; - BasicFileAttributes fileAttributes = - Files.readAttributes(file.toPath(), BasicFileAttributes.class); - if (fileAttributes.isDirectory() && !isSymlink(file)) { + BasicFileAttributes fileAttributes = readFileAttributes(file); + // SPARK-50716: If the file attributes are null, that is, the file attributes cannot be read, + // or if the file does not exist and is not a broken symbolic link, then return directly. + if (fileAttributes == null || (!file.exists() && !fileAttributes.isSymbolicLink())) return; + if (fileAttributes.isDirectory()) { IOException savedIOException = null; for (File child : listFilesSafely(file, filter)) { try { @@ -143,8 +145,8 @@ private static void deleteRecursivelyUsingJavaIO( } } - // Delete file only when it's a normal file or an empty directory. - if (fileAttributes.isRegularFile() || + // Delete file only when it's a normal file, a symbolic link, or an empty directory. + if (fileAttributes.isRegularFile() || fileAttributes.isSymbolicLink() || (fileAttributes.isDirectory() && listFilesSafely(file, null).length == 0)) { boolean deleted = file.delete(); // Delete can also fail if the file simply did not exist. @@ -154,6 +156,18 @@ private static void deleteRecursivelyUsingJavaIO( } } + /** + * Reads basic attributes of a given file, of return null if an I/O error occurs. + */ + private static BasicFileAttributes readFileAttributes(File file) { + try { + return Files.readAttributes( + file.toPath(), BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + } catch (IOException e) { + return null; + } + } + private static void deleteRecursivelyUsingUnixNative(File file) throws IOException { ProcessBuilder builder = new ProcessBuilder("rm", "-rf", file.getAbsolutePath()); Process process = null; @@ -192,17 +206,6 @@ private static File[] listFilesSafely(File file, FilenameFilter filter) throws I } } - private static boolean isSymlink(File file) throws IOException { - Objects.requireNonNull(file); - File fileInCanonicalDir = null; - if (file.getParent() == null) { - fileInCanonicalDir = file; - } else { - fileInCanonicalDir = new File(file.getParentFile().getCanonicalFile(), file.getName()); - } - return !fileInCanonicalDir.getCanonicalFile().equals(fileInCanonicalDir.getAbsoluteFile()); - } - private static final Map timeSuffixes; private static final Map byteSuffixes; diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 26b3de7f50890..52c0315bd0732 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1229,6 +1229,18 @@ }, "sqlState" : "4274K" }, + "DUPLICATE_ROUTINE_PARAMETER_NAMES" : { + "message" : [ + "Found duplicate name(s) in the parameter list of the user-defined routine : ." + ], + "sqlState" : "42734" + }, + "DUPLICATE_ROUTINE_RETURNS_COLUMNS" : { + "message" : [ + "Found duplicate column(s) in the RETURNS clause column list of the user-defined routine : ." + ], + "sqlState" : "42711" + }, "EMITTING_ROWS_OLDER_THAN_WATERMARK_NOT_ALLOWED" : { "message" : [ "Previous node emitted a row with eventTime= which is older than current_watermark_value=", @@ -4146,6 +4158,18 @@ ], "sqlState" : "38000" }, + "RECURSIVE_CTE_IN_LEGACY_MODE" : { + "message" : [ + "Recursive definitions cannot be used in legacy CTE precedence mode (spark.sql.legacy.ctePrecedencePolicy=LEGACY)." + ], + "sqlState" : "42836" + }, + "RECURSIVE_CTE_WHEN_INLINING_IS_FORCED" : { + "message" : [ + "Recursive definitions cannot be used when CTE inlining is forced." + ], + "sqlState" : "42836" + }, "RECURSIVE_PROTOBUF_SCHEMA" : { "message" : [ "Found recursive reference in Protobuf schema, which can not be processed by Spark by default: . try setting the option `recursive.fields.max.depth` 1 to 10. Going beyond 10 levels of recursion is not allowed." @@ -4683,6 +4707,12 @@ ], "sqlState" : "42P01" }, + "TABLE_VALUED_ARGUMENTS_NOT_YET_IMPLEMENTED_FOR_SQL_FUNCTIONS" : { + "message" : [ + "Cannot SQL user-defined function with TABLE arguments because this functionality is not yet implemented." + ], + "sqlState" : "0A000" + }, "TABLE_VALUED_FUNCTION_FAILED_TO_ANALYZE_IN_PYTHON" : { "message" : [ "Failed to analyze the Python user defined table function: " @@ -5815,6 +5845,54 @@ ], "sqlState" : "42K0E" }, + "USER_DEFINED_FUNCTIONS" : { + "message" : [ + "User defined function is invalid:" + ], + "subClass" : { + "CANNOT_CONTAIN_COMPLEX_FUNCTIONS" : { + "message" : [ + "SQL scalar function cannot contain aggregate/window/generate functions: " + ] + }, + "CANNOT_REPLACE_NON_SQL_UDF_WITH_SQL_UDF" : { + "message" : [ + "Cannot replace the non-SQL function with a SQL function." + ] + }, + "NOT_A_VALID_DEFAULT_EXPRESSION" : { + "message" : [ + "The DEFAULT expression of ``.`` is not supported because it contains a subquery." + ] + }, + "NOT_A_VALID_DEFAULT_PARAMETER_POSITION" : { + "message" : [ + "In routine `` parameter `` with DEFAULT must not be followed by parameter `` without DEFAULT." + ] + }, + "NOT_NULL_ON_FUNCTION_PARAMETERS" : { + "message" : [ + "Cannot specify NOT NULL on function parameters: " + ] + }, + "RETURN_COLUMN_COUNT_MISMATCH" : { + "message" : [ + "The number of columns produced by the RETURN clause (num: ``) does not match the number of column names specified by the RETURNS clause (num: ``) of ." + ] + }, + "SQL_TABLE_UDF_BODY_MUST_BE_A_QUERY" : { + "message" : [ + "SQL table function body must be a query." + ] + }, + "SQL_TABLE_UDF_MISSING_COLUMN_NAMES" : { + "message" : [ + "The relation returned by the query in the CREATE FUNCTION statement for with RETURNS TABLE clause lacks explicit names for one or more output columns; please rewrite the function body to provide explicit column names or add column names to the RETURNS TABLE clause, and re-run the command." + ] + } + }, + "sqlState" : "42601" + }, "USER_RAISED_EXCEPTION" : { "message" : [ "" @@ -5935,7 +6013,7 @@ }, "XML_ROW_TAG_MISSING" : { "message" : [ - " option is required for reading files in XML format." + " option is required for reading/writing files in XML format." ], "sqlState" : "42KDF" }, diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala index 63b5f27c4745e..d5505d2222c4f 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/KeyValueGroupedDataset.scala @@ -27,7 +27,7 @@ import org.apache.spark.connect.proto import org.apache.spark.sql.catalyst.encoders.AgnosticEncoder import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{agnosticEncoderFor, ProductEncoder} import org.apache.spark.sql.connect.ConnectConversions._ -import org.apache.spark.sql.connect.common.UdfUtils +import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, UdfUtils} import org.apache.spark.sql.expressions.SparkUserDefinedFunction import org.apache.spark.sql.functions.col import org.apache.spark.sql.internal.ColumnNodeToProtoConverter.toExpr @@ -502,6 +502,7 @@ private class KeyValueGroupedDatasetImpl[K, V, IK, IV]( } val outputEncoder = agnosticEncoderFor[U] + val stateEncoder = agnosticEncoderFor[S] val nf = UDFAdaptors.flatMapGroupsWithStateWithMappedValues(func, valueMapFunc) sparkSession.newDataset[U](outputEncoder) { builder => @@ -509,11 +510,12 @@ private class KeyValueGroupedDatasetImpl[K, V, IK, IV]( groupMapBuilder .setInput(plan.getRoot) .addAllGroupingExpressions(groupingExprs) - .setFunc(getUdf(nf, outputEncoder)(ivEncoder)) + .setFunc(getUdf(nf, outputEncoder, stateEncoder)(ivEncoder)) .setIsMapGroupsWithState(isMapGroupWithState) .setOutputMode(if (outputMode.isEmpty) OutputMode.Update.toString else outputMode.get.toString) .setTimeoutConf(timeoutConf.toString) + .setStateSchema(DataTypeProtoConverter.toConnectProtoType(stateEncoder.schema)) if (initialStateImpl != null) { groupMapBuilder @@ -533,6 +535,21 @@ private class KeyValueGroupedDatasetImpl[K, V, IK, IV]( udf.apply(inputEncoders.map(_ => col("*")): _*).expr.getCommonInlineUserDefinedFunction } + private def getUdf[U: Encoder, S: Encoder]( + nf: AnyRef, + outputEncoder: AgnosticEncoder[U], + stateEncoder: AgnosticEncoder[S])( + inEncoders: AgnosticEncoder[_]*): proto.CommonInlineUserDefinedFunction = { + // Apply keyAs changes by setting kEncoder + // Add the state encoder to the inputEncoders. + val inputEncoders = kEncoder +: stateEncoder +: inEncoders + val udf = SparkUserDefinedFunction( + function = nf, + inputEncoders = inputEncoders, + outputEncoder = outputEncoder) + udf.apply(inputEncoders.map(_ => col("*")): _*).expr.getCommonInlineUserDefinedFunction + } + /** * We cannot deserialize a connect [[KeyValueGroupedDataset]] because of a class clash on the * server side. We null out the instance for now. diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/internal/columnNodeSupport.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/internal/columnNodeSupport.scala index 8d57a8d3efd44..7802d9750bbc3 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/internal/columnNodeSupport.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/internal/columnNodeSupport.scala @@ -73,13 +73,19 @@ object ColumnNodeToProtoConverter extends (ColumnNode => proto.Expression) { .setColName(regex) planId.foreach(b.setPlanId) - case UnresolvedFunction(functionName, arguments, isDistinct, isUserDefinedFunction, _, _) => - // TODO(SPARK-49087) use internal namespace. + case UnresolvedFunction( + functionName, + arguments, + isDistinct, + isUserDefinedFunction, + isInternal, + _) => builder.getUnresolvedFunctionBuilder .setFunctionName(functionName) .setIsUserDefinedFunction(isUserDefinedFunction) .setIsDistinct(isDistinct) .addAllArguments(arguments.map(apply(_, e)).asJava) + .setIsInternal(isInternal) case Alias(child, name, metadata, _) => val b = builder.getAliasBuilder.setExpr(apply(child, e)) @@ -156,6 +162,7 @@ object ColumnNodeToProtoConverter extends (ColumnNode => proto.Expression) { case CaseWhenOtherwise(branches, otherwise, _) => val b = builder.getUnresolvedFunctionBuilder .setFunctionName("when") + .setIsInternal(false) branches.foreach { case (condition, value) => b.addArguments(apply(condition, e)) b.addArguments(apply(value, e)) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala index fc37444f77197..4b36d36983a5d 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala @@ -354,6 +354,28 @@ class DataFrameSubquerySuite extends QueryTest with RemoteSparkSession { } } + test("lateral join with star expansion") { + withView("t1", "t2") { + val t1 = table1() + val t2 = table2() + + checkAnswer( + t1.lateralJoin(spark.range(1).select().select($"*")), + sql("SELECT * FROM t1, LATERAL (SELECT *)")) + checkAnswer( + t1.lateralJoin(t2.select($"*")).toDF("c1", "c2", "c3", "c4"), + sql("SELECT * FROM t1, LATERAL (SELECT * FROM t2)").toDF("c1", "c2", "c3", "c4")) + checkAnswer( + t1.lateralJoin(t2.select($"t1.*".outer(), $"t2.*")) + .toDF("c1", "c2", "c3", "c4", "c5", "c6"), + sql("SELECT * FROM t1, LATERAL (SELECT t1.*, t2.* FROM t2)") + .toDF("c1", "c2", "c3", "c4", "c5", "c6")) + checkAnswer( + t1.lateralJoin(t2.alias("t1").select($"t1.*")).toDF("c1", "c2", "c3", "c4"), + sql("SELECT * FROM t1, LATERAL (SELECT t1.* FROM t2 AS t1)").toDF("c1", "c2", "c3", "c4")) + } + } + test("lateral join with different join types") { withView("t1") { val t1 = table1() @@ -375,6 +397,17 @@ class DataFrameSubquerySuite extends QueryTest with RemoteSparkSession { } } + test("lateral join with subquery alias") { + withView("t1") { + val t1 = table1() + + checkAnswer( + t1.lateralJoin(spark.range(1).select($"c1".outer(), $"c2".outer()).toDF("a", "b").as("s")) + .select("a", "b"), + sql("SELECT a, b FROM t1, LATERAL (SELECT c1, c2) s(a, b)")) + } + } + test("lateral join with correlated equality / non-equality predicates") { withView("t1", "t2") { val t1 = table1() @@ -441,8 +474,8 @@ class DataFrameSubquerySuite extends QueryTest with RemoteSparkSession { val t2 = table2() checkAnswer( - t1.lateralJoin(t2.where($"t1.c1".outer() === $"t2.c1").select($"c2"), "left") - .join(t1.as("t3"), $"t2.c2" === $"t3.c2", "left") + t1.lateralJoin(t2.where($"t1.c1".outer() === $"t2.c1").select($"c2").as("s"), "left") + .join(t1.as("t3"), $"s.c2" === $"t3.c2", "left") .toDF("c1", "c2", "c3", "c4", "c5"), sql(""" |SELECT * FROM t1 diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala index aeef2e8f0fcf2..12a49ad21676e 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala @@ -61,10 +61,11 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi val t3 = spark.table("t3") checkAnswer( - t1.lateralJoin(spark.tvf.explode(array($"c1".outer(), $"c2".outer()))), + t1.lateralJoin( + spark.tvf.explode(array($"c1".outer(), $"c2".outer())).toDF("c3").as("t2")), sql("SELECT * FROM t1, LATERAL EXPLODE(ARRAY(c1, c2)) t2(c3)")) checkAnswer( - t3.lateralJoin(spark.tvf.explode($"c2".outer())), + t3.lateralJoin(spark.tvf.explode($"c2".outer()).toDF("v").as("t2")), sql("SELECT * FROM t3, LATERAL EXPLODE(c2) t2(v)")) checkAnswer( spark.tvf @@ -113,10 +114,11 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi val t3 = spark.table("t3") checkAnswer( - t1.lateralJoin(spark.tvf.explode_outer(array($"c1".outer(), $"c2".outer()))), + t1.lateralJoin( + spark.tvf.explode_outer(array($"c1".outer(), $"c2".outer())).toDF("c3").as("t2")), sql("SELECT * FROM t1, LATERAL EXPLODE_OUTER(ARRAY(c1, c2)) t2(c3)")) checkAnswer( - t3.lateralJoin(spark.tvf.explode_outer($"c2".outer())), + t3.lateralJoin(spark.tvf.explode_outer($"c2".outer()).toDF("v").as("t2")), sql("SELECT * FROM t3, LATERAL EXPLODE_OUTER(c2) t2(v)")) checkAnswer( spark.tvf @@ -161,7 +163,10 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi arrayStruct.lateralJoin(spark.tvf.inline($"arr".outer())), sql("SELECT * FROM array_struct JOIN LATERAL INLINE(arr)")) checkAnswer( - arrayStruct.lateralJoin(spark.tvf.inline($"arr".outer()), $"id" === $"col1", "left"), + arrayStruct.lateralJoin( + spark.tvf.inline($"arr".outer()).toDF("k", "v").as("t"), + $"id" === $"k", + "left"), sql("SELECT * FROM array_struct LEFT JOIN LATERAL INLINE(arr) t(k, v) ON id = k")) } } @@ -202,8 +207,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi sql("SELECT * FROM array_struct JOIN LATERAL INLINE_OUTER(arr)")) checkAnswer( arrayStruct.lateralJoin( - spark.tvf.inline_outer($"arr".outer()), - $"id" === $"col1", + spark.tvf.inline_outer($"arr".outer()).toDF("k", "v").as("t"), + $"id" === $"k", "left"), sql("SELECT * FROM array_struct LEFT JOIN LATERAL INLINE_OUTER(arr) t(k, v) ON id = k")) } @@ -238,30 +243,27 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi jsonTable .as("t1") .lateralJoin( - spark.tvf.json_tuple( - $"t1.jstring".outer(), - lit("f1"), - lit("f2"), - lit("f3"), - lit("f4"), - lit("f5"))) - .select($"key", $"c0", $"c1", $"c2", $"c3", $"c4"), + spark.tvf + .json_tuple( + $"t1.jstring".outer(), + lit("f1"), + lit("f2"), + lit("f3"), + lit("f4"), + lit("f5")) + .as("t2")) + .select($"t1.key", $"t2.*"), sql( "SELECT t1.key, t2.* FROM json_table t1, " + "LATERAL json_tuple(t1.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') t2")) checkAnswer( jsonTable .as("t1") - .lateralJoin( - spark.tvf.json_tuple( - $"jstring".outer(), - lit("f1"), - lit("f2"), - lit("f3"), - lit("f4"), - lit("f5"))) - .where($"c0".isNotNull) - .select($"key", $"c0", $"c1", $"c2", $"c3", $"c4"), + .lateralJoin(spark.tvf + .json_tuple($"jstring".outer(), lit("f1"), lit("f2"), lit("f3"), lit("f4"), lit("f5")) + .as("t2")) + .where($"t2.c0".isNotNull) + .select($"t1.key", $"t2.*"), sql( "SELECT t1.key, t2.* FROM json_table t1, " + "LATERAL json_tuple(t1.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') t2 " + @@ -390,17 +392,18 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi checkAnswer( t1.lateralJoin( - spark.tvf.stack(lit(2), lit("Key"), $"c1".outer(), lit("Value"), $"c2".outer())) - .select($"col0", $"col1"), + spark.tvf.stack(lit(2), lit("Key"), $"c1".outer(), lit("Value"), $"c2".outer()).as("t")) + .select($"t.*"), sql("SELECT t.* FROM t1, LATERAL stack(2, 'Key', c1, 'Value', c2) t")) checkAnswer( - t1.lateralJoin(spark.tvf.stack(lit(1), $"c1".outer(), $"c2".outer())) - .select($"col0".as("x"), $"col1".as("y")), + t1.lateralJoin( + spark.tvf.stack(lit(1), $"c1".outer(), $"c2".outer()).toDF("x", "y").as("t")) + .select($"t.*"), sql("SELECT t.* FROM t1 JOIN LATERAL stack(1, c1, c2) t(x, y)")) checkAnswer( t1.join(t3, $"t1.c1" === $"t3.c1") - .lateralJoin(spark.tvf.stack(lit(1), $"t1.c2".outer(), $"t3.c2".outer())) - .select($"col0", $"col1"), + .lateralJoin(spark.tvf.stack(lit(1), $"t1.c2".outer(), $"t3.c2".outer()).as("t")) + .select($"t.*"), sql("SELECT t.* FROM t1 JOIN t3 ON t1.c1 = t3.c1 JOIN LATERAL stack(1, t1.c2, t3.c2) t")) } } @@ -463,8 +466,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi checkAnswer( variantTable .as("t1") - .lateralJoin(spark.tvf.variant_explode($"v".outer())) - .select($"id", $"pos", $"key", $"value"), + .lateralJoin(spark.tvf.variant_explode($"v".outer()).as("t")) + .select($"t1.id", $"t.*"), sql("SELECT t1.id, t.* FROM variant_table AS t1, LATERAL variant_explode(v) AS t")) } } @@ -515,8 +518,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with RemoteSparkSessi checkAnswer( variantTable .as("t1") - .lateralJoin(spark.tvf.variant_explode_outer($"v".outer())) - .select($"id", $"pos", $"key", $"value"), + .lateralJoin(spark.tvf.variant_explode_outer($"v".outer()).as("t")) + .select($"t1.id", $"t.*"), sql("SELECT t1.id, t.* FROM variant_table AS t1, LATERAL variant_explode_outer(v) AS t")) } } diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala index 988774d5eec94..6fd664d905408 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/KeyValueGroupedDatasetE2ETestSuite.scala @@ -479,6 +479,25 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with RemoteSparkSessi "(c,1,1)") } + test("SPARK-50693: groupby on unresolved plan") { + val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) + .toDF("key", "seq", "value") + val grouped = ds.select("*").groupBy($"key").as[String, (String, Int, Int)] + val aggregated = grouped + .flatMapSortedGroups($"seq", expr("length(key)"), $"value") { (g, iter) => + Iterator(g, iter.mkString(", ")) + } + + checkDatasetUnorderly( + aggregated, + "a", + "(a,1,10), (a,2,20)", + "b", + "(b,1,2), (b,2,1)", + "c", + "(c,1,1)") + } + test("groupby - keyAs, keys") { val ds = Seq(("a", 1, 10), ("a", 2, 20), ("b", 2, 1), ("b", 1, 2), ("c", 1, 1)) .toDF("key", "seq", "value") @@ -597,6 +616,16 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with RemoteSparkSessi ("c", 1L)) } + test("SPARK-50693: RowEncoder in udf on unresolved plan") { + val ds = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") + + checkDatasetUnorderly( + ds.select("*").groupByKey(k => k.getAs[String](0)).agg(sum("c2").as[Long]), + ("a", 30L), + ("b", 3L), + ("c", 1L)) + } + test("mapGroups with row encoder") { val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") @@ -611,6 +640,21 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with RemoteSparkSessi 1) } + test("SPARK-50693: mapGroups with row encoder on unresolved plan") { + val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") + + checkDataset( + df.select("*") + .groupByKey(r => r.getAs[String]("c1")) + .mapGroups((_, it) => + it.map(r => { + r.getAs[Int]("c2") + }).sum), + 30, + 3, + 1) + } + test("coGroup with row encoder") { val df1 = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") val df2 = Seq(("x", 10), ("x", 20), ("y", 1), ("y", 2), ("a", 1)).toDF("c1", "c2") @@ -632,6 +676,30 @@ class KeyValueGroupedDatasetE2ETestSuite extends QueryTest with RemoteSparkSessi 3) } + test("SPARK-50693: coGroup with row encoder on unresolved plan") { + val df1 = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") + val df2 = Seq(("x", 10), ("x", 20), ("y", 1), ("y", 2), ("a", 1)).toDF("c1", "c2") + + Seq((df1.select("*"), df2), (df1, df2.select("*")), (df1.select("*"), df2.select("*"))) + .foreach { case (df1, df2) => + val ds1: KeyValueGroupedDataset[String, Row] = + df1.groupByKey(r => r.getAs[String]("c1")) + val ds2: KeyValueGroupedDataset[String, Row] = + df2.groupByKey(r => r.getAs[String]("c1")) + checkDataset( + ds1.cogroup(ds2)((_, it, it2) => { + val sum1 = it.map(r => r.getAs[Int]("c2")).sum + val sum2 = it2.map(r => r.getAs[Int]("c2")).sum + Iterator(sum1 + sum2) + }), + 31, + 3, + 1, + 30, + 3) + } + } + test("serialize as null") { val kvgds = session.range(10).groupByKey(_ % 2) val bytes = SparkSerDeUtils.serialize(kvgds) diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala index ca754c7b542f7..8415444c10aac 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/UserDefinedFunctionE2ETestSuite.scala @@ -301,6 +301,14 @@ class UserDefinedFunctionE2ETestSuite extends QueryTest with RemoteSparkSession checkDataset(df.filter(r => r.getInt(1) > 5), Row("a", 10), Row("a", 20)) } + test("SPARK-50693: Filter with row input encoder on unresolved plan") { + val session: SparkSession = spark + import session.implicits._ + val df = Seq(("a", 10), ("a", 20), ("b", 1), ("b", 2), ("c", 1)).toDF("c1", "c2") + + checkDataset(df.select("*").filter(r => r.getInt(1) > 5), Row("a", 10), Row("a", 20)) + } + test("mapPartitions with row input encoder") { val session: SparkSession = spark import session.implicits._ diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala index 4ec84a4087eb7..7bac10e79d0b4 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala @@ -183,6 +183,11 @@ object CheckConnectJvmClientCompatibility { "org.apache.spark.sql.ExtendedExplainGenerator"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.UDTFRegistration"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.DataSourceRegistration"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.TableArg"), + ProblemFilters.exclude[MissingClassProblem]( + "org.apache.spark.sql.artifact.ArtifactStateForCleanup"), + ProblemFilters.exclude[MissingClassProblem]( + "org.apache.spark.sql.artifact.ArtifactStateForCleanup$"), // DataFrameNaFunctions ProblemFilters.exclude[Problem]("org.apache.spark.sql.DataFrameNaFunctions.fillValue"), diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/internal/ColumnNodeToProtoConverterSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/internal/ColumnNodeToProtoConverterSuite.scala index 4cb03420c4d07..94729d34f37b5 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/internal/ColumnNodeToProtoConverterSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/internal/ColumnNodeToProtoConverterSuite.scala @@ -128,19 +128,22 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite { .setFunctionName("+") .setIsDistinct(false) .addArguments(attribute("a")) - .addArguments(expr(_.getLiteralBuilder.setInteger(1))))) + .addArguments(expr(_.getLiteralBuilder.setInteger(1))) + .setIsInternal(false))) testConversion( UnresolvedFunction( "db1.myAgg", Seq(UnresolvedAttribute("a")), isDistinct = true, - isUserDefinedFunction = true), + isUserDefinedFunction = true, + isInternal = true), expr( _.getUnresolvedFunctionBuilder .setFunctionName("db1.myAgg") .setIsDistinct(true) .setIsUserDefinedFunction(true) - .addArguments(attribute("a")))) + .addArguments(attribute("a")) + .setIsInternal(true))) } test("alias") { @@ -247,10 +250,12 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite { expr( _.getWindowBuilder .setWindowFunction( - expr(_.getUnresolvedFunctionBuilder - .setFunctionName("sum") - .setIsDistinct(false) - .addArguments(attribute("a")))) + expr( + _.getUnresolvedFunctionBuilder + .setFunctionName("sum") + .setIsDistinct(false) + .addArguments(attribute("a")) + .setIsInternal(false))) .addPartitionSpec(attribute("b")) .addPartitionSpec(attribute("c")) .addOrderSpec(proto.Expression.SortOrder @@ -276,7 +281,8 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite { _.getUnresolvedFunctionBuilder .setFunctionName("sum") .setIsDistinct(false) - .addArguments(attribute("a")))) + .addArguments(attribute("a")) + .setIsInternal(false))) .addPartitionSpec(attribute("b")) .addPartitionSpec(attribute("c")))) testWindowFrame( @@ -310,7 +316,8 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite { _.getUnresolvedFunctionBuilder .setFunctionName("+") .addArguments(expr(_.setUnresolvedNamedLambdaVariable(catX))) - .addArguments(attribute("y")))) + .addArguments(attribute("y")) + .setIsInternal(false))) .addArguments(catX))) } @@ -330,7 +337,8 @@ class ColumnNodeToProtoConverterSuite extends ConnectFunSuite { .setFunctionName("when") .addArguments(attribute("c1")) .addArguments(expr(_.getLiteralBuilder.setString("r1"))) - .addArguments(expr(_.getLiteralBuilder.setString("fallback"))))) + .addArguments(expr(_.getLiteralBuilder.setString("fallback"))) + .setIsInternal(false))) } test("extract field") { diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala index dc74463f1a25b..9bd6614028cbf 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/streaming/FlatMapGroupsWithStateStreamingSuite.scala @@ -55,7 +55,9 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with RemoteSparkSes val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") - Iterator(ClickState(key, values.size)) + val newState = ClickState(key, values.size) + state.update(newState) + Iterator(newState) } spark.sql("DROP TABLE IF EXISTS my_sink") @@ -96,7 +98,9 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with RemoteSparkSes val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) - Iterator(ClickState(key, currState.count + values.size)) + val newState = ClickState(key, currState.count + values.size) + state.update(newState) + Iterator(newState) } val initialState = flatMapGroupsWithStateInitialStateData .toDS() @@ -141,7 +145,9 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with RemoteSparkSes val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { if (state.exists) throw new IllegalArgumentException("state.exists should be false") - ClickState(key, values.size) + val newState = ClickState(key, values.size) + state.update(newState) + newState } spark.sql("DROP TABLE IF EXISTS my_sink") @@ -183,7 +189,9 @@ class FlatMapGroupsWithStateStreamingSuite extends QueryTest with RemoteSparkSes val stateFunc = (key: String, values: Iterator[ClickEvent], state: GroupState[ClickState]) => { val currState = state.getOption.getOrElse(ClickState(key, 0)) - ClickState(key, currState.count + values.size) + val newState = ClickState(key, currState.count + values.size) + state.update(newState) + newState } val initialState = flatMapGroupsWithStateInitialStateData .toDS() diff --git a/connector/protobuf/pom.xml b/connector/protobuf/pom.xml index e85481ef9e1c8..22d24a7cdb62d 100644 --- a/connector/protobuf/pom.xml +++ b/connector/protobuf/pom.xml @@ -189,7 +189,7 @@ src/test/resources/protobuf - true + direct java diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 5d3a9c2690c41..30d772bd62d77 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -423,9 +423,6 @@ class SparkContext(config: SparkConf) extends Logging { if (!_conf.contains("spark.app.name")) { throw new SparkException("An application name must be set in your configuration") } - // HADOOP-19097 Set fs.s3a.connection.establish.timeout to 30s - // We can remove this after Apache Hadoop 3.4.1 releases - conf.setIfMissing("spark.hadoop.fs.s3a.connection.establish.timeout", "30000") // This should be set as early as possible. SparkContext.fillMissingMagicCommitterConfsIfNeeded(_conf) diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala index 4f7338f74e298..e06b7d86e1db0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala @@ -2937,7 +2937,8 @@ private[spark] class DAGScheduler( } else { // This stage is only used by the job, so finish the stage if it is running. val stage = stageIdToStage(stageId) - if (runningStages.contains(stage)) { + // Stages with failedAttemptIds may have tasks that are running + if (runningStages.contains(stage) || stage.failedAttemptIds.nonEmpty) { try { // killAllTaskAttempts will fail if a SchedulerBackend does not implement killTask taskScheduler.killAllTaskAttempts(stageId, shouldInterruptTaskThread(job), reason) if (legacyAbortStageAfterKillTasks) { diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index 243d33fe55a79..3e507df706ba5 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -185,6 +185,8 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti private var firstInit: Boolean = _ /** Set of TaskSets the DAGScheduler has requested executed. */ val taskSets = scala.collection.mutable.Buffer[TaskSet]() + /** Track running tasks, the key is the task's stageId , the value is the task's partitionId */ + var runningTaskInfos = new HashMap[Int, HashSet[Int]]() /** Stages for which the DAGScheduler has called TaskScheduler.killAllTaskAttempts(). */ val cancelledStages = new HashSet[Int]() @@ -206,12 +208,14 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti // normally done by TaskSetManager taskSet.tasks.foreach(_.epoch = mapOutputTracker.getEpoch) taskSets += taskSet + runningTaskInfos.put(taskSet.stageId, new HashSet[Int]() ++ taskSet.tasks.map(_.partitionId)) } override def killTaskAttempt( taskId: Long, interruptThread: Boolean, reason: String): Boolean = false override def killAllTaskAttempts( stageId: Int, interruptThread: Boolean, reason: String): Unit = { cancelledStages += stageId + runningTaskInfos.remove(stageId) } override def notifyPartitionCompletion(stageId: Int, partitionId: Int): Unit = { taskSets.filter(_.stageId == stageId).lastOption.foreach { ts => @@ -393,6 +397,14 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti handleShuffleMergeFinalized(shuffleMapStage, shuffleMapStage.shuffleDep.shuffleMergeId) } } + + override private[scheduler] def handleTaskCompletion(event: CompletionEvent): Unit = { + super.handleTaskCompletion(event) + runningTaskInfos.get(event.task.stageId).foreach{ partitions => + partitions -= event.task.partitionId + if (partitions.isEmpty) runningTaskInfos.remove(event.task.stageId) + } + } } override def beforeEach(): Unit = { @@ -2252,6 +2264,46 @@ class DAGSchedulerSuite extends SparkFunSuite with TempLocalSparkContext with Ti assert(scheduler.activeJobs.isEmpty) } + test("SPARK-50648: when job is cancelled during shuffle retry in parent stage, " + + "should kill all running tasks") { + val shuffleMapRdd = new MyRDD(sc, 2, Nil) + val shuffleDep = new ShuffleDependency(shuffleMapRdd, new HashPartitioner(2)) + val reduceRdd = new MyRDD(sc, 2, List(shuffleDep)) + submit(reduceRdd, Array(0, 1)) + completeShuffleMapStageSuccessfully(0, 0, 2) + sc.listenerBus.waitUntilEmpty() + + val info = new TaskInfo( + 3, index = 1, attemptNumber = 1, + partitionId = taskSets(1).tasks(0).partitionId, 0L, "", "", TaskLocality.ANY, true) + // result task 0.0 fetch failed, but result task 1.0 is still running + runEvent(makeCompletionEvent(taskSets(1).tasks(0), + FetchFailed(makeBlockManagerId("hostA"), shuffleDep.shuffleId, 0L, 0, 1, "ignored"), + null, + Seq.empty, + Array.empty, + info)) + sc.listenerBus.waitUntilEmpty() + + Thread.sleep(DAGScheduler.RESUBMIT_TIMEOUT * 2) + // map stage is running by resubmitted, result stage is waiting + // map tasks and the origin result task 1.0 are running + assert(scheduler.runningStages.size == 1, "Map stage should be running") + val mapStage = scheduler.runningStages.head + assert(mapStage.id === 0) + assert(mapStage.latestInfo.failureReason.isEmpty) + assert(scheduler.waitingStages.size == 1, "Result stage should be waiting") + assert(runningTaskInfos.size == 2) + assert(runningTaskInfos(taskSets(1).stageId).size == 1, + "origin result task 1.0 should be running") + + scheduler.doCancelAllJobs() + // all tasks should be killed + assert(runningTaskInfos.isEmpty) + assert(scheduler.runningStages.isEmpty) + assert(scheduler.waitingStages.isEmpty) + } + test("misbehaved accumulator should not crash DAGScheduler and SparkContext") { val acc = new LongAccumulator { override def add(v: java.lang.Long): Unit = throw new DAGSchedulerSuiteDummyException diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index febc59e5c5369..baa748573d75b 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -22,6 +22,7 @@ import java.lang.reflect.Field import java.net.{BindException, ServerSocket, URI} import java.nio.{ByteBuffer, ByteOrder} import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files => JFiles} import java.text.DecimalFormatSymbols import java.util.Locale import java.util.concurrent.TimeUnit @@ -731,6 +732,43 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties { assert(!sourceFile2.exists()) } + test("SPARK-50716: deleteRecursively - SymbolicLink To File") { + val tempDir = Utils.createTempDir() + val sourceFile = new File(tempDir, "foo.txt") + JFiles.write(sourceFile.toPath, "Some content".getBytes) + assert(sourceFile.exists()) + + val symlinkFile = new File(tempDir, "bar.txt") + JFiles.createSymbolicLink(symlinkFile.toPath, sourceFile.toPath) + + // Check that the symlink was created successfully + assert(JFiles.isSymbolicLink(symlinkFile.toPath)) + Utils.deleteRecursively(tempDir) + + // Verify that everything is deleted + assert(!tempDir.exists) + } + + test("SPARK-50716: deleteRecursively - SymbolicLink To Dir") { + val tempDir = Utils.createTempDir() + val sourceDir = new File(tempDir, "sourceDir") + assert(sourceDir.mkdir()) + val sourceFile = new File(sourceDir, "file.txt") + JFiles.write(sourceFile.toPath, "Some content".getBytes) + + val symlinkDir = new File(tempDir, "targetDir") + JFiles.createSymbolicLink(symlinkDir.toPath, sourceDir.toPath) + + // Check that the symlink was created successfully + assert(JFiles.isSymbolicLink(symlinkDir.toPath)) + + // Now delete recursively + Utils.deleteRecursively(tempDir) + + // Verify that everything is deleted + assert(!tempDir.exists) + } + test("loading properties from file") { withTempDir { tmpDir => val outFile = File.createTempFile("test-load-spark-properties", "test", tmpDir) diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml index 4285028109419..6c50718e27fe5 100644 --- a/dev/checkstyle.xml +++ b/dev/checkstyle.xml @@ -172,32 +172,10 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile index 190c82429e29e..7b97aba21205b 100644 --- a/dev/create-release/spark-rm/Dockerfile +++ b/dev/create-release/spark-rm/Dockerfile @@ -130,7 +130,7 @@ RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CON # See 'docutils<0.18.0' in SPARK-39421 RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \ -'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.9.1' \ +'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \ 'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' RUN python3.9 -m pip list diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 8ce2b4d362990..aa171f0cbfffd 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -158,31 +158,31 @@ jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar jul-to-slf4j/2.0.16//jul-to-slf4j-2.0.16.jar kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar -kubernetes-client-api/6.13.4//kubernetes-client-api-6.13.4.jar -kubernetes-client/6.13.4//kubernetes-client-6.13.4.jar -kubernetes-httpclient-okhttp/6.13.4//kubernetes-httpclient-okhttp-6.13.4.jar -kubernetes-model-admissionregistration/6.13.4//kubernetes-model-admissionregistration-6.13.4.jar -kubernetes-model-apiextensions/6.13.4//kubernetes-model-apiextensions-6.13.4.jar -kubernetes-model-apps/6.13.4//kubernetes-model-apps-6.13.4.jar -kubernetes-model-autoscaling/6.13.4//kubernetes-model-autoscaling-6.13.4.jar -kubernetes-model-batch/6.13.4//kubernetes-model-batch-6.13.4.jar -kubernetes-model-certificates/6.13.4//kubernetes-model-certificates-6.13.4.jar -kubernetes-model-common/6.13.4//kubernetes-model-common-6.13.4.jar -kubernetes-model-coordination/6.13.4//kubernetes-model-coordination-6.13.4.jar -kubernetes-model-core/6.13.4//kubernetes-model-core-6.13.4.jar -kubernetes-model-discovery/6.13.4//kubernetes-model-discovery-6.13.4.jar -kubernetes-model-events/6.13.4//kubernetes-model-events-6.13.4.jar -kubernetes-model-extensions/6.13.4//kubernetes-model-extensions-6.13.4.jar -kubernetes-model-flowcontrol/6.13.4//kubernetes-model-flowcontrol-6.13.4.jar -kubernetes-model-gatewayapi/6.13.4//kubernetes-model-gatewayapi-6.13.4.jar -kubernetes-model-metrics/6.13.4//kubernetes-model-metrics-6.13.4.jar -kubernetes-model-networking/6.13.4//kubernetes-model-networking-6.13.4.jar -kubernetes-model-node/6.13.4//kubernetes-model-node-6.13.4.jar -kubernetes-model-policy/6.13.4//kubernetes-model-policy-6.13.4.jar -kubernetes-model-rbac/6.13.4//kubernetes-model-rbac-6.13.4.jar -kubernetes-model-resource/6.13.4//kubernetes-model-resource-6.13.4.jar -kubernetes-model-scheduling/6.13.4//kubernetes-model-scheduling-6.13.4.jar -kubernetes-model-storageclass/6.13.4//kubernetes-model-storageclass-6.13.4.jar +kubernetes-client-api/7.0.1//kubernetes-client-api-7.0.1.jar +kubernetes-client/7.0.1//kubernetes-client-7.0.1.jar +kubernetes-httpclient-vertx/7.0.1//kubernetes-httpclient-vertx-7.0.1.jar +kubernetes-model-admissionregistration/7.0.1//kubernetes-model-admissionregistration-7.0.1.jar +kubernetes-model-apiextensions/7.0.1//kubernetes-model-apiextensions-7.0.1.jar +kubernetes-model-apps/7.0.1//kubernetes-model-apps-7.0.1.jar +kubernetes-model-autoscaling/7.0.1//kubernetes-model-autoscaling-7.0.1.jar +kubernetes-model-batch/7.0.1//kubernetes-model-batch-7.0.1.jar +kubernetes-model-certificates/7.0.1//kubernetes-model-certificates-7.0.1.jar +kubernetes-model-common/7.0.1//kubernetes-model-common-7.0.1.jar +kubernetes-model-coordination/7.0.1//kubernetes-model-coordination-7.0.1.jar +kubernetes-model-core/7.0.1//kubernetes-model-core-7.0.1.jar +kubernetes-model-discovery/7.0.1//kubernetes-model-discovery-7.0.1.jar +kubernetes-model-events/7.0.1//kubernetes-model-events-7.0.1.jar +kubernetes-model-extensions/7.0.1//kubernetes-model-extensions-7.0.1.jar +kubernetes-model-flowcontrol/7.0.1//kubernetes-model-flowcontrol-7.0.1.jar +kubernetes-model-gatewayapi/7.0.1//kubernetes-model-gatewayapi-7.0.1.jar +kubernetes-model-metrics/7.0.1//kubernetes-model-metrics-7.0.1.jar +kubernetes-model-networking/7.0.1//kubernetes-model-networking-7.0.1.jar +kubernetes-model-node/7.0.1//kubernetes-model-node-7.0.1.jar +kubernetes-model-policy/7.0.1//kubernetes-model-policy-7.0.1.jar +kubernetes-model-rbac/7.0.1//kubernetes-model-rbac-7.0.1.jar +kubernetes-model-resource/7.0.1//kubernetes-model-resource-7.0.1.jar +kubernetes-model-scheduling/7.0.1//kubernetes-model-scheduling-7.0.1.jar +kubernetes-model-storageclass/7.0.1//kubernetes-model-storageclass-7.0.1.jar lapack/3.0.3//lapack-3.0.3.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar @@ -193,7 +193,6 @@ log4j-api/2.24.3//log4j-api-2.24.3.jar log4j-core/2.24.3//log4j-core-2.24.3.jar log4j-layout-template-json/2.24.3//log4j-layout-template-json-2.24.3.jar log4j-slf4j2-impl/2.24.3//log4j-slf4j2-impl-2.24.3.jar -logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar lz4-java/1.8.0//lz4-java-1.8.0.jar metrics-core/4.2.29//metrics-core-4.2.29.jar metrics-graphite/4.2.29//metrics-graphite-4.2.29.jar @@ -203,6 +202,7 @@ metrics-jvm/4.2.29//metrics-jvm-4.2.29.jar minlog/1.3.0//minlog-1.3.0.jar netty-all/4.1.115.Final//netty-all-4.1.115.Final.jar netty-buffer/4.1.115.Final//netty-buffer-4.1.115.Final.jar +netty-codec-dns/4.1.115.Final//netty-codec-dns-4.1.115.Final.jar netty-codec-http/4.1.115.Final//netty-codec-http-4.1.115.Final.jar netty-codec-http2/4.1.115.Final//netty-codec-http2-4.1.115.Final.jar netty-codec-socks/4.1.115.Final//netty-codec-socks-4.1.115.Final.jar @@ -210,6 +210,7 @@ netty-codec/4.1.115.Final//netty-codec-4.1.115.Final.jar netty-common/4.1.115.Final//netty-common-4.1.115.Final.jar netty-handler-proxy/4.1.115.Final//netty-handler-proxy-4.1.115.Final.jar netty-handler/4.1.115.Final//netty-handler-4.1.115.Final.jar +netty-resolver-dns/4.1.115.Final//netty-resolver-dns-4.1.115.Final.jar netty-resolver/4.1.115.Final//netty-resolver-4.1.115.Final.jar netty-tcnative-boringssl-static/2.0.69.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar netty-tcnative-boringssl-static/2.0.69.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar @@ -258,7 +259,7 @@ scala-parser-combinators_2.13/2.4.0//scala-parser-combinators_2.13-2.4.0.jar scala-reflect/2.13.15//scala-reflect-2.13.15.jar scala-xml_2.13/2.3.0//scala-xml_2.13-2.3.0.jar slf4j-api/2.0.16//slf4j-api-2.0.16.jar -snakeyaml-engine/2.7//snakeyaml-engine-2.7.jar +snakeyaml-engine/2.8//snakeyaml-engine-2.8.jar snakeyaml/2.3//snakeyaml-2.3.jar snappy-java/1.1.10.7//snappy-java-1.1.10.7.jar spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar @@ -272,11 +273,15 @@ threeten-extra/1.7.1//threeten-extra-1.7.1.jar tink/1.16.0//tink-1.16.0.jar transaction-api/1.1//transaction-api-1.1.jar univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar +vertx-auth-common/4.5.11//vertx-auth-common-4.5.11.jar +vertx-core/4.5.11//vertx-core-4.5.11.jar +vertx-web-client/4.5.11//vertx-web-client-4.5.11.jar +vertx-web-common/4.5.11//vertx-web-common-4.5.11.jar wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar xbean-asm9-shaded/4.26//xbean-asm9-shaded-4.26.jar xmlschema-core/2.3.1//xmlschema-core-2.3.1.jar xz/1.10//xz-1.10.jar -zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar +zjsonpatch/7.0.1//zjsonpatch-7.0.1.jar zookeeper-jute/3.9.3//zookeeper-jute-3.9.3.jar zookeeper/3.9.3//zookeeper-3.9.3.jar -zstd-jni/1.5.6-8//zstd-jni-1.5.6-8.jar +zstd-jni/1.5.6-9//zstd-jni-1.5.6-9.jar diff --git a/dev/pyproject.toml b/dev/pyproject.toml index f19107b3782a6..8b91943009555 100644 --- a/dev/pyproject.toml +++ b/dev/pyproject.toml @@ -27,7 +27,7 @@ testpaths = [ [tool.black] # When changing the version, we have to update # GitHub workflow version and dev/reformat-python -required-version = "23.9.1" +required-version = "23.12.1" line-length = 100 target-version = ['py39'] include = '\.pyi?$' diff --git a/dev/reformat-python b/dev/reformat-python index 46b7efc931aae..9a1199faa938e 100755 --- a/dev/reformat-python +++ b/dev/reformat-python @@ -22,7 +22,7 @@ FWDIR="$( cd "$DIR"/.. && pwd )" cd "$FWDIR" BLACK_BUILD="${PYTHON_EXECUTABLE} -m black" -BLACK_VERSION="23.9.1" +BLACK_VERSION="23.12.1" $PYTHON_EXECUTABLE -c 'import black' 2> /dev/null if [ $? -ne 0 ]; then echo "The Python library providing the 'black' module was not found. Please install Black, for example, via 'pip install black==$BLACK_VERSION'." diff --git a/dev/requirements.txt b/dev/requirements.txt index 33300cc28d3ca..845778f0e9b99 100644 --- a/dev/requirements.txt +++ b/dev/requirements.txt @@ -1,11 +1,11 @@ # PySpark dependencies (required) -py4j>=0.10.9.7 +py4j>=0.10.9.8 # PySpark dependencies (optional) numpy>=1.21 pyarrow>=11.0.0 six==1.16.0 -pandas>=2.0.0 +pandas>=2.2.0 scipy plotly>=4.8 mlflow>=2.3.1 @@ -54,7 +54,7 @@ jira>=3.5.2 PyGithub # pandas API on Spark Code formatter. -black==23.9.1 +black==23.12.1 py # Spark Connect (required) diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile index 6b087f08281c3..f1e33763df468 100644 --- a/dev/spark-test-image/docs/Dockerfile +++ b/dev/spark-test-image/docs/Dockerfile @@ -85,7 +85,7 @@ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9 # See 'docutils<0.18.0' in SPARK-39421 RUN python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \ ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.20.0' pyarrow pandas 'plotly>=4.8' 'docutils<0.18.0' \ - 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.9.1' \ + 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \ 'pandas-stubs==1.2.0.53' 'grpcio==1.67.0' 'grpcio-status==1.67.0' 'protobuf==5.29.1' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \ 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' \ && python3.9 -m pip cache purge diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index c663c3dac3b9f..c3ffd7ba4e4b2 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -72,7 +72,7 @@ RUN apt-get update && apt-get install -y python3.9 python3.9-distutils \ RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9 RUN python3.9 -m pip install \ - 'black==23.9.1' \ + 'black==23.12.1' \ 'flake8==3.9.0' \ 'googleapis-common-protos-stubs==2.2.0' \ 'grpc-stubs==1.24.11' \ diff --git a/dev/spark-test-image/python-ps-minimum/Dockerfile b/dev/spark-test-image/python-ps-minimum/Dockerfile new file mode 100644 index 0000000000000..913da06c551ca --- /dev/null +++ b/dev/spark-test-image/python-ps-minimum/Dockerfile @@ -0,0 +1,81 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Image for building and testing Spark branches. Based on Ubuntu 22.04. +# See also in https://hub.docker.com/_/ubuntu +FROM ubuntu:jammy-20240911.1 +LABEL org.opencontainers.image.authors="Apache Spark project " +LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image For Pandas API on Spark with old dependencies" +# Overwrite this label to avoid exposing the underlying Ubuntu OS version label +LABEL org.opencontainers.image.version="" + +ENV FULL_REFRESH_DATE=20250102 + +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NONINTERACTIVE_SEEN=true + +RUN apt-get update && apt-get install -y \ + build-essential \ + ca-certificates \ + curl \ + gfortran \ + git \ + gnupg \ + libcurl4-openssl-dev \ + libfontconfig1-dev \ + libfreetype6-dev \ + libfribidi-dev \ + libgit2-dev \ + libharfbuzz-dev \ + libjpeg-dev \ + liblapack-dev \ + libopenblas-dev \ + libpng-dev \ + libpython3-dev \ + libssl-dev \ + libtiff5-dev \ + libxml2-dev \ + openjdk-17-jdk-headless \ + pkg-config \ + qpdf \ + tzdata \ + software-properties-common \ + wget \ + zlib1g-dev + + +# Should keep the installation consistent with https://apache.github.io/spark/api/python/getting_started/install.html + +# Install Python 3.9 +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update && apt-get install -y \ + python3.9 \ + python3.9-distutils \ + && apt-get autoremove --purge -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +ARG BASIC_PIP_PKGS="pyarrow==11.0.0 pandas==2.2.0 six==1.16.0 numpy scipy coverage unittest-xml-reporting" +# Python deps for Spark Connect +ARG CONNECT_PIP_PKGS="grpcio==1.67.0 grpcio-status==1.67.0 googleapis-common-protos==1.65.0 graphviz==0.20 protobuf" + +# Install Python 3.9 packages +RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.9 +RUN python3.9 -m pip install --force $BASIC_PIP_PKGS $CONNECT_PIP_PKGS && \ + python3.9 -m pip cache purge diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 2a19439bd92f9..71817a64b3581 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -1438,7 +1438,7 @@ def __hash__(self): ], ) -pyspark_logging = Module( +pyspark_logger = Module( name="pyspark-logger", dependencies=[], source_file_regexes=["python/pyspark/logger"], diff --git a/docs/sql-data-sources-xml.md b/docs/sql-data-sources-xml.md index 6168f570a81a9..949e6239e52ba 100644 --- a/docs/sql-data-sources-xml.md +++ b/docs/sql-data-sources-xml.md @@ -19,7 +19,7 @@ license: | limitations under the License. --- -Spark SQL provides `spark.read().xml("file_1_path","file_2_path")` to read a file or directory of files in XML format into a Spark DataFrame, and `dataframe.write().xml("path")` to write to a xml file. When reading a XML file, the `rowTag` option must be specified to indicate the XML element that maps to a `DataFrame row`. The option() function can be used to customize the behavior of reading or writing, such as controlling behavior of the XML attributes, XSD validation, compression, and so on. +Spark SQL provides `spark.read().xml("file_1_path","file_2_path")` to read a file or directory of files in XML format into a Spark DataFrame, and `dataframe.write().xml("path")` to write to a xml file. The `rowTag` option must be specified to indicate the XML element that maps to a `DataFrame row`. The option() function can be used to customize the behavior of reading or writing, such as controlling behavior of the XML attributes, XSD validation, compression, and so on.
@@ -61,7 +61,7 @@ Data source options of XML can be set via: <books><book></book>...</books> the appropriate value would be book. This is a required option for both read and write. - read + read/write diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index 7af54850f5da7..50cdcd6d09790 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -651,6 +651,7 @@ Below is a list of all the keywords in Spark SQL. |RECORDREADER|non-reserved|non-reserved|non-reserved| |RECORDWRITER|non-reserved|non-reserved|non-reserved| |RECOVER|non-reserved|non-reserved|non-reserved| +|RECURSIVE|reserved|non-reserved|reserved| |REDUCE|non-reserved|non-reserved|non-reserved| |REFERENCES|reserved|non-reserved|reserved| |REFRESH|non-reserved|non-reserved|non-reserved| diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml index 9c2f21e7ab617..3361081549242 100644 --- a/hadoop-cloud/pom.xml +++ b/hadoop-cloud/pom.xml @@ -34,6 +34,8 @@ hadoop-cloud + 3.12.12 + 1.17.6 @@ -191,6 +193,16 @@ ${jetty.version} ${hadoop.deps.scope} + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + com.squareup.okio + okio + ${okio.version} + diff --git a/pom.xml b/pom.xml index 05ebba16270de..3a8cbd1f033c2 100644 --- a/pom.xml +++ b/pom.xml @@ -235,8 +235,7 @@ org.fusesource.leveldbjni - 6.13.4 - 1.17.6 + 7.0.1 ${java.home} @@ -839,7 +838,7 @@ com.github.luben zstd-jni - 1.5.6-8 + 1.5.6-9 com.clearspring.analytics @@ -2594,11 +2593,6 @@ javax.servlet-api ${javaxservlet.version} - - com.squareup.okio - okio - ${okio.version} - diff --git a/python/docs/source/getting_started/install.rst b/python/docs/source/getting_started/install.rst index b35588a618acf..515224b52e09a 100644 --- a/python/docs/source/getting_started/install.rst +++ b/python/docs/source/getting_started/install.rst @@ -239,7 +239,7 @@ Installable with ``pip install "pyspark[pandas_on_spark]"``. ========= ================= ================================ Package Supported version Note ========= ================= ================================ -`pandas` >=2.0.0 Required for Pandas API on Spark +`pandas` >=2.2.0 Required for Pandas API on Spark `pyarrow` >=11.0.0 Required for Pandas API on Spark ========= ================= ================================ diff --git a/python/docs/source/reference/pyspark.sql/spark_session.rst b/python/docs/source/reference/pyspark.sql/spark_session.rst index 1677d3e8e0209..a35fccbcffe99 100644 --- a/python/docs/source/reference/pyspark.sql/spark_session.rst +++ b/python/docs/source/reference/pyspark.sql/spark_session.rst @@ -52,6 +52,8 @@ See also :class:`SparkSession`. SparkSession.dataSource SparkSession.getActiveSession SparkSession.getTags + SparkSession.interruptAll + SparkSession.interruptTag SparkSession.newSession SparkSession.profile SparkSession.removeTag @@ -86,8 +88,6 @@ Spark Connect Only SparkSession.clearProgressHandlers SparkSession.client SparkSession.copyFromLocalToFs - SparkSession.interruptAll SparkSession.interruptOperation - SparkSession.interruptTag SparkSession.registerProgressHandler SparkSession.removeProgressHandler diff --git a/python/pyspark/core/context.py b/python/pyspark/core/context.py index 9ed4699c4b5ba..42a368555ae9e 100644 --- a/python/pyspark/core/context.py +++ b/python/pyspark/core/context.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: from pyspark.accumulators import AccumulatorParam + from pyspark.sql.types import DataType, StructType __all__ = ["SparkContext"] @@ -2623,6 +2624,16 @@ def _assert_on_driver() -> None: messageParameters={}, ) + def _to_ddl(self, struct: "StructType") -> str: + assert self._jvm is not None + return self._jvm.PythonSQLUtils.jsonToDDL(struct.json()) + + def _parse_ddl(self, ddl: str) -> "DataType": + from pyspark.sql.types import _parse_datatype_json_string + + assert self._jvm is not None + return _parse_datatype_json_string(self._jvm.PythonSQLUtils.ddlToJson(ddl)) + def _test() -> None: import doctest diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py index 5488940645a13..0d01cbb961bb6 100644 --- a/python/pyspark/errors/utils.py +++ b/python/pyspark/errors/utils.py @@ -259,8 +259,6 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: if spark is not None and hasattr(func, "__name__") and is_debugging_enabled(): if is_remote(): - global current_origin - # Getting the configuration requires RPC call. Uses the default value for now. depth = 1 set_current_origin(func.__name__, _capture_call_site(depth)) diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py index 35b96543b9ebb..86820573344ea 100644 --- a/python/pyspark/pandas/frame.py +++ b/python/pyspark/pandas/frame.py @@ -7292,8 +7292,6 @@ def select_dtypes( 4 1 True 1.0 5 2 False 2.0 """ - from pyspark.sql.types import _parse_datatype_string - include_list: List[str] if not is_list_like(include): include_list = [cast(str, include)] if include is not None else [] @@ -7320,14 +7318,14 @@ def select_dtypes( include_spark_type = [] for inc in include_list: try: - include_spark_type.append(_parse_datatype_string(inc)) + include_spark_type.append(self._internal.spark_frame._session._parse_ddl(inc)) except BaseException: pass exclude_spark_type = [] for exc in exclude_list: try: - exclude_spark_type.append(_parse_datatype_string(exc)) + exclude_spark_type.append(self._internal.spark_frame._session._parse_ddl(exc)) except BaseException: pass diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py index c77cdf51a2f6d..d31bc1f48d112 100644 --- a/python/pyspark/pandas/namespace.py +++ b/python/pyspark/pandas/namespace.py @@ -138,14 +138,44 @@ def from_pandas(pobj: Union[pd.DataFrame, pd.Series, pd.Index]) -> Union[Series, Parameters ---------- - pobj : pandas.DataFrame or pandas.Series - pandas DataFrame or Series to read. + pobj : pandas.DataFrame, pandas.Series or pandas.Index + pandas DataFrame, Series or Index to read. Returns ------- - Series or DataFrame - If a pandas Series is passed in, this function returns a pandas-on-Spark Series. + DataFrame, Series or Index If a pandas DataFrame is passed in, this function returns a pandas-on-Spark DataFrame. + If a pandas Series is passed in, this function returns a pandas-on-Spark Series. + If a pandas Index is passed in, this function returns a pandas-on-Spark Index. + + Examples + -------- + >>> import pandas as pd + >>> import pyspark.pandas as ps + + Convert a pandas DataFrame: + >>> pdf = pd.DataFrame({'a': [1, 2, 3]}) + >>> psdf = ps.from_pandas(pdf) + >>> psdf + a + 0 1 + 1 2 + 2 3 + + Convert a pandas Series: + >>> pser = pd.Series([1, 2, 3]) + >>> psser = ps.from_pandas(pser) + >>> psser + 0 1 + 1 2 + 2 3 + dtype: int64 + + Convert a pandas Index: + >>> pidx = pd.Index([1, 2, 3]) + >>> psidx = ps.from_pandas(pidx) + >>> psidx + Index([1, 2, 3], dtype='int64') """ if isinstance(pobj, pd.Series): return Series(pobj) diff --git a/python/pyspark/pandas/tests/io/test_io.py b/python/pyspark/pandas/tests/io/test_io.py index 6fbdc366dd76a..da5817b86b984 100644 --- a/python/pyspark/pandas/tests/io/test_io.py +++ b/python/pyspark/pandas/tests/io/test_io.py @@ -24,7 +24,12 @@ from pyspark import pandas as ps from pyspark.testing.pandasutils import PandasOnSparkTestCase from pyspark.testing.sqlutils import SQLTestUtils -from pyspark.testing.utils import have_tabulate, tabulate_requirement_message +from pyspark.testing.utils import ( + have_jinja2, + jinja2_requirement_message, + have_tabulate, + tabulate_requirement_message, +) # This file contains test cases for 'Serialization / IO / Conversion' @@ -91,6 +96,7 @@ def test_from_dict(self): psdf = ps.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"]) self.assert_eq(pdf, psdf) + @unittest.skipIf(not have_jinja2, jinja2_requirement_message) def test_style(self): # Currently, the `style` function returns a pandas object `Styler` as it is, # processing only the number of rows declared in `compute.max_rows`. diff --git a/python/pyspark/pandas/tests/io/test_series_conversion.py b/python/pyspark/pandas/tests/io/test_series_conversion.py index 2ae40e92b489a..06d923816633d 100644 --- a/python/pyspark/pandas/tests/io/test_series_conversion.py +++ b/python/pyspark/pandas/tests/io/test_series_conversion.py @@ -23,6 +23,7 @@ from pyspark import pandas as ps from pyspark.testing.pandasutils import PandasOnSparkTestCase from pyspark.testing.sqlutils import SQLTestUtils +from pyspark.testing.utils import have_jinja2, jinja2_requirement_message class SeriesConversionTestsMixin: @@ -48,6 +49,7 @@ def test_to_clipboard(self): psser.to_clipboard(sep=",", index=False), pser.to_clipboard(sep=",", index=False) ) + @unittest.skipIf(not have_jinja2, jinja2_requirement_message) def test_to_latex(self): pser = self.pser psser = self.psser diff --git a/python/pyspark/sql/_typing.pyi b/python/pyspark/sql/_typing.pyi index 4969268939adf..27fa0f2a90133 100644 --- a/python/pyspark/sql/_typing.pyi +++ b/python/pyspark/sql/_typing.pyi @@ -36,8 +36,10 @@ from pyspark._typing import PrimitiveType from pyspark.profiler import CodeMapDict import pyspark.sql.types from pyspark.sql.column import Column +from pyspark.sql.tvf_argument import TableValuedFunctionArgument ColumnOrName = Union[Column, str] +TVFArgumentOrName = Union[TableValuedFunctionArgument, str] ColumnOrNameOrOrdinal = Union[Column, str, int] DecimalLiteral = decimal.Decimal DateTimeLiteral = Union[datetime.datetime, datetime.date] diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py index cc44d2c9942d2..8beabda6c135d 100644 --- a/python/pyspark/sql/classic/dataframe.py +++ b/python/pyspark/sql/classic/dataframe.py @@ -74,6 +74,7 @@ from pyspark.sql.utils import get_active_spark_context, to_java_array, to_scala_map from pyspark.sql.pandas.conversion import PandasConversionMixin from pyspark.sql.pandas.map_ops import PandasMapOpsMixin +from pyspark.sql.table_arg import TableArg if TYPE_CHECKING: @@ -1787,6 +1788,9 @@ def transpose(self, indexColumn: Optional["ColumnOrName"] = None) -> ParentDataF else: return DataFrame(self._jdf.transpose(), self.sparkSession) + def asTable(self) -> TableArg: + return TableArg(self._jdf.asTable()) + def scalar(self) -> Column: return Column(self._jdf.scalar()) diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py index 285d30fad3bc2..e5640dd81b1fb 100644 --- a/python/pyspark/sql/column.py +++ b/python/pyspark/sql/column.py @@ -25,6 +25,7 @@ Union, ) +from pyspark.sql.tvf_argument import TableValuedFunctionArgument from pyspark.sql.utils import dispatch_col_method from pyspark.sql.types import DataType from pyspark.errors import PySparkValueError @@ -37,7 +38,7 @@ __all__ = ["Column"] -class Column: +class Column(TableValuedFunctionArgument): """ A column in a DataFrame. diff --git a/python/pyspark/sql/connect/conf.py b/python/pyspark/sql/connect/conf.py index 1ef72ee3cfa43..84d7ad34fb360 100644 --- a/python/pyspark/sql/connect/conf.py +++ b/python/pyspark/sql/connect/conf.py @@ -49,6 +49,20 @@ def set(self, key: str, value: Union[str, int, bool]) -> None: set.__doc__ = PySparkRuntimeConfig.set.__doc__ + def _set_all(self, configs: Dict[str, Union[str, int, bool]], silent: bool) -> None: + conf_list = [] + for key, value in configs.items(): + if isinstance(value, bool): + value = "true" if value else "false" + elif isinstance(value, int): + value = str(value) + conf_list.append(proto.KeyValue(key=key, value=value)) + op_set = proto.ConfigRequest.Set(pairs=conf_list, silent=silent) + operation = proto.ConfigRequest.Operation(set=op_set) + result = self._client.config(operation) + for warn in result.warnings: + warnings.warn(warn) + def get( self, key: str, default: Union[Optional[str], _NoValueType] = _NoValue ) -> Optional[str]: diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py index 33956c867669c..789292bdd56f6 100644 --- a/python/pyspark/sql/connect/dataframe.py +++ b/python/pyspark/sql/connect/dataframe.py @@ -54,7 +54,7 @@ from pyspark import _NoValue from pyspark._globals import _NoValueType from pyspark.util import is_remote_only -from pyspark.sql.types import Row, StructType, _create_row, _parse_datatype_string +from pyspark.sql.types import Row, StructType, _create_row from pyspark.sql.dataframe import ( DataFrame as ParentDataFrame, DataFrameNaFunctions as ParentDataFrameNaFunctions, @@ -86,6 +86,7 @@ from pyspark.sql.connect.functions import builtin as F from pyspark.sql.pandas.types import from_arrow_schema, to_arrow_schema from pyspark.sql.pandas.functions import _validate_pandas_udf # type: ignore[attr-defined] +from pyspark.sql.table_arg import TableArg if TYPE_CHECKING: @@ -1801,6 +1802,13 @@ def transpose(self, indexColumn: Optional["ColumnOrName"] = None) -> ParentDataF self._session, ) + def asTable(self) -> TableArg: + # TODO(SPARK-50393): Support DataFrame conversion to table argument in Spark Connect + raise PySparkNotImplementedError( + errorClass="NOT_IMPLEMENTED", + messageParameters={"feature": "asTable()"}, + ) + def scalar(self) -> Column: from pyspark.sql.connect.column import Column as ConnectColumn @@ -2037,7 +2045,7 @@ def _map_partitions( _validate_pandas_udf(func, evalType) if isinstance(schema, str): - schema = cast(StructType, _parse_datatype_string(schema)) + schema = cast(StructType, self._session._parse_ddl(schema)) udf_obj = UserDefinedFunction( func, returnType=schema, diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py index 006af8756e639..11adc8850fec1 100644 --- a/python/pyspark/sql/connect/group.py +++ b/python/pyspark/sql/connect/group.py @@ -35,7 +35,7 @@ from pyspark.sql.group import GroupedData as PySparkGroupedData from pyspark.sql.pandas.group_ops import PandasCogroupedOps as PySparkPandasCogroupedOps from pyspark.sql.pandas.functions import _validate_pandas_udf # type: ignore[attr-defined] -from pyspark.sql.types import NumericType, StructType, _parse_datatype_string +from pyspark.sql.types import NumericType, StructType import pyspark.sql.connect.plan as plan from pyspark.sql.column import Column @@ -295,7 +295,7 @@ def applyInPandas( _validate_pandas_udf(func, PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF) if isinstance(schema, str): - schema = cast(StructType, _parse_datatype_string(schema)) + schema = cast(StructType, self._df._session._parse_ddl(schema)) udf_obj = UserDefinedFunction( func, returnType=schema, @@ -369,7 +369,7 @@ def applyInArrow( _validate_pandas_udf(func, PythonEvalType.SQL_GROUPED_MAP_ARROW_UDF) if isinstance(schema, str): - schema = cast(StructType, _parse_datatype_string(schema)) + schema = cast(StructType, self._df._session._parse_ddl(schema)) udf_obj = UserDefinedFunction( func, returnType=schema, @@ -414,7 +414,7 @@ def applyInPandas( _validate_pandas_udf(func, PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF) if isinstance(schema, str): - schema = cast(StructType, _parse_datatype_string(schema)) + schema = cast(StructType, self._gd1._df._session._parse_ddl(schema)) udf_obj = UserDefinedFunction( func, returnType=schema, @@ -445,7 +445,7 @@ def applyInArrow( _validate_pandas_udf(func, PythonEvalType.SQL_COGROUPED_MAP_ARROW_UDF) if isinstance(schema, str): - schema = cast(StructType, _parse_datatype_string(schema)) + schema = cast(StructType, self._gd1._df._session._parse_ddl(schema)) udf_obj = UserDefinedFunction( func, returnType=schema, diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py index 97694c33abeb2..64c549ffe9565 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.py +++ b/python/pyspark/sql/connect/proto/base_pb2.py @@ -43,7 +43,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\xa3\x05\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\xa5\x01\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xe6\x16\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1av\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x42\x0f\n\r_start_offset\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x8d\x01\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\x87\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\x34\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xab\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x42\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x93\x0c\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\x99\x03\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_state\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation2\xb2\x07\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"t\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.connect.CommandH\x00R\x07\x63ommandB\t\n\x07op_type"z\n\x0bUserContext\x12\x17\n\x07user_id\x18\x01 \x01(\tR\x06userId\x12\x1b\n\tuser_name\x18\x02 \x01(\tR\x08userName\x12\x35\n\nextensions\x18\xe7\x07 \x03(\x0b\x32\x14.google.protobuf.AnyR\nextensions"\xf5\x14\n\x12\x41nalyzePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x11 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12\x42\n\x06schema\x18\x04 \x01(\x0b\x32(.spark.connect.AnalyzePlanRequest.SchemaH\x00R\x06schema\x12\x45\n\x07\x65xplain\x18\x05 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.ExplainH\x00R\x07\x65xplain\x12O\n\x0btree_string\x18\x06 \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.TreeStringH\x00R\ntreeString\x12\x46\n\x08is_local\x18\x07 \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.IsLocalH\x00R\x07isLocal\x12R\n\x0cis_streaming\x18\x08 \x01(\x0b\x32-.spark.connect.AnalyzePlanRequest.IsStreamingH\x00R\x0bisStreaming\x12O\n\x0binput_files\x18\t \x01(\x0b\x32,.spark.connect.AnalyzePlanRequest.InputFilesH\x00R\ninputFiles\x12U\n\rspark_version\x18\n \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SparkVersionH\x00R\x0csparkVersion\x12I\n\tddl_parse\x18\x0b \x01(\x0b\x32*.spark.connect.AnalyzePlanRequest.DDLParseH\x00R\x08\x64\x64lParse\x12X\n\x0esame_semantics\x18\x0c \x01(\x0b\x32/.spark.connect.AnalyzePlanRequest.SameSemanticsH\x00R\rsameSemantics\x12U\n\rsemantic_hash\x18\r \x01(\x0b\x32..spark.connect.AnalyzePlanRequest.SemanticHashH\x00R\x0csemanticHash\x12\x45\n\x07persist\x18\x0e \x01(\x0b\x32).spark.connect.AnalyzePlanRequest.PersistH\x00R\x07persist\x12K\n\tunpersist\x18\x0f \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.UnpersistH\x00R\tunpersist\x12_\n\x11get_storage_level\x18\x10 \x01(\x0b\x32\x31.spark.connect.AnalyzePlanRequest.GetStorageLevelH\x00R\x0fgetStorageLevel\x12M\n\x0bjson_to_ddl\x18\x12 \x01(\x0b\x32+.spark.connect.AnalyzePlanRequest.JsonToDDLH\x00R\tjsonToDdl\x1a\x31\n\x06Schema\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\xbb\x02\n\x07\x45xplain\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12X\n\x0c\x65xplain_mode\x18\x02 \x01(\x0e\x32\x35.spark.connect.AnalyzePlanRequest.Explain.ExplainModeR\x0b\x65xplainMode"\xac\x01\n\x0b\x45xplainMode\x12\x1c\n\x18\x45XPLAIN_MODE_UNSPECIFIED\x10\x00\x12\x17\n\x13\x45XPLAIN_MODE_SIMPLE\x10\x01\x12\x19\n\x15\x45XPLAIN_MODE_EXTENDED\x10\x02\x12\x18\n\x14\x45XPLAIN_MODE_CODEGEN\x10\x03\x12\x15\n\x11\x45XPLAIN_MODE_COST\x10\x04\x12\x1a\n\x16\x45XPLAIN_MODE_FORMATTED\x10\x05\x1aZ\n\nTreeString\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12\x19\n\x05level\x18\x02 \x01(\x05H\x00R\x05level\x88\x01\x01\x42\x08\n\x06_level\x1a\x32\n\x07IsLocal\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x36\n\x0bIsStreaming\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x35\n\nInputFiles\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x0e\n\x0cSparkVersion\x1a)\n\x08\x44\x44LParse\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlString\x1ay\n\rSameSemantics\x12\x34\n\x0btarget_plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\ntargetPlan\x12\x32\n\nother_plan\x18\x02 \x01(\x0b\x32\x13.spark.connect.PlanR\totherPlan\x1a\x37\n\x0cSemanticHash\x12\'\n\x04plan\x18\x01 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x1a\x97\x01\n\x07Persist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x45\n\rstorage_level\x18\x02 \x01(\x0b\x32\x1b.spark.connect.StorageLevelH\x00R\x0cstorageLevel\x88\x01\x01\x42\x10\n\x0e_storage_level\x1an\n\tUnpersist\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x12\x1f\n\x08\x62locking\x18\x02 \x01(\x08H\x00R\x08\x62locking\x88\x01\x01\x42\x0b\n\t_blocking\x1a\x46\n\x0fGetStorageLevel\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1a,\n\tJsonToDDL\x12\x1f\n\x0bjson_string\x18\x01 \x01(\tR\njsonStringB\t\n\x07\x61nalyzeB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xca\x0e\n\x13\x41nalyzePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12\x43\n\x06schema\x18\x02 \x01(\x0b\x32).spark.connect.AnalyzePlanResponse.SchemaH\x00R\x06schema\x12\x46\n\x07\x65xplain\x18\x03 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.ExplainH\x00R\x07\x65xplain\x12P\n\x0btree_string\x18\x04 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.TreeStringH\x00R\ntreeString\x12G\n\x08is_local\x18\x05 \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.IsLocalH\x00R\x07isLocal\x12S\n\x0cis_streaming\x18\x06 \x01(\x0b\x32..spark.connect.AnalyzePlanResponse.IsStreamingH\x00R\x0bisStreaming\x12P\n\x0binput_files\x18\x07 \x01(\x0b\x32-.spark.connect.AnalyzePlanResponse.InputFilesH\x00R\ninputFiles\x12V\n\rspark_version\x18\x08 \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SparkVersionH\x00R\x0csparkVersion\x12J\n\tddl_parse\x18\t \x01(\x0b\x32+.spark.connect.AnalyzePlanResponse.DDLParseH\x00R\x08\x64\x64lParse\x12Y\n\x0esame_semantics\x18\n \x01(\x0b\x32\x30.spark.connect.AnalyzePlanResponse.SameSemanticsH\x00R\rsameSemantics\x12V\n\rsemantic_hash\x18\x0b \x01(\x0b\x32/.spark.connect.AnalyzePlanResponse.SemanticHashH\x00R\x0csemanticHash\x12\x46\n\x07persist\x18\x0c \x01(\x0b\x32*.spark.connect.AnalyzePlanResponse.PersistH\x00R\x07persist\x12L\n\tunpersist\x18\r \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.UnpersistH\x00R\tunpersist\x12`\n\x11get_storage_level\x18\x0e \x01(\x0b\x32\x32.spark.connect.AnalyzePlanResponse.GetStorageLevelH\x00R\x0fgetStorageLevel\x12N\n\x0bjson_to_ddl\x18\x10 \x01(\x0b\x32,.spark.connect.AnalyzePlanResponse.JsonToDDLH\x00R\tjsonToDdl\x1a\x39\n\x06Schema\x12/\n\x06schema\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1a\x30\n\x07\x45xplain\x12%\n\x0e\x65xplain_string\x18\x01 \x01(\tR\rexplainString\x1a-\n\nTreeString\x12\x1f\n\x0btree_string\x18\x01 \x01(\tR\ntreeString\x1a$\n\x07IsLocal\x12\x19\n\x08is_local\x18\x01 \x01(\x08R\x07isLocal\x1a\x30\n\x0bIsStreaming\x12!\n\x0cis_streaming\x18\x01 \x01(\x08R\x0bisStreaming\x1a"\n\nInputFiles\x12\x14\n\x05\x66iles\x18\x01 \x03(\tR\x05\x66iles\x1a(\n\x0cSparkVersion\x12\x18\n\x07version\x18\x01 \x01(\tR\x07version\x1a;\n\x08\x44\x44LParse\x12/\n\x06parsed\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06parsed\x1a\'\n\rSameSemantics\x12\x16\n\x06result\x18\x01 \x01(\x08R\x06result\x1a&\n\x0cSemanticHash\x12\x16\n\x06result\x18\x01 \x01(\x05R\x06result\x1a\t\n\x07Persist\x1a\x0b\n\tUnpersist\x1aS\n\x0fGetStorageLevel\x12@\n\rstorage_level\x18\x01 \x01(\x0b\x32\x1b.spark.connect.StorageLevelR\x0cstorageLevel\x1a*\n\tJsonToDDL\x12\x1d\n\nddl_string\x18\x01 \x01(\tR\tddlStringB\x08\n\x06result"\xa3\x05\n\x12\x45xecutePlanRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12&\n\x0coperation_id\x18\x06 \x01(\tH\x01R\x0boperationId\x88\x01\x01\x12\'\n\x04plan\x18\x03 \x01(\x0b\x32\x13.spark.connect.PlanR\x04plan\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12X\n\x0frequest_options\x18\x05 \x03(\x0b\x32/.spark.connect.ExecutePlanRequest.RequestOptionR\x0erequestOptions\x12\x12\n\x04tags\x18\x07 \x03(\tR\x04tags\x1a\xa5\x01\n\rRequestOption\x12K\n\x10reattach_options\x18\x01 \x01(\x0b\x32\x1e.spark.connect.ReattachOptionsH\x00R\x0freattachOptions\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x10\n\x0erequest_optionB)\n\'_client_observed_server_side_session_idB\x0f\n\r_operation_idB\x0e\n\x0c_client_type"\xe6\x16\n\x13\x45xecutePlanResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x0f \x01(\tR\x13serverSideSessionId\x12!\n\x0coperation_id\x18\x0c \x01(\tR\x0boperationId\x12\x1f\n\x0bresponse_id\x18\r \x01(\tR\nresponseId\x12P\n\x0b\x61rrow_batch\x18\x02 \x01(\x0b\x32-.spark.connect.ExecutePlanResponse.ArrowBatchH\x00R\narrowBatch\x12\x63\n\x12sql_command_result\x18\x05 \x01(\x0b\x32\x33.spark.connect.ExecutePlanResponse.SqlCommandResultH\x00R\x10sqlCommandResult\x12~\n#write_stream_operation_start_result\x18\x08 \x01(\x0b\x32..spark.connect.WriteStreamOperationStartResultH\x00R\x1fwriteStreamOperationStartResult\x12q\n\x1estreaming_query_command_result\x18\t \x01(\x0b\x32*.spark.connect.StreamingQueryCommandResultH\x00R\x1bstreamingQueryCommandResult\x12k\n\x1cget_resources_command_result\x18\n \x01(\x0b\x32(.spark.connect.GetResourcesCommandResultH\x00R\x19getResourcesCommandResult\x12\x87\x01\n&streaming_query_manager_command_result\x18\x0b \x01(\x0b\x32\x31.spark.connect.StreamingQueryManagerCommandResultH\x00R"streamingQueryManagerCommandResult\x12\x87\x01\n&streaming_query_listener_events_result\x18\x10 \x01(\x0b\x32\x31.spark.connect.StreamingQueryListenerEventsResultH\x00R"streamingQueryListenerEventsResult\x12\\\n\x0fresult_complete\x18\x0e \x01(\x0b\x32\x31.spark.connect.ExecutePlanResponse.ResultCompleteH\x00R\x0eresultComplete\x12\x87\x01\n&create_resource_profile_command_result\x18\x11 \x01(\x0b\x32\x31.spark.connect.CreateResourceProfileCommandResultH\x00R"createResourceProfileCommandResult\x12\x65\n\x12\x65xecution_progress\x18\x12 \x01(\x0b\x32\x34.spark.connect.ExecutePlanResponse.ExecutionProgressH\x00R\x11\x65xecutionProgress\x12\x64\n\x19\x63heckpoint_command_result\x18\x13 \x01(\x0b\x32&.spark.connect.CheckpointCommandResultH\x00R\x17\x63heckpointCommandResult\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x44\n\x07metrics\x18\x04 \x01(\x0b\x32*.spark.connect.ExecutePlanResponse.MetricsR\x07metrics\x12]\n\x10observed_metrics\x18\x06 \x03(\x0b\x32\x32.spark.connect.ExecutePlanResponse.ObservedMetricsR\x0fobservedMetrics\x12/\n\x06schema\x18\x07 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema\x1aG\n\x10SqlCommandResult\x12\x33\n\x08relation\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x08relation\x1av\n\nArrowBatch\x12\x1b\n\trow_count\x18\x01 \x01(\x03R\x08rowCount\x12\x12\n\x04\x64\x61ta\x18\x02 \x01(\x0cR\x04\x64\x61ta\x12&\n\x0cstart_offset\x18\x03 \x01(\x03H\x00R\x0bstartOffset\x88\x01\x01\x42\x0f\n\r_start_offset\x1a\x85\x04\n\x07Metrics\x12Q\n\x07metrics\x18\x01 \x03(\x0b\x32\x37.spark.connect.ExecutePlanResponse.Metrics.MetricObjectR\x07metrics\x1a\xcc\x02\n\x0cMetricObject\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x17\n\x07plan_id\x18\x02 \x01(\x03R\x06planId\x12\x16\n\x06parent\x18\x03 \x01(\x03R\x06parent\x12z\n\x11\x65xecution_metrics\x18\x04 \x03(\x0b\x32M.spark.connect.ExecutePlanResponse.Metrics.MetricObject.ExecutionMetricsEntryR\x10\x65xecutionMetrics\x1a{\n\x15\x45xecutionMetricsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ExecutePlanResponse.Metrics.MetricValueR\x05value:\x02\x38\x01\x1aX\n\x0bMetricValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n\x05value\x18\x02 \x01(\x03R\x05value\x12\x1f\n\x0bmetric_type\x18\x03 \x01(\tR\nmetricType\x1a\x8d\x01\n\x0fObservedMetrics\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x12\x12\n\x04keys\x18\x03 \x03(\tR\x04keys\x12\x17\n\x07plan_id\x18\x04 \x01(\x03R\x06planId\x1a\x10\n\x0eResultComplete\x1a\xcd\x02\n\x11\x45xecutionProgress\x12V\n\x06stages\x18\x01 \x03(\x0b\x32>.spark.connect.ExecutePlanResponse.ExecutionProgress.StageInfoR\x06stages\x12,\n\x12num_inflight_tasks\x18\x02 \x01(\x03R\x10numInflightTasks\x1a\xb1\x01\n\tStageInfo\x12\x19\n\x08stage_id\x18\x01 \x01(\x03R\x07stageId\x12\x1b\n\tnum_tasks\x18\x02 \x01(\x03R\x08numTasks\x12.\n\x13num_completed_tasks\x18\x03 \x01(\x03R\x11numCompletedTasks\x12(\n\x10input_bytes_read\x18\x04 \x01(\x03R\x0einputBytesRead\x12\x12\n\x04\x64one\x18\x05 \x01(\x08R\x04\x64oneB\x0f\n\rresponse_type"A\n\x08KeyValue\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x19\n\x05value\x18\x02 \x01(\tH\x00R\x05value\x88\x01\x01\x42\x08\n\x06_value"\xaf\t\n\rConfigRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x08 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x44\n\toperation\x18\x03 \x01(\x0b\x32&.spark.connect.ConfigRequest.OperationR\toperation\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x1a\xf2\x03\n\tOperation\x12\x34\n\x03set\x18\x01 \x01(\x0b\x32 .spark.connect.ConfigRequest.SetH\x00R\x03set\x12\x34\n\x03get\x18\x02 \x01(\x0b\x32 .spark.connect.ConfigRequest.GetH\x00R\x03get\x12W\n\x10get_with_default\x18\x03 \x01(\x0b\x32+.spark.connect.ConfigRequest.GetWithDefaultH\x00R\x0egetWithDefault\x12G\n\nget_option\x18\x04 \x01(\x0b\x32&.spark.connect.ConfigRequest.GetOptionH\x00R\tgetOption\x12>\n\x07get_all\x18\x05 \x01(\x0b\x32#.spark.connect.ConfigRequest.GetAllH\x00R\x06getAll\x12:\n\x05unset\x18\x06 \x01(\x0b\x32".spark.connect.ConfigRequest.UnsetH\x00R\x05unset\x12P\n\ris_modifiable\x18\x07 \x01(\x0b\x32).spark.connect.ConfigRequest.IsModifiableH\x00R\x0cisModifiableB\t\n\x07op_type\x1a\\\n\x03Set\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1b\n\x06silent\x18\x02 \x01(\x08H\x00R\x06silent\x88\x01\x01\x42\t\n\x07_silent\x1a\x19\n\x03Get\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a?\n\x0eGetWithDefault\x12-\n\x05pairs\x18\x01 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x1a\x1f\n\tGetOption\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a\x30\n\x06GetAll\x12\x1b\n\x06prefix\x18\x01 \x01(\tH\x00R\x06prefix\x88\x01\x01\x42\t\n\x07_prefix\x1a\x1b\n\x05Unset\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keys\x1a"\n\x0cIsModifiable\x12\x12\n\x04keys\x18\x01 \x03(\tR\x04keysB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xaf\x01\n\x0e\x43onfigResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x04 \x01(\tR\x13serverSideSessionId\x12-\n\x05pairs\x18\x02 \x03(\x0b\x32\x17.spark.connect.KeyValueR\x05pairs\x12\x1a\n\x08warnings\x18\x03 \x03(\tR\x08warnings"\xea\x07\n\x13\x41\x64\x64\x41rtifactsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12$\n\x0b\x63lient_type\x18\x06 \x01(\tH\x02R\nclientType\x88\x01\x01\x12@\n\x05\x62\x61tch\x18\x03 \x01(\x0b\x32(.spark.connect.AddArtifactsRequest.BatchH\x00R\x05\x62\x61tch\x12Z\n\x0b\x62\x65gin_chunk\x18\x04 \x01(\x0b\x32\x37.spark.connect.AddArtifactsRequest.BeginChunkedArtifactH\x00R\nbeginChunk\x12H\n\x05\x63hunk\x18\x05 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkH\x00R\x05\x63hunk\x1a\x35\n\rArtifactChunk\x12\x12\n\x04\x64\x61ta\x18\x01 \x01(\x0cR\x04\x64\x61ta\x12\x10\n\x03\x63rc\x18\x02 \x01(\x03R\x03\x63rc\x1ao\n\x13SingleChunkArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x44\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x04\x64\x61ta\x1a]\n\x05\x42\x61tch\x12T\n\tartifacts\x18\x01 \x03(\x0b\x32\x36.spark.connect.AddArtifactsRequest.SingleChunkArtifactR\tartifacts\x1a\xc1\x01\n\x14\x42\x65ginChunkedArtifact\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1f\n\x0btotal_bytes\x18\x02 \x01(\x03R\ntotalBytes\x12\x1d\n\nnum_chunks\x18\x03 \x01(\x03R\tnumChunks\x12U\n\rinitial_chunk\x18\x04 \x01(\x0b\x32\x30.spark.connect.AddArtifactsRequest.ArtifactChunkR\x0cinitialChunkB\t\n\x07payloadB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x02\n\x14\x41\x64\x64\x41rtifactsResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\tartifacts\x18\x01 \x03(\x0b\x32\x33.spark.connect.AddArtifactsResponse.ArtifactSummaryR\tartifacts\x1aQ\n\x0f\x41rtifactSummary\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x11is_crc_successful\x18\x02 \x01(\x08R\x0fisCrcSuccessful"\xc6\x02\n\x17\x41rtifactStatusesRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x01R\nclientType\x88\x01\x01\x12\x14\n\x05names\x18\x04 \x03(\tR\x05namesB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xe0\x02\n\x18\x41rtifactStatusesResponse\x12\x1d\n\nsession_id\x18\x02 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12Q\n\x08statuses\x18\x01 \x03(\x0b\x32\x35.spark.connect.ArtifactStatusesResponse.StatusesEntryR\x08statuses\x1as\n\rStatusesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12L\n\x05value\x18\x02 \x01(\x0b\x32\x36.spark.connect.ArtifactStatusesResponse.ArtifactStatusR\x05value:\x02\x38\x01\x1a(\n\x0e\x41rtifactStatus\x12\x16\n\x06\x65xists\x18\x01 \x01(\x08R\x06\x65xists"\xdb\x04\n\x10InterruptRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x02R\nclientType\x88\x01\x01\x12T\n\x0einterrupt_type\x18\x04 \x01(\x0e\x32-.spark.connect.InterruptRequest.InterruptTypeR\rinterruptType\x12%\n\roperation_tag\x18\x05 \x01(\tH\x00R\x0coperationTag\x12#\n\x0coperation_id\x18\x06 \x01(\tH\x00R\x0boperationId"\x80\x01\n\rInterruptType\x12\x1e\n\x1aINTERRUPT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12INTERRUPT_TYPE_ALL\x10\x01\x12\x16\n\x12INTERRUPT_TYPE_TAG\x10\x02\x12\x1f\n\x1bINTERRUPT_TYPE_OPERATION_ID\x10\x03\x42\x0b\n\tinterruptB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x90\x01\n\x11InterruptResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\'\n\x0finterrupted_ids\x18\x02 \x03(\tR\x0einterruptedIds"5\n\x0fReattachOptions\x12"\n\x0creattachable\x18\x01 \x01(\x08R\x0creattachable"\x96\x03\n\x16ReattachExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x06 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x12-\n\x10last_response_id\x18\x05 \x01(\tH\x02R\x0elastResponseId\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_typeB\x13\n\x11_last_response_id"\xc9\x04\n\x15ReleaseExecuteRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x07 \x01(\tH\x01R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12!\n\x0coperation_id\x18\x03 \x01(\tR\x0boperationId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x02R\nclientType\x88\x01\x01\x12R\n\x0brelease_all\x18\x05 \x01(\x0b\x32/.spark.connect.ReleaseExecuteRequest.ReleaseAllH\x00R\nreleaseAll\x12X\n\rrelease_until\x18\x06 \x01(\x0b\x32\x31.spark.connect.ReleaseExecuteRequest.ReleaseUntilH\x00R\x0creleaseUntil\x1a\x0c\n\nReleaseAll\x1a/\n\x0cReleaseUntil\x12\x1f\n\x0bresponse_id\x18\x01 \x01(\tR\nresponseIdB\t\n\x07releaseB)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\xa5\x01\n\x16ReleaseExecuteResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12&\n\x0coperation_id\x18\x02 \x01(\tH\x00R\x0boperationId\x88\x01\x01\x42\x0f\n\r_operation_id"\xd4\x01\n\x15ReleaseSessionRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12$\n\x0b\x63lient_type\x18\x03 \x01(\tH\x00R\nclientType\x88\x01\x01\x12\'\n\x0f\x61llow_reconnect\x18\x04 \x01(\x08R\x0e\x61llowReconnectB\x0e\n\x0c_client_type"l\n\x16ReleaseSessionResponse\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12\x33\n\x16server_side_session_id\x18\x02 \x01(\tR\x13serverSideSessionId"\xcc\x02\n\x18\x46\x65tchErrorDetailsRequest\x12\x1d\n\nsession_id\x18\x01 \x01(\tR\tsessionId\x12V\n&client_observed_server_side_session_id\x18\x05 \x01(\tH\x00R!clientObservedServerSideSessionId\x88\x01\x01\x12=\n\x0cuser_context\x18\x02 \x01(\x0b\x32\x1a.spark.connect.UserContextR\x0buserContext\x12\x19\n\x08\x65rror_id\x18\x03 \x01(\tR\x07\x65rrorId\x12$\n\x0b\x63lient_type\x18\x04 \x01(\tH\x01R\nclientType\x88\x01\x01\x42)\n\'_client_observed_server_side_session_idB\x0e\n\x0c_client_type"\x93\x0c\n\x19\x46\x65tchErrorDetailsResponse\x12\x33\n\x16server_side_session_id\x18\x03 \x01(\tR\x13serverSideSessionId\x12\x1d\n\nsession_id\x18\x04 \x01(\tR\tsessionId\x12)\n\x0eroot_error_idx\x18\x01 \x01(\x05H\x00R\x0crootErrorIdx\x88\x01\x01\x12\x46\n\x06\x65rrors\x18\x02 \x03(\x0b\x32..spark.connect.FetchErrorDetailsResponse.ErrorR\x06\x65rrors\x1a\xae\x01\n\x11StackTraceElement\x12\'\n\x0f\x64\x65\x63laring_class\x18\x01 \x01(\tR\x0e\x64\x65\x63laringClass\x12\x1f\n\x0bmethod_name\x18\x02 \x01(\tR\nmethodName\x12 \n\tfile_name\x18\x03 \x01(\tH\x00R\x08\x66ileName\x88\x01\x01\x12\x1f\n\x0bline_number\x18\x04 \x01(\x05R\nlineNumberB\x0c\n\n_file_name\x1a\xf0\x02\n\x0cQueryContext\x12\x64\n\x0c\x63ontext_type\x18\n \x01(\x0e\x32\x41.spark.connect.FetchErrorDetailsResponse.QueryContext.ContextTypeR\x0b\x63ontextType\x12\x1f\n\x0bobject_type\x18\x01 \x01(\tR\nobjectType\x12\x1f\n\x0bobject_name\x18\x02 \x01(\tR\nobjectName\x12\x1f\n\x0bstart_index\x18\x03 \x01(\x05R\nstartIndex\x12\x1d\n\nstop_index\x18\x04 \x01(\x05R\tstopIndex\x12\x1a\n\x08\x66ragment\x18\x05 \x01(\tR\x08\x66ragment\x12\x1b\n\tcall_site\x18\x06 \x01(\tR\x08\x63\x61llSite\x12\x18\n\x07summary\x18\x07 \x01(\tR\x07summary"%\n\x0b\x43ontextType\x12\x07\n\x03SQL\x10\x00\x12\r\n\tDATAFRAME\x10\x01\x1a\x99\x03\n\x0eSparkThrowable\x12$\n\x0b\x65rror_class\x18\x01 \x01(\tH\x00R\nerrorClass\x88\x01\x01\x12}\n\x12message_parameters\x18\x02 \x03(\x0b\x32N.spark.connect.FetchErrorDetailsResponse.SparkThrowable.MessageParametersEntryR\x11messageParameters\x12\\\n\x0equery_contexts\x18\x03 \x03(\x0b\x32\x35.spark.connect.FetchErrorDetailsResponse.QueryContextR\rqueryContexts\x12 \n\tsql_state\x18\x04 \x01(\tH\x01R\x08sqlState\x88\x01\x01\x1a\x44\n\x16MessageParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_error_classB\x0c\n\n_sql_state\x1a\xdb\x02\n\x05\x45rror\x12\x30\n\x14\x65rror_type_hierarchy\x18\x01 \x03(\tR\x12\x65rrorTypeHierarchy\x12\x18\n\x07message\x18\x02 \x01(\tR\x07message\x12[\n\x0bstack_trace\x18\x03 \x03(\x0b\x32:.spark.connect.FetchErrorDetailsResponse.StackTraceElementR\nstackTrace\x12 \n\tcause_idx\x18\x04 \x01(\x05H\x00R\x08\x63\x61useIdx\x88\x01\x01\x12\x65\n\x0fspark_throwable\x18\x05 \x01(\x0b\x32\x37.spark.connect.FetchErrorDetailsResponse.SparkThrowableH\x01R\x0esparkThrowable\x88\x01\x01\x42\x0c\n\n_cause_idxB\x12\n\x10_spark_throwableB\x11\n\x0f_root_error_idx"Z\n\x17\x43heckpointCommandResult\x12?\n\x08relation\x18\x01 \x01(\x0b\x32#.spark.connect.CachedRemoteRelationR\x08relation2\xb2\x07\n\x13SparkConnectService\x12X\n\x0b\x45xecutePlan\x12!.spark.connect.ExecutePlanRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12V\n\x0b\x41nalyzePlan\x12!.spark.connect.AnalyzePlanRequest\x1a".spark.connect.AnalyzePlanResponse"\x00\x12G\n\x06\x43onfig\x12\x1c.spark.connect.ConfigRequest\x1a\x1d.spark.connect.ConfigResponse"\x00\x12[\n\x0c\x41\x64\x64\x41rtifacts\x12".spark.connect.AddArtifactsRequest\x1a#.spark.connect.AddArtifactsResponse"\x00(\x01\x12\x63\n\x0e\x41rtifactStatus\x12&.spark.connect.ArtifactStatusesRequest\x1a\'.spark.connect.ArtifactStatusesResponse"\x00\x12P\n\tInterrupt\x12\x1f.spark.connect.InterruptRequest\x1a .spark.connect.InterruptResponse"\x00\x12`\n\x0fReattachExecute\x12%.spark.connect.ReattachExecuteRequest\x1a".spark.connect.ExecutePlanResponse"\x00\x30\x01\x12_\n\x0eReleaseExecute\x12$.spark.connect.ReleaseExecuteRequest\x1a%.spark.connect.ReleaseExecuteResponse"\x00\x12_\n\x0eReleaseSession\x12$.spark.connect.ReleaseSessionRequest\x1a%.spark.connect.ReleaseSessionResponse"\x00\x12h\n\x11\x46\x65tchErrorDetails\x12\'.spark.connect.FetchErrorDetailsRequest\x1a(.spark.connect.FetchErrorDetailsResponse"\x00\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -167,91 +167,91 @@ _globals["_KEYVALUE"]._serialized_start = 8609 _globals["_KEYVALUE"]._serialized_end = 8674 _globals["_CONFIGREQUEST"]._serialized_start = 8677 - _globals["_CONFIGREQUEST"]._serialized_end = 9836 + _globals["_CONFIGREQUEST"]._serialized_end = 9876 _globals["_CONFIGREQUEST_OPERATION"]._serialized_start = 8985 _globals["_CONFIGREQUEST_OPERATION"]._serialized_end = 9483 _globals["_CONFIGREQUEST_SET"]._serialized_start = 9485 - _globals["_CONFIGREQUEST_SET"]._serialized_end = 9537 - _globals["_CONFIGREQUEST_GET"]._serialized_start = 9539 - _globals["_CONFIGREQUEST_GET"]._serialized_end = 9564 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9566 - _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9629 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9631 - _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9662 - _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9664 - _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9712 - _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9714 - _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9741 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9743 - _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9777 - _globals["_CONFIGRESPONSE"]._serialized_start = 9839 - _globals["_CONFIGRESPONSE"]._serialized_end = 10014 - _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10017 - _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11019 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10492 - _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10545 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10547 - _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 10658 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10660 - _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10753 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 10756 - _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 10949 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11022 - _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11294 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11213 - _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11294 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11297 - _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11623 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11626 - _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 11978 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 11821 - _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11936 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 11938 - _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 11978 - _globals["_INTERRUPTREQUEST"]._serialized_start = 11981 - _globals["_INTERRUPTREQUEST"]._serialized_end = 12584 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12384 - _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12512 - _globals["_INTERRUPTRESPONSE"]._serialized_start = 12587 - _globals["_INTERRUPTRESPONSE"]._serialized_end = 12731 - _globals["_REATTACHOPTIONS"]._serialized_start = 12733 - _globals["_REATTACHOPTIONS"]._serialized_end = 12786 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12789 - _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13195 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13198 - _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13783 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13652 - _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13664 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13666 - _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13713 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13786 - _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13951 - _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13954 - _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14125 - _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14127 - _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14235 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14238 - _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14570 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14573 - _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16128 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 14802 - _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 14976 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 14979 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15347 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15310 - _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15347 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15350 - _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 15759 + _globals["_CONFIGREQUEST_SET"]._serialized_end = 9577 + _globals["_CONFIGREQUEST_GET"]._serialized_start = 9579 + _globals["_CONFIGREQUEST_GET"]._serialized_end = 9604 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_start = 9606 + _globals["_CONFIGREQUEST_GETWITHDEFAULT"]._serialized_end = 9669 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_start = 9671 + _globals["_CONFIGREQUEST_GETOPTION"]._serialized_end = 9702 + _globals["_CONFIGREQUEST_GETALL"]._serialized_start = 9704 + _globals["_CONFIGREQUEST_GETALL"]._serialized_end = 9752 + _globals["_CONFIGREQUEST_UNSET"]._serialized_start = 9754 + _globals["_CONFIGREQUEST_UNSET"]._serialized_end = 9781 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_start = 9783 + _globals["_CONFIGREQUEST_ISMODIFIABLE"]._serialized_end = 9817 + _globals["_CONFIGRESPONSE"]._serialized_start = 9879 + _globals["_CONFIGRESPONSE"]._serialized_end = 10054 + _globals["_ADDARTIFACTSREQUEST"]._serialized_start = 10057 + _globals["_ADDARTIFACTSREQUEST"]._serialized_end = 11059 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_start = 10532 + _globals["_ADDARTIFACTSREQUEST_ARTIFACTCHUNK"]._serialized_end = 10585 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_start = 10587 + _globals["_ADDARTIFACTSREQUEST_SINGLECHUNKARTIFACT"]._serialized_end = 10698 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_start = 10700 + _globals["_ADDARTIFACTSREQUEST_BATCH"]._serialized_end = 10793 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_start = 10796 + _globals["_ADDARTIFACTSREQUEST_BEGINCHUNKEDARTIFACT"]._serialized_end = 10989 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_start = 11062 + _globals["_ADDARTIFACTSRESPONSE"]._serialized_end = 11334 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_start = 11253 + _globals["_ADDARTIFACTSRESPONSE_ARTIFACTSUMMARY"]._serialized_end = 11334 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_start = 11337 + _globals["_ARTIFACTSTATUSESREQUEST"]._serialized_end = 11663 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_start = 11666 + _globals["_ARTIFACTSTATUSESRESPONSE"]._serialized_end = 12018 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_start = 11861 + _globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_end = 11976 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_start = 11978 + _globals["_ARTIFACTSTATUSESRESPONSE_ARTIFACTSTATUS"]._serialized_end = 12018 + _globals["_INTERRUPTREQUEST"]._serialized_start = 12021 + _globals["_INTERRUPTREQUEST"]._serialized_end = 12624 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_start = 12424 + _globals["_INTERRUPTREQUEST_INTERRUPTTYPE"]._serialized_end = 12552 + _globals["_INTERRUPTRESPONSE"]._serialized_start = 12627 + _globals["_INTERRUPTRESPONSE"]._serialized_end = 12771 + _globals["_REATTACHOPTIONS"]._serialized_start = 12773 + _globals["_REATTACHOPTIONS"]._serialized_end = 12826 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_start = 12829 + _globals["_REATTACHEXECUTEREQUEST"]._serialized_end = 13235 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_start = 13238 + _globals["_RELEASEEXECUTEREQUEST"]._serialized_end = 13823 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_start = 13692 + _globals["_RELEASEEXECUTEREQUEST_RELEASEALL"]._serialized_end = 13704 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_start = 13706 + _globals["_RELEASEEXECUTEREQUEST_RELEASEUNTIL"]._serialized_end = 13753 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_start = 13826 + _globals["_RELEASEEXECUTERESPONSE"]._serialized_end = 13991 + _globals["_RELEASESESSIONREQUEST"]._serialized_start = 13994 + _globals["_RELEASESESSIONREQUEST"]._serialized_end = 14206 + _globals["_RELEASESESSIONRESPONSE"]._serialized_start = 14208 + _globals["_RELEASESESSIONRESPONSE"]._serialized_end = 14316 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_start = 14319 + _globals["_FETCHERRORDETAILSREQUEST"]._serialized_end = 14651 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_start = 14654 + _globals["_FETCHERRORDETAILSRESPONSE"]._serialized_end = 16209 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_start = 14883 + _globals["_FETCHERRORDETAILSRESPONSE_STACKTRACEELEMENT"]._serialized_end = 15057 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_start = 15060 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT"]._serialized_end = 15428 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_start = 15391 + _globals["_FETCHERRORDETAILSRESPONSE_QUERYCONTEXT_CONTEXTTYPE"]._serialized_end = 15428 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_start = 15431 + _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE"]._serialized_end = 15840 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_start = 15661 + ]._serialized_start = 15742 _globals[ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY" - ]._serialized_end = 15729 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15762 - _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16109 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16130 - _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16220 - _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16223 - _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17169 + ]._serialized_end = 15810 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_start = 15843 + _globals["_FETCHERRORDETAILSRESPONSE_ERROR"]._serialized_end = 16190 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_start = 16211 + _globals["_CHECKPOINTCOMMANDRESULT"]._serialized_end = 16301 + _globals["_SPARKCONNECTSERVICE"]._serialized_start = 16304 + _globals["_SPARKCONNECTSERVICE"]._serialized_end = 17250 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi index 253f8a58166a6..adea62ed6b805 100644 --- a/python/pyspark/sql/connect/proto/base_pb2.pyi +++ b/python/pyspark/sql/connect/proto/base_pb2.pyi @@ -1921,17 +1921,32 @@ class ConfigRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor PAIRS_FIELD_NUMBER: builtins.int + SILENT_FIELD_NUMBER: builtins.int @property def pairs( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___KeyValue]: """(Required) The config key-value pairs to set.""" + silent: builtins.bool + """(Optional) Whether to ignore failures.""" def __init__( self, *, pairs: collections.abc.Iterable[global___KeyValue] | None = ..., + silent: builtins.bool | None = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["pairs", b"pairs"]) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["_silent", b"_silent", "silent", b"silent"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "_silent", b"_silent", "pairs", b"pairs", "silent", b"silent" + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_silent", b"_silent"] + ) -> typing_extensions.Literal["silent"] | None: ... class Get(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -3216,6 +3231,7 @@ class ReleaseSessionRequest(google.protobuf.message.Message): SESSION_ID_FIELD_NUMBER: builtins.int USER_CONTEXT_FIELD_NUMBER: builtins.int CLIENT_TYPE_FIELD_NUMBER: builtins.int + ALLOW_RECONNECT_FIELD_NUMBER: builtins.int session_id: builtins.str """(Required) @@ -3234,12 +3250,27 @@ class ReleaseSessionRequest(google.protobuf.message.Message): can be used for language or version specific information and is only intended for logging purposes and will not be interpreted by the server. """ + allow_reconnect: builtins.bool + """Signals the server to allow the client to reconnect to the session after it is released. + + By default, the server tombstones the session upon release, preventing reconnections and + fully cleaning the session state. + + If this flag is set to true, the server may permit the client to reconnect to the session + post-release, even if the session state has been cleaned. This can result in missing state, + such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session. + + Use this option sparingly and only when the client fully understands the implications of + reconnecting to a released session. The client must ensure that any queries executed do not + rely on the session state prior to its release. + """ def __init__( self, *, session_id: builtins.str = ..., user_context: global___UserContext | None = ..., client_type: builtins.str | None = ..., + allow_reconnect: builtins.bool = ..., ) -> None: ... def HasField( self, @@ -3257,6 +3288,8 @@ class ReleaseSessionRequest(google.protobuf.message.Message): field_name: typing_extensions.Literal[ "_client_type", b"_client_type", + "allow_reconnect", + b"allow_reconnect", "client_type", b"client_type", "session_id", diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py index 093997a0d0c56..7edcbcac15c73 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.py +++ b/python/pyspark/sql/connect/proto/expressions_pb2.py @@ -40,7 +40,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\xe1\x31\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12H\n\x0flazy_expression\x18\x15 \x01(\x0b\x32\x1d.spark.connect.LazyExpressionH\x00R\x0elazyExpression\x12T\n\x13subquery_expression\x18\x16 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9b\x0c\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x82\x01\n\x05\x41rray\x12:\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xe3\x01\n\x03Map\x12\x32\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x07keyType\x12\x36\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x81\x01\n\x06Struct\x12\x38\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lementsB\x0e\n\x0cliteral_type\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\xcc\x01\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\xec\x02\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdfB\n\n\x08\x66unction"\xcc\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"A\n\x0eLazyExpression\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild"\xe1\x01\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType"]\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x97\x32\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolvedAttribute\x12_\n\x13unresolved_function\x18\x03 \x01(\x0b\x32,.spark.connect.Expression.UnresolvedFunctionH\x00R\x12unresolvedFunction\x12Y\n\x11\x65xpression_string\x18\x04 \x01(\x0b\x32*.spark.connect.Expression.ExpressionStringH\x00R\x10\x65xpressionString\x12S\n\x0funresolved_star\x18\x05 \x01(\x0b\x32(.spark.connect.Expression.UnresolvedStarH\x00R\x0eunresolvedStar\x12\x37\n\x05\x61lias\x18\x06 \x01(\x0b\x32\x1f.spark.connect.Expression.AliasH\x00R\x05\x61lias\x12\x34\n\x04\x63\x61st\x18\x07 \x01(\x0b\x32\x1e.spark.connect.Expression.CastH\x00R\x04\x63\x61st\x12V\n\x10unresolved_regex\x18\x08 \x01(\x0b\x32).spark.connect.Expression.UnresolvedRegexH\x00R\x0funresolvedRegex\x12\x44\n\nsort_order\x18\t \x01(\x0b\x32#.spark.connect.Expression.SortOrderH\x00R\tsortOrder\x12S\n\x0flambda_function\x18\n \x01(\x0b\x32(.spark.connect.Expression.LambdaFunctionH\x00R\x0elambdaFunction\x12:\n\x06window\x18\x0b \x01(\x0b\x32 .spark.connect.Expression.WindowH\x00R\x06window\x12l\n\x18unresolved_extract_value\x18\x0c \x01(\x0b\x32\x30.spark.connect.Expression.UnresolvedExtractValueH\x00R\x16unresolvedExtractValue\x12M\n\rupdate_fields\x18\r \x01(\x0b\x32&.spark.connect.Expression.UpdateFieldsH\x00R\x0cupdateFields\x12\x82\x01\n unresolved_named_lambda_variable\x18\x0e \x01(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableH\x00R\x1dunresolvedNamedLambdaVariable\x12~\n#common_inline_user_defined_function\x18\x0f \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x1f\x63ommonInlineUserDefinedFunction\x12\x42\n\rcall_function\x18\x10 \x01(\x0b\x32\x1b.spark.connect.CallFunctionH\x00R\x0c\x63\x61llFunction\x12\x64\n\x19named_argument_expression\x18\x11 \x01(\x0b\x32&.spark.connect.NamedArgumentExpressionH\x00R\x17namedArgumentExpression\x12?\n\x0cmerge_action\x18\x13 \x01(\x0b\x32\x1a.spark.connect.MergeActionH\x00R\x0bmergeAction\x12g\n\x1atyped_aggregate_expression\x18\x14 \x01(\x0b\x32\'.spark.connect.TypedAggregateExpressionH\x00R\x18typedAggregateExpression\x12H\n\x0flazy_expression\x18\x15 \x01(\x0b\x32\x1d.spark.connect.LazyExpressionH\x00R\x0elazyExpression\x12T\n\x13subquery_expression\x18\x16 \x01(\x0b\x32!.spark.connect.SubqueryExpressionH\x00R\x12subqueryExpression\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x1a\x8f\x06\n\x06Window\x12\x42\n\x0fwindow_function\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0ewindowFunction\x12@\n\x0epartition_spec\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\rpartitionSpec\x12\x42\n\norder_spec\x18\x03 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\torderSpec\x12K\n\nframe_spec\x18\x04 \x01(\x0b\x32,.spark.connect.Expression.Window.WindowFrameR\tframeSpec\x1a\xed\x03\n\x0bWindowFrame\x12U\n\nframe_type\x18\x01 \x01(\x0e\x32\x36.spark.connect.Expression.Window.WindowFrame.FrameTypeR\tframeType\x12P\n\x05lower\x18\x02 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05lower\x12P\n\x05upper\x18\x03 \x01(\x0b\x32:.spark.connect.Expression.Window.WindowFrame.FrameBoundaryR\x05upper\x1a\x91\x01\n\rFrameBoundary\x12!\n\x0b\x63urrent_row\x18\x01 \x01(\x08H\x00R\ncurrentRow\x12\x1e\n\tunbounded\x18\x02 \x01(\x08H\x00R\tunbounded\x12\x31\n\x05value\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\x05valueB\n\n\x08\x62oundary"O\n\tFrameType\x12\x18\n\x14\x46RAME_TYPE_UNDEFINED\x10\x00\x12\x12\n\x0e\x46RAME_TYPE_ROW\x10\x01\x12\x14\n\x10\x46RAME_TYPE_RANGE\x10\x02\x1a\xa9\x03\n\tSortOrder\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12O\n\tdirection\x18\x02 \x01(\x0e\x32\x31.spark.connect.Expression.SortOrder.SortDirectionR\tdirection\x12U\n\rnull_ordering\x18\x03 \x01(\x0e\x32\x30.spark.connect.Expression.SortOrder.NullOrderingR\x0cnullOrdering"l\n\rSortDirection\x12\x1e\n\x1aSORT_DIRECTION_UNSPECIFIED\x10\x00\x12\x1c\n\x18SORT_DIRECTION_ASCENDING\x10\x01\x12\x1d\n\x19SORT_DIRECTION_DESCENDING\x10\x02"U\n\x0cNullOrdering\x12\x1a\n\x16SORT_NULLS_UNSPECIFIED\x10\x00\x12\x14\n\x10SORT_NULLS_FIRST\x10\x01\x12\x13\n\x0fSORT_NULLS_LAST\x10\x02\x1a\xbb\x02\n\x04\x43\x61st\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12-\n\x04type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04type\x12\x1b\n\x08type_str\x18\x03 \x01(\tH\x00R\x07typeStr\x12\x44\n\teval_mode\x18\x04 \x01(\x0e\x32\'.spark.connect.Expression.Cast.EvalModeR\x08\x65valMode"b\n\x08\x45valMode\x12\x19\n\x15\x45VAL_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10\x45VAL_MODE_LEGACY\x10\x01\x12\x12\n\x0e\x45VAL_MODE_ANSI\x10\x02\x12\x11\n\rEVAL_MODE_TRY\x10\x03\x42\x0e\n\x0c\x63\x61st_to_type\x1a\x9b\x0c\n\x07Literal\x12-\n\x04null\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x04null\x12\x18\n\x06\x62inary\x18\x02 \x01(\x0cH\x00R\x06\x62inary\x12\x1a\n\x07\x62oolean\x18\x03 \x01(\x08H\x00R\x07\x62oolean\x12\x14\n\x04\x62yte\x18\x04 \x01(\x05H\x00R\x04\x62yte\x12\x16\n\x05short\x18\x05 \x01(\x05H\x00R\x05short\x12\x1a\n\x07integer\x18\x06 \x01(\x05H\x00R\x07integer\x12\x14\n\x04long\x18\x07 \x01(\x03H\x00R\x04long\x12\x16\n\x05\x66loat\x18\n \x01(\x02H\x00R\x05\x66loat\x12\x18\n\x06\x64ouble\x18\x0b \x01(\x01H\x00R\x06\x64ouble\x12\x45\n\x07\x64\x65\x63imal\x18\x0c \x01(\x0b\x32).spark.connect.Expression.Literal.DecimalH\x00R\x07\x64\x65\x63imal\x12\x18\n\x06string\x18\r \x01(\tH\x00R\x06string\x12\x14\n\x04\x64\x61te\x18\x10 \x01(\x05H\x00R\x04\x64\x61te\x12\x1e\n\ttimestamp\x18\x11 \x01(\x03H\x00R\ttimestamp\x12%\n\rtimestamp_ntz\x18\x12 \x01(\x03H\x00R\x0ctimestampNtz\x12\x61\n\x11\x63\x61lendar_interval\x18\x13 \x01(\x0b\x32\x32.spark.connect.Expression.Literal.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12\x30\n\x13year_month_interval\x18\x14 \x01(\x05H\x00R\x11yearMonthInterval\x12,\n\x11\x64\x61y_time_interval\x18\x15 \x01(\x03H\x00R\x0f\x64\x61yTimeInterval\x12?\n\x05\x61rray\x18\x16 \x01(\x0b\x32\'.spark.connect.Expression.Literal.ArrayH\x00R\x05\x61rray\x12\x39\n\x03map\x18\x17 \x01(\x0b\x32%.spark.connect.Expression.Literal.MapH\x00R\x03map\x12\x42\n\x06struct\x18\x18 \x01(\x0b\x32(.spark.connect.Expression.Literal.StructH\x00R\x06struct\x1au\n\x07\x44\x65\x63imal\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12!\n\tprecision\x18\x02 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x19\n\x05scale\x18\x03 \x01(\x05H\x01R\x05scale\x88\x01\x01\x42\x0c\n\n_precisionB\x08\n\x06_scale\x1a\x62\n\x10\x43\x61lendarInterval\x12\x16\n\x06months\x18\x01 \x01(\x05R\x06months\x12\x12\n\x04\x64\x61ys\x18\x02 \x01(\x05R\x04\x64\x61ys\x12"\n\x0cmicroseconds\x18\x03 \x01(\x03R\x0cmicroseconds\x1a\x82\x01\n\x05\x41rray\x12:\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x0b\x65lementType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lements\x1a\xe3\x01\n\x03Map\x12\x32\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x07keyType\x12\x36\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\tvalueType\x12\x35\n\x04keys\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x04keys\x12\x39\n\x06values\x18\x04 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1a\x81\x01\n\x06Struct\x12\x38\n\x0bstruct_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\nstructType\x12=\n\x08\x65lements\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x08\x65lementsB\x0e\n\x0cliteral_type\x1a\xba\x01\n\x13UnresolvedAttribute\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12\x31\n\x12is_metadata_column\x18\x03 \x01(\x08H\x01R\x10isMetadataColumn\x88\x01\x01\x42\n\n\x08_plan_idB\x15\n\x13_is_metadata_column\x1a\x82\x02\n\x12UnresolvedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x1f\n\x0bis_distinct\x18\x03 \x01(\x08R\nisDistinct\x12\x37\n\x18is_user_defined_function\x18\x04 \x01(\x08R\x15isUserDefinedFunction\x12$\n\x0bis_internal\x18\x05 \x01(\x08H\x00R\nisInternal\x88\x01\x01\x42\x0e\n\x0c_is_internal\x1a\x32\n\x10\x45xpressionString\x12\x1e\n\nexpression\x18\x01 \x01(\tR\nexpression\x1a|\n\x0eUnresolvedStar\x12,\n\x0funparsed_target\x18\x01 \x01(\tH\x00R\x0eunparsedTarget\x88\x01\x01\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x01R\x06planId\x88\x01\x01\x42\x12\n\x10_unparsed_targetB\n\n\x08_plan_id\x1aV\n\x0fUnresolvedRegex\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id\x1a\x84\x01\n\x16UnresolvedExtractValue\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12\x39\n\nextraction\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\nextraction\x1a\xbb\x01\n\x0cUpdateFields\x12\x46\n\x11struct_expression\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x10structExpression\x12\x1d\n\nfield_name\x18\x02 \x01(\tR\tfieldName\x12\x44\n\x10value_expression\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x0fvalueExpression\x1ax\n\x05\x41lias\x12-\n\x04\x65xpr\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x04\x65xpr\x12\x12\n\x04name\x18\x02 \x03(\tR\x04name\x12\x1f\n\x08metadata\x18\x03 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x9e\x01\n\x0eLambdaFunction\x12\x35\n\x08\x66unction\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08\x66unction\x12U\n\targuments\x18\x02 \x03(\x0b\x32\x37.spark.connect.Expression.UnresolvedNamedLambdaVariableR\targuments\x1a>\n\x1dUnresolvedNamedLambdaVariable\x12\x1d\n\nname_parts\x18\x01 \x03(\tR\tnamePartsB\x0b\n\texpr_type"A\n\x10\x45xpressionCommon\x12-\n\x06origin\x18\x01 \x01(\x0b\x32\x15.spark.connect.OriginR\x06origin"\xec\x02\n\x1f\x43ommonInlineUserDefinedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12\x39\n\npython_udf\x18\x04 \x01(\x0b\x32\x18.spark.connect.PythonUDFH\x00R\tpythonUdf\x12I\n\x10scalar_scala_udf\x18\x05 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFH\x00R\x0escalarScalaUdf\x12\x33\n\x08java_udf\x18\x06 \x01(\x0b\x32\x16.spark.connect.JavaUDFH\x00R\x07javaUdfB\n\n\x08\x66unction"\xcc\x01\n\tPythonUDF\x12\x38\n\x0boutput_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVer\x12/\n\x13\x61\x64\x64itional_includes\x18\x05 \x03(\tR\x12\x61\x64\x64itionalIncludes"\xd6\x01\n\x0eScalarScalaUDF\x12\x18\n\x07payload\x18\x01 \x01(\x0cR\x07payload\x12\x37\n\ninputTypes\x18\x02 \x03(\x0b\x32\x17.spark.connect.DataTypeR\ninputTypes\x12\x37\n\noutputType\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeR\noutputType\x12\x1a\n\x08nullable\x18\x04 \x01(\x08R\x08nullable\x12\x1c\n\taggregate\x18\x05 \x01(\x08R\taggregate"\x95\x01\n\x07JavaUDF\x12\x1d\n\nclass_name\x18\x01 \x01(\tR\tclassName\x12=\n\x0boutput_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\noutputType\x88\x01\x01\x12\x1c\n\taggregate\x18\x03 \x01(\x08R\taggregateB\x0e\n\x0c_output_type"c\n\x18TypedAggregateExpression\x12G\n\x10scalar_scala_udf\x18\x01 \x01(\x0b\x32\x1d.spark.connect.ScalarScalaUDFR\x0escalarScalaUdf"l\n\x0c\x43\x61llFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"\\\n\x17NamedArgumentExpression\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\x80\x04\n\x0bMergeAction\x12\x46\n\x0b\x61\x63tion_type\x18\x01 \x01(\x0e\x32%.spark.connect.MergeAction.ActionTypeR\nactionType\x12<\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionH\x00R\tcondition\x88\x01\x01\x12G\n\x0b\x61ssignments\x18\x03 \x03(\x0b\x32%.spark.connect.MergeAction.AssignmentR\x0b\x61ssignments\x1aj\n\nAssignment\x12+\n\x03key\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value"\xa7\x01\n\nActionType\x12\x17\n\x13\x41\x43TION_TYPE_INVALID\x10\x00\x12\x16\n\x12\x41\x43TION_TYPE_DELETE\x10\x01\x12\x16\n\x12\x41\x43TION_TYPE_INSERT\x10\x02\x12\x1b\n\x17\x41\x43TION_TYPE_INSERT_STAR\x10\x03\x12\x16\n\x12\x41\x43TION_TYPE_UPDATE\x10\x04\x12\x1b\n\x17\x41\x43TION_TYPE_UPDATE_STAR\x10\x05\x42\x0c\n\n_condition"A\n\x0eLazyExpression\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild"\xe1\x01\n\x12SubqueryExpression\x12\x17\n\x07plan_id\x18\x01 \x01(\x03R\x06planId\x12S\n\rsubquery_type\x18\x02 \x01(\x0e\x32..spark.connect.SubqueryExpression.SubqueryTypeR\x0csubqueryType"]\n\x0cSubqueryType\x12\x19\n\x15SUBQUERY_TYPE_UNKNOWN\x10\x00\x12\x18\n\x14SUBQUERY_TYPE_SCALAR\x10\x01\x12\x18\n\x14SUBQUERY_TYPE_EXISTS\x10\x02\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -54,7 +54,7 @@ "DESCRIPTOR" ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated" _globals["_EXPRESSION"]._serialized_start = 133 - _globals["_EXPRESSION"]._serialized_end = 6502 + _globals["_EXPRESSION"]._serialized_end = 6556 _globals["_EXPRESSION_WINDOW"]._serialized_start = 2060 _globals["_EXPRESSION_WINDOW"]._serialized_end = 2843 _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_start = 2350 @@ -88,49 +88,49 @@ _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_start = 5158 _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_end = 5344 _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_start = 5347 - _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 5551 - _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 5553 - _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 5603 - _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 5605 - _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 5729 - _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 5731 - _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 5817 - _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 5820 - _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 5952 - _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 5955 - _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6142 - _globals["_EXPRESSION_ALIAS"]._serialized_start = 6144 - _globals["_EXPRESSION_ALIAS"]._serialized_end = 6264 - _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6267 - _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6425 - _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6427 - _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6489 - _globals["_EXPRESSIONCOMMON"]._serialized_start = 6504 - _globals["_EXPRESSIONCOMMON"]._serialized_end = 6569 - _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 6572 - _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 6936 - _globals["_PYTHONUDF"]._serialized_start = 6939 - _globals["_PYTHONUDF"]._serialized_end = 7143 - _globals["_SCALARSCALAUDF"]._serialized_start = 7146 - _globals["_SCALARSCALAUDF"]._serialized_end = 7360 - _globals["_JAVAUDF"]._serialized_start = 7363 - _globals["_JAVAUDF"]._serialized_end = 7512 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 7514 - _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 7613 - _globals["_CALLFUNCTION"]._serialized_start = 7615 - _globals["_CALLFUNCTION"]._serialized_end = 7723 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 7725 - _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 7817 - _globals["_MERGEACTION"]._serialized_start = 7820 - _globals["_MERGEACTION"]._serialized_end = 8332 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8042 - _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8148 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8151 - _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8318 - _globals["_LAZYEXPRESSION"]._serialized_start = 8334 - _globals["_LAZYEXPRESSION"]._serialized_end = 8399 - _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8402 - _globals["_SUBQUERYEXPRESSION"]._serialized_end = 8627 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 8534 - _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 8627 + _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 5605 + _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 5607 + _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 5657 + _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 5659 + _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 5783 + _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 5785 + _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 5871 + _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 5874 + _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 6006 + _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 6009 + _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6196 + _globals["_EXPRESSION_ALIAS"]._serialized_start = 6198 + _globals["_EXPRESSION_ALIAS"]._serialized_end = 6318 + _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6321 + _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6479 + _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6481 + _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6543 + _globals["_EXPRESSIONCOMMON"]._serialized_start = 6558 + _globals["_EXPRESSIONCOMMON"]._serialized_end = 6623 + _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 6626 + _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 6990 + _globals["_PYTHONUDF"]._serialized_start = 6993 + _globals["_PYTHONUDF"]._serialized_end = 7197 + _globals["_SCALARSCALAUDF"]._serialized_start = 7200 + _globals["_SCALARSCALAUDF"]._serialized_end = 7414 + _globals["_JAVAUDF"]._serialized_start = 7417 + _globals["_JAVAUDF"]._serialized_end = 7566 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 7568 + _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 7667 + _globals["_CALLFUNCTION"]._serialized_start = 7669 + _globals["_CALLFUNCTION"]._serialized_end = 7777 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 7779 + _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 7871 + _globals["_MERGEACTION"]._serialized_start = 7874 + _globals["_MERGEACTION"]._serialized_end = 8386 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8096 + _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8202 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8205 + _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8372 + _globals["_LAZYEXPRESSION"]._serialized_start = 8388 + _globals["_LAZYEXPRESSION"]._serialized_end = 8453 + _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8456 + _globals["_SUBQUERYEXPRESSION"]._serialized_end = 8681 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 8588 + _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 8681 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi index 0a6f3caee8b54..1a8c60f673054 100644 --- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi +++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi @@ -847,6 +847,7 @@ class Expression(google.protobuf.message.Message): ARGUMENTS_FIELD_NUMBER: builtins.int IS_DISTINCT_FIELD_NUMBER: builtins.int IS_USER_DEFINED_FUNCTION_FIELD_NUMBER: builtins.int + IS_INTERNAL_FIELD_NUMBER: builtins.int function_name: builtins.str """(Required) name (or unparsed name for user defined function) for the unresolved function.""" @property @@ -864,6 +865,11 @@ class Expression(google.protobuf.message.Message): When it is not a user defined function, Connect will use the function name directly. When it is a user defined function, Connect will parse the function name first. """ + is_internal: builtins.bool + """(Optional) Indicate if this function is defined in the internal function registry. + If not set, the server will try to look up the function in the internal function registry + and decide appropriately. + """ def __init__( self, *, @@ -871,20 +877,34 @@ class Expression(google.protobuf.message.Message): arguments: collections.abc.Iterable[global___Expression] | None = ..., is_distinct: builtins.bool = ..., is_user_defined_function: builtins.bool = ..., + is_internal: builtins.bool | None = ..., ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "_is_internal", b"_is_internal", "is_internal", b"is_internal" + ], + ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ + "_is_internal", + b"_is_internal", "arguments", b"arguments", "function_name", b"function_name", "is_distinct", b"is_distinct", + "is_internal", + b"is_internal", "is_user_defined_function", b"is_user_defined_function", ], ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_is_internal", b"_is_internal"] + ) -> typing_extensions.Literal["is_internal"] | None: ... class ExpressionString(google.protobuf.message.Message): """Expression as string.""" diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py index 506b266f60145..b7248d4b17080 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.py +++ b/python/pyspark/sql/connect/proto/relations_pb2.py @@ -42,7 +42,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto"\xdd\x1c\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12\x34\n\x06set_op\x18\x06 \x01(\x0b\x32\x1b.spark.connect.SetOperationH\x00R\x05setOp\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12,\n\x05range\x18\x0f \x01(\x0b\x32\x14.spark.connect.RangeH\x00R\x05range\x12\x45\n\x0esubquery_alias\x18\x10 \x01(\x0b\x32\x1c.spark.connect.SubqueryAliasH\x00R\rsubqueryAlias\x12>\n\x0brepartition\x18\x11 \x01(\x0b\x32\x1a.spark.connect.RepartitionH\x00R\x0brepartition\x12*\n\x05to_df\x18\x12 \x01(\x0b\x32\x13.spark.connect.ToDFH\x00R\x04toDf\x12U\n\x14with_columns_renamed\x18\x13 \x01(\x0b\x32!.spark.connect.WithColumnsRenamedH\x00R\x12withColumnsRenamed\x12<\n\x0bshow_string\x18\x14 \x01(\x0b\x32\x19.spark.connect.ShowStringH\x00R\nshowString\x12)\n\x04\x64rop\x18\x15 \x01(\x0b\x32\x13.spark.connect.DropH\x00R\x04\x64rop\x12)\n\x04tail\x18\x16 \x01(\x0b\x32\x13.spark.connect.TailH\x00R\x04tail\x12?\n\x0cwith_columns\x18\x17 \x01(\x0b\x32\x1a.spark.connect.WithColumnsH\x00R\x0bwithColumns\x12)\n\x04hint\x18\x18 \x01(\x0b\x32\x13.spark.connect.HintH\x00R\x04hint\x12\x32\n\x07unpivot\x18\x19 \x01(\x0b\x32\x16.spark.connect.UnpivotH\x00R\x07unpivot\x12\x36\n\tto_schema\x18\x1a \x01(\x0b\x32\x17.spark.connect.ToSchemaH\x00R\x08toSchema\x12\x64\n\x19repartition_by_expression\x18\x1b \x01(\x0b\x32&.spark.connect.RepartitionByExpressionH\x00R\x17repartitionByExpression\x12\x45\n\x0emap_partitions\x18\x1c \x01(\x0b\x32\x1c.spark.connect.MapPartitionsH\x00R\rmapPartitions\x12H\n\x0f\x63ollect_metrics\x18\x1d \x01(\x0b\x32\x1d.spark.connect.CollectMetricsH\x00R\x0e\x63ollectMetrics\x12,\n\x05parse\x18\x1e \x01(\x0b\x32\x14.spark.connect.ParseH\x00R\x05parse\x12\x36\n\tgroup_map\x18\x1f \x01(\x0b\x32\x17.spark.connect.GroupMapH\x00R\x08groupMap\x12=\n\x0c\x63o_group_map\x18 \x01(\x0b\x32\x19.spark.connect.CoGroupMapH\x00R\ncoGroupMap\x12\x45\n\x0ewith_watermark\x18! \x01(\x0b\x32\x1c.spark.connect.WithWatermarkH\x00R\rwithWatermark\x12\x63\n\x1a\x61pply_in_pandas_with_state\x18" \x01(\x0b\x32%.spark.connect.ApplyInPandasWithStateH\x00R\x16\x61pplyInPandasWithState\x12<\n\x0bhtml_string\x18# \x01(\x0b\x32\x19.spark.connect.HtmlStringH\x00R\nhtmlString\x12X\n\x15\x63\x61\x63hed_local_relation\x18$ \x01(\x0b\x32".spark.connect.CachedLocalRelationH\x00R\x13\x63\x61\x63hedLocalRelation\x12[\n\x16\x63\x61\x63hed_remote_relation\x18% \x01(\x0b\x32#.spark.connect.CachedRemoteRelationH\x00R\x14\x63\x61\x63hedRemoteRelation\x12\x8e\x01\n)common_inline_user_defined_table_function\x18& \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R$commonInlineUserDefinedTableFunction\x12\x37\n\nas_of_join\x18\' \x01(\x0b\x32\x17.spark.connect.AsOfJoinH\x00R\x08\x61sOfJoin\x12\x85\x01\n&common_inline_user_defined_data_source\x18( \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R!commonInlineUserDefinedDataSource\x12\x45\n\x0ewith_relations\x18) \x01(\x0b\x32\x1c.spark.connect.WithRelationsH\x00R\rwithRelations\x12\x38\n\ttranspose\x18* \x01(\x0b\x32\x18.spark.connect.TransposeH\x00R\ttranspose\x12w\n unresolved_table_valued_function\x18+ \x01(\x0b\x32,.spark.connect.UnresolvedTableValuedFunctionH\x00R\x1dunresolvedTableValuedFunction\x12?\n\x0clateral_join\x18, \x01(\x0b\x32\x1a.spark.connect.LateralJoinH\x00R\x0blateralJoin\x12\x30\n\x07\x66ill_na\x18Z \x01(\x0b\x32\x15.spark.connect.NAFillH\x00R\x06\x66illNa\x12\x30\n\x07\x64rop_na\x18[ \x01(\x0b\x32\x15.spark.connect.NADropH\x00R\x06\x64ropNa\x12\x34\n\x07replace\x18\\ \x01(\x0b\x32\x18.spark.connect.NAReplaceH\x00R\x07replace\x12\x36\n\x07summary\x18\x64 \x01(\x0b\x32\x1a.spark.connect.StatSummaryH\x00R\x07summary\x12\x39\n\x08\x63rosstab\x18\x65 \x01(\x0b\x32\x1b.spark.connect.StatCrosstabH\x00R\x08\x63rosstab\x12\x39\n\x08\x64\x65scribe\x18\x66 \x01(\x0b\x32\x1b.spark.connect.StatDescribeH\x00R\x08\x64\x65scribe\x12*\n\x03\x63ov\x18g \x01(\x0b\x32\x16.spark.connect.StatCovH\x00R\x03\x63ov\x12-\n\x04\x63orr\x18h \x01(\x0b\x32\x17.spark.connect.StatCorrH\x00R\x04\x63orr\x12L\n\x0f\x61pprox_quantile\x18i \x01(\x0b\x32!.spark.connect.StatApproxQuantileH\x00R\x0e\x61pproxQuantile\x12=\n\nfreq_items\x18j \x01(\x0b\x32\x1c.spark.connect.StatFreqItemsH\x00R\tfreqItems\x12:\n\tsample_by\x18k \x01(\x0b\x32\x1b.spark.connect.StatSampleByH\x00R\x08sampleBy\x12\x33\n\x07\x63\x61talog\x18\xc8\x01 \x01(\x0b\x32\x16.spark.connect.CatalogH\x00R\x07\x63\x61talog\x12\x35\n\textension\x18\xe6\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"\x8e\x01\n\x0eRelationCommon\x12#\n\x0bsource_info\x18\x01 \x01(\tB\x02\x18\x01R\nsourceInfo\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12-\n\x06origin\x18\x03 \x01(\x0b\x32\x15.spark.connect.OriginR\x06originB\n\n\x08_plan_id"\xde\x03\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query\x12\x34\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.spark.connect.SQL.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12O\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32&.spark.connect.SQL.NamedArgumentsEntryR\x0enamedArguments\x12>\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cposArguments\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"u\n\rWithRelations\x12+\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04root\x12\x37\n\nreferences\x18\x02 \x03(\x0b\x32\x17.spark.connect.RelationR\nreferences"\x97\x05\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x12!\n\x0cis_streaming\x18\x03 \x01(\x08R\x0bisStreaming\x1a\xc0\x01\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x45\n\x07options\x18\x02 \x03(\x0b\x32+.spark.connect.Read.NamedTable.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x95\x02\n\nDataSource\x12\x1b\n\x06\x66ormat\x18\x01 \x01(\tH\x00R\x06\x66ormat\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x12\x14\n\x05paths\x18\x04 \x03(\tR\x05paths\x12\x1e\n\npredicates\x18\x05 \x03(\tR\npredicates\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07_formatB\t\n\x07_schemaB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x95\x05\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType\x12#\n\rusing_columns\x18\x05 \x03(\tR\x0cusingColumns\x12K\n\x0ejoin_data_type\x18\x06 \x01(\x0b\x32 .spark.connect.Join.JoinDataTypeH\x00R\x0cjoinDataType\x88\x01\x01\x1a\\\n\x0cJoinDataType\x12$\n\x0eis_left_struct\x18\x01 \x01(\x08R\x0cisLeftStruct\x12&\n\x0fis_right_struct\x18\x02 \x01(\x08R\risRightStruct"\xd0\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06\x12\x13\n\x0fJOIN_TYPE_CROSS\x10\x07\x42\x11\n\x0f_join_data_type"\xdf\x03\n\x0cSetOperation\x12\x36\n\nleft_input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\tleftInput\x12\x38\n\x0bright_input\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\nrightInput\x12\x45\n\x0bset_op_type\x18\x03 \x01(\x0e\x32%.spark.connect.SetOperation.SetOpTypeR\tsetOpType\x12\x1a\n\x06is_all\x18\x04 \x01(\x08H\x00R\x05isAll\x88\x01\x01\x12\x1c\n\x07\x62y_name\x18\x05 \x01(\x08H\x01R\x06\x62yName\x88\x01\x01\x12\x37\n\x15\x61llow_missing_columns\x18\x06 \x01(\x08H\x02R\x13\x61llowMissingColumns\x88\x01\x01"r\n\tSetOpType\x12\x1b\n\x17SET_OP_TYPE_UNSPECIFIED\x10\x00\x12\x19\n\x15SET_OP_TYPE_INTERSECT\x10\x01\x12\x15\n\x11SET_OP_TYPE_UNION\x10\x02\x12\x16\n\x12SET_OP_TYPE_EXCEPT\x10\x03\x42\t\n\x07_is_allB\n\n\x08_by_nameB\x18\n\x16_allow_missing_columns"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"K\n\x04Tail\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"\xfe\x05\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x41\n\ngroup_type\x18\x02 \x01(\x0e\x32".spark.connect.Aggregate.GroupTypeR\tgroupType\x12L\n\x14grouping_expressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12N\n\x15\x61ggregate_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x14\x61ggregateExpressions\x12\x34\n\x05pivot\x18\x05 \x01(\x0b\x32\x1e.spark.connect.Aggregate.PivotR\x05pivot\x12J\n\rgrouping_sets\x18\x06 \x03(\x0b\x32%.spark.connect.Aggregate.GroupingSetsR\x0cgroupingSets\x1ao\n\x05Pivot\x12+\n\x03\x63ol\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1aL\n\x0cGroupingSets\x12<\n\x0cgrouping_set\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0bgroupingSet"\x9f\x01\n\tGroupType\x12\x1a\n\x16GROUP_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12GROUP_TYPE_GROUPBY\x10\x01\x12\x15\n\x11GROUP_TYPE_ROLLUP\x10\x02\x12\x13\n\x0fGROUP_TYPE_CUBE\x10\x03\x12\x14\n\x10GROUP_TYPE_PIVOT\x10\x04\x12\x1c\n\x18GROUP_TYPE_GROUPING_SETS\x10\x05"\xa0\x01\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x05order\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\x05order\x12 \n\tis_global\x18\x03 \x01(\x08H\x00R\x08isGlobal\x88\x01\x01\x42\x0c\n\n_is_global"\x8d\x01\n\x04\x44rop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x33\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07\x63olumns\x12!\n\x0c\x63olumn_names\x18\x03 \x03(\tR\x0b\x63olumnNames"\xf0\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12\x32\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08H\x00R\x10\x61llColumnsAsKeys\x88\x01\x01\x12.\n\x10within_watermark\x18\x04 \x01(\x08H\x01R\x0fwithinWatermark\x88\x01\x01\x42\x16\n\x14_all_columns_as_keysB\x13\n\x11_within_watermark"Y\n\rLocalRelation\x12\x17\n\x04\x64\x61ta\x18\x01 \x01(\x0cH\x00R\x04\x64\x61ta\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x42\x07\n\x05_dataB\t\n\x07_schema"H\n\x13\x43\x61\x63hedLocalRelation\x12\x12\n\x04hash\x18\x03 \x01(\tR\x04hashJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03R\x06userIdR\tsessionId"7\n\x14\x43\x61\x63hedRemoteRelation\x12\x1f\n\x0brelation_id\x18\x01 \x01(\tR\nrelationId"\x91\x02\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12.\n\x10with_replacement\x18\x04 \x01(\x08H\x00R\x0fwithReplacement\x88\x01\x01\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x01R\x04seed\x88\x01\x01\x12/\n\x13\x64\x65terministic_order\x18\x06 \x01(\x08R\x12\x64\x65terministicOrderB\x13\n\x11_with_replacementB\x07\n\x05_seed"\x91\x01\n\x05Range\x12\x19\n\x05start\x18\x01 \x01(\x03H\x00R\x05start\x88\x01\x01\x12\x10\n\x03\x65nd\x18\x02 \x01(\x03R\x03\x65nd\x12\x12\n\x04step\x18\x03 \x01(\x03R\x04step\x12*\n\x0enum_partitions\x18\x04 \x01(\x05H\x01R\rnumPartitions\x88\x01\x01\x42\x08\n\x06_startB\x11\n\x0f_num_partitions"r\n\rSubqueryAlias\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\x12\x1c\n\tqualifier\x18\x03 \x03(\tR\tqualifier"\x8e\x01\n\x0bRepartition\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12%\n\x0enum_partitions\x18\x02 \x01(\x05R\rnumPartitions\x12\x1d\n\x07shuffle\x18\x03 \x01(\x08H\x00R\x07shuffle\x88\x01\x01\x42\n\n\x08_shuffle"\x8e\x01\n\nShowString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate\x12\x1a\n\x08vertical\x18\x04 \x01(\x08R\x08vertical"r\n\nHtmlString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate"\\\n\x0bStatSummary\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1e\n\nstatistics\x18\x02 \x03(\tR\nstatistics"Q\n\x0cStatDescribe\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols"e\n\x0cStatCrosstab\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"`\n\x07StatCov\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"\x89\x01\n\x08StatCorr\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2\x12\x1b\n\x06method\x18\x04 \x01(\tH\x00R\x06method\x88\x01\x01\x42\t\n\x07_method"\xa4\x01\n\x12StatApproxQuantile\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12$\n\rprobabilities\x18\x03 \x03(\x01R\rprobabilities\x12%\n\x0erelative_error\x18\x04 \x01(\x01R\rrelativeError"}\n\rStatFreqItems\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x1d\n\x07support\x18\x03 \x01(\x01H\x00R\x07support\x88\x01\x01\x42\n\n\x08_support"\xb5\x02\n\x0cStatSampleBy\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03\x63ol\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x42\n\tfractions\x18\x03 \x03(\x0b\x32$.spark.connect.StatSampleBy.FractionR\tfractions\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x00R\x04seed\x88\x01\x01\x1a\x63\n\x08\x46raction\x12;\n\x07stratum\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x07stratum\x12\x1a\n\x08\x66raction\x18\x02 \x01(\x01R\x08\x66ractionB\x07\n\x05_seed"\x86\x01\n\x06NAFill\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x39\n\x06values\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x86\x01\n\x06NADrop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\'\n\rmin_non_nulls\x18\x03 \x01(\x05H\x00R\x0bminNonNulls\x88\x01\x01\x42\x10\n\x0e_min_non_nulls"\xa8\x02\n\tNAReplace\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12H\n\x0creplacements\x18\x03 \x03(\x0b\x32$.spark.connect.NAReplace.ReplacementR\x0creplacements\x1a\x8d\x01\n\x0bReplacement\x12>\n\told_value\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08oldValue\x12>\n\tnew_value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08newValue"X\n\x04ToDF\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames"\xfe\x02\n\x12WithColumnsRenamed\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12i\n\x12rename_columns_map\x18\x02 \x03(\x0b\x32\x37.spark.connect.WithColumnsRenamed.RenameColumnsMapEntryB\x02\x18\x01R\x10renameColumnsMap\x12\x42\n\x07renames\x18\x03 \x03(\x0b\x32(.spark.connect.WithColumnsRenamed.RenameR\x07renames\x1a\x43\n\x15RenameColumnsMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x45\n\x06Rename\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12 \n\x0cnew_col_name\x18\x02 \x01(\tR\nnewColName"w\n\x0bWithColumns\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x07\x61liases\x18\x02 \x03(\x0b\x32\x1f.spark.connect.Expression.AliasR\x07\x61liases"\x86\x01\n\rWithWatermark\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\nevent_time\x18\x02 \x01(\tR\teventTime\x12\'\n\x0f\x64\x65lay_threshold\x18\x03 \x01(\tR\x0e\x64\x65layThreshold"\x84\x01\n\x04Hint\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x39\n\nparameters\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\nparameters"\xc7\x02\n\x07Unpivot\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03ids\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x03ids\x12:\n\x06values\x18\x03 \x01(\x0b\x32\x1d.spark.connect.Unpivot.ValuesH\x00R\x06values\x88\x01\x01\x12\x30\n\x14variable_column_name\x18\x04 \x01(\tR\x12variableColumnName\x12*\n\x11value_column_name\x18\x05 \x01(\tR\x0fvalueColumnName\x1a;\n\x06Values\x12\x31\n\x06values\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x06valuesB\t\n\x07_values"z\n\tTranspose\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\rindex_columns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cindexColumns"}\n\x1dUnresolvedTableValuedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"j\n\x08ToSchema\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12/\n\x06schema\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema"\xcb\x01\n\x17RepartitionByExpression\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x0fpartition_exprs\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0epartitionExprs\x12*\n\x0enum_partitions\x18\x03 \x01(\x05H\x00R\rnumPartitions\x88\x01\x01\x42\x11\n\x0f_num_partitions"\xe8\x01\n\rMapPartitions\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x04\x66unc\x18\x02 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12"\n\nis_barrier\x18\x03 \x01(\x08H\x00R\tisBarrier\x88\x01\x01\x12"\n\nprofile_id\x18\x04 \x01(\x05H\x01R\tprofileId\x88\x01\x01\x42\r\n\x0b_is_barrierB\r\n\x0b_profile_id"\xfb\x04\n\x08GroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12J\n\x13sorting_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x12sortingExpressions\x12<\n\rinitial_input\x18\x05 \x01(\x0b\x32\x17.spark.connect.RelationR\x0cinitialInput\x12[\n\x1cinitial_grouping_expressions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x1ainitialGroupingExpressions\x12;\n\x18is_map_groups_with_state\x18\x07 \x01(\x08H\x00R\x14isMapGroupsWithState\x88\x01\x01\x12$\n\x0boutput_mode\x18\x08 \x01(\tH\x01R\noutputMode\x88\x01\x01\x12&\n\x0ctimeout_conf\x18\t \x01(\tH\x02R\x0btimeoutConf\x88\x01\x01\x42\x1b\n\x19_is_map_groups_with_stateB\x0e\n\x0c_output_modeB\x0f\n\r_timeout_conf"\x8e\x04\n\nCoGroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12W\n\x1ainput_grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18inputGroupingExpressions\x12-\n\x05other\x18\x03 \x01(\x0b\x32\x17.spark.connect.RelationR\x05other\x12W\n\x1aother_grouping_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18otherGroupingExpressions\x12\x42\n\x04\x66unc\x18\x05 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12U\n\x19input_sorting_expressions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x17inputSortingExpressions\x12U\n\x19other_sorting_expressions\x18\x07 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x17otherSortingExpressions"\xe5\x02\n\x16\x41pplyInPandasWithState\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12#\n\routput_schema\x18\x04 \x01(\tR\x0coutputSchema\x12!\n\x0cstate_schema\x18\x05 \x01(\tR\x0bstateSchema\x12\x1f\n\x0boutput_mode\x18\x06 \x01(\tR\noutputMode\x12!\n\x0ctimeout_conf\x18\x07 \x01(\tR\x0btimeoutConf"\xf4\x01\n$CommonInlineUserDefinedTableFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12<\n\x0bpython_udtf\x18\x04 \x01(\x0b\x32\x19.spark.connect.PythonUDTFH\x00R\npythonUdtfB\n\n\x08\x66unction"\xb1\x01\n\nPythonUDTF\x12=\n\x0breturn_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\nreturnType\x88\x01\x01\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVerB\x0e\n\x0c_return_type"\x97\x01\n!CommonInlineUserDefinedDataSource\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12O\n\x12python_data_source\x18\x02 \x01(\x0b\x32\x1f.spark.connect.PythonDataSourceH\x00R\x10pythonDataSourceB\r\n\x0b\x64\x61ta_source"K\n\x10PythonDataSource\x12\x18\n\x07\x63ommand\x18\x01 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x02 \x01(\tR\tpythonVer"\x88\x01\n\x0e\x43ollectMetrics\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x33\n\x07metrics\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07metrics"\x84\x03\n\x05Parse\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x38\n\x06\x66ormat\x18\x02 \x01(\x0e\x32 .spark.connect.Parse.ParseFormatR\x06\x66ormat\x12\x34\n\x06schema\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x06schema\x88\x01\x01\x12;\n\x07options\x18\x04 \x03(\x0b\x32!.spark.connect.Parse.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"X\n\x0bParseFormat\x12\x1c\n\x18PARSE_FORMAT_UNSPECIFIED\x10\x00\x12\x14\n\x10PARSE_FORMAT_CSV\x10\x01\x12\x15\n\x11PARSE_FORMAT_JSON\x10\x02\x42\t\n\x07_schema"\xdb\x03\n\x08\x41sOfJoin\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12\x37\n\nleft_as_of\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08leftAsOf\x12\x39\n\x0bright_as_of\x18\x04 \x01(\x0b\x32\x19.spark.connect.ExpressionR\trightAsOf\x12\x36\n\tjoin_expr\x18\x05 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08joinExpr\x12#\n\rusing_columns\x18\x06 \x03(\tR\x0cusingColumns\x12\x1b\n\tjoin_type\x18\x07 \x01(\tR\x08joinType\x12\x37\n\ttolerance\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\ttolerance\x12.\n\x13\x61llow_exact_matches\x18\t \x01(\x08R\x11\x61llowExactMatches\x12\x1c\n\tdirection\x18\n \x01(\tR\tdirection"\xe6\x01\n\x0bLateralJoin\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinTypeB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' + b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto"\xdd\x1c\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12\x34\n\x06set_op\x18\x06 \x01(\x0b\x32\x1b.spark.connect.SetOperationH\x00R\x05setOp\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12,\n\x05range\x18\x0f \x01(\x0b\x32\x14.spark.connect.RangeH\x00R\x05range\x12\x45\n\x0esubquery_alias\x18\x10 \x01(\x0b\x32\x1c.spark.connect.SubqueryAliasH\x00R\rsubqueryAlias\x12>\n\x0brepartition\x18\x11 \x01(\x0b\x32\x1a.spark.connect.RepartitionH\x00R\x0brepartition\x12*\n\x05to_df\x18\x12 \x01(\x0b\x32\x13.spark.connect.ToDFH\x00R\x04toDf\x12U\n\x14with_columns_renamed\x18\x13 \x01(\x0b\x32!.spark.connect.WithColumnsRenamedH\x00R\x12withColumnsRenamed\x12<\n\x0bshow_string\x18\x14 \x01(\x0b\x32\x19.spark.connect.ShowStringH\x00R\nshowString\x12)\n\x04\x64rop\x18\x15 \x01(\x0b\x32\x13.spark.connect.DropH\x00R\x04\x64rop\x12)\n\x04tail\x18\x16 \x01(\x0b\x32\x13.spark.connect.TailH\x00R\x04tail\x12?\n\x0cwith_columns\x18\x17 \x01(\x0b\x32\x1a.spark.connect.WithColumnsH\x00R\x0bwithColumns\x12)\n\x04hint\x18\x18 \x01(\x0b\x32\x13.spark.connect.HintH\x00R\x04hint\x12\x32\n\x07unpivot\x18\x19 \x01(\x0b\x32\x16.spark.connect.UnpivotH\x00R\x07unpivot\x12\x36\n\tto_schema\x18\x1a \x01(\x0b\x32\x17.spark.connect.ToSchemaH\x00R\x08toSchema\x12\x64\n\x19repartition_by_expression\x18\x1b \x01(\x0b\x32&.spark.connect.RepartitionByExpressionH\x00R\x17repartitionByExpression\x12\x45\n\x0emap_partitions\x18\x1c \x01(\x0b\x32\x1c.spark.connect.MapPartitionsH\x00R\rmapPartitions\x12H\n\x0f\x63ollect_metrics\x18\x1d \x01(\x0b\x32\x1d.spark.connect.CollectMetricsH\x00R\x0e\x63ollectMetrics\x12,\n\x05parse\x18\x1e \x01(\x0b\x32\x14.spark.connect.ParseH\x00R\x05parse\x12\x36\n\tgroup_map\x18\x1f \x01(\x0b\x32\x17.spark.connect.GroupMapH\x00R\x08groupMap\x12=\n\x0c\x63o_group_map\x18 \x01(\x0b\x32\x19.spark.connect.CoGroupMapH\x00R\ncoGroupMap\x12\x45\n\x0ewith_watermark\x18! \x01(\x0b\x32\x1c.spark.connect.WithWatermarkH\x00R\rwithWatermark\x12\x63\n\x1a\x61pply_in_pandas_with_state\x18" \x01(\x0b\x32%.spark.connect.ApplyInPandasWithStateH\x00R\x16\x61pplyInPandasWithState\x12<\n\x0bhtml_string\x18# \x01(\x0b\x32\x19.spark.connect.HtmlStringH\x00R\nhtmlString\x12X\n\x15\x63\x61\x63hed_local_relation\x18$ \x01(\x0b\x32".spark.connect.CachedLocalRelationH\x00R\x13\x63\x61\x63hedLocalRelation\x12[\n\x16\x63\x61\x63hed_remote_relation\x18% \x01(\x0b\x32#.spark.connect.CachedRemoteRelationH\x00R\x14\x63\x61\x63hedRemoteRelation\x12\x8e\x01\n)common_inline_user_defined_table_function\x18& \x01(\x0b\x32\x33.spark.connect.CommonInlineUserDefinedTableFunctionH\x00R$commonInlineUserDefinedTableFunction\x12\x37\n\nas_of_join\x18\' \x01(\x0b\x32\x17.spark.connect.AsOfJoinH\x00R\x08\x61sOfJoin\x12\x85\x01\n&common_inline_user_defined_data_source\x18( \x01(\x0b\x32\x30.spark.connect.CommonInlineUserDefinedDataSourceH\x00R!commonInlineUserDefinedDataSource\x12\x45\n\x0ewith_relations\x18) \x01(\x0b\x32\x1c.spark.connect.WithRelationsH\x00R\rwithRelations\x12\x38\n\ttranspose\x18* \x01(\x0b\x32\x18.spark.connect.TransposeH\x00R\ttranspose\x12w\n unresolved_table_valued_function\x18+ \x01(\x0b\x32,.spark.connect.UnresolvedTableValuedFunctionH\x00R\x1dunresolvedTableValuedFunction\x12?\n\x0clateral_join\x18, \x01(\x0b\x32\x1a.spark.connect.LateralJoinH\x00R\x0blateralJoin\x12\x30\n\x07\x66ill_na\x18Z \x01(\x0b\x32\x15.spark.connect.NAFillH\x00R\x06\x66illNa\x12\x30\n\x07\x64rop_na\x18[ \x01(\x0b\x32\x15.spark.connect.NADropH\x00R\x06\x64ropNa\x12\x34\n\x07replace\x18\\ \x01(\x0b\x32\x18.spark.connect.NAReplaceH\x00R\x07replace\x12\x36\n\x07summary\x18\x64 \x01(\x0b\x32\x1a.spark.connect.StatSummaryH\x00R\x07summary\x12\x39\n\x08\x63rosstab\x18\x65 \x01(\x0b\x32\x1b.spark.connect.StatCrosstabH\x00R\x08\x63rosstab\x12\x39\n\x08\x64\x65scribe\x18\x66 \x01(\x0b\x32\x1b.spark.connect.StatDescribeH\x00R\x08\x64\x65scribe\x12*\n\x03\x63ov\x18g \x01(\x0b\x32\x16.spark.connect.StatCovH\x00R\x03\x63ov\x12-\n\x04\x63orr\x18h \x01(\x0b\x32\x17.spark.connect.StatCorrH\x00R\x04\x63orr\x12L\n\x0f\x61pprox_quantile\x18i \x01(\x0b\x32!.spark.connect.StatApproxQuantileH\x00R\x0e\x61pproxQuantile\x12=\n\nfreq_items\x18j \x01(\x0b\x32\x1c.spark.connect.StatFreqItemsH\x00R\tfreqItems\x12:\n\tsample_by\x18k \x01(\x0b\x32\x1b.spark.connect.StatSampleByH\x00R\x08sampleBy\x12\x33\n\x07\x63\x61talog\x18\xc8\x01 \x01(\x0b\x32\x16.spark.connect.CatalogH\x00R\x07\x63\x61talog\x12\x35\n\textension\x18\xe6\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"\x8e\x01\n\x0eRelationCommon\x12#\n\x0bsource_info\x18\x01 \x01(\tB\x02\x18\x01R\nsourceInfo\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x12-\n\x06origin\x18\x03 \x01(\x0b\x32\x15.spark.connect.OriginR\x06originB\n\n\x08_plan_id"\xde\x03\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query\x12\x34\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.spark.connect.SQL.ArgsEntryB\x02\x18\x01R\x04\x61rgs\x12@\n\x08pos_args\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralB\x02\x18\x01R\x07posArgs\x12O\n\x0fnamed_arguments\x18\x04 \x03(\x0b\x32&.spark.connect.SQL.NamedArgumentsEntryR\x0enamedArguments\x12>\n\rpos_arguments\x18\x05 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cposArguments\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01\x1a\\\n\x13NamedArgumentsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12/\n\x05value\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05value:\x02\x38\x01"u\n\rWithRelations\x12+\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04root\x12\x37\n\nreferences\x18\x02 \x03(\x0b\x32\x17.spark.connect.RelationR\nreferences"\x97\x05\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x12!\n\x0cis_streaming\x18\x03 \x01(\x08R\x0bisStreaming\x1a\xc0\x01\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x45\n\x07options\x18\x02 \x03(\x0b\x32+.spark.connect.Read.NamedTable.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x95\x02\n\nDataSource\x12\x1b\n\x06\x66ormat\x18\x01 \x01(\tH\x00R\x06\x66ormat\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x12\x14\n\x05paths\x18\x04 \x03(\tR\x05paths\x12\x1e\n\npredicates\x18\x05 \x03(\tR\npredicates\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07_formatB\t\n\x07_schemaB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\x95\x05\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType\x12#\n\rusing_columns\x18\x05 \x03(\tR\x0cusingColumns\x12K\n\x0ejoin_data_type\x18\x06 \x01(\x0b\x32 .spark.connect.Join.JoinDataTypeH\x00R\x0cjoinDataType\x88\x01\x01\x1a\\\n\x0cJoinDataType\x12$\n\x0eis_left_struct\x18\x01 \x01(\x08R\x0cisLeftStruct\x12&\n\x0fis_right_struct\x18\x02 \x01(\x08R\risRightStruct"\xd0\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06\x12\x13\n\x0fJOIN_TYPE_CROSS\x10\x07\x42\x11\n\x0f_join_data_type"\xdf\x03\n\x0cSetOperation\x12\x36\n\nleft_input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\tleftInput\x12\x38\n\x0bright_input\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\nrightInput\x12\x45\n\x0bset_op_type\x18\x03 \x01(\x0e\x32%.spark.connect.SetOperation.SetOpTypeR\tsetOpType\x12\x1a\n\x06is_all\x18\x04 \x01(\x08H\x00R\x05isAll\x88\x01\x01\x12\x1c\n\x07\x62y_name\x18\x05 \x01(\x08H\x01R\x06\x62yName\x88\x01\x01\x12\x37\n\x15\x61llow_missing_columns\x18\x06 \x01(\x08H\x02R\x13\x61llowMissingColumns\x88\x01\x01"r\n\tSetOpType\x12\x1b\n\x17SET_OP_TYPE_UNSPECIFIED\x10\x00\x12\x19\n\x15SET_OP_TYPE_INTERSECT\x10\x01\x12\x15\n\x11SET_OP_TYPE_UNION\x10\x02\x12\x16\n\x12SET_OP_TYPE_EXCEPT\x10\x03\x42\t\n\x07_is_allB\n\n\x08_by_nameB\x18\n\x16_allow_missing_columns"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"K\n\x04Tail\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"\xfe\x05\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x41\n\ngroup_type\x18\x02 \x01(\x0e\x32".spark.connect.Aggregate.GroupTypeR\tgroupType\x12L\n\x14grouping_expressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12N\n\x15\x61ggregate_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x14\x61ggregateExpressions\x12\x34\n\x05pivot\x18\x05 \x01(\x0b\x32\x1e.spark.connect.Aggregate.PivotR\x05pivot\x12J\n\rgrouping_sets\x18\x06 \x03(\x0b\x32%.spark.connect.Aggregate.GroupingSetsR\x0cgroupingSets\x1ao\n\x05Pivot\x12+\n\x03\x63ol\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values\x1aL\n\x0cGroupingSets\x12<\n\x0cgrouping_set\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0bgroupingSet"\x9f\x01\n\tGroupType\x12\x1a\n\x16GROUP_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12GROUP_TYPE_GROUPBY\x10\x01\x12\x15\n\x11GROUP_TYPE_ROLLUP\x10\x02\x12\x13\n\x0fGROUP_TYPE_CUBE\x10\x03\x12\x14\n\x10GROUP_TYPE_PIVOT\x10\x04\x12\x1c\n\x18GROUP_TYPE_GROUPING_SETS\x10\x05"\xa0\x01\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x05order\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\x05order\x12 \n\tis_global\x18\x03 \x01(\x08H\x00R\x08isGlobal\x88\x01\x01\x42\x0c\n\n_is_global"\x8d\x01\n\x04\x44rop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x33\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07\x63olumns\x12!\n\x0c\x63olumn_names\x18\x03 \x03(\tR\x0b\x63olumnNames"\xf0\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12\x32\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08H\x00R\x10\x61llColumnsAsKeys\x88\x01\x01\x12.\n\x10within_watermark\x18\x04 \x01(\x08H\x01R\x0fwithinWatermark\x88\x01\x01\x42\x16\n\x14_all_columns_as_keysB\x13\n\x11_within_watermark"Y\n\rLocalRelation\x12\x17\n\x04\x64\x61ta\x18\x01 \x01(\x0cH\x00R\x04\x64\x61ta\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x42\x07\n\x05_dataB\t\n\x07_schema"H\n\x13\x43\x61\x63hedLocalRelation\x12\x12\n\x04hash\x18\x03 \x01(\tR\x04hashJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03R\x06userIdR\tsessionId"7\n\x14\x43\x61\x63hedRemoteRelation\x12\x1f\n\x0brelation_id\x18\x01 \x01(\tR\nrelationId"\x91\x02\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12.\n\x10with_replacement\x18\x04 \x01(\x08H\x00R\x0fwithReplacement\x88\x01\x01\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x01R\x04seed\x88\x01\x01\x12/\n\x13\x64\x65terministic_order\x18\x06 \x01(\x08R\x12\x64\x65terministicOrderB\x13\n\x11_with_replacementB\x07\n\x05_seed"\x91\x01\n\x05Range\x12\x19\n\x05start\x18\x01 \x01(\x03H\x00R\x05start\x88\x01\x01\x12\x10\n\x03\x65nd\x18\x02 \x01(\x03R\x03\x65nd\x12\x12\n\x04step\x18\x03 \x01(\x03R\x04step\x12*\n\x0enum_partitions\x18\x04 \x01(\x05H\x01R\rnumPartitions\x88\x01\x01\x42\x08\n\x06_startB\x11\n\x0f_num_partitions"r\n\rSubqueryAlias\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\x12\x1c\n\tqualifier\x18\x03 \x03(\tR\tqualifier"\x8e\x01\n\x0bRepartition\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12%\n\x0enum_partitions\x18\x02 \x01(\x05R\rnumPartitions\x12\x1d\n\x07shuffle\x18\x03 \x01(\x08H\x00R\x07shuffle\x88\x01\x01\x42\n\n\x08_shuffle"\x8e\x01\n\nShowString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate\x12\x1a\n\x08vertical\x18\x04 \x01(\x08R\x08vertical"r\n\nHtmlString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate"\\\n\x0bStatSummary\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1e\n\nstatistics\x18\x02 \x03(\tR\nstatistics"Q\n\x0cStatDescribe\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols"e\n\x0cStatCrosstab\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"`\n\x07StatCov\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"\x89\x01\n\x08StatCorr\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2\x12\x1b\n\x06method\x18\x04 \x01(\tH\x00R\x06method\x88\x01\x01\x42\t\n\x07_method"\xa4\x01\n\x12StatApproxQuantile\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12$\n\rprobabilities\x18\x03 \x03(\x01R\rprobabilities\x12%\n\x0erelative_error\x18\x04 \x01(\x01R\rrelativeError"}\n\rStatFreqItems\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x1d\n\x07support\x18\x03 \x01(\x01H\x00R\x07support\x88\x01\x01\x42\n\n\x08_support"\xb5\x02\n\x0cStatSampleBy\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03\x63ol\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x42\n\tfractions\x18\x03 \x03(\x0b\x32$.spark.connect.StatSampleBy.FractionR\tfractions\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x00R\x04seed\x88\x01\x01\x1a\x63\n\x08\x46raction\x12;\n\x07stratum\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x07stratum\x12\x1a\n\x08\x66raction\x18\x02 \x01(\x01R\x08\x66ractionB\x07\n\x05_seed"\x86\x01\n\x06NAFill\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x39\n\x06values\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x86\x01\n\x06NADrop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\'\n\rmin_non_nulls\x18\x03 \x01(\x05H\x00R\x0bminNonNulls\x88\x01\x01\x42\x10\n\x0e_min_non_nulls"\xa8\x02\n\tNAReplace\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12H\n\x0creplacements\x18\x03 \x03(\x0b\x32$.spark.connect.NAReplace.ReplacementR\x0creplacements\x1a\x8d\x01\n\x0bReplacement\x12>\n\told_value\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08oldValue\x12>\n\tnew_value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08newValue"X\n\x04ToDF\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames"\xfe\x02\n\x12WithColumnsRenamed\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12i\n\x12rename_columns_map\x18\x02 \x03(\x0b\x32\x37.spark.connect.WithColumnsRenamed.RenameColumnsMapEntryB\x02\x18\x01R\x10renameColumnsMap\x12\x42\n\x07renames\x18\x03 \x03(\x0b\x32(.spark.connect.WithColumnsRenamed.RenameR\x07renames\x1a\x43\n\x15RenameColumnsMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x45\n\x06Rename\x12\x19\n\x08\x63ol_name\x18\x01 \x01(\tR\x07\x63olName\x12 \n\x0cnew_col_name\x18\x02 \x01(\tR\nnewColName"w\n\x0bWithColumns\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x07\x61liases\x18\x02 \x03(\x0b\x32\x1f.spark.connect.Expression.AliasR\x07\x61liases"\x86\x01\n\rWithWatermark\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\nevent_time\x18\x02 \x01(\tR\teventTime\x12\'\n\x0f\x64\x65lay_threshold\x18\x03 \x01(\tR\x0e\x64\x65layThreshold"\x84\x01\n\x04Hint\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x39\n\nparameters\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\nparameters"\xc7\x02\n\x07Unpivot\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03ids\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x03ids\x12:\n\x06values\x18\x03 \x01(\x0b\x32\x1d.spark.connect.Unpivot.ValuesH\x00R\x06values\x88\x01\x01\x12\x30\n\x14variable_column_name\x18\x04 \x01(\tR\x12variableColumnName\x12*\n\x11value_column_name\x18\x05 \x01(\tR\x0fvalueColumnName\x1a;\n\x06Values\x12\x31\n\x06values\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x06valuesB\t\n\x07_values"z\n\tTranspose\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12>\n\rindex_columns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0cindexColumns"}\n\x1dUnresolvedTableValuedFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12\x37\n\targuments\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments"j\n\x08ToSchema\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12/\n\x06schema\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema"\xcb\x01\n\x17RepartitionByExpression\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x0fpartition_exprs\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0epartitionExprs\x12*\n\x0enum_partitions\x18\x03 \x01(\x05H\x00R\rnumPartitions\x88\x01\x01\x42\x11\n\x0f_num_partitions"\xe8\x01\n\rMapPartitions\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x04\x66unc\x18\x02 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12"\n\nis_barrier\x18\x03 \x01(\x08H\x00R\tisBarrier\x88\x01\x01\x12"\n\nprofile_id\x18\x04 \x01(\x05H\x01R\tprofileId\x88\x01\x01\x42\r\n\x0b_is_barrierB\r\n\x0b_profile_id"\xcd\x05\n\x08GroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12J\n\x13sorting_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x12sortingExpressions\x12<\n\rinitial_input\x18\x05 \x01(\x0b\x32\x17.spark.connect.RelationR\x0cinitialInput\x12[\n\x1cinitial_grouping_expressions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x1ainitialGroupingExpressions\x12;\n\x18is_map_groups_with_state\x18\x07 \x01(\x08H\x00R\x14isMapGroupsWithState\x88\x01\x01\x12$\n\x0boutput_mode\x18\x08 \x01(\tH\x01R\noutputMode\x88\x01\x01\x12&\n\x0ctimeout_conf\x18\t \x01(\tH\x02R\x0btimeoutConf\x88\x01\x01\x12?\n\x0cstate_schema\x18\n \x01(\x0b\x32\x17.spark.connect.DataTypeH\x03R\x0bstateSchema\x88\x01\x01\x42\x1b\n\x19_is_map_groups_with_stateB\x0e\n\x0c_output_modeB\x0f\n\r_timeout_confB\x0f\n\r_state_schema"\x8e\x04\n\nCoGroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12W\n\x1ainput_grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18inputGroupingExpressions\x12-\n\x05other\x18\x03 \x01(\x0b\x32\x17.spark.connect.RelationR\x05other\x12W\n\x1aother_grouping_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18otherGroupingExpressions\x12\x42\n\x04\x66unc\x18\x05 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12U\n\x19input_sorting_expressions\x18\x06 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x17inputSortingExpressions\x12U\n\x19other_sorting_expressions\x18\x07 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x17otherSortingExpressions"\xe5\x02\n\x16\x41pplyInPandasWithState\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc\x12#\n\routput_schema\x18\x04 \x01(\tR\x0coutputSchema\x12!\n\x0cstate_schema\x18\x05 \x01(\tR\x0bstateSchema\x12\x1f\n\x0boutput_mode\x18\x06 \x01(\tR\noutputMode\x12!\n\x0ctimeout_conf\x18\x07 \x01(\tR\x0btimeoutConf"\xf4\x01\n$CommonInlineUserDefinedTableFunction\x12#\n\rfunction_name\x18\x01 \x01(\tR\x0c\x66unctionName\x12$\n\rdeterministic\x18\x02 \x01(\x08R\rdeterministic\x12\x37\n\targuments\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\targuments\x12<\n\x0bpython_udtf\x18\x04 \x01(\x0b\x32\x19.spark.connect.PythonUDTFH\x00R\npythonUdtfB\n\n\x08\x66unction"\xb1\x01\n\nPythonUDTF\x12=\n\x0breturn_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\nreturnType\x88\x01\x01\x12\x1b\n\teval_type\x18\x02 \x01(\x05R\x08\x65valType\x12\x18\n\x07\x63ommand\x18\x03 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x04 \x01(\tR\tpythonVerB\x0e\n\x0c_return_type"\x97\x01\n!CommonInlineUserDefinedDataSource\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12O\n\x12python_data_source\x18\x02 \x01(\x0b\x32\x1f.spark.connect.PythonDataSourceH\x00R\x10pythonDataSourceB\r\n\x0b\x64\x61ta_source"K\n\x10PythonDataSource\x12\x18\n\x07\x63ommand\x18\x01 \x01(\x0cR\x07\x63ommand\x12\x1d\n\npython_ver\x18\x02 \x01(\tR\tpythonVer"\x88\x01\n\x0e\x43ollectMetrics\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x33\n\x07metrics\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07metrics"\x84\x03\n\x05Parse\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x38\n\x06\x66ormat\x18\x02 \x01(\x0e\x32 .spark.connect.Parse.ParseFormatR\x06\x66ormat\x12\x34\n\x06schema\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x06schema\x88\x01\x01\x12;\n\x07options\x18\x04 \x03(\x0b\x32!.spark.connect.Parse.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"X\n\x0bParseFormat\x12\x1c\n\x18PARSE_FORMAT_UNSPECIFIED\x10\x00\x12\x14\n\x10PARSE_FORMAT_CSV\x10\x01\x12\x15\n\x11PARSE_FORMAT_JSON\x10\x02\x42\t\n\x07_schema"\xdb\x03\n\x08\x41sOfJoin\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12\x37\n\nleft_as_of\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08leftAsOf\x12\x39\n\x0bright_as_of\x18\x04 \x01(\x0b\x32\x19.spark.connect.ExpressionR\trightAsOf\x12\x36\n\tjoin_expr\x18\x05 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x08joinExpr\x12#\n\rusing_columns\x18\x06 \x03(\tR\x0cusingColumns\x12\x1b\n\tjoin_type\x18\x07 \x01(\tR\x08joinType\x12\x37\n\ttolerance\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\ttolerance\x12.\n\x13\x61llow_exact_matches\x18\t \x01(\x08R\x11\x61llowExactMatches\x12\x1c\n\tdirection\x18\n \x01(\tR\tdirection"\xe6\x01\n\x0bLateralJoin\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinTypeB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3' ) _globals = globals() @@ -208,29 +208,29 @@ _globals["_MAPPARTITIONS"]._serialized_start = 12857 _globals["_MAPPARTITIONS"]._serialized_end = 13089 _globals["_GROUPMAP"]._serialized_start = 13092 - _globals["_GROUPMAP"]._serialized_end = 13727 - _globals["_COGROUPMAP"]._serialized_start = 13730 - _globals["_COGROUPMAP"]._serialized_end = 14256 - _globals["_APPLYINPANDASWITHSTATE"]._serialized_start = 14259 - _globals["_APPLYINPANDASWITHSTATE"]._serialized_end = 14616 - _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_start = 14619 - _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_end = 14863 - _globals["_PYTHONUDTF"]._serialized_start = 14866 - _globals["_PYTHONUDTF"]._serialized_end = 15043 - _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_start = 15046 - _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_end = 15197 - _globals["_PYTHONDATASOURCE"]._serialized_start = 15199 - _globals["_PYTHONDATASOURCE"]._serialized_end = 15274 - _globals["_COLLECTMETRICS"]._serialized_start = 15277 - _globals["_COLLECTMETRICS"]._serialized_end = 15413 - _globals["_PARSE"]._serialized_start = 15416 - _globals["_PARSE"]._serialized_end = 15804 + _globals["_GROUPMAP"]._serialized_end = 13809 + _globals["_COGROUPMAP"]._serialized_start = 13812 + _globals["_COGROUPMAP"]._serialized_end = 14338 + _globals["_APPLYINPANDASWITHSTATE"]._serialized_start = 14341 + _globals["_APPLYINPANDASWITHSTATE"]._serialized_end = 14698 + _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_start = 14701 + _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_end = 14945 + _globals["_PYTHONUDTF"]._serialized_start = 14948 + _globals["_PYTHONUDTF"]._serialized_end = 15125 + _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_start = 15128 + _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_end = 15279 + _globals["_PYTHONDATASOURCE"]._serialized_start = 15281 + _globals["_PYTHONDATASOURCE"]._serialized_end = 15356 + _globals["_COLLECTMETRICS"]._serialized_start = 15359 + _globals["_COLLECTMETRICS"]._serialized_end = 15495 + _globals["_PARSE"]._serialized_start = 15498 + _globals["_PARSE"]._serialized_end = 15886 _globals["_PARSE_OPTIONSENTRY"]._serialized_start = 4941 _globals["_PARSE_OPTIONSENTRY"]._serialized_end = 4999 - _globals["_PARSE_PARSEFORMAT"]._serialized_start = 15705 - _globals["_PARSE_PARSEFORMAT"]._serialized_end = 15793 - _globals["_ASOFJOIN"]._serialized_start = 15807 - _globals["_ASOFJOIN"]._serialized_end = 16282 - _globals["_LATERALJOIN"]._serialized_start = 16285 - _globals["_LATERALJOIN"]._serialized_end = 16515 + _globals["_PARSE_PARSEFORMAT"]._serialized_start = 15787 + _globals["_PARSE_PARSEFORMAT"]._serialized_end = 15875 + _globals["_ASOFJOIN"]._serialized_start = 15889 + _globals["_ASOFJOIN"]._serialized_end = 16364 + _globals["_LATERALJOIN"]._serialized_start = 16367 + _globals["_LATERALJOIN"]._serialized_end = 16597 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi index e5a6bff9e430a..371d735b9e871 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.pyi +++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi @@ -3409,6 +3409,7 @@ class GroupMap(google.protobuf.message.Message): IS_MAP_GROUPS_WITH_STATE_FIELD_NUMBER: builtins.int OUTPUT_MODE_FIELD_NUMBER: builtins.int TIMEOUT_CONF_FIELD_NUMBER: builtins.int + STATE_SCHEMA_FIELD_NUMBER: builtins.int @property def input(self) -> global___Relation: """(Required) Input relation for Group Map API: apply, applyInPandas.""" @@ -3447,6 +3448,9 @@ class GroupMap(google.protobuf.message.Message): """(Optional) The output mode of the function.""" timeout_conf: builtins.str """(Optional) Timeout configuration for groups that do not receive data for a while.""" + @property + def state_schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType: + """(Optional) The schema for the grouped state.""" def __init__( self, *, @@ -3469,6 +3473,7 @@ class GroupMap(google.protobuf.message.Message): is_map_groups_with_state: builtins.bool | None = ..., output_mode: builtins.str | None = ..., timeout_conf: builtins.str | None = ..., + state_schema: pyspark.sql.connect.proto.types_pb2.DataType | None = ..., ) -> None: ... def HasField( self, @@ -3477,6 +3482,8 @@ class GroupMap(google.protobuf.message.Message): b"_is_map_groups_with_state", "_output_mode", b"_output_mode", + "_state_schema", + b"_state_schema", "_timeout_conf", b"_timeout_conf", "func", @@ -3489,6 +3496,8 @@ class GroupMap(google.protobuf.message.Message): b"is_map_groups_with_state", "output_mode", b"output_mode", + "state_schema", + b"state_schema", "timeout_conf", b"timeout_conf", ], @@ -3500,6 +3509,8 @@ class GroupMap(google.protobuf.message.Message): b"_is_map_groups_with_state", "_output_mode", b"_output_mode", + "_state_schema", + b"_state_schema", "_timeout_conf", b"_timeout_conf", "func", @@ -3518,6 +3529,8 @@ class GroupMap(google.protobuf.message.Message): b"output_mode", "sorting_expressions", b"sorting_expressions", + "state_schema", + b"state_schema", "timeout_conf", b"timeout_conf", ], @@ -3534,6 +3547,10 @@ class GroupMap(google.protobuf.message.Message): self, oneof_group: typing_extensions.Literal["_output_mode", b"_output_mode"] ) -> typing_extensions.Literal["output_mode"] | None: ... @typing.overload + def WhichOneof( + self, oneof_group: typing_extensions.Literal["_state_schema", b"_state_schema"] + ) -> typing_extensions.Literal["state_schema"] | None: ... + @typing.overload def WhichOneof( self, oneof_group: typing_extensions.Literal["_timeout_conf", b"_timeout_conf"] ) -> typing_extensions.Literal["timeout_conf"] | None: ... diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py index 925eaaeabf605..59349a17886bb 100644 --- a/python/pyspark/sql/connect/session.py +++ b/python/pyspark/sql/connect/session.py @@ -200,34 +200,26 @@ def _apply_options(self, session: "SparkSession") -> None: for i in range(int(os.environ.get("PYSPARK_REMOTE_INIT_CONF_LEN", "0"))): init_opts = json.loads(os.environ[f"PYSPARK_REMOTE_INIT_CONF_{i}"]) + # The options are applied after session creation, + # so options ["spark.remote", "spark.master"] always take no effect. + invalid_opts = ["spark.remote", "spark.master"] + with self._lock: + opts = {} + + # Only attempts to set Spark SQL configurations. + # If the configurations are static, it might throw an exception so + # simply ignore it for now. for k, v in init_opts.items(): - # the options are applied after session creation, - # so following options always take no effect - if k not in [ - "spark.remote", - "spark.master", - ] and k.startswith("spark.sql."): - # Only attempts to set Spark SQL configurations. - # If the configurations are static, it might throw an exception so - # simply ignore it for now. - try: - session.conf.set(k, v) - except Exception as e: - logger.warn(f"Failed to set configuration {k} due to {e}") + if k not in invalid_opts and k.startswith("spark.sql."): + opts[k] = v - with self._lock: for k, v in self._options.items(): - # the options are applied after session creation, - # so following options always take no effect - if k not in [ - "spark.remote", - "spark.master", - ]: - try: - session.conf.set(k, v) - except Exception as e: - logger.warn(f"Failed to set configuration {k} due to {e}") + if k not in invalid_opts: + opts[k] = v + + if len(opts) > 0: + session.conf._set_all(configs=opts, silent=True) def create(self) -> "SparkSession": has_channel_builder = self._channel_builder is not None @@ -1111,6 +1103,16 @@ def creator(old_session_id: str) -> "SparkSession": return creator, (self._session_id,) + def _to_ddl(self, struct: StructType) -> str: + ddl = self._client._analyze(method="json_to_ddl", json_string=struct.json()).ddl_string + assert ddl is not None + return ddl + + def _parse_ddl(self, ddl: str) -> DataType: + dt = self._client._analyze(method="ddl_parse", ddl_string=ddl).parsed + assert dt is not None + return dt + SparkSession.__doc__ = PySparkSession.__doc__ diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py index 660f577f56f8b..394ac6bdb69c9 100644 --- a/python/pyspark/sql/dataframe.py +++ b/python/pyspark/sql/dataframe.py @@ -42,6 +42,7 @@ from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2 from pyspark.sql.merge import MergeIntoWriter from pyspark.sql.streaming import DataStreamWriter +from pyspark.sql.table_arg import TableArg from pyspark.sql.types import StructType, Row from pyspark.sql.utils import dispatch_df_method @@ -2713,11 +2714,10 @@ def lateralJoin( >>> customers.alias("c").lateralJoin( ... orders.alias("o") ... .where(sf.col("o.customer_id") == sf.col("c.customer_id").outer()) + ... .select("order_id", "order_date") ... .orderBy(sf.col("order_date").desc()) ... .limit(2), ... how="left" - ... ).select( - ... "c.customer_id", "name", "order_id", "order_date" ... ).orderBy("customer_id", "order_id").show() +-----------+-------+--------+----------+ |customer_id| name|order_id|order_date| @@ -6579,6 +6579,29 @@ def transpose(self, indexColumn: Optional["ColumnOrName"] = None) -> "DataFrame" """ ... + def asTable(self) -> TableArg: + """ + Converts the DataFrame into a `TableArg` object, which can be used as a table argument + in a user-defined table function (UDTF). + + After obtaining a TableArg from a DataFrame using this method, you can specify partitioning + and ordering for the table argument by calling methods such as `partitionBy`, `orderBy`, and + `withSinglePartition` on the `TableArg` instance. + - partitionBy(*cols): Partitions the data based on the specified columns. This method cannot + be called after withSinglePartition() has been called. + - orderBy(*cols): Orders the data within partitions based on the specified columns. + - withSinglePartition(): Indicates that the data should be treated as a single partition. + This method cannot be called after partitionBy() has been called. + + .. versionadded:: 4.0.0 + + Returns + ------- + :class:`TableArg` + A `TableArg` object representing a table argument. + """ + ... + def scalar(self) -> Column: """ Return a `Column` object for a SCALAR Subquery containing exactly one row and one column. diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 7b14598a0ef4e..0ff9b17d68a2c 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -15329,9 +15329,9 @@ def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: Parameters ---------- - str : :class:`~pyspark.sql.Column` or str + str : :class:`~pyspark.sql.Column` or column name target column to work on. - regexp : :class:`~pyspark.sql.Column` or str + regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. Returns @@ -15341,13 +15341,35 @@ def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: Examples -------- + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - >>> df.select(regexp_count('str', lit(r'\d+')).alias('d')).collect() - [Row(d=3)] - >>> df.select(regexp_count('str', lit(r'mmm')).alias('d')).collect() - [Row(d=0)] - >>> df.select(regexp_count("str", col("regexp")).alias('d')).collect() - [Row(d=3)] + >>> df.select('*', sf.regexp_count('str', sf.lit(r'\d+'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, \d+)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+----------------------+ + + >>> df.select('*', sf.regexp_count('str', sf.lit(r'mmm'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, mmm)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 0| + +---------+------+----------------------+ + + >>> df.select('*', sf.regexp_count("str", sf.col("regexp"))).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ + + >>> df.select('*', sf.regexp_count(sf.col('str'), "regexp")).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ """ return _invoke_function_over_columns("regexp_count", str, regexp) @@ -15364,7 +15386,7 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: Parameters ---------- - str : :class:`~pyspark.sql.Column` or str + str : :class:`~pyspark.sql.Column` or column name target column to work on. pattern : str regex pattern to apply. @@ -15376,17 +15398,36 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: :class:`~pyspark.sql.Column` matched value specified by `idx` group id. + See Also + -------- + :meth:`pyspark.sql.functions.regexp_extract_all` + Examples -------- + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('100-200',)], ['str']) - >>> df.select(regexp_extract('str', r'(\d+)-(\d+)', 1).alias('d')).collect() - [Row(d='100')] + >>> df.select('*', sf.regexp_extract('str', r'(\d+)-(\d+)', 1)).show() + +-------+-----------------------------------+ + | str|regexp_extract(str, (\d+)-(\d+), 1)| + +-------+-----------------------------------+ + |100-200| 100| + +-------+-----------------------------------+ + >>> df = spark.createDataFrame([('foo',)], ['str']) - >>> df.select(regexp_extract('str', r'(\d+)', 1).alias('d')).collect() - [Row(d='')] + >>> df.select('*', sf.regexp_extract('str', r'(\d+)', 1)).show() + +---+-----------------------------+ + |str|regexp_extract(str, (\d+), 1)| + +---+-----------------------------+ + |foo| | + +---+-----------------------------+ + >>> df = spark.createDataFrame([('aaaac',)], ['str']) - >>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect() - [Row(d='')] + >>> df.select('*', sf.regexp_extract(sf.col('str'), '(a+)(b)?(c)', 2)).show() + +-----+-----------------------------------+ + | str|regexp_extract(str, (a+)(b)?(c), 2)| + +-----+-----------------------------------+ + |aaaac| | + +-----+-----------------------------------+ """ from pyspark.sql.classic.column import _to_java_column @@ -15406,11 +15447,11 @@ def regexp_extract_all( Parameters ---------- - str : :class:`~pyspark.sql.Column` or str + str : :class:`~pyspark.sql.Column` or column name target column to work on. - regexp : :class:`~pyspark.sql.Column` or str + regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. - idx : int, optional + idx : :class:`~pyspark.sql.Column` or int, optional matched group id. Returns @@ -15418,17 +15459,48 @@ def regexp_extract_all( :class:`~pyspark.sql.Column` all strings in the `str` that match a Java regex and corresponding to the regex group index. + See Also + -------- + :meth:`pyspark.sql.functions.regexp_extract` + Examples -------- + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([("100-200, 300-400", r"(\d+)-(\d+)")], ["str", "regexp"]) - >>> df.select(regexp_extract_all('str', lit(r'(\d+)-(\d+)')).alias('d')).collect() - [Row(d=['100', '300'])] - >>> df.select(regexp_extract_all('str', lit(r'(\d+)-(\d+)'), 1).alias('d')).collect() - [Row(d=['100', '300'])] - >>> df.select(regexp_extract_all('str', lit(r'(\d+)-(\d+)'), 2).alias('d')).collect() - [Row(d=['200', '400'])] - >>> df.select(regexp_extract_all('str', col("regexp")).alias('d')).collect() - [Row(d=['100', '300'])] + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ + + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), sf.lit(1))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ + + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), 2)).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 2)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [200, 400]| + +----------------+-----------+---------------------------------------+ + + >>> df.select('*', sf.regexp_extract_all('str', sf.col("regexp"))).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ + + >>> df.select('*', sf.regexp_extract_all(sf.col('str'), "regexp")).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ """ if idx is None: return _invoke_function_over_columns("regexp_extract_all", str, regexp) @@ -15463,43 +15535,102 @@ def regexp_replace( Examples -------- - >>> df = spark.createDataFrame([("100-200", r"(\d+)", "--")], ["str", "pattern", "replacement"]) - >>> df.select(regexp_replace('str', r'(\d+)', '--').alias('d')).collect() - [Row(d='-----')] - >>> df.select(regexp_replace("str", col("pattern"), col("replacement")).alias('d')).collect() - [Row(d='-----')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("100-200", r"(\d+)", "--")], + ... ["str", "pattern", "replacement"] + ... ) + + Example 1: Replaces all the substrings in the `str` column name that + match the regex pattern `(\d+)` (one or more digits) with the replacement + string "--". + + >>> df.select('*', sf.regexp_replace('str', r'(\d+)', '--')).show() + +-------+-------+-----------+---------------------------------+ + | str|pattern|replacement|regexp_replace(str, (\d+), --, 1)| + +-------+-------+-----------+---------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+---------------------------------+ + + Example 2: Replaces all the substrings in the `str` Column that match + the regex pattern in the `pattern` Column with the string in the `replacement` + column. + + >>> df.select('*', \ + ... sf.regexp_replace(sf.col("str"), sf.col("pattern"), sf.col("replacement")) \ + ... ).show() + +-------+-------+-----------+--------------------------------------------+ + | str|pattern|replacement|regexp_replace(str, pattern, replacement, 1)| + +-------+-------+-----------+--------------------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+--------------------------------------------+ """ return _invoke_function_over_columns("regexp_replace", string, lit(pattern), lit(replacement)) @_try_remote_functions def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns the substring that matches the Java regex `regexp` within the string `str`. + r"""Returns the first substring that matches the Java regex `regexp` within the string `str`. If the regular expression is not found, the result is null. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str + str : :class:`~pyspark.sql.Column` or column name target column to work on. - regexp : :class:`~pyspark.sql.Column` or str + regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. Returns ------- :class:`~pyspark.sql.Column` - the substring that matches a Java regex within the string `str`. + the first substring that matches a Java regex within the string `str`. Examples -------- + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - >>> df.select(regexp_substr('str', lit(r'\d+')).alias('d')).collect() - [Row(d='1')] - >>> df.select(regexp_substr('str', lit(r'mmm')).alias('d')).collect() - [Row(d=None)] - >>> df.select(regexp_substr("str", col("regexp")).alias('d')).collect() - [Row(d='1')] + + Example 1: Returns the first substring in the `str` column name that + matches the regex pattern `(\d+)` (one or more digits). + + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'\d+'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, \d+)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+-----------------------+ + + Example 2: Returns the first substring in the `str` column name that + matches the regex pattern `(mmm)` (three consecutive 'm' characters) + + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'mmm'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, mmm)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| NULL| + +---------+------+-----------------------+ + + Example 3: Returns the first substring in the `str` column name that + matches the regex pattern in `regexp` Column. + + >>> df.select('*', sf.regexp_substr("str", sf.col("regexp"))).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ + + Example 4: Returns the first substring in the `str` Column that + matches the regex pattern in `regexp` column name. + + >>> df.select('*', sf.regexp_substr(sf.col("str"), "regexp")).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ """ return _invoke_function_over_columns("regexp_substr", str, regexp) @@ -15508,36 +15639,70 @@ def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: def regexp_instr( str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None ) -> Column: - r"""Extract all strings in the `str` that match the Java regex `regexp` + r"""Returns the position of the first substring in the `str` that match the Java regex `regexp` and corresponding to the regex group index. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str + str : :class:`~pyspark.sql.Column` or column name target column to work on. - regexp : :class:`~pyspark.sql.Column` or str + regexp : :class:`~pyspark.sql.Column` or column name regex pattern to apply. - idx : int, optional + idx : :class:`~pyspark.sql.Column` or int, optional matched group id. Returns ------- :class:`~pyspark.sql.Column` - all strings in the `str` that match a Java regex and corresponding to the regex group index. + the position of the first substring in the `str` that match a Java regex and corresponding + to the regex group index. Examples -------- + >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+(a|b|m)")], ["str", "regexp"]) - >>> df.select(regexp_instr('str', lit(r'\d+(a|b|m)')).alias('d')).collect() - [Row(d=1)] - >>> df.select(regexp_instr('str', lit(r'\d+(a|b|m)'), 1).alias('d')).collect() - [Row(d=1)] - >>> df.select(regexp_instr('str', lit(r'\d+(a|b|m)'), 2).alias('d')).collect() - [Row(d=1)] - >>> df.select(regexp_instr('str', col("regexp")).alias('d')).collect() - [Row(d=1)] + + Example 1: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'). + + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 0)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ + + Example 2: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'), + + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'), sf.lit(1))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 1)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ + + Example 3: Returns the position of the first substring in the `str` column name that + match the regex pattern in `regexp` Column. + + >>> df.select('*', sf.regexp_instr('str', sf.col("regexp"))).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ + + Example 4: Returns the position of the first substring in the `str` Column that + match the regex pattern in `regexp` column name. + + >>> df.select('*', sf.regexp_instr(sf.col("str"), "regexp")).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ """ if idx is None: return _invoke_function_over_columns("regexp_instr", str, regexp) diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py index bd12b41b34361..343a68bf010bf 100644 --- a/python/pyspark/sql/pandas/group_ops.py +++ b/python/pyspark/sql/pandas/group_ops.py @@ -36,7 +36,7 @@ ) from pyspark.sql.streaming.stateful_processor import StatefulProcessor, StatefulProcessorHandle from pyspark.sql.streaming.stateful_processor_util import TransformWithStateInPandasFuncMode -from pyspark.sql.types import StructType, _parse_datatype_string +from pyspark.sql.types import StructType if TYPE_CHECKING: from pyspark.sql.pandas._typing import ( @@ -348,9 +348,9 @@ def applyInPandasWithState( ] if isinstance(outputStructType, str): - outputStructType = cast(StructType, _parse_datatype_string(outputStructType)) + outputStructType = cast(StructType, self._df._session._parse_ddl(outputStructType)) if isinstance(stateStructType, str): - stateStructType = cast(StructType, _parse_datatype_string(stateStructType)) + stateStructType = cast(StructType, self._df._session._parse_ddl(stateStructType)) udf = pandas_udf( func, # type: ignore[call-overload] @@ -502,7 +502,7 @@ def transformWithStateInPandas( if initialState is not None: assert isinstance(initialState, GroupedData) if isinstance(outputStructType, str): - outputStructType = cast(StructType, _parse_datatype_string(outputStructType)) + outputStructType = cast(StructType, self._df._session._parse_ddl(outputStructType)) def handle_pre_init( statefulProcessorApiClient: StatefulProcessorApiClient, @@ -681,7 +681,7 @@ def transformWithStateWithInitStateUDF( return result if isinstance(outputStructType, str): - outputStructType = cast(StructType, _parse_datatype_string(outputStructType)) + outputStructType = cast(StructType, self._df._session._parse_ddl(outputStructType)) df = self._df diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 00fa60442b410..fc434cd16bfbd 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -58,7 +58,6 @@ _has_nulltype, _merge_type, _create_converter, - _parse_datatype_string, _from_numpy_type, ) from pyspark.errors.exceptions.captured import install_exception_handler @@ -1501,7 +1500,7 @@ def createDataFrame( # type: ignore[misc] ) if isinstance(schema, str): - schema = cast(Union[AtomicType, StructType, str], _parse_datatype_string(schema)) + schema = cast(Union[AtomicType, StructType, str], self._parse_ddl(schema)) elif isinstance(schema, (list, tuple)): # Must re-encode any unicode strings to be consistent with StructField names schema = [x.encode("utf-8") if not isinstance(x, str) else x for x in schema] @@ -2198,13 +2197,15 @@ def copyFromLocalToFs(self, local_path: str, dest_path: str) -> None: messageParameters={"feature": "SparkSession.copyFromLocalToFs"}, ) - @remote_only def interruptAll(self) -> List[str]: """ Interrupt all operations of this session currently running on the connected server. .. versionadded:: 3.5.0 + .. versionchanged:: 4.0.0 + Supports Spark Classic. + Returns ------- list of str @@ -2214,18 +2215,25 @@ def interruptAll(self) -> List[str]: ----- There is still a possibility of operation finishing just as it is interrupted. """ - raise PySparkRuntimeError( - errorClass="ONLY_SUPPORTED_WITH_SPARK_CONNECT", - messageParameters={"feature": "SparkSession.interruptAll"}, - ) + java_list = self._jsparkSession.interruptAll() + python_list = list() + + # Use iterator to manually iterate through Java list + java_iterator = java_list.iterator() + while java_iterator.hasNext(): + python_list.append(str(java_iterator.next())) + + return python_list - @remote_only def interruptTag(self, tag: str) -> List[str]: """ Interrupt all operations of this session with the given operation tag. .. versionadded:: 3.5.0 + .. versionchanged:: 4.0.0 + Supports Spark Classic. + Returns ------- list of str @@ -2235,10 +2243,15 @@ def interruptTag(self, tag: str) -> List[str]: ----- There is still a possibility of operation finishing just as it is interrupted. """ - raise PySparkRuntimeError( - errorClass="ONLY_SUPPORTED_WITH_SPARK_CONNECT", - messageParameters={"feature": "SparkSession.interruptTag"}, - ) + java_list = self._jsparkSession.interruptTag(tag) + python_list = list() + + # Use iterator to manually iterate through Java list + java_iterator = java_list.iterator() + while java_iterator.hasNext(): + python_list.append(str(java_iterator.next())) + + return python_list @remote_only def interruptOperation(self, op_id: str) -> List[str]: @@ -2338,6 +2351,12 @@ def clearTags(self) -> None: """ self._jsparkSession.clearTags() + def _to_ddl(self, struct: StructType) -> str: + return self._sc._to_ddl(struct) + + def _parse_ddl(self, ddl: str) -> DataType: + return self._sc._parse_ddl(ddl) + def _test() -> None: import os diff --git a/python/pyspark/sql/table_arg.py b/python/pyspark/sql/table_arg.py new file mode 100644 index 0000000000000..d4b5e1653c7a1 --- /dev/null +++ b/python/pyspark/sql/table_arg.py @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import TYPE_CHECKING + +from pyspark.sql.classic.column import _to_java_column, _to_seq +from pyspark.sql.tvf_argument import TableValuedFunctionArgument +from pyspark.sql.utils import get_active_spark_context + + +if TYPE_CHECKING: + from py4j.java_gateway import JavaObject + from pyspark.sql._typing import ColumnOrName + + +class TableArg(TableValuedFunctionArgument): + def __init__(self, j_table_arg: "JavaObject"): + self._j_table_arg = j_table_arg + + def partitionBy(self, *cols: "ColumnOrName") -> "TableArg": + sc = get_active_spark_context() + if len(cols) == 1 and isinstance(cols[0], list): + cols = cols[0] + j_cols = _to_seq(sc, cols, _to_java_column) + new_j_table_arg = self._j_table_arg.partitionBy(j_cols) + return TableArg(new_j_table_arg) + + def orderBy(self, *cols: "ColumnOrName") -> "TableArg": + sc = get_active_spark_context() + if len(cols) == 1 and isinstance(cols[0], list): + cols = cols[0] + j_cols = _to_seq(sc, cols, _to_java_column) + new_j_table_arg = self._j_table_arg.orderBy(j_cols) + return TableArg(new_j_table_arg) + + def withSinglePartition(self) -> "TableArg": + new_j_table_arg = self._j_table_arg.withSinglePartition() + return TableArg(new_j_table_arg) diff --git a/python/pyspark/sql/tests/connect/test_parity_job_cancellation.py b/python/pyspark/sql/tests/connect/test_parity_job_cancellation.py index c7f50495af15b..ddb4554afa55a 100644 --- a/python/pyspark/sql/tests/connect/test_parity_job_cancellation.py +++ b/python/pyspark/sql/tests/connect/test_parity_job_cancellation.py @@ -32,32 +32,10 @@ def func(target): create_thread=lambda target, session: threading.Thread(target=func, args=(target,)) ) - def test_interrupt_tag(self): - thread_ids = range(4) - self.check_job_cancellation( - lambda job_group: self.spark.addTag(job_group), - lambda job_group: self.spark.interruptTag(job_group), - thread_ids, - [i for i in thread_ids if i % 2 == 0], - [i for i in thread_ids if i % 2 != 0], - ) - self.spark.clearTags() - - def test_interrupt_all(self): - thread_ids = range(4) - self.check_job_cancellation( - lambda job_group: None, - lambda job_group: self.spark.interruptAll(), - thread_ids, - thread_ids, - [], - ) - self.spark.clearTags() - if __name__ == "__main__": import unittest - from pyspark.sql.tests.connect.test_parity_serde import * # noqa: F401 + from pyspark.sql.tests.connect.test_parity_job_cancellation import * # noqa: F401 try: import xmlrunner # type: ignore[import] diff --git a/python/pyspark/sql/tests/connect/test_parity_udtf.py b/python/pyspark/sql/tests/connect/test_parity_udtf.py index 6955e7377b4c4..6f4e4133335eb 100644 --- a/python/pyspark/sql/tests/connect/test_parity_udtf.py +++ b/python/pyspark/sql/tests/connect/test_parity_udtf.py @@ -76,6 +76,14 @@ def test_udtf_with_analyze_using_file(self): def test_udtf_access_spark_session(self): super().test_udtf_access_spark_session() + @unittest.skip("TODO(SPARK-50393): support df.asTable() in Spark Connect") + def test_df_asTable(self): + super().test_df_asTable() + + @unittest.skip("TODO(SPARK-50393): support df.asTable() in Spark Connect") + def test_df_asTable_chaining_methods(self): + super().test_df_asTable_chaining_methods() + def _add_pyfile(self, path): self.spark.addArtifacts(path, pyfile=True) diff --git a/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py b/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py index 183e93a871bd8..516a95a91a5e7 100644 --- a/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py +++ b/python/pyspark/sql/tests/pandas/test_pandas_transform_with_state.py @@ -314,6 +314,9 @@ def check_results(batch_df, batch_id): SimpleTTLStatefulProcessor(), check_results, False, "processingTime" ) + @unittest.skipIf( + "COVERAGE_PROCESS_START" in os.environ, "Flaky with coverage enabled, skipping for now." + ) def test_value_state_ttl_expiration(self): def check_results(batch_df, batch_id): if batch_id == 0: @@ -922,6 +925,9 @@ def test_transform_with_state_in_pandas_batch_query_initial_state(self): # This test covers mapState with TTL, an empty state variable # and additional test against initial state python runner + @unittest.skipIf( + "COVERAGE_PROCESS_START" in os.environ, "Flaky with coverage enabled, skipping for now." + ) def test_transform_with_map_state_metadata(self): checkpoint_path = tempfile.mktemp() diff --git a/python/pyspark/sql/tests/test_connect_compatibility.py b/python/pyspark/sql/tests/test_connect_compatibility.py index ef83dc3834d0c..25b8be1f9ac7a 100644 --- a/python/pyspark/sql/tests/test_connect_compatibility.py +++ b/python/pyspark/sql/tests/test_connect_compatibility.py @@ -266,9 +266,7 @@ def test_spark_session_compatibility(self): "addArtifacts", "clearProgressHandlers", "copyFromLocalToFs", - "interruptAll", "interruptOperation", - "interruptTag", "newSession", "registerProgressHandler", "removeProgressHandler", diff --git a/python/pyspark/sql/tests/test_job_cancellation.py b/python/pyspark/sql/tests/test_job_cancellation.py index 6fc404b0a0b3a..3f30f78808892 100644 --- a/python/pyspark/sql/tests/test_job_cancellation.py +++ b/python/pyspark/sql/tests/test_job_cancellation.py @@ -166,13 +166,35 @@ def get_outer_local_prop(): self.assertEqual(first, {"a", "b"}) self.assertEqual(second, {"a", "b", "c"}) + def test_interrupt_tag(self): + thread_ids = range(4) + self.check_job_cancellation( + lambda job_group: self.spark.addTag(job_group), + lambda job_group: self.spark.interruptTag(job_group), + thread_ids, + [i for i in thread_ids if i % 2 == 0], + [i for i in thread_ids if i % 2 != 0], + ) + self.spark.clearTags() + + def test_interrupt_all(self): + thread_ids = range(4) + self.check_job_cancellation( + lambda job_group: None, + lambda job_group: self.spark.interruptAll(), + thread_ids, + thread_ids, + [], + ) + self.spark.clearTags() + class JobCancellationTests(JobCancellationTestsMixin, ReusedSQLTestCase): pass if __name__ == "__main__": - from pyspark.sql.tests.test_session import * # noqa: F401 + from pyspark.sql.tests.test_job_cancellation import * # noqa: F401 try: import xmlrunner diff --git a/python/pyspark/sql/tests/test_session.py b/python/pyspark/sql/tests/test_session.py index 3fbc0be943e45..a22fe777e3c9a 100644 --- a/python/pyspark/sql/tests/test_session.py +++ b/python/pyspark/sql/tests/test_session.py @@ -227,7 +227,6 @@ def test_unsupported_api(self): (lambda: session.client, "client"), (session.addArtifacts, "addArtifact(s)"), (lambda: session.copyFromLocalToFs("", ""), "copyFromLocalToFs"), - (lambda: session.interruptTag(""), "interruptTag"), (lambda: session.interruptOperation(""), "interruptOperation"), ] diff --git a/python/pyspark/sql/tests/test_subquery.py b/python/pyspark/sql/tests/test_subquery.py index 0f431589b4616..99a22d7c29664 100644 --- a/python/pyspark/sql/tests/test_subquery.py +++ b/python/pyspark/sql/tests/test_subquery.py @@ -518,6 +518,28 @@ def test_lateral_join_with_single_column_select(self): self.spark.sql("""SELECT * FROM t1, LATERAL (SELECT t1.c1 + t2.c1 FROM t2)"""), ) + def test_lateral_join_with_star_expansion(self): + with self.tempView("t1", "t2"): + t1 = self.table1() + t2 = self.table2() + + assertDataFrameEqual( + t1.lateralJoin(self.spark.range(1).select().select(sf.col("*"))), + self.spark.sql("""SELECT * FROM t1, LATERAL (SELECT *)"""), + ) + assertDataFrameEqual( + t1.lateralJoin(t2.select(sf.col("*"))), + self.spark.sql("""SELECT * FROM t1, LATERAL (SELECT * FROM t2)"""), + ) + assertDataFrameEqual( + t1.lateralJoin(t2.select(sf.col("t1.*").outer(), sf.col("t2.*"))), + self.spark.sql("""SELECT * FROM t1, LATERAL (SELECT t1.*, t2.* FROM t2)"""), + ) + assertDataFrameEqual( + t1.lateralJoin(t2.alias("t1").select(sf.col("t1.*"))), + self.spark.sql("""SELECT * FROM t1, LATERAL (SELECT t1.* FROM t2 AS t1)"""), + ) + def test_lateral_join_with_different_join_types(self): with self.tempView("t1"): t1 = self.table1() @@ -572,6 +594,20 @@ def test_lateral_join_with_different_join_types(self): }, ) + def test_lateral_join_with_subquery_alias(self): + with self.tempView("t1"): + t1 = self.table1() + + assertDataFrameEqual( + t1.lateralJoin( + self.spark.range(1) + .select(sf.col("c1").outer(), sf.col("c2").outer()) + .toDF("a", "b") + .alias("s") + ).select("a", "b"), + self.spark.sql("""SELECT a, b FROM t1, LATERAL (SELECT c1, c2) s(a, b)"""), + ) + def test_lateral_join_with_correlated_predicates(self): with self.tempView("t1", "t2"): t1 = self.table1() @@ -661,9 +697,11 @@ def test_lateral_join_in_between_regular_joins(self): assertDataFrameEqual( t1.lateralJoin( - t2.where(sf.col("t1.c1").outer() == sf.col("t2.c1")).select(sf.col("c2")), + t2.where(sf.col("t1.c1").outer() == sf.col("t2.c1")) + .select(sf.col("c2")) + .alias("s"), how="left", - ).join(t1.alias("t3"), sf.col("t2.c2") == sf.col("t3.c2"), how="left"), + ).join(t1.alias("t3"), sf.col("s.c2") == sf.col("t3.c2"), how="left"), self.spark.sql( """ SELECT * FROM t1 diff --git a/python/pyspark/sql/tests/test_tvf.py b/python/pyspark/sql/tests/test_tvf.py index ea20cbf9b8f3a..c7274c0810cfb 100644 --- a/python/pyspark/sql/tests/test_tvf.py +++ b/python/pyspark/sql/tests/test_tvf.py @@ -65,11 +65,13 @@ def test_explode_with_lateral_join(self): assertDataFrameEqual( t1.lateralJoin( self.spark.tvf.explode(sf.array(sf.col("c1").outer(), sf.col("c2").outer())) - ).toDF("c1", "c2", "c3"), + .toDF("c3") + .alias("t2") + ), self.spark.sql("""SELECT * FROM t1, LATERAL EXPLODE(ARRAY(c1, c2)) t2(c3)"""), ) assertDataFrameEqual( - t3.lateralJoin(self.spark.tvf.explode(sf.col("c2").outer())).toDF("c1", "c2", "v"), + t3.lateralJoin(self.spark.tvf.explode(sf.col("c2").outer()).toDF("v").alias("t2")), self.spark.sql("""SELECT * FROM t3, LATERAL EXPLODE(c2) t2(v)"""), ) assertDataFrameEqual( @@ -127,12 +129,14 @@ def test_explode_outer_with_lateral_join(self): self.spark.tvf.explode_outer( sf.array(sf.col("c1").outer(), sf.col("c2").outer()) ) - ).toDF("c1", "c2", "c3"), + .toDF("c3") + .alias("t2") + ), self.spark.sql("""SELECT * FROM t1, LATERAL EXPLODE_OUTER(ARRAY(c1, c2)) t2(c3)"""), ) assertDataFrameEqual( - t3.lateralJoin(self.spark.tvf.explode_outer(sf.col("c2").outer())).toDF( - "c1", "c2", "v" + t3.lateralJoin( + self.spark.tvf.explode_outer(sf.col("c2").outer()).toDF("v").alias("t2") ), self.spark.sql("""SELECT * FROM t3, LATERAL EXPLODE_OUTER(c2) t2(v)"""), ) @@ -193,10 +197,10 @@ def test_inline_with_lateral_join(self): ) assertDataFrameEqual( array_struct.lateralJoin( - self.spark.tvf.inline(sf.col("arr").outer()), - sf.col("id") == sf.col("col1"), + self.spark.tvf.inline(sf.col("arr").outer()).toDF("k", "v").alias("t"), + sf.col("id") == sf.col("k"), "left", - ).toDF("id", "arr", "k", "v"), + ), self.spark.sql( """ SELECT * FROM array_struct LEFT JOIN LATERAL INLINE(arr) t(k, v) ON id = k @@ -252,10 +256,10 @@ def test_inline_outer_with_lateral_join(self): ) assertDataFrameEqual( array_struct.lateralJoin( - self.spark.tvf.inline_outer(sf.col("arr").outer()), - sf.col("id") == sf.col("col1"), + self.spark.tvf.inline_outer(sf.col("arr").outer()).toDF("k", "v").alias("t"), + sf.col("id") == sf.col("k"), "left", - ).toDF("id", "arr", "k", "v"), + ), self.spark.sql( """ SELECT * FROM array_struct LEFT JOIN LATERAL INLINE_OUTER(arr) t(k, v) ON id = k @@ -302,9 +306,9 @@ def test_json_tuple_with_lateral_join(self): sf.lit("f3"), sf.lit("f4"), sf.lit("f5"), - ) + ).alias("t2") ) - .select("key", "c0", "c1", "c2", "c3", "c4"), + .select("t1.key", "t2.*"), self.spark.sql( """ SELECT t1.key, t2.* FROM json_table t1, @@ -322,10 +326,10 @@ def test_json_tuple_with_lateral_join(self): sf.lit("f3"), sf.lit("f4"), sf.lit("f5"), - ) + ).alias("t2") ) - .where(sf.col("c0").isNotNull()) - .select("key", "c0", "c1", "c2", "c3", "c4"), + .where(sf.col("t2.c0").isNotNull()) + .select("t1.key", "t2.*"), self.spark.sql( """ SELECT t1.key, t2.* FROM json_table t1, @@ -485,8 +489,8 @@ def test_stack_with_lateral_join(self): sf.col("c1").outer(), sf.lit("Value"), sf.col("c2").outer(), - ) - ).select("col0", "col1"), + ).alias("t") + ).select("t.*"), self.spark.sql( """SELECT t.* FROM t1, LATERAL stack(2, 'Key', c1, 'Value', c2) t""" ), @@ -494,17 +498,19 @@ def test_stack_with_lateral_join(self): assertDataFrameEqual( t1.lateralJoin( self.spark.tvf.stack(sf.lit(1), sf.col("c1").outer(), sf.col("c2").outer()) - ).select("col0", "col1"), - self.spark.sql("""SELECT t.* FROM t1 JOIN LATERAL stack(1, c1, c2) t"""), + .toDF("x", "y") + .alias("t") + ).select("t.*"), + self.spark.sql("""SELECT t.* FROM t1 JOIN LATERAL stack(1, c1, c2) t(x, y)"""), ) assertDataFrameEqual( t1.join(t3, sf.col("t1.c1") == sf.col("t3.c1")) .lateralJoin( self.spark.tvf.stack( sf.lit(1), sf.col("t1.c2").outer(), sf.col("t3.c2").outer() - ) + ).alias("t") ) - .select("col0", "col1"), + .select("t.*"), self.spark.sql( """ SELECT t.* FROM t1 JOIN t3 ON t1.c1 = t3.c1 @@ -570,8 +576,8 @@ def test_variant_explode_with_lateral_join(self): assertDataFrameEqual( variant_table.alias("t1") - .lateralJoin(self.spark.tvf.variant_explode(sf.col("v").outer())) - .select("id", "pos", "key", "value"), + .lateralJoin(self.spark.tvf.variant_explode(sf.col("v").outer()).alias("t")) + .select("t1.id", "t.*"), self.spark.sql( """ SELECT t1.id, t.* FROM variant_table AS t1, @@ -629,8 +635,8 @@ def test_variant_explode_outer_with_lateral_join(self): assertDataFrameEqual( variant_table.alias("t1") - .lateralJoin(self.spark.tvf.variant_explode_outer(sf.col("v").outer())) - .select("id", "pos", "key", "value"), + .lateralJoin(self.spark.tvf.variant_explode_outer(sf.col("v").outer()).alias("t")) + .select("t1.id", "t.*"), self.spark.sql( """ SELECT t1.id, t.* FROM variant_table AS t1, diff --git a/python/pyspark/sql/tests/test_udtf.py b/python/pyspark/sql/tests/test_udtf.py index 2424f74f93d36..eca3ab0013b92 100644 --- a/python/pyspark/sql/tests/test_udtf.py +++ b/python/pyspark/sql/tests/test_udtf.py @@ -27,6 +27,7 @@ PySparkTypeError, AnalysisException, PySparkPicklingError, + IllegalArgumentException, ) from pyspark.util import PythonEvalType from pyspark.sql.functions import ( @@ -1055,6 +1056,15 @@ def test_udtf_with_table_argument_query(self): [Row(a=6), Row(a=7)], ) + def test_df_asTable(self): + func = self.udtf_for_table_argument() + self.spark.udtf.register("test_udtf", func) + df = self.spark.range(8) + assertDataFrameEqual( + func(df.asTable()), + self.spark.sql("SELECT * FROM test_udtf(TABLE (SELECT id FROM range(0, 8)))"), + ) + def udtf_for_table_argument(self): class TestUDTF: def eval(self, row: Row): @@ -1064,6 +1074,108 @@ def eval(self, row: Row): func = udtf(TestUDTF, returnType="a: int") return func + def test_df_asTable_chaining_methods(self): + class TestUDTF: + def eval(self, row: Row): + yield row["key"], row["value"] + + func = udtf(TestUDTF, returnType="key: int, value: string") + df = self.spark.createDataFrame( + [(1, "a", 3), (1, "b", 3), (2, "c", 4), (2, "d", 4)], ["key", "value", "number"] + ) + assertDataFrameEqual( + func(df.asTable().partitionBy("key").orderBy(df.value)), + [ + Row(key=1, value="a"), + Row(key=1, value="b"), + Row(key=2, value="c"), + Row(key=2, value="d"), + ], + checkRowOrder=True, + ) + assertDataFrameEqual( + func(df.asTable().partitionBy(["key", "number"]).orderBy(df.value)), + [ + Row(key=1, value="a"), + Row(key=1, value="b"), + Row(key=2, value="c"), + Row(key=2, value="d"), + ], + checkRowOrder=True, + ) + assertDataFrameEqual( + func(df.asTable().partitionBy("key").orderBy(df.value.desc())), + [ + Row(key=1, value="b"), + Row(key=1, value="a"), + Row(key=2, value="d"), + Row(key=2, value="c"), + ], + checkRowOrder=True, + ) + assertDataFrameEqual( + func(df.asTable().partitionBy("key").orderBy(["number", "value"])), + [ + Row(key=1, value="a"), + Row(key=1, value="b"), + Row(key=2, value="c"), + Row(key=2, value="d"), + ], + checkRowOrder=True, + ) + assertDataFrameEqual( + func(df.asTable().withSinglePartition()), + [ + Row(key=1, value="a"), + Row(key=1, value="b"), + Row(key=2, value="c"), + Row(key=2, value="d"), + ], + ) + + assertDataFrameEqual( + func(df.asTable().withSinglePartition().orderBy("value")), + [ + Row(key=1, value="a"), + Row(key=1, value="b"), + Row(key=2, value="c"), + Row(key=2, value="d"), + ], + ) + + with self.assertRaisesRegex( + IllegalArgumentException, + r"Cannot call withSinglePartition\(\) after partitionBy\(\)" + r" or withSinglePartition\(\) has been called", + ): + df.asTable().partitionBy(df.key).withSinglePartition() + + with self.assertRaisesRegex( + IllegalArgumentException, + r"Cannot call partitionBy\(\) after partitionBy\(\)" + r" or withSinglePartition\(\) has been called", + ): + df.asTable().withSinglePartition().partitionBy(df.key) + + with self.assertRaisesRegex( + IllegalArgumentException, + r"Please call partitionBy\(\) or withSinglePartition\(\) before orderBy\(\)", + ): + df.asTable().orderBy(df.key) + + with self.assertRaisesRegex( + IllegalArgumentException, + r"Please call partitionBy\(\) or withSinglePartition\(\) before orderBy\(\)", + ): + df.asTable().partitionBy().orderBy(df.key) + + with self.assertRaisesRegex( + IllegalArgumentException, + r"Cannot call partitionBy\(\) after partitionBy\(\)" + r" or withSinglePartition\(\) has been called", + ): + df.asTable().partitionBy(df.key).partitionBy() + def test_udtf_with_int_and_table_argument_query(self): class TestUDTF: def eval(self, i: int, row: Row): diff --git a/python/pyspark/sql/tvf_argument.py b/python/pyspark/sql/tvf_argument.py new file mode 100644 index 0000000000000..cb373d35d9ec2 --- /dev/null +++ b/python/pyspark/sql/tvf_argument.py @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class TableValuedFunctionArgument: + """ + Base class for arguments passed to Table Valued Functions. + + This class is intentionally left empty and serves as a marker to achieve + parity with the Scala `TableValuedFunctionArgument` trait. + """ + + pass diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py index 303c0c3c8bd07..f40a8bf62b290 100644 --- a/python/pyspark/sql/types.py +++ b/python/pyspark/sql/types.py @@ -1563,16 +1563,9 @@ def toDDL(self) -> str: session = SparkSession.getActiveSession() assert session is not None - return session._client._analyze( # type: ignore[return-value] - method="json_to_ddl", json_string=self.json() - ).ddl_string - + return session._to_ddl(self) else: - from py4j.java_gateway import JVMView - - sc = get_active_spark_context() - assert sc._jvm is not None - return cast(JVMView, sc._jvm).PythonSQLUtils.jsonToDDL(self.json()) + return get_active_spark_context()._to_ddl(self) class VariantType(AtomicType): @@ -1907,43 +1900,9 @@ def _parse_datatype_string(s: str) -> DataType: if is_remote(): from pyspark.sql.connect.session import SparkSession - return cast( - DataType, - SparkSession.active()._client._analyze(method="ddl_parse", ddl_string=s).parsed, - ) - + return SparkSession.active()._parse_ddl(s) else: - from py4j.java_gateway import JVMView - - sc = get_active_spark_context() - - def from_ddl_schema(type_str: str) -> DataType: - return _parse_datatype_json_string( - cast(JVMView, sc._jvm) - .org.apache.spark.sql.types.StructType.fromDDL(type_str) - .json() - ) - - def from_ddl_datatype(type_str: str) -> DataType: - return _parse_datatype_json_string( - cast(JVMView, sc._jvm) - .org.apache.spark.sql.api.python.PythonSQLUtils.parseDataType(type_str) - .json() - ) - - try: - # DDL format, "fieldname datatype, fieldname datatype". - return from_ddl_schema(s) - except Exception as e: - try: - # For backwards compatibility, "integer", "struct" and etc. - return from_ddl_datatype(s) - except BaseException: - try: - # For backwards compatibility, "fieldname: datatype, fieldname: datatype" case. - return from_ddl_datatype("struct<%s>" % s.strip()) - except BaseException: - raise e + return get_active_spark_context()._parse_ddl(s) def _parse_datatype_json_string(json_string: str) -> DataType: diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py index 12c3e985fb89d..3ea32349d81d2 100644 --- a/python/pyspark/sql/udtf.py +++ b/python/pyspark/sql/udtf.py @@ -32,7 +32,7 @@ if TYPE_CHECKING: from py4j.java_gateway import JavaObject - from pyspark.sql._typing import ColumnOrName + from pyspark.sql._typing import TVFArgumentOrName from pyspark.sql.dataframe import DataFrame from pyspark.sql.session import SparkSession @@ -372,22 +372,41 @@ def _create_judtf(self, func: Type) -> "JavaObject": )(self._name, wrapped_func, jdt, self.evalType, self.deterministic) return judtf - def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> "DataFrame": + def __call__(self, *args: "TVFArgumentOrName", **kwargs: "TVFArgumentOrName") -> "DataFrame": from pyspark.sql.classic.column import _to_java_column, _to_seq from pyspark.sql import DataFrame, SparkSession + from pyspark.sql.table_arg import TableArg spark = SparkSession._getActiveSessionOrCreate() sc = spark.sparkContext assert sc._jvm is not None - jcols = [_to_java_column(arg) for arg in args] + [ - sc._jvm.PythonSQLUtils.namedArgumentExpression(key, _to_java_column(value)) - for key, value in kwargs.items() - ] + # Process positional arguments + jargs = [] + for arg in args: + if isinstance(arg, TableArg): + # If the argument is a TableArg, get the Java TableArg object + jargs.append(arg._j_table_arg) + else: + # Otherwise, convert it to a Java column + jargs.append(_to_java_column(arg)) # type: ignore[arg-type] + + # Process keyword arguments + jkwargs = [] + for key, value in kwargs.items(): + if isinstance(value, TableArg): + # If the value is a TableArg, get the Java TableArg object + j_arg = value._j_table_arg + else: + # Otherwise, convert it to a Java column + j_arg = _to_java_column(value) # type: ignore[arg-type] + # Create a named argument expression + j_named_arg = sc._jvm.PythonSQLUtils.namedArgumentExpression(key, j_arg) + jkwargs.append(j_named_arg) judtf = self._judtf - jPythonUDTF = judtf.apply(spark._jsparkSession, _to_seq(sc, jcols)) + jPythonUDTF = judtf.apply(spark._jsparkSession, _to_seq(sc, jargs + jkwargs)) return DataFrame(jPythonUDTF, spark) def asDeterministic(self) -> "UserDefinedTableFunction": diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py index 3cacc5b9d021a..fbe0b274b496c 100644 --- a/python/pyspark/sql/utils.py +++ b/python/pyspark/sql/utils.py @@ -118,18 +118,34 @@ def require_test_compiled() -> None: def require_minimum_plotly_version() -> None: """Raise ImportError if plotly is not installed""" + from pyspark.loose_version import LooseVersion + minimum_plotly_version = "4.8" try: - import plotly # noqa: F401 + import plotly + + have_plotly = True except ImportError as error: + have_plotly = False + raised_error = error + if not have_plotly: raise PySparkImportError( errorClass="PACKAGE_NOT_INSTALLED", messageParameters={ - "package_name": "plotly", + "package_name": "Plotly", "minimum_version": str(minimum_plotly_version), }, - ) from error + ) from raised_error + if LooseVersion(plotly.__version__) < LooseVersion(minimum_plotly_version): + raise PySparkImportError( + errorClass="UNSUPPORTED_PACKAGE_VERSION", + messageParameters={ + "package_name": "Plotly", + "minimum_version": str(minimum_plotly_version), + "current_version": str(plotly.__version__), + }, + ) class ForeachBatchFunction: diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 211c6c93b9674..814a3e1c595b8 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -37,7 +37,7 @@ io.fabric8 - volcano-model-v1beta1 + volcano-model ${kubernetes-client.version} @@ -105,11 +105,6 @@ test - - io.fabric8 - kubernetes-httpclient-okhttp - ${kubernetes-client.version} - io.fabric8 kubernetes-client diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala index 2c28dc380046c..557bf01cbdbae 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala @@ -24,10 +24,7 @@ import com.google.common.io.Files import io.fabric8.kubernetes.client.{ConfigBuilder, KubernetesClient, KubernetesClientBuilder} import io.fabric8.kubernetes.client.Config.KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY import io.fabric8.kubernetes.client.Config.autoConfigure -import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory import io.fabric8.kubernetes.client.utils.Utils.getSystemPropertyOrEnvVar -import okhttp3.Dispatcher -import okhttp3.OkHttpClient import org.apache.spark.SparkConf import org.apache.spark.annotation.{DeveloperApi, Since, Stable} @@ -35,7 +32,6 @@ import org.apache.spark.deploy.k8s.Config._ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.internal.LogKeys.K8S_CONTEXT import org.apache.spark.internal.config.ConfigEntry -import org.apache.spark.util.ThreadUtils /** * :: DeveloperApi :: @@ -78,10 +74,6 @@ object SparkKubernetesClientFactory extends Logging { .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_KEY_FILE_CONF_SUFFIX") val clientCertFile = sparkConf .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX") - // TODO(SPARK-37687): clean up direct usage of OkHttpClient, see also: - // https://github.com/fabric8io/kubernetes-client/issues/3547 - val dispatcher = new Dispatcher( - ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher")) // Allow for specifying a context used to auto-configure from the users K8S config file val kubeContext = sparkConf.get(KUBERNETES_CONTEXT).filter(_.nonEmpty) @@ -117,17 +109,9 @@ object SparkKubernetesClientFactory extends Logging { }.withOption(namespace) { (ns, configBuilder) => configBuilder.withNamespace(ns) }.build() - val factoryWithCustomDispatcher = new OkHttpClientFactory() { - override protected def additionalConfig(builder: OkHttpClient.Builder): Unit = { - builder.dispatcher(dispatcher) - } - } logDebug("Kubernetes client config: " + new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config)) - new KubernetesClientBuilder() - .withHttpClientFactory(factoryWithCustomDispatcher) - .withConfig(config) - .build() + new KubernetesClientBuilder().withConfig(config).build() } private implicit class OptionConfigurableConfigBuilder(val configBuilder: ConfigBuilder) diff --git a/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala b/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala index 314550713ef16..046d268df4e29 100644 --- a/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala +++ b/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala @@ -17,8 +17,8 @@ package org.apache.spark.deploy.k8s.features import io.fabric8.kubernetes.api.model._ +import io.fabric8.volcano.api.model.scheduling.v1beta1.{PodGroup, PodGroupSpec} import io.fabric8.volcano.client.DefaultVolcanoClient -import io.fabric8.volcano.scheduling.v1beta1.{PodGroup, PodGroupSpec} import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf, KubernetesExecutorConf, SparkPod} import org.apache.spark.internal.Logging diff --git a/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala b/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala index dab414e0e19e7..c92164993ef80 100644 --- a/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features import java.io.File -import io.fabric8.volcano.scheduling.v1beta1.PodGroup +import io.fabric8.volcano.api.model.scheduling.v1beta1.PodGroup import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s._ diff --git a/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala b/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala index 2f414b72ee195..4a9dc135ecfdc 100644 --- a/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala @@ -28,8 +28,8 @@ import scala.concurrent.Future import scala.jdk.CollectionConverters._ import io.fabric8.kubernetes.api.model.{HasMetadata, Pod, Quantity} +import io.fabric8.volcano.api.model.scheduling.v1beta1.{Queue, QueueBuilder} import io.fabric8.volcano.client.VolcanoClient -import io.fabric8.volcano.scheduling.v1beta1.{Queue, QueueBuilder} import org.scalatest.BeforeAndAfterEach import org.scalatest.concurrent.Eventually diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index b593d73f85721..911ce2e27f96d 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -820,6 +820,7 @@ private[yarn] class YarnAllocator( logInfo(log"Skip launching executorRunnable as running executors count: " + log"${MDC(LogKeys.COUNT, rpRunningExecs)} reached target executors count: " + log"${MDC(LogKeys.NUM_EXECUTOR_TARGET, getOrUpdateTargetNumExecutorsForRPId(rpId))}.") + internalReleaseContainer(container) } } } diff --git a/scalastyle-config.xml b/scalastyle-config.xml index 05b3f6a268985..7e64dc9be6731 100644 --- a/scalastyle-config.xml +++ b/scalastyle-config.xml @@ -460,33 +460,6 @@ This file is divided into 3 sections: -1,0,1,2,3 - - Objects.toStringHelper - Avoid using Object.toStringHelper. Use ToStringBuilder instead. - - - - Files\.createTempDir\( - Avoid using com.google.common.io.Files.createTempDir due to CVE-2020-8908. - Use org.apache.spark.util.Utils.createTempDir instead. - - - - - FileBackedOutputStream - Avoid using FileBackedOutputStream due to CVE-2023-2976. - - - - AtomicDoubleArray - Avoid using AtomicDoubleArray due to CVE-2018-10237. - - - - CompoundOrdering - Avoid using CompoundOrdering due to CVE-2018-10237. - - byteCountToDisplaySize Use Utils.bytesToString instead of byteCountToDisplaySize for consistency. diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index eeebe89de8ff1..91a267364216c 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -365,6 +365,7 @@ REAL: 'REAL'; RECORDREADER: 'RECORDREADER'; RECORDWRITER: 'RECORDWRITER'; RECOVER: 'RECOVER'; +RECURSIVE: 'RECURSIVE'; REDUCE: 'REDUCE'; REFERENCES: 'REFERENCES'; REFRESH: 'REFRESH'; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index a5d217486bf20..3ca120da98dd4 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -509,7 +509,7 @@ describeColName ; ctes - : WITH namedQuery (COMMA namedQuery)* + : WITH RECURSIVE? namedQuery (COMMA namedQuery)* ; namedQuery @@ -2118,6 +2118,7 @@ nonReserved | RECORDREADER | RECORDWRITER | RECOVER + | RECURSIVE | REDUCE | REFERENCES | REFRESH diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala index 50ef61d4a7a19..f13b340e5e9c8 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala @@ -137,7 +137,7 @@ class TypedColumn[-T, U](node: ColumnNode, private[sql] val encoder: Encoder[U]) * @since 1.3.0 */ @Stable -class Column(val node: ColumnNode) extends Logging { +class Column(val node: ColumnNode) extends Logging with TableValuedFunctionArgument { private[sql] def this(name: String, planId: Option[Long]) = this(withOrigin { name match { case "*" => internal.UnresolvedStar(None, planId) diff --git a/sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala b/sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala new file mode 100644 index 0000000000000..f99c4ecd48554 --- /dev/null +++ b/sql/api/src/main/scala/org/apache/spark/sql/TableValuedFunctionArgument.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +trait TableValuedFunctionArgument diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala index 9c043320dc812..8dff1ceccfcfe 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/RebaseDateTime.scala @@ -22,7 +22,7 @@ import java.time.temporal.ChronoField import java.util.{Calendar, TimeZone} import java.util.Calendar.{DAY_OF_MONTH, DST_OFFSET, ERA, HOUR_OF_DAY, MINUTE, MONTH, SECOND, YEAR, ZONE_OFFSET} -import scala.collection.mutable.AnyRefMap +import scala.collection.mutable.HashMap import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} @@ -285,12 +285,12 @@ object RebaseDateTime { } // Loads rebasing info from an JSON file. JSON records in the files should conform to - // `JsonRebaseRecord`. AnyRefMap is used here instead of Scala's immutable map because - // it is 2 times faster in DateTimeRebaseBenchmark. - private[sql] def loadRebaseRecords(fileName: String): AnyRefMap[String, RebaseInfo] = { + // `JsonRebaseRecord`. Mutable HashMap is used here instead of AnyRefMap due to SPARK-49491. + private[sql] def loadRebaseRecords(fileName: String): HashMap[String, RebaseInfo] = { val file = SparkClassUtils.getSparkClassLoader.getResource(fileName) val jsonRebaseRecords = mapper.readValue[Seq[JsonRebaseRecord]](file) - val anyRefMap = new AnyRefMap[String, RebaseInfo]((3 * jsonRebaseRecords.size) / 2) + val hashMap = new HashMap[String, RebaseInfo] + hashMap.sizeHint(jsonRebaseRecords.size) jsonRebaseRecords.foreach { jsonRecord => val rebaseInfo = RebaseInfo(jsonRecord.switches, jsonRecord.diffs) var i = 0 @@ -299,9 +299,9 @@ object RebaseDateTime { rebaseInfo.diffs(i) = rebaseInfo.diffs(i) * MICROS_PER_SECOND i += 1 } - anyRefMap.update(jsonRecord.tz, rebaseInfo) + hashMap.update(jsonRecord.tz, rebaseInfo) } - anyRefMap + hashMap } /** @@ -313,7 +313,7 @@ object RebaseDateTime { */ private val gregJulianRebaseMap = loadRebaseRecords("gregorian-julian-rebase-micros.json") - private def getLastSwitchTs(rebaseMap: AnyRefMap[String, RebaseInfo]): Long = { + private def getLastSwitchTs(rebaseMap: HashMap[String, RebaseInfo]): Long = { val latestTs = rebaseMap.values.map(_.switches.last).max require( rebaseMap.values.forall(_.diffs.last == 0), @@ -404,7 +404,7 @@ object RebaseDateTime { if (micros >= lastSwitchGregorianTs) { micros } else { - val rebaseRecord = gregJulianRebaseMap.getOrNull(timeZoneId) + val rebaseRecord = gregJulianRebaseMap.get(timeZoneId).orNull if (rebaseRecord == null || micros < rebaseRecord.switches(0)) { rebaseGregorianToJulianMicros(TimeZone.getTimeZone(timeZoneId), micros) } else { @@ -526,7 +526,7 @@ object RebaseDateTime { if (micros >= lastSwitchJulianTs) { micros } else { - val rebaseRecord = julianGregRebaseMap.getOrNull(timeZoneId) + val rebaseRecord = julianGregRebaseMap.get(timeZoneId).orNull if (rebaseRecord == null || micros < rebaseRecord.switches(0)) { rebaseJulianToGregorianMicros(TimeZone.getTimeZone(timeZoneId), micros) } else { diff --git a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala index d4e590629921c..f33a49e686a59 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala @@ -147,6 +147,18 @@ case class StructField( if (metadata.contains("comment")) Option(metadata.getString("comment")) else None } + /** + * Return the default value of this StructField. This is used for storing the default value of a + * function parameter. + */ + private[sql] def getDefault(): Option[String] = { + if (metadata.contains("default")) { + Option(metadata.getString("default")) + } else { + None + } + } + /** * Updates the StructField with a new current default value. */ diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java index bd2dec9e27be0..49afcd5ebcd50 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/util/V2ExpressionSQLBuilder.java @@ -118,7 +118,7 @@ yield visitBinaryArithmetic( "COT", "ASIN", "ASINH", "ACOS", "ACOSH", "ATAN", "ATANH", "ATAN2", "CBRT", "DEGREES", "RADIANS", "SIGN", "WIDTH_BUCKET", "SUBSTRING", "UPPER", "LOWER", "TRANSLATE", "DATE_ADD", "DATE_DIFF", "TRUNC", "AES_ENCRYPT", "AES_DECRYPT", "SHA1", "SHA2", "MD5", - "CRC32", "BIT_LENGTH", "CHAR_LENGTH", "CONCAT" -> + "CRC32", "BIT_LENGTH", "CHAR_LENGTH", "CONCAT", "RPAD", "LPAD" -> visitSQLFunction(name, expressionsToStringArray(e.children())); case "CASE_WHEN" -> visitCaseWhen(expressionsToStringArray(e.children())); case "TRIM" -> visitTrim("BOTH", expressionsToStringArray(e.children())); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala index 84f6d04eeb1a4..50f149bb28064 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CTESubstitution.scala @@ -149,10 +149,15 @@ object CTESubstitution extends Rule[LogicalPlan] { plan: LogicalPlan, cteDefs: ArrayBuffer[CTERelationDef]): LogicalPlan = { plan.resolveOperatorsUp { - case UnresolvedWith(child, relations, _) => - val resolvedCTERelations = - resolveCTERelations(relations, isLegacy = true, forceInline = false, Seq.empty, cteDefs) - substituteCTE(child, alwaysInline = true, resolvedCTERelations) + case cte @ UnresolvedWith(child, relations, allowRecursion) => + if (allowRecursion) { + cte.failAnalysis( + errorClass = "RECURSIVE_CTE_IN_LEGACY_MODE", + messageParameters = Map.empty) + } + val resolvedCTERelations = resolveCTERelations(relations, isLegacy = true, + forceInline = false, Seq.empty, cteDefs, allowRecursion) + substituteCTE(child, alwaysInline = true, resolvedCTERelations, None) } } @@ -202,14 +207,21 @@ object CTESubstitution extends Rule[LogicalPlan] { var firstSubstituted: Option[LogicalPlan] = None val newPlan = plan.resolveOperatorsDownWithPruning( _.containsAnyPattern(UNRESOLVED_WITH, PLAN_EXPRESSION)) { - case UnresolvedWith(child: LogicalPlan, relations, _) => + // allowRecursion flag is set to `True` by the parser if the `RECURSIVE` keyword is used. + case cte @ UnresolvedWith(child: LogicalPlan, relations, allowRecursion) => + if (allowRecursion && forceInline) { + cte.failAnalysis( + errorClass = "RECURSIVE_CTE_WHEN_INLINING_IS_FORCED", + messageParameters = Map.empty) + } val resolvedCTERelations = - resolveCTERelations(relations, isLegacy = false, forceInline, outerCTEDefs, cteDefs) ++ - outerCTEDefs + resolveCTERelations(relations, isLegacy = false, forceInline, outerCTEDefs, cteDefs, + allowRecursion) ++ outerCTEDefs val substituted = substituteCTE( traverseAndSubstituteCTE(child, forceInline, resolvedCTERelations, cteDefs)._1, forceInline, - resolvedCTERelations) + resolvedCTERelations, + None) if (firstSubstituted.isEmpty) { firstSubstituted = Some(substituted) } @@ -228,7 +240,8 @@ object CTESubstitution extends Rule[LogicalPlan] { isLegacy: Boolean, forceInline: Boolean, outerCTEDefs: Seq[(String, CTERelationDef)], - cteDefs: ArrayBuffer[CTERelationDef]): Seq[(String, CTERelationDef)] = { + cteDefs: ArrayBuffer[CTERelationDef], + allowRecursion: Boolean): Seq[(String, CTERelationDef)] = { val alwaysInline = isLegacy || forceInline var resolvedCTERelations = if (alwaysInline) { Seq.empty @@ -247,49 +260,116 @@ object CTESubstitution extends Rule[LogicalPlan] { // NOTE: we must call `traverseAndSubstituteCTE` before `substituteCTE`, as the relations // in the inner CTE have higher priority over the relations in the outer CTE when resolving // inner CTE relations. For example: - // WITH t1 AS (SELECT 1) - // t2 AS ( - // WITH t1 AS (SELECT 2) - // WITH t3 AS (SELECT * FROM t1) - // ) - // t3 should resolve the t1 to `SELECT 2` instead of `SELECT 1`. - traverseAndSubstituteCTE(relation, forceInline, resolvedCTERelations, cteDefs)._1 + // WITH + // t1 AS (SELECT 1), + // t2 AS ( + // WITH + // t1 AS (SELECT 2), + // t3 AS (SELECT * FROM t1) + // SELECT * FROM t1 + // ) + // SELECT * FROM t2 + // t3 should resolve the t1 to `SELECT 2` ("inner" t1) instead of `SELECT 1`. + // + // When recursion allowed (RECURSIVE keyword used): + // Consider following example: + // WITH + // t1 AS (SELECT 1), + // t2 AS ( + // WITH RECURSIVE + // t1 AS ( + // SELECT 1 AS level + // UNION ( + // WITH t3 AS (SELECT level + 1 FROM t1 WHERE level < 10) + // SELECT * FROM t3 + // ) + // ) + // SELECT * FROM t1 + // ) + // SELECT * FROM t2 + // t1 reference within t3 would initially resolve to outer `t1` (SELECT 1), as the inner t1 + // is not yet known. Therefore, we need to remove definitions that conflict with current + // relation `name` from the list of `outerCTEDefs` entering `traverseAndSubstituteCTE()`. + // NOTE: It will be recognized later in the code that this is actually a self-reference + // (reference to the inner t1). + val nonConflictingCTERelations = if (allowRecursion) { + resolvedCTERelations.filterNot { + case (cteName, cteDef) => cteDef.conf.resolver(cteName, name) + } + } else { + resolvedCTERelations + } + traverseAndSubstituteCTE(relation, forceInline, nonConflictingCTERelations, cteDefs)._1 } - // CTE definition can reference a previous one - val substituted = substituteCTE(innerCTEResolved, alwaysInline, resolvedCTERelations) + + // If recursion is allowed (RECURSIVE keyword specified) + // then it has higher priority than outer or previous relations. + // Therefore, we construct a `CTERelationDef` for the current relation. + // Later if we encounter unresolved relation which we need to find which CTE Def it is + // referencing to, we first check if it is a reference to this one. If yes, then we set the + // reference as being recursive. + val recursiveCTERelation = if (allowRecursion) { + Some(name -> CTERelationDef(relation)) + } else { + None + } + // CTE definition can reference a previous one or itself if recursion allowed. + val substituted = substituteCTE(innerCTEResolved, alwaysInline, + resolvedCTERelations, recursiveCTERelation) val cteRelation = CTERelationDef(substituted) if (!alwaysInline) { cteDefs += cteRelation } + // Prepending new CTEs makes sure that those have higher priority over outer ones. resolvedCTERelations +:= (name -> cteRelation) } resolvedCTERelations } + /** + * This function is called from `substituteCTE` to actually substitute unresolved relations + * with CTE references. + */ private def resolveWithCTERelations( table: String, alwaysInline: Boolean, cteRelations: Seq[(String, CTERelationDef)], + recursiveCTERelation: Option[(String, CTERelationDef)], unresolvedRelation: UnresolvedRelation): LogicalPlan = { - cteRelations - .find(r => conf.resolver(r._1, table)) - .map { + if (recursiveCTERelation.isDefined && conf.resolver(recursiveCTERelation.get._1, table)) { + // self-reference is found + recursiveCTERelation.map { case (_, d) => - if (alwaysInline) { - d.child - } else { - // Add a `SubqueryAlias` for hint-resolving rules to match relation names. - SubqueryAlias(table, CTERelationRef(d.id, d.resolved, d.output, d.isStreaming)) - } - } - .getOrElse(unresolvedRelation) + SubqueryAlias(table, + CTERelationRef(d.id, d.resolved, d.output, d.isStreaming, recursive = true)) + }.get + } else { + cteRelations + .find(r => conf.resolver(r._1, table)) + .map { + case (_, d) => + if (alwaysInline) { + d.child + } else { + // Add a `SubqueryAlias` for hint-resolving rules to match relation names. + // This is a non-recursive reference, recursive parameter is by default set to false + SubqueryAlias(table, + CTERelationRef(d.id, d.resolved, d.output, d.isStreaming)) + } + } + .getOrElse(unresolvedRelation) + } } + /** + * Substitute unresolved relations in the plan with CTE references (CTERelationRef). + */ private def substituteCTE( plan: LogicalPlan, alwaysInline: Boolean, - cteRelations: Seq[(String, CTERelationDef)]): LogicalPlan = { + cteRelations: Seq[(String, CTERelationDef)], + recursiveCTERelation: Option[(String, CTERelationDef)]): LogicalPlan = { plan.resolveOperatorsUpWithPruning( _.containsAnyPattern(RELATION_TIME_TRAVEL, UNRESOLVED_RELATION, PLAN_EXPRESSION, UNRESOLVED_IDENTIFIER)) { @@ -298,7 +378,8 @@ object CTESubstitution extends Rule[LogicalPlan] { throw QueryCompilationErrors.timeTravelUnsupportedError(toSQLId(table)) case u @ UnresolvedRelation(Seq(table), _, _) => - resolveWithCTERelations(table, alwaysInline, cteRelations, u) + resolveWithCTERelations(table, alwaysInline, cteRelations, + recursiveCTERelation, u) case p: PlanWithUnresolvedIdentifier => // We must look up CTE relations first when resolving `UnresolvedRelation`s, @@ -308,7 +389,8 @@ object CTESubstitution extends Rule[LogicalPlan] { p.copy(planBuilder = (nameParts, children) => { p.planBuilder.apply(nameParts, children) match { case u @ UnresolvedRelation(Seq(table), _, _) => - resolveWithCTERelations(table, alwaysInline, cteRelations, u) + resolveWithCTERelations(table, alwaysInline, cteRelations, + recursiveCTERelation, u) case other => other } }) @@ -317,7 +399,8 @@ object CTESubstitution extends Rule[LogicalPlan] { // This cannot be done in ResolveSubquery because ResolveSubquery does not know the CTE. other.transformExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION)) { case e: SubqueryExpression => - e.withNewPlan(apply(substituteCTE(e.plan, alwaysInline, cteRelations))) + e.withNewPlan( + apply(substituteCTE(e.plan, alwaysInline, cteRelations, None))) } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveLateralColumnAliasReference.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveLateralColumnAliasReference.scala index 677d852ebad57..cb26820a0c79d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveLateralColumnAliasReference.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveLateralColumnAliasReference.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.analysis import java.util.LinkedHashSet +import scala.jdk.CollectionConverters._ + import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.WindowExpression.hasWindowExpression import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression @@ -224,7 +226,7 @@ object ResolveLateralColumnAliasReference extends Rule[LogicalPlan] { if (!aggregateExpressions.forall(eligibleToLiftUp)) { agg } else { - val newAggExprs = collection.mutable.Set.empty[NamedExpression] + val newAggExprs = new LinkedHashSet[NamedExpression] val expressionMap = collection.mutable.LinkedHashMap.empty[Expression, NamedExpression] // Extract the expressions to keep in the Aggregate. Return the transformed expression // fully substituted with the attribute reference to the extracted expressions. @@ -251,11 +253,11 @@ object ResolveLateralColumnAliasReference extends Rule[LogicalPlan] { } } val ne = expressionMap.getOrElseUpdate(aggExpr.canonicalized, assignAlias(aggExpr)) - newAggExprs += ne + newAggExprs.add(ne) ne.toAttribute case e if groupingExpressions.exists(_.semanticEquals(e)) => val ne = expressionMap.getOrElseUpdate(e.canonicalized, assignAlias(e)) - newAggExprs += ne + newAggExprs.add(ne) ne.toAttribute case e => e.mapChildren(extractExpressions) } @@ -264,7 +266,7 @@ object ResolveLateralColumnAliasReference extends Rule[LogicalPlan] { extractExpressions(_).asInstanceOf[NamedExpression]) Project( projectList = projectExprs, - child = agg.copy(aggregateExpressions = newAggExprs.toSeq) + child = agg.copy(aggregateExpressions = newAggExprs.asScala.toSeq) ) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala index 87a5e94d9f630..b47af90c651a6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala @@ -206,7 +206,8 @@ case class ResolvedInlineTable(rows: Seq[Seq[Expression]], output: Seq[Attribute */ case class UnresolvedTableValuedFunction( name: Seq[String], - functionArgs: Seq[Expression]) + functionArgs: Seq[Expression], + override val isStreaming: Boolean = false) extends UnresolvedLeafNode { final override val nodePatterns: Seq[TreePattern] = Seq(UNRESOLVED_TABLE_VALUED_FUNCTION) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunctionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunctionErrors.scala index a5381669caea8..e8cfa8d74e83f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunctionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunctionErrors.scala @@ -18,10 +18,12 @@ package org.apache.spark.sql.catalyst.catalog import org.apache.spark.SparkException +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.errors.QueryErrorsBase /** - * Errors during registering and executing [[UserDefinedFunction]]s. + * Errors during registering and executing + * [[org.apache.spark.sql.expressions.UserDefinedFunction]]s. */ object UserDefinedFunctionErrors extends QueryErrorsBase { def unsupportedUserDefinedFunction(language: RoutineLanguage): Throwable = { @@ -31,4 +33,68 @@ object UserDefinedFunctionErrors extends QueryErrorsBase { def unsupportedUserDefinedFunction(language: String): Throwable = { SparkException.internalError(s"Unsupported user defined function type: $language") } + + def duplicateParameterNames(routineName: String, names: String): Throwable = { + new AnalysisException( + errorClass = "DUPLICATE_ROUTINE_PARAMETER_NAMES", + messageParameters = Map("routineName" -> routineName, "names" -> names)) + } + + def duplicateReturnsColumns(routineName: String, columns: String): Throwable = { + new AnalysisException( + errorClass = "DUPLICATE_ROUTINE_RETURNS_COLUMNS", + messageParameters = Map("routineName" -> routineName, "columns" -> columns)) + } + + def cannotSpecifyNotNullOnFunctionParameters(input: String): Throwable = { + new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.NOT_NULL_ON_FUNCTION_PARAMETERS", + messageParameters = Map("input" -> input)) + } + + def bodyIsNotAQueryForSqlTableUdf(functionName: String): Throwable = { + new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.SQL_TABLE_UDF_BODY_MUST_BE_A_QUERY", + messageParameters = Map("name" -> functionName)) + } + + def missingColumnNamesForSqlTableUdf(functionName: String): Throwable = { + new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.SQL_TABLE_UDF_MISSING_COLUMN_NAMES", + messageParameters = Map("functionName" -> toSQLId(functionName))) + } + + def invalidTempViewReference(routineName: Seq[String], tempViewName: Seq[String]): Throwable = { + new AnalysisException( + errorClass = "INVALID_TEMP_OBJ_REFERENCE", + messageParameters = Map( + "obj" -> "FUNCTION", + "objName" -> toSQLId(routineName), + "tempObj" -> "VIEW", + "tempObjName" -> toSQLId(tempViewName) + ) + ) + } + + def invalidTempFuncReference(routineName: Seq[String], tempFuncName: String): Throwable = { + new AnalysisException( + errorClass = "INVALID_TEMP_OBJ_REFERENCE", + messageParameters = Map( + "obj" -> "FUNCTION", + "objName" -> toSQLId(routineName), + "tempObj" -> "FUNCTION", + "tempObjName" -> toSQLId(tempFuncName) + ) + ) + } + + def invalidTempVarReference(routineName: Seq[String], varName: Seq[String]): Throwable = { + new AnalysisException( + errorClass = "INVALID_TEMP_OBJ_REFERENCE", + messageParameters = Map( + "obj" -> "FUNCTION", + "objName" -> toSQLId(routineName), + "tempObj" -> "VARIABLE", + "tempObjName" -> toSQLId(varName))) + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala index ba910b8c7e5fd..ff8b168793b5d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala @@ -184,33 +184,37 @@ case class ToVariantObject(child: Expression) } } -object VariantPathParser extends RegexParsers { - // A path segment in the `VariantGet` expression represents either an object key access or an - // array index access. - type PathSegment = Either[String, Int] +// A path segment in the `VariantGet` expression represents either an object key access or an array +// index access. +sealed abstract class VariantPathSegment extends Serializable + +case class ObjectExtraction(key: String) extends VariantPathSegment +case class ArrayExtraction(index: Int) extends VariantPathSegment + +object VariantPathParser extends RegexParsers { private def root: Parser[Char] = '$' // Parse index segment like `[123]`. - private def index: Parser[PathSegment] = + private def index: Parser[VariantPathSegment] = for { index <- '[' ~> "\\d+".r <~ ']' } yield { - scala.util.Right(index.toInt) + ArrayExtraction(index.toInt) } // Parse key segment like `.name`, `['name']`, or `["name"]`. - private def key: Parser[PathSegment] = + private def key: Parser[VariantPathSegment] = for { key <- '.' ~> "[^\\.\\[]+".r | "['" ~> "[^\\'\\?]+".r <~ "']" | "[\"" ~> "[^\\\"\\?]+".r <~ "\"]" } yield { - scala.util.Left(key) + ObjectExtraction(key) } - private val parser: Parser[List[PathSegment]] = phrase(root ~> rep(key | index)) + private val parser: Parser[List[VariantPathSegment]] = phrase(root ~> rep(key | index)) - def parse(str: String): Option[Array[PathSegment]] = { + def parse(str: String): Option[Array[VariantPathSegment]] = { this.parseAll(parser, str) match { case Success(result, _) => Some(result.toArray) case _ => None @@ -349,14 +353,14 @@ case object VariantGet { /** The actual implementation of the `VariantGet` expression. */ def variantGet( input: VariantVal, - parsedPath: Array[VariantPathParser.PathSegment], + parsedPath: Array[VariantPathSegment], dataType: DataType, castArgs: VariantCastArgs): Any = { var v = new Variant(input.getValue, input.getMetadata) for (path <- parsedPath) { v = path match { - case scala.util.Left(key) if v.getType == Type.OBJECT => v.getFieldByKey(key) - case scala.util.Right(index) if v.getType == Type.ARRAY => v.getElementAtIndex(index) + case ObjectExtraction(key) if v.getType == Type.OBJECT => v.getFieldByKey(key) + case ArrayExtraction(index) if v.getType == Type.ARRAY => v.getElementAtIndex(index) case _ => null } if (v == null) return null diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushdownPredicatesAndPruneColumnsForCTEDef.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushdownPredicatesAndPruneColumnsForCTEDef.scala index 838827a0e2e1e..59b3d83c55162 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushdownPredicatesAndPruneColumnsForCTEDef.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PushdownPredicatesAndPruneColumnsForCTEDef.scala @@ -122,7 +122,7 @@ object PushdownPredicatesAndPruneColumnsForCTEDef extends Rule[LogicalPlan] { private def pushdownPredicatesAndAttributes( plan: LogicalPlan, cteMap: CTEMap): LogicalPlan = plan.transformWithSubqueries { - case cteDef @ CTERelationDef(child, id, originalPlanWithPredicates, _, _, _) => + case cteDef @ CTERelationDef(child, id, originalPlanWithPredicates, _, _) => val (_, _, newPreds, newAttrSet) = cteMap(id) val originalPlan = originalPlanWithPredicates.map(_._1).getOrElse(child) val preds = originalPlanWithPredicates.map(_._2).getOrElse(Seq.empty) @@ -170,7 +170,7 @@ object PushdownPredicatesAndPruneColumnsForCTEDef extends Rule[LogicalPlan] { object CleanUpTempCTEInfo extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan.transformWithPruning(_.containsPattern(CTE)) { - case cteDef @ CTERelationDef(_, _, Some(_), _, _, _) => + case cteDef @ CTERelationDef(_, _, Some(_), _, _) => cteDef.copy(originalPlanWithPredicates = None) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala index 40189a9f61021..5d85e89e1eabe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpression.scala @@ -68,9 +68,15 @@ object RewriteWithExpression extends Rule[LogicalPlan] { private def applyInternal(p: LogicalPlan): LogicalPlan = { val inputPlans = p.children + val commonExprIdSet = p.expressions + .flatMap(_.collect { case r: CommonExpressionRef => r.id }) + .groupBy(identity) + .transform((_, v) => v.size) + .filter(_._2 > 1) + .keySet val commonExprsPerChild = Array.fill(inputPlans.length)(mutable.ListBuffer.empty[(Alias, Long)]) var newPlan: LogicalPlan = p.mapExpressions { expr => - rewriteWithExprAndInputPlans(expr, inputPlans, commonExprsPerChild) + rewriteWithExprAndInputPlans(expr, inputPlans, commonExprsPerChild, commonExprIdSet) } val newChildren = inputPlans.zip(commonExprsPerChild).map { case (inputPlan, commonExprs) => if (commonExprs.isEmpty) { @@ -96,6 +102,7 @@ object RewriteWithExpression extends Rule[LogicalPlan] { e: Expression, inputPlans: Seq[LogicalPlan], commonExprsPerChild: Array[mutable.ListBuffer[(Alias, Long)]], + commonExprIdSet: Set[CommonExpressionId], isNestedWith: Boolean = false): Expression = { if (!e.containsPattern(WITH_EXPRESSION)) return e e match { @@ -103,9 +110,9 @@ object RewriteWithExpression extends Rule[LogicalPlan] { case w: With if !isNestedWith => // Rewrite nested With expressions first val child = rewriteWithExprAndInputPlans( - w.child, inputPlans, commonExprsPerChild, isNestedWith = true) + w.child, inputPlans, commonExprsPerChild, commonExprIdSet, isNestedWith = true) val defs = w.defs.map(rewriteWithExprAndInputPlans( - _, inputPlans, commonExprsPerChild, isNestedWith = true)) + _, inputPlans, commonExprsPerChild, commonExprIdSet, isNestedWith = true)) val refToExpr = mutable.HashMap.empty[CommonExpressionId, Expression] defs.zipWithIndex.foreach { case (CommonExpressionDef(child, id), index) => @@ -114,7 +121,7 @@ object RewriteWithExpression extends Rule[LogicalPlan] { "Cannot rewrite canonicalized Common expression definitions") } - if (CollapseProject.isCheap(child)) { + if (CollapseProject.isCheap(child) || !commonExprIdSet.contains(id)) { refToExpr(id) = child } else { val childPlanIndex = inputPlans.indexWhere( @@ -171,7 +178,8 @@ object RewriteWithExpression extends Rule[LogicalPlan] { case c: ConditionalExpression => val newAlwaysEvaluatedInputs = c.alwaysEvaluatedInputs.map( - rewriteWithExprAndInputPlans(_, inputPlans, commonExprsPerChild, isNestedWith)) + rewriteWithExprAndInputPlans( + _, inputPlans, commonExprsPerChild, commonExprIdSet, isNestedWith)) val newExpr = c.withNewAlwaysEvaluatedInputs(newAlwaysEvaluatedInputs) // Use transformUp to handle nested With. newExpr.transformUpWithPruning(_.containsPattern(WITH_EXPRESSION)) { @@ -185,7 +193,8 @@ object RewriteWithExpression extends Rule[LogicalPlan] { } case other => other.mapChildren( - rewriteWithExprAndInputPlans(_, inputPlans, commonExprsPerChild, isNestedWith) + rewriteWithExprAndInputPlans( + _, inputPlans, commonExprsPerChild, commonExprIdSet, isNestedWith) ) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index aa32cc9100515..0a300cea03ffe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -551,7 +551,7 @@ class AstBuilder extends DataTypeAstBuilder throw QueryParsingErrors.duplicateCteDefinitionNamesError( duplicates.map(toSQLId).mkString(", "), ctx) } - UnresolvedWith(plan, ctes.toSeq) + UnresolvedWith(plan, ctes.toSeq, ctx.RECURSIVE() != null) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/NormalizePlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/NormalizePlan.scala index ee68e433fbeab..38cf2730e9acd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/NormalizePlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/NormalizePlan.scala @@ -21,7 +21,6 @@ import org.apache.spark.sql.catalyst.analysis.GetViewColumnByNameAndOrdinal import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.connector.read.streaming.SparkDataStream object NormalizePlan extends PredicateHelper { def apply(plan: LogicalPlan): LogicalPlan = @@ -105,8 +104,15 @@ object NormalizePlan extends PredicateHelper { case Project(projectList, child) => Project(normalizeProjectList(projectList), child) case c: KeepAnalyzedQuery => c.storeAnalyzedQuery() - case localRelation: LocalRelation => - ComparableLocalRelation.fromLocalRelation(localRelation) + case localRelation: LocalRelation if !localRelation.data.isEmpty => + /** + * A substitute for the [[LocalRelation.data]]. [[GenericInternalRow]] is incomparable for + * maps, because [[ArrayBasedMapData]] doesn't define [[equals]]. + */ + val unsafeProjection = UnsafeProjection.create(localRelation.schema) + localRelation.copy(data = localRelation.data.map { row => + unsafeProjection(row) + }) } } @@ -137,33 +143,3 @@ object NormalizePlan extends PredicateHelper { case _ => condition // Don't reorder. } } - -/** - * A substitute for the [[LocalRelation]] that has comparable `data` field. [[LocalRelation]]'s - * `data` is incomparable for maps, because [[ArrayBasedMapData]] doesn't define [[equals]]. - */ -case class ComparableLocalRelation( - override val output: Seq[Attribute], - data: Seq[Seq[Expression]], - override val isStreaming: Boolean, - stream: Option[SparkDataStream]) extends LeafNode - -object ComparableLocalRelation { - def fromLocalRelation(localRelation: LocalRelation): ComparableLocalRelation = { - val dataTypes = localRelation.output.map(_.dataType) - ComparableLocalRelation( - output = localRelation.output, - data = localRelation.data.map { row => - if (row != null) { - row.toSeq(dataTypes).zip(dataTypes).map { - case (value, dataType) => Literal(value, dataType) - } - } else { - Seq.empty - } - }, - isStreaming = localRelation.isStreaming, - stream = localRelation.stream - ) - } -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index b897026a0df46..c1261f2b5fac5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -866,7 +866,6 @@ case class UnresolvedWith( * pushdown to help ensure rule idempotency. * @param underSubquery If true, it means we don't need to add a shuffle for this CTE relation as * subquery reuse will be applied to reuse CTE relation output. - * @param recursive If true, then this CTE Definition is recursive - it contains a self-reference. * @param recursionAnchor A helper plan node that temporary stores the anchor term of recursive * definitions. In the beginning of recursive resolution the `ResolveWithCTE` * rule updates this parameter and once it is resolved the same rule resolves @@ -877,7 +876,6 @@ case class CTERelationDef( id: Long = CTERelationDef.newId, originalPlanWithPredicates: Option[(LogicalPlan, Seq[Expression])] = None, underSubquery: Boolean = false, - recursive: Boolean = false, recursionAnchor: Option[LogicalPlan] = None) extends UnaryNode { final override val nodePatterns: Seq[TreePattern] = Seq(CTE) @@ -886,6 +884,13 @@ case class CTERelationDef( copy(child = newChild) override def output: Seq[Attribute] = if (resolved) child.output else Nil + + lazy val recursive: Boolean = child.exists{ + // if the reference is found inside the child, referencing to this CTE definition, + // and already marked as recursive, then this CTE definition is recursive. + case CTERelationRef(this.id, _, _, _, _, true) => true + case _ => false + } } object CTERelationDef { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala index 85b5e8379d3d5..58c62a90225aa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala @@ -21,8 +21,8 @@ import org.apache.spark.{SparkIllegalArgumentException, SparkUnsupportedOperatio import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.analysis.{AnalysisContext, AssignmentUtils, EliminateSubqueryAliases, FieldName, NamedRelation, PartitionSpec, ResolvedIdentifier, ResolvedProcedure, TypeCheckResult, UnresolvedException, UnresolvedProcedure, ViewSchemaMode} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.catalog.{FunctionResource, RoutineLanguage} import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec -import org.apache.spark.sql.catalyst.catalog.FunctionResource import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, Expression, MetadataAttribute, NamedExpression, UnaryExpression, Unevaluable, V2ExpressionUtils} import org.apache.spark.sql.catalyst.plans.DescribeCommandSchema import org.apache.spark.sql.catalyst.trees.BinaryLike @@ -1072,6 +1072,26 @@ case class CreateFunction( copy(child = newChild) } +/** + * The logical plan of the CREATE FUNCTION command for SQL Functions. + */ +case class CreateUserDefinedFunction( + child: LogicalPlan, + inputParamText: Option[String], + returnTypeText: String, + exprText: Option[String], + queryText: Option[String], + comment: Option[String], + isDeterministic: Option[Boolean], + containsSQL: Option[Boolean], + language: RoutineLanguage, + isTableFunc: Boolean, + ignoreIfExists: Boolean, + replace: Boolean) extends UnaryCommand { + override protected def withNewChildInternal(newChild: LogicalPlan): CreateUserDefinedFunction = + copy(child = newChild) +} + /** * The logical plan of the DROP FUNCTION command. */ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala index db94659b1033b..9b8584604d32f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala @@ -85,9 +85,10 @@ class CatalogManager( * in the fallback configuration, spark.sql.sources.useV1SourceList */ private[sql] def v2SessionCatalog: CatalogPlugin = { - conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).map { _ => - catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, loadV2SessionCatalog()) - }.getOrElse(defaultSessionCatalog) + conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION) match { + case "builtin" => defaultSessionCatalog + case _ => catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, loadV2SessionCatalog()) + } } private var _currentNamespace: Option[Array[String]] = None diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index d38c7a01e1c44..65ae8da3c4da1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -2172,6 +2172,15 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat "ability" -> ability)) } + def tableValuedArgumentsNotYetImplementedForSqlFunctions( + action: String, functionName: String): Throwable = { + new AnalysisException( + errorClass = "TABLE_VALUED_ARGUMENTS_NOT_YET_IMPLEMENTED_FOR_SQL_FUNCTIONS", + messageParameters = Map( + "action" -> action, + "functionName" -> functionName)) + } + def tableValuedFunctionTooManyTableArgumentsError(num: Int): Throwable = { new AnalysisException( errorClass = "TABLE_VALUED_FUNCTION_TOO_MANY_TABLE_ARGUMENTS", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 5e630577638a0..7bc4051b45d4c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4404,10 +4404,16 @@ object SQLConf { s"the $SESSION_CATALOG_NAME and must be consistent with it; for example, if a table can " + s"be loaded by the $SESSION_CATALOG_NAME, this catalog must also return the table " + s"metadata. To delegate operations to the $SESSION_CATALOG_NAME, implementations can " + - "extend 'CatalogExtension'.") + "extend 'CatalogExtension'. The value should be either 'builtin' which represents the " + + "spark's builit-in V2SessionCatalog, or a fully qualified class name of the catalog " + + "implementation.") .version("3.0.0") .stringConf - .createOptional + .transform { + case builtin if builtin.equalsIgnoreCase("builtin") => "builtin" + case fullClassName => fullClassName + } + .createWithDefault("builtin") object MapKeyDedupPolicy extends Enumeration { val EXCEPTION, LAST_WIN = Value @@ -4659,6 +4665,22 @@ object SQLConf { .booleanConf .createWithDefault(false) + val VARIANT_WRITE_SHREDDING_ENABLED = + buildConf("spark.sql.variant.writeShredding.enabled") + .internal() + .doc("When true, the Parquet writer is allowed to write shredded variant. ") + .version("4.0.0") + .booleanConf + .createWithDefault(false) + + val VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST = + buildConf("spark.sql.variant.forceShreddingSchemaForTest") + .internal() + .doc("FOR INTERNAL TESTING ONLY. Sets shredding schema for Variant.") + .version("4.0.0") + .stringConf + .createWithDefault("") + val LEGACY_CSV_ENABLE_DATE_TIME_PARSING_FALLBACK = buildConf("spark.sql.legacy.csv.enableDateTimeParsingFallback") .internal() diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala index 71744f4d15105..58e6cd7fe1695 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala @@ -198,6 +198,21 @@ trait AnalysisTest extends PlanTest { } } + protected def assertParseErrorClass( + parser: String => Any, + sqlCommand: String, + errorClass: String, + parameters: Map[String, String], + queryContext: Array[ExpectedContext] = Array.empty): Unit = { + val e = parseException(parser)(sqlCommand) + checkError( + exception = e, + condition = errorClass, + parameters = parameters, + queryContext = queryContext + ) + } + protected def interceptParseException(parser: String => Any)( sqlCommand: String, messages: String*)(condition: Option[String] = None): Unit = { val e = parseException(parser)(sqlCommand) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala index 891e2d048b7a8..b0ed1ecabb8d6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/codegen/BufferHolderSparkSubmitSuite.scala @@ -18,7 +18,9 @@ package org.apache.spark.sql.catalyst.expressions.codegen import org.scalatest.{Assertions, BeforeAndAfterEach} +import org.scalatest.concurrent.Eventually.{eventually, interval, timeout} import org.scalatest.matchers.must.Matchers +import org.scalatest.time.SpanSugar._ import org.apache.spark.{SparkIllegalArgumentException, TestUtils} import org.apache.spark.deploy.SparkSubmitTestUtils @@ -46,7 +48,10 @@ class BufferHolderSparkSubmitSuite "--conf", "spark.master.rest.enabled=false", "--conf", "spark.driver.extraJavaOptions=-ea", unusedJar.toString) - runSparkSubmit(argsForSparkSubmit) + // Given that the default timeout of runSparkSubmit is 60 seconds, try 3 times in total. + eventually(timeout(210.seconds), interval(70.seconds)) { + runSparkSubmit(argsForSparkSubmit) + } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index fc2697d55f6d0..4cc2ee99284a5 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -46,7 +46,7 @@ class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper { $"e".boolean, $"f".boolean, $"g".boolean, $"h".boolean) val testRelationWithData = LocalRelation.fromExternalRows( - testRelation.output, Seq(Row(1, 2, 3, "abc")) + testRelation.output, Seq(Row(1, 2, 3, "abc", true, true, true, true)) ) val testNotNullableRelation = LocalRelation($"a".int.notNull, $"b".int.notNull, $"c".int.notNull, @@ -54,7 +54,7 @@ class BooleanSimplificationSuite extends PlanTest with ExpressionEvalHelper { $"h".boolean.notNull) val testNotNullableRelationWithData = LocalRelation.fromExternalRows( - testNotNullableRelation.output, Seq(Row(1, 2, 3, "abc")) + testNotNullableRelation.output, Seq(Row(1, 2, 3, "abc", true, true, true, true)) ) private def checkCondition(input: Expression, expected: LogicalPlan): Unit = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala index 02631c4cf61c9..2dcab5cfd29c4 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.dsl.plans._ -import org.apache.spark.sql.catalyst.expressions.Add +import org.apache.spark.sql.catalyst.expressions.{Add, GenericInternalRow} import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules._ @@ -189,7 +189,9 @@ class LimitPushdownSuite extends PlanTest { } test("full outer join where neither side is limited and left side has larger statistics") { - val xBig = testRelation.copy(data = Seq.fill(10)(null)).subquery("x") + val nulls = new GenericInternalRow( + Seq.fill(testRelation.output.length)(null).toArray.asInstanceOf[Array[Any]]) + val xBig = testRelation.copy(data = Seq.fill(10)(nulls)).subquery("x") assert(xBig.stats.sizeInBytes > y.stats.sizeInBytes) Seq(Some("x.a".attr === "y.b".attr), None).foreach { condition => val originalQuery = xBig.join(y, FullOuter, condition).limit(1).analyze @@ -204,7 +206,9 @@ class LimitPushdownSuite extends PlanTest { } test("full outer join where neither side is limited and right side has larger statistics") { - val yBig = testRelation.copy(data = Seq.fill(10)(null)).subquery("y") + val nulls = new GenericInternalRow( + Seq.fill(testRelation.output.length)(null).toArray.asInstanceOf[Array[Any]]) + val yBig = testRelation.copy(data = Seq.fill(10)(nulls)).subquery("y") assert(x.stats.sizeInBytes < yBig.stats.sizeInBytes) Seq(Some("x.a".attr === "y.b".attr), None).foreach { condition => val originalQuery = x.join(yBig, FullOuter, condition).limit(1).analyze diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpressionSuite.scala index 9f0a7fdaf3152..8918b58ca1b56 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/RewriteWithExpressionSuite.scala @@ -140,7 +140,7 @@ class RewriteWithExpressionSuite extends PlanTest { val commonExprDef2 = CommonExpressionDef(a + a, CommonExpressionId(2)) val ref2 = new CommonExpressionRef(commonExprDef2) // The inner main expression references the outer expression - val innerExpr2 = With(ref2 + outerRef, Seq(commonExprDef2)) + val innerExpr2 = With(ref2 + ref2 + outerRef, Seq(commonExprDef2)) val outerExpr2 = With(outerRef + innerExpr2, Seq(outerCommonExprDef)) comparePlans( Optimizer.execute(testRelation.select(outerExpr2.as("col"))), @@ -152,7 +152,8 @@ class RewriteWithExpressionSuite extends PlanTest { .select(star(), (a + a).as("_common_expr_2")) // The final Project contains the final result expression, which references both common // expressions. - .select(($"_common_expr_0" + ($"_common_expr_2" + $"_common_expr_0")).as("col")) + .select(($"_common_expr_0" + + ($"_common_expr_2" + $"_common_expr_2" + $"_common_expr_0")).as("col")) .analyze ) } @@ -490,4 +491,13 @@ class RewriteWithExpressionSuite extends PlanTest { val wrongPlan = testRelation.select(expr1.as("c1"), expr3.as("c3")).analyze intercept[AssertionError](Optimizer.execute(wrongPlan)) } + + test("SPARK-50683: inline the common expression in With if used once") { + val a = testRelation.output.head + val exprDef = CommonExpressionDef(a + a) + val exprRef = new CommonExpressionRef(exprDef) + val expr = With(exprRef + 1, Seq(exprDef)) + val plan = testRelation.select(expr.as("col")) + comparePlans(Optimizer.execute(plan), testRelation.select((a + a + 1).as("col"))) + } } diff --git a/sql/connect/common/src/main/protobuf/spark/connect/base.proto b/sql/connect/common/src/main/protobuf/spark/connect/base.proto index e27049d2114d7..74413509ebc14 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/base.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/base.proto @@ -525,6 +525,9 @@ message ConfigRequest { message Set { // (Required) The config key-value pairs to set. repeated KeyValue pairs = 1; + + // (Optional) Whether to ignore failures. + optional bool silent = 2; } message Get { @@ -924,6 +927,20 @@ message ReleaseSessionRequest { // can be used for language or version specific information and is only intended for // logging purposes and will not be interpreted by the server. optional string client_type = 3; + + // Signals the server to allow the client to reconnect to the session after it is released. + // + // By default, the server tombstones the session upon release, preventing reconnections and + // fully cleaning the session state. + // + // If this flag is set to true, the server may permit the client to reconnect to the session + // post-release, even if the session state has been cleaned. This can result in missing state, + // such as Temporary Views, Temporary UDFs, or the Current Catalog, in the reconnected session. + // + // Use this option sparingly and only when the client fully understands the implications of + // reconnecting to a released session. The client must ensure that any queries executed do not + // rely on the session state prior to its release. + bool allow_reconnect = 4; } // Next ID: 3 diff --git a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto index a01b5229a7b74..bbe605a47f4ff 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto @@ -261,6 +261,11 @@ message Expression { // When it is not a user defined function, Connect will use the function name directly. // When it is a user defined function, Connect will parse the function name first. bool is_user_defined_function = 4; + + // (Optional) Indicate if this function is defined in the internal function registry. + // If not set, the server will try to look up the function in the internal function registry + // and decide appropriately. + optional bool is_internal = 5; } // Expression as string. diff --git a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto index 7a86db2799149..5ab9f64149f51 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto @@ -975,6 +975,9 @@ message GroupMap { // (Optional) Timeout configuration for groups that do not receive data for a while. optional string timeout_conf = 9; + + // (Optional) The schema for the grouped state. + optional DataType state_schema = 10; } message CoGroupMap { diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_add.json b/sql/connect/common/src/test/resources/query-tests/queries/column_add.json index cfa40fac8c6f9..3b8219884aa0b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_add.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_add.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_add.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_add.proto.bin index 10b410b5b08b5..9b9889cf775fe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_add.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_add.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_and.json b/sql/connect/common/src/test/resources/query-tests/queries/column_and.json index d3f8cd0e73cbc..2fa4c654cce1d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_and.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_and.json @@ -25,7 +25,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -38,9 +39,11 @@ "literal": { "double": 0.5 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_and.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_and.proto.bin index 241f1a9303b2c..99111eba7191c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_and.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_and.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_between.json b/sql/connect/common/src/test/resources/query-tests/queries/column_between.json index 20927b93d8438..de970b1cdf343 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_between.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_between.json @@ -25,7 +25,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -38,9 +39,11 @@ "literal": { "integer": 20 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_between.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_between.proto.bin index d03dd02a2f36a..324e26b35750a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_between.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_between.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.json b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.json index bd3ac671fca33..71f6d6b3ec3ab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.json @@ -22,7 +22,8 @@ "literal": { "integer": 255 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.proto.bin index 4815bc7dd1a20..13e798120873a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseAND.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.json b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.json index eaa27ffa46164..e2771dc543b12 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.json @@ -22,7 +22,8 @@ "literal": { "integer": 7 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.proto.bin index 9cf110da4ad61..f823efd7551a1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseOR.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.json b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.json index c51eb3140c339..108d66745d34d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.json @@ -22,7 +22,8 @@ "literal": { "integer": 78 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.proto.bin index 70c61f9620576..994283d7a9431 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_bitwiseXOR.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_contains.json b/sql/connect/common/src/test/resources/query-tests/queries/column_contains.json index 05d6ccf38b367..d8aef66bdf546 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_contains.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_contains.json @@ -22,7 +22,8 @@ "literal": { "string": "baz" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_contains.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_contains.proto.bin index 9c796f9470c31..1126c759b0332 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_contains.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_contains.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_divide.json b/sql/connect/common/src/test/resources/query-tests/queries/column_divide.json index 8d71061b151ca..be7a5f3c851d1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_divide.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_divide.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_divide.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_divide.proto.bin index 49b5d8d2590dd..22b010a97dd58 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_divide.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_divide.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.json b/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.json index f4171c2792fbd..5ee6cfe40b1ec 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.json @@ -22,7 +22,8 @@ "literal": { "string": "suffix_" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.proto.bin index 03f41a339f00c..3f3db0c90bc19 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_endsWith.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.json b/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.json index eea1da49bc59e..44e11ad2b8942 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.proto.bin index 22de941ad44b0..0614560048a9b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_eqNullSafe.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_equals.json b/sql/connect/common/src/test/resources/query-tests/queries/column_equals.json index 7397f4fb46acd..1f05b249eb00b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_equals.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_equals.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_equals.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_equals.proto.bin index e226de59ddcd4..cad0e9b14a814 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_equals.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_equals.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_geq.json b/sql/connect/common/src/test/resources/query-tests/queries/column_geq.json index 9f24bc251739f..4c7f5339409f0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_geq.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_geq.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_geq.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_geq.proto.bin index 1c4af866109ab..a68ee6cc8b6a7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_geq.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_geq.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_gt.json b/sql/connect/common/src/test/resources/query-tests/queries/column_gt.json index 4bb8fb41f249d..74be85e709ed5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_gt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_gt.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_gt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_gt.proto.bin index 44ca37fbb4048..e85f5a3e23552 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_gt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_gt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.json b/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.json index 47c1b63abe319..60a62c4595ac8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.json @@ -22,7 +22,8 @@ "literal": { "string": "%fOb%" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.proto.bin index 285400db7daf5..368bebd9ea48d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_ilike.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.json b/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.json index f594918ed930a..12d3d19d7797a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.proto.bin index 1030abda5b8c2..8c2fad75be346 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_isNaN.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.json b/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.json index f34d3f4eac552..6af0e5bfdb4e3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.proto.bin index e8cccdf024934..cdc382e44ee22 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_isNotNull.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.json b/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.json index 74e990622a3a7..a6ac6534ecd55 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.proto.bin index 8fc24a9e21b38..0ea4d6f2ffe89 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_isNull.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isin.json b/sql/connect/common/src/test/resources/query-tests/queries/column_isin.json index d8811a4e780b5..b34abc941cce8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_isin.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_isin.json @@ -30,7 +30,8 @@ "literal": { "string": "foo" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_isin.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_isin.proto.bin index 365e07f35bb48..cde6686dd4064 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_isin.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_isin.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_leq.json b/sql/connect/common/src/test/resources/query-tests/queries/column_leq.json index cda8694c0439e..55388f667c448 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_leq.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_leq.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_leq.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_leq.proto.bin index e8463292e4040..692ccad0aa9c3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_leq.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_leq.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_like.json b/sql/connect/common/src/test/resources/query-tests/queries/column_like.json index 1390451af55ab..be7b4e8dfeb34 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_like.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_like.json @@ -22,7 +22,8 @@ "literal": { "string": "%bob%" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_like.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_like.proto.bin index 07382ec1643cb..a1856511eb1b2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_like.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_like.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_lt.json b/sql/connect/common/src/test/resources/query-tests/queries/column_lt.json index c927e75de181b..1264a0e43a54a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_lt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_lt.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_lt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_lt.proto.bin index f4c3a110b126b..083c8d46611d1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_lt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_lt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.json b/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.json index 0c5a78eea2dff..d11494f159a5a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.proto.bin index 55bfeba04ed66..a86b5e5de63e4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_modulo.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.json b/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.json index 8c17581c67d1c..7b12a8850d5f2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.proto.bin index 8fd1b3941d1f7..b4958d84c5a05 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_multiply.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_not.json b/sql/connect/common/src/test/resources/query-tests/queries/column_not.json index 2f873196ba1d0..3fa58e874d75d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_not.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_not.json @@ -18,7 +18,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_not.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_not.proto.bin index 19609b6ee85a5..51ea1c4d20bc7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_not.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_not.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.json b/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.json index 589d57a18b94b..093770f4563be 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.json @@ -25,9 +25,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.proto.bin index cdf0b4290e61e..cc13a11b48013 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_not_equals.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_or.json b/sql/connect/common/src/test/resources/query-tests/queries/column_or.json index ae1424f763feb..324bfc850d2dc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_or.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_or.json @@ -25,7 +25,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -38,9 +39,11 @@ "literal": { "double": 0.5 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_or.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_or.proto.bin index 69f219e938a4e..a52ba0707a755 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_or.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_or.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.json b/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.json index e53403db41cd0..6ff88b1ea6560 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.json @@ -22,7 +22,8 @@ "literal": { "string": "^[0-9]*$" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.proto.bin index 7dd56baf04213..b4cd080c61aad 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_rlike.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.json b/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.json index 431e13d818639..9a9036b3cf963 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.json @@ -22,7 +22,8 @@ "literal": { "string": "prefix_" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.proto.bin index fa1132c73de7b..366011b3c3968 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_startsWith.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_substr.json b/sql/connect/common/src/test/resources/query-tests/queries/column_substr.json index 3b02117cc6e5b..5beaf7c4c3711 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_substr.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_substr.json @@ -26,7 +26,8 @@ "literal": { "integer": 3 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_substr.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_substr.proto.bin index 636a46a480626..5eedae63ea95c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_substr.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_substr.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.json b/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.json index d15c2941ee1bd..68faab9cbb05a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.proto.bin index f5716427588ed..2a341fb5201ed 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_subtract.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.json b/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.json index 0db558e49e38c..b4c76e2c6719b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.json @@ -18,7 +18,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.proto.bin index 66343bea4e29b..53277e9dd2452 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_unary_minus.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.json b/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.json index db2ceccfd22ab..f2223c20e569c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.json @@ -25,7 +25,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, { "literal": { @@ -42,7 +43,8 @@ "literal": { "integer": 20 } - }] + }], + "isInternal": false } }, { "literal": { @@ -52,7 +54,8 @@ "literal": { "string": "high" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.proto.bin index 031c3683c5e6d..e22f469ea2490 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/column_when_otherwise.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/cube_column.json b/sql/connect/common/src/test/resources/query-tests/queries/cube_column.json index 5b9709ff06576..b4d884568354b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/cube_column.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/cube_column.json @@ -30,7 +30,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "name": ["count"] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/cube_column.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/cube_column.proto.bin index d46e40b39dcfe..99a704c0c7f07 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/cube_column.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/cube_column.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/cube_string.json b/sql/connect/common/src/test/resources/query-tests/queries/cube_string.json index 03625861d88f2..1c9fb7b264664 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/cube_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/cube_string.json @@ -32,7 +32,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "name": ["count"] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/cube_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/cube_string.proto.bin index 59c7a55571201..2e092aa640278 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/cube_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/cube_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/filter.json b/sql/connect/common/src/test/resources/query-tests/queries/filter.json index 1046e1262150e..d40f8031884a5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/filter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/filter.json @@ -22,7 +22,8 @@ "literal": { "long": "10" } - }] + }], + "isInternal": false } } } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/filter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/filter.proto.bin index 069171ead3233..36bb753fff234 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/filter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/filter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.json b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.json index 662aa746af243..adbc647c186de 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.json @@ -41,9 +41,11 @@ "literal": { "string": "zstandard" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.proto.bin index 5da5c48b41153..eba3a4648ca60 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_with_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.json b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.json index da2840f2d3a0b..0ef3262f1eb4a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\": \"string\", \"name\": \"name\"}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.proto.bin index 4dd12e2dbe1dd..629804e8608aa 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/from_avro_without_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.json b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.json index 375c0f9324c3f..fe2efd928ccf4 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.json @@ -26,7 +26,8 @@ "literal": { "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.proto.bin index 07d4c6c5b286f..db667ef8ee6b2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.json b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.json index db9371b64ef72..93974afec3566 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.json @@ -37,9 +37,11 @@ "literal": { "string": "2" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.proto.bin index 00fd58da6be84..a7262d64522c6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_descFilePath_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_abs.json b/sql/connect/common/src/test/resources/query-tests/queries/function_abs.json index 13df3437ddabe..aa589275670b8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_abs.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_abs.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_abs.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_abs.proto.bin index 86cfbc09a8f91..6bda0e50aa837 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_abs.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_abs.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_acos.json b/sql/connect/common/src/test/resources/query-tests/queries/function_acos.json index 7506c0f6cb630..82543692456c1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_acos.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_acos.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_acos.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_acos.proto.bin index cc6a279cb188e..98bc0d821d7d0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_acos.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_acos.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.json index 6a83b4ab008bc..82a69e9f74166 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.proto.bin index e16ed2ba92e3f..48c57cb1c9f9b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_acosh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.json b/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.json index b1b2e78a08435..97b9a00d5ea3e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.proto.bin index 6abacc9cc2b40..c396f24928cf5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_add_months.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json index 4204a44b44ce0..28beb401cd650 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin index f635e1fc689b1..40687059a8c45 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json index 9c630e1253494..0436dd1a60c85 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin index 41d024cdb7eed..8e9a324c2fde5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json index 8f5be474ab4b3..56ad10f6f74bf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin index cd6764581f2ca..97091b52e6c59 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json index 9381042b71886..305cf0b317a23 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.json @@ -34,7 +34,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin index ca789f04ce1d4..0d4c98e59ce0a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_decrypt_with_mode_padding_aad.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.json index 06469d4840547..89d07a44e8440 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.proto.bin index c7a70b51707f3..0089323b6bbe1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.json index 7eb9b4ed8b4ed..afef2dba9aad9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.proto.bin index ecd81ae44fcbd..3d89f200e609c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.json index 59a6a5e35fd42..8617d2d9d928a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.proto.bin index 9de01ddc5ea69..3888e9a1d075d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.json index 285c67289d30a..b8b8e66787848 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.json @@ -34,7 +34,8 @@ "literal": { "binary": "Q0RF" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.proto.bin index 812426f3c00d3..0ff56c7c74372 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.json index eb0e178fd3534..cb790e822a52f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.proto.bin index ee39beb07cee4..7ff11b9868b99 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aes_encrypt_with_mode_padding_iv_aad.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.json index df1813aed64c5..8e113b8874a5f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.json @@ -35,7 +35,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -55,7 +56,8 @@ "nameParts": ["x_3"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.proto.bin index c43f4e6dbbc1b..12456f54ab438 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.json b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.json index 956b42db65639..b46810e63a304 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.json @@ -35,7 +35,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -57,14 +58,16 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }, "arguments": [{ "nameParts": ["x_3"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.proto.bin index cf32ea4ddd3e7..70fbe778cb715 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_aggregate_with_finish_lambda.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any.json b/sql/connect/common/src/test/resources/query-tests/queries/function_any.json index 4512c060d703b..73332625c2995 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_any.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_any.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "flag" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_any.proto.bin index 9b014b58da57c..ff038010b22c2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_any.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_any.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.json index 011d43a91d080..35cc2daade7a8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.proto.bin index 546c696ecfdf3..29c9a0d93723d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_ignore_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.json index 7d4f5a2de38e8..9307c56feb4af 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.proto.bin index 124a7ad7efe09..fc2e75c2015b9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_any_value_with_respect_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.json index 5579faf119647..773fd7df0ab30 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.proto.bin index bac82f670b298..9c7bd463ab943 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.json b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.json index 851862082ca04..0224877077bc5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.json @@ -22,7 +22,8 @@ "literal": { "double": 0.1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.proto.bin index fd61420fd1e45..79b285eb0a28f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_count_distinct_rsd.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.json b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.json index 490a2dcd86967..925f95e618930 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.json @@ -26,7 +26,8 @@ "literal": { "integer": 20 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.proto.bin index ae73716fa4319..478af7d5ff53a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_approx_percentile.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array.json index 20fe495bb9bf4..99152d4e998d1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array.proto.bin index 2b679eb4c6db1..25fd26ed8ec81 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.json index a3197ce95068a..0e4e0fe4dd504 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.proto.bin index c7306df86214e..4b30c105a1078 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.json index cabd44c063dec..b6af59d5a1cc1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.json @@ -22,7 +22,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.proto.bin index 76f2f0255bf25..3e2ac2115ff67 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_append.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.json index c3ebf313190c2..93b449217eb51 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.proto.bin index 949d66cb951f0..5c244efd0258f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_compact.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.json index a362d66d9d64d..349927b7cfd9e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.json @@ -22,7 +22,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.proto.bin index d8764f60364c2..af333721d6944 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_contains.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.json index d38f4194bcd2b..00b65fc1665d5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.proto.bin index e6359c074bf23..98dfa75ac01b3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_distinct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.json index 17d50c87161d6..81ed93a29524d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.json @@ -33,9 +33,11 @@ "literal": { "integer": 4 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.proto.bin index 692511b2f74a6..1037a1ffe38db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_except.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.json index f4540edbf4108..5bd114b61ad40 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.json @@ -26,7 +26,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.proto.bin index 6e2178ad124e9..a44ca96ace24a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_insert.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.json index 1b95a6724f86d..daa94e5aed678 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.json @@ -29,9 +29,11 @@ "literal": { "integer": 4 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.proto.bin index 67fb497cf270c..dca31097549b8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_intersect.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.json index 94e8c176cefbf..cbecc842d0b2c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.json @@ -22,7 +22,8 @@ "literal": { "string": ";" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.proto.bin index fbab1b208605d..0b2959acac448 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.json index ad580c33e476c..c5c5dce8cda9e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.json @@ -26,7 +26,8 @@ "literal": { "string": "null" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.proto.bin index e3fb6b3bf67c3..f0045cbeb529e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_join_with_null_replacement.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.json index ba67984758a5a..2bf3706f92b88 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.proto.bin index f7a98c08cd175..b8009a1fb6193 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_max.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.json index a342ae18f9ef7..6485fb4cf3691 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.proto.bin index 02cfdfeb215d6..45ac77d97e837 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_min.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.json index 4c212cb028273..20c7794a7de6b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.proto.bin index 4ef2b11273f25..ee2811ae0de0f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_position.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.json index ededeb015a227..ff6bd2b0e33fc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.json @@ -22,7 +22,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.proto.bin index 837710597e7b6..8ad00dfca7a04 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_prepend.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.json index 8c562247714a4..f769471cd9791 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.json @@ -22,7 +22,8 @@ "literal": { "integer": 314 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.proto.bin index 95e2872ad77bd..fd44cfb3372ad 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_remove.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.json index c9d9f1f9ca79d..0d218470c1ec1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.proto.bin index e370db16e977c..6302ae6ee3f0c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_repeat.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.json index c1c618bc7f11f..ac279580a09e1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.proto.bin index 47949dfbbda29..97554f7ecc930 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_size.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.json index 406dc54c8cd2f..74a038895b36a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.proto.bin index 2074caae16384..6ac4fd09dc0ca 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.json index f8178ddd64aaf..7a36c03476279 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.json @@ -31,7 +31,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -40,7 +41,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.proto.bin index c506889388c97..77719f7334985 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_sort_with_comparator.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.json b/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.json index 7d54079cdb47e..841888bcb1497 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.json @@ -33,9 +33,11 @@ "literal": { "integer": 3 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.proto.bin index fc3d9d7cd0fd1..ee743e69702ae 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_array_union.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.json b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.json index ce1d288e00d78..73b49b729edfa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.json @@ -29,9 +29,11 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.proto.bin index 216f306507d40..b555679123842 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_overlap.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.json b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.json index f24ee44835eb4..ce0c0ce75ab9c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.json @@ -29,9 +29,11 @@ "literal": { "integer": 20 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.proto.bin index 67c867e6d450c..c1baf7e190a15 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_arrays_zip.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.json index 3c4dcb70fead3..4c98e62f9dbb9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.proto.bin index 5989bd3b5c606..cd46fe994aedd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ascii.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_asin.json b/sql/connect/common/src/test/resources/query-tests/queries/function_asin.json index 4bf89be753458..6d06a28966590 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_asin.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_asin.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_asin.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_asin.proto.bin index 737ad789da268..6365765a3a45a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_asin.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_asin.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.json index 238571b0231c6..95900eaf761fd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.proto.bin index 01ea4675b22eb..f23beb2848fae 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_asinh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.json b/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.json index 5520b70a0250b..83cd89e5b9afa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.json @@ -25,13 +25,15 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }, { "literal": { "string": "id negative!" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.proto.bin index 6992604efe1b3..923478e910580 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_assert_true_with_message.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atan.json b/sql/connect/common/src/test/resources/query-tests/queries/function_atan.json index 3ae4e7ef188ec..2a873025e6254 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_atan.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_atan.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atan.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_atan.proto.bin index b932086941f45..c458d693ca127 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_atan.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_atan.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.json b/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.json index 7d08116c40ae6..53a03d1324f25 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.proto.bin index 372ae8358494e..25a25871185e7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_atan2.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.json index 8daec8813917e..f78b9f6421e89 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.proto.bin index 0aa2f3527ae9c..eebe635bc33e0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_atanh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_avg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_avg.json index b433f1ea89c29..a3d8868671520 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_avg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_avg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_avg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_avg.proto.bin index 9d9bd296dbdda..eb1078cd83ba7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_avg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_avg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_base64.json b/sql/connect/common/src/test/resources/query-tests/queries/function_base64.json index 97739dca283ef..8ba9e38dd538e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_base64.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_base64.json @@ -26,7 +26,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_base64.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_base64.proto.bin index fc854d974752b..0ab0e038829c3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_base64.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_base64.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bin.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bin.json index 304e56504bad9..56a5ce889d93d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bin.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bin.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bin.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bin.proto.bin index e8d55fb8d6149..64790b75bc090 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bin.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bin.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.json index 83b2bcf599f7c..3d156cec87a17 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin index ad81bec6f0865..af643d406b41e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_and.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.json index d5c6b698f7f16..277244f769144 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin index 875e17d974e21..1eb39a6fb7346 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.json index 39425c5e3ffbc..9e2b2d901308f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.json @@ -22,7 +22,8 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin index cd0f4098374c0..e3769a795a2f4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_get.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.json index df21871cb535d..1f09a52756b80 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.proto.bin index 860c2eaec0e85..13617e95afd73 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.json index c8e1b2acfe4e0..be21d9557e076 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin index a52907474fb96..9e4ef6d43a175 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_or.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json index 463e6fc5322f2..250d2d97c5b0b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin index c4a9a5e654f2b..33f0570e72f10 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bit_xor.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.json index 8956c3f303fea..1e8a60bae7e3d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.proto.bin index 8dcf7b1718d4b..849fa465d39db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bit_position.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.json index 8956c3f303fea..1e8a60bae7e3d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.proto.bin index 8dcf7b1718d4b..849fa465d39db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_bucket_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.json index 910702e90e0ed..aac1f57b5d9b6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.proto.bin index deae4a3aea072..8c57d776b1d63 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_construct_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.json index b2e9f11efbcbd..95095c7c28922 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.proto.bin index a568b2dae4208..216c51d3c4d29 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.json index 23e37246647b9..041ed9cb6330f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.proto.bin index d27edc3766288..41309fc648cda 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitmap_or_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.json index 7ddf73253e0a3..736751212fa96 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.proto.bin index bfaefb2a20075..5cfb94c1da38f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bitwise_not.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.json index 9b58c898242d9..3e013976a418f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "flag" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.proto.bin index 6ea3860027c86..e6e52551364e1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_and.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.json index 763b019b05f1f..7cf57c57812fd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "flag" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.proto.bin index dd928c8dc8a31..207083a8a12a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bool_or.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bround.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bround.json index 585a0befb224d..d42711c424c46 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bround.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bround.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bround.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bround.proto.bin index 8625ccb1a58f1..40e173d9df4a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bround.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bround.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.json b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.json index 3f35d627f9a54..8c3ec028f0a54 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.proto.bin index 200dac07a0bb7..e2cbbefb551a4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.json b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.json index cf0476340ccb3..9f8b77f176f72 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.proto.bin index d7669c93b2b89..25e48c663b97e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_btrim_with_specified_trim_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.json b/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.json index 971660144a5bc..002222d2f7765 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.proto.bin index 1b389401f15e6..874d68be22a53 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_bucket.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.json b/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.json index 0d78dd471f20c..a4c83ac456721 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.json @@ -19,7 +19,8 @@ "unparsedIdentifier": "g" } }], - "isUserDefinedFunction": true + "isUserDefinedFunction": true, + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.proto.bin index aee05767813f9..bde6f21637bea 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_call_function.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.json b/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.json index e2b3dd0428793..82b5dadd84073 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.proto.bin index 54c8cfe843433..232a89d09c6e8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_cardinality.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.json index 5a9961ab47f55..d6f4769366fe7 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.proto.bin index 3761deb1663a2..2b18da5724b81 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.json index bda5e85924c30..7413bb77656f5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.proto.bin index 8db402ac167e0..a658e6f879f89 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ceil_scale.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.json index 99726305e8524..878003697424f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin index cc91ac246a57c..57b7616c17930 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json index c0b0742b12157..c1fdfb5dfad50 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin index 30efc42b9d2bc..ba54d5d4d1edc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_char.json b/sql/connect/common/src/test/resources/query-tests/queries/function_char.json index 593139a0a584a..70f6fbcd475f6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_char.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_char.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_char.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_char.proto.bin index 21c3dad55657b..df7fb55ab4ab0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_char.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_char.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.json index 3e408260d7020..94323cf8e727e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.proto.bin index 7f290c6ddc623..93096e7ff4051 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_char_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.json index ad12dde8a956c..286b1e9d6040f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.proto.bin index f1762971d4eca..12932bdb3087d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_character_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_chr.json b/sql/connect/common/src/test/resources/query-tests/queries/function_chr.json index 28366f87e10d7..a7d551a94c194 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_chr.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_chr.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_chr.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_chr.proto.bin index dc665d294ecb6..7b51a92f63aae 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_chr.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_chr.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.json b/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.json index 497922b5df75c..523f853552b5e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.json @@ -22,7 +22,8 @@ "literal": { "integer": 3 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.proto.bin index ec871018489c2..6ceb52f4de47e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_coalesce.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_collate.json index 8bb6c2c4c3726..f1af032978d9e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_collate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_collate.json @@ -22,7 +22,8 @@ "literal": { "string": "UNICODE" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_collate.proto.bin index dda4a00a395b6..17ce43c3de332 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_collate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_collate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collation.json b/sql/connect/common/src/test/resources/query-tests/queries/function_collation.json index dac04b3b9858f..50ad1268ea9d6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_collation.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_collation.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collation.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_collation.proto.bin index 739994a486026..3cb87236be855 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_collation.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_collation.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.json b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.json index c5bae4baef352..81210507bc861 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.proto.bin index e3827b9f650ae..a66294b2c475d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_list.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.json b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.json index 615386d050e14..4ebf94c214391 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.proto.bin index 5fb97f27d25b6..abbd935086a33 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_collect_set.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_concat.json b/sql/connect/common/src/test/resources/query-tests/queries/function_concat.json index bad1ad6f3b90e..cd3702e86b3dc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_concat.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_concat.json @@ -29,7 +29,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -42,9 +43,11 @@ "literal": { "integer": 40 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_concat.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_concat.proto.bin index 7411f55f14747..87076f350fe42 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_concat.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_concat.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.json b/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.json index b9ba89b42185c..9468c641c0a81 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.proto.bin index 2fbc4f7090448..34d0bbdeb8794 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_concat_ws.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_contains.json b/sql/connect/common/src/test/resources/query-tests/queries/function_contains.json index b7cb12d9aa9a3..3a10154044dab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_contains.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_contains.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_contains.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_contains.proto.bin index 8864968a9dc3b..80eeae9cd02c2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_contains.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_contains.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_conv.json b/sql/connect/common/src/test/resources/query-tests/queries/function_conv.json index c6734936bfcd1..9e65edfed6ee6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_conv.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_conv.json @@ -26,7 +26,8 @@ "literal": { "integer": 16 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_conv.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_conv.proto.bin index 373b997b79240..3161da1cbca42 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_conv.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_conv.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.json index b27d7e2b55fae..6d0b24f2c8c72 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.proto.bin index 8ef4e3bdce29e..26e096c4c3f20 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_with_source_time_zone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.json index b072c89d42bd1..40f599741a9d2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.proto.bin index c6d1db9b8fb1d..a7f5317512a32 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_convert_timezone_without_source_time_zone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_corr.json b/sql/connect/common/src/test/resources/query-tests/queries/function_corr.json index 6fadb0385622b..c7c44457d4e0d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_corr.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_corr.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_corr.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_corr.proto.bin index fdeeb4fd12d19..5aac92504c0bd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_corr.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_corr.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cos.json b/sql/connect/common/src/test/resources/query-tests/queries/function_cos.json index f7072dff03404..f9ee2077f33a0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_cos.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_cos.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cos.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_cos.proto.bin index 09fd198b097c0..f7592d6ab63c9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_cos.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_cos.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.json index 3bcab61d37a0d..78b0d73b977ab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.proto.bin index 54d5da8fabfa6..5619d48d20397 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_cosh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cot.json b/sql/connect/common/src/test/resources/query-tests/queries/function_cot.json index 62ce963fa8737..1e521c2e6d7f6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_cot.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_cot.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cot.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_cot.proto.bin index e79c32660a772..e8ba807d9506a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_cot.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_cot.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count.json b/sql/connect/common/src/test/resources/query-tests/queries/function_count.json index 126a0ca242c52..613ba5510828f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_count.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_count.proto.bin index 6c87a809ad0c4..1e9b69069e05d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.json index eb211ceb239aa..6a844d694ae8f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.json @@ -23,7 +23,8 @@ "unparsedIdentifier": "g" } }], - "isDistinct": true + "isDistinct": true, + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.proto.bin index 591e2300ec689..b25c393cea048 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_countDistinct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.json b/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.json index 669477bbc5dd8..f19ce9ced1a5d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.json @@ -25,9 +25,11 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.proto.bin index 07c65ebaa4293..97306ae5253e4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_count_if.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.json b/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.json index 94be79dcc33e5..badc965380235 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.json @@ -30,7 +30,8 @@ "literal": { "integer": 11 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.proto.bin index 11bcae8062e82..d2bc25513fde3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_count_min_sketch.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.json b/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.json index 1c5df90b79cd1..db67d7e86bd46 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.proto.bin index 44b613eb40c6f..852290d484be3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_count_typed.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.json b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.json index 3c4df70a5fbfc..294d055796e08 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.proto.bin index 4a7202f15e768..09939663ce3bf 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_pop.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.json index 7c723069e4671..f366b2f62ae50 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.proto.bin index ebff687730e35..c8ef18e85fe67 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_covar_samp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.json b/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.json index 1892a9af85d97..7ab879e648a6c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.json @@ -26,7 +26,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.proto.bin index 54ad14dedae4e..07cb4138077d1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_crc32.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_csc.json b/sql/connect/common/src/test/resources/query-tests/queries/function_csc.json index 88504ed9c5280..68cc4e0096f4b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_csc.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_csc.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_csc.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_csc.proto.bin index 0ed5022a73adf..8bf716cbab8a1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_csc.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_csc.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.json b/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.json index ac48841199075..a2978d3242ca6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.json @@ -15,7 +15,8 @@ "window": { "windowFunction": { "unresolvedFunction": { - "functionName": "cume_dist" + "functionName": "cume_dist", + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.proto.bin index 7578245aabe3a..1dea3404d5a55 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_cume_dist.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.json index c344f5271704d..5b49d10fb5c56 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "curdate" + "functionName": "curdate", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin index 6ec58b57c2a7a..b307dc12a647e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_curdate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json index 27c7b23111908..ff962a03b8053 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_catalog" + "functionName": "current_catalog", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin index bb25a49935482..b1db0f9bd74d8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_catalog.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.json index dfa59fd5fe50a..1624c30ce4000 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_database" + "functionName": "current_database", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin index fdb11c9c8bd06..541891bc4418b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_database.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.json index 6dab8c39d626c..3f3c9c7dba5c0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_date" + "functionName": "current_date", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.proto.bin index f32c3f541c4c7..d6b94df786293 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_date.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.json index 01d5126a74b39..f8ac56d6398f0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_schema" + "functionName": "current_schema", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin index 9687afe89a50b..e083896cdfeae 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_schema.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.json index 16af5eb9ba084..bea08c2ee6af0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_timestamp" + "functionName": "current_timestamp", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.proto.bin index 5a1f3de6c3a9a..8691a0468fa82 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json index 082d7f5bae6f6..d3807c4dbbe92 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_timezone" + "functionName": "current_timezone", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin index a780830516bc0..bf06690e07afc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.json b/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.json index 30ddb80f884c2..f7bf8d9f91509 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "current_user" + "functionName": "current_user", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin index 0a79078fd7097..df4ea4e2cc4da 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_current_user.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.json index f81ad3335242c..1c022cdafbc3b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.proto.bin index f4dbc16b05c1d..d1e296f485cb7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_add.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.json index b6094ff8734d6..a4fc25f1bf0db 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.json @@ -33,9 +33,11 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.proto.bin index 5621af09474d8..bd37bb76df8e8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_diff.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.json index 9b3d469ed4e98..6ef48b7f015a3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.json @@ -22,7 +22,8 @@ "literal": { "string": "yyyy-MM-dd" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.proto.bin index 7226c20974b2a..f28036fdcc2d9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.json index ada0747743b78..08769597e4d7c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.json @@ -18,7 +18,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.proto.bin index 5cbd76eda9074..ba9b9047893d7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_from_unix_date.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.json index 5e8d075c4e2d0..51590b395c19a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.proto.bin index 368ecb676c1fe..cddd5c1954888 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_part.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.json index f1dde0902a20a..2d74935b65010 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.proto.bin index 43b630c27ed45..a033be1ab6e41 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_sub.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.json b/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.json index 363da9b9b9006..649c211af8465 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.json @@ -22,7 +22,8 @@ "literal": { "string": "minute" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.proto.bin index f037fb8d34a56..fad72e5dc4eb8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_date_trunc.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.json index 2658c724d287a..e8d272bafa922 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.proto.bin index e72a77f0e2394..1dcbd5fc64ca2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dateadd.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.json b/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.json index b5ef560486d0d..e9e9dccf52f95 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.json @@ -33,9 +33,11 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.proto.bin index 02e917b406838..bbcd3b621848a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_datediff.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.json b/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.json index cc4dca8674264..42b21d6e9c831 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.proto.bin index 9d58fd3474d70..470e4796515fe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_datepart.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_day.json b/sql/connect/common/src/test/resources/query-tests/queries/function_day.json index c1e4b4d13fb7a..712ad56df109d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_day.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_day.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_day.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_day.proto.bin index e72a4a354c31d..9115ae09ce8fc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_day.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_day.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.json index 7898aa53deb89..a27513443fe4a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.proto.bin index 2809f3b9b7a11..370737bbc1fb7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dayname.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.json index 3e453c1f7a652..76b0a6e03f27c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.proto.bin index 3a2973e21e5a0..8a63ee68777e8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofmonth.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.json index 74715de151e77..7cea8fdc7745d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.proto.bin index fceea203c790e..6c9a6252a48ed 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofweek.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.json index d23c6790a47dd..3579113015448 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.proto.bin index a526b449ae0a4..ef1d60c231599 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dayofyear.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_days.json b/sql/connect/common/src/test/resources/query-tests/queries/function_days.json index 9e20c48729a30..b9a73878304b6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_days.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_days.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_days.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_days.proto.bin index b0a8472f8c4ff..b81c13479a36f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_days.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_days.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_decode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_decode.json index 6be60808e64f3..2aaae4466b5fc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_decode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_decode.json @@ -30,7 +30,8 @@ "literal": { "string": "UTF-8" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_decode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_decode.proto.bin index 18b8bbcf6a01d..d71f018e81ee8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_decode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_decode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.json b/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.json index e096b07e4dc6e..f366faf9ac7c5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.proto.bin index e2d264bb2e108..efe6bb3a93cca 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_degrees.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.json b/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.json index 46c5e1eaddfc0..2699a863e9755 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.json @@ -15,7 +15,8 @@ "window": { "windowFunction": { "unresolvedFunction": { - "functionName": "dense_rank" + "functionName": "dense_rank", + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.proto.bin index 4597e63be8379..bd6861d0514e4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_dense_rank.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_e.json b/sql/connect/common/src/test/resources/query-tests/queries/function_e.json index c99c04a6befdb..12f1ed4146bb1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_e.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_e.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "e" + "functionName": "e", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_e.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_e.proto.bin index 49f6c12fbcc72..e08ad39fe02f7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_e.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_e.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.json b/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.json index ef5551440934c..562f84a0abfb3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.json @@ -22,7 +22,8 @@ "literal": { "string": "bob" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.proto.bin index 993818c6cb4bf..6cbf58ebaaf63 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_element_at.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_elt.json b/sql/connect/common/src/test/resources/query-tests/queries/function_elt.json index fe7dd29f91a33..0573d25e70879 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_elt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_elt.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_elt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_elt.proto.bin index d719db6f89c37..429c39ca81b26 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_elt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_elt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_encode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_encode.json index 92e95f2c946d0..51d0c871f3494 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_encode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_encode.json @@ -22,7 +22,8 @@ "literal": { "string": "UTF-8" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_encode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_encode.proto.bin index 9644825af470b..a996b0e73435b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_encode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_encode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.json b/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.json index 1f7943f5116ba..2577b62ccbca8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.proto.bin index 2dfef1c6d86bc..89619bc8c38ba 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_endswith.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.json b/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.json index bc53edfa25f5e..50963dd1c9d0a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.proto.bin index 2855f3ebbf3c3..ad21a1e96b9de 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_equal_null.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_every.json b/sql/connect/common/src/test/resources/query-tests/queries/function_every.json index ffefb8cf1103f..31d78cf56c07e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_every.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_every.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "flag" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_every.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_every.proto.bin index 1b28782200223..6ccfd9c879923 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_every.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_every.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_exists.json b/sql/connect/common/src/test/resources/query-tests/queries/function_exists.json index 3ae49d13c5fc6..caa2c6170f251 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_exists.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_exists.json @@ -31,14 +31,16 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, "arguments": [{ "nameParts": ["x_1"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_exists.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_exists.proto.bin index d808227fdc659..ac30b86a396da 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_exists.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_exists.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_exp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_exp.json index d317efef75eee..a814a7f3a4937 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_exp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_exp.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_exp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_exp.proto.bin index 7def20c94df00..0803af7c262c1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_exp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_exp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_explode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_explode.json index 35ad40ccdd04f..8c0094b3a39d5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_explode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_explode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_explode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_explode.proto.bin index 9c15f942bb11d..fb5d06b544709 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_explode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_explode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.json b/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.json index efd7f4b524d47..12c753a38be70 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.proto.bin index 9f2cf9554dd15..8e9ecc3cfacb0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_explode_outer.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.json b/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.json index d425a6de709b7..70cd7b9284654 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.proto.bin index 3c310cb04ce3d..daa03048ab5b5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_expm1.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_extract.json b/sql/connect/common/src/test/resources/query-tests/queries/function_extract.json index 6ccdb2f9d7531..851ae5de2741a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_extract.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_extract.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_extract.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_extract.proto.bin index 91553c3b94bcf..1d47c480d7708 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_extract.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_extract.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.json b/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.json index 7f13a10480915..e762531e0be86 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.json @@ -25,9 +25,11 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.proto.bin index ac776eb60d2b0..4863eee97ac1c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_factorial.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_filter.json b/sql/connect/common/src/test/resources/query-tests/queries/function_filter.json index 1c71362f75247..6963b28386740 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_filter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_filter.json @@ -31,14 +31,16 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, "arguments": [{ "nameParts": ["x_1"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_filter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_filter.proto.bin index aa776b474a4d6..b669b9ff69c2b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_filter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_filter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.json b/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.json index f2b85c21af755..94f04a9111a88 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.json @@ -34,7 +34,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -47,9 +48,11 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -58,7 +61,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.proto.bin index 8cf5f2d65cf29..b927f1c623a44 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_filter_with_pair_input.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.json b/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.json index 538651b52c424..424357c82f13d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.proto.bin index 26abfa0e394c3..662aec53059a7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_find_in_set.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.json index af55fe44ae8ca..bab44ade0cfaa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.proto.bin index 7121c820aa737..6c72610c94265 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_ignore_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.json index 8276e35893feb..85ef747ef81ba 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.proto.bin index b843d52111528..b6c8d358976a8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_first_value_with_respect_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.json index dc33bad3c506a..ec4c9da731d65 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.proto.bin index cb029dfd26be9..a995bb05e6f26 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_ignore_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.json index 0e315cc6b1bce..244f2d1f9575e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.proto.bin index bf1d48903dfab..eb52e48309944 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_first_with_respect_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.json b/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.json index 1f04630fd5f31..3118b0620c0e3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.json @@ -32,11 +32,14 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.proto.bin index 9a684850f9cfa..1d930ab05303c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_flatten.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_floor.json b/sql/connect/common/src/test/resources/query-tests/queries/function_floor.json index 78924f5f33627..9bf149ea295d9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_floor.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_floor.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_floor.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_floor.proto.bin index b52696ca4d00a..1f529ad20219f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_floor.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_floor.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.json b/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.json index 394621e4dd314..af37c822a21df 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.proto.bin index ee0665bab644c..f2510d8540374 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_floor_scale.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_forall.json b/sql/connect/common/src/test/resources/query-tests/queries/function_forall.json index 4a4914d6a9b1d..a9084fe59bdb6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_forall.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_forall.json @@ -31,14 +31,16 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }, "arguments": [{ "nameParts": ["x_1"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_forall.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_forall.proto.bin index 7fc2821694589..b72863b1db28b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_forall.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_forall.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.json index daa648c0a599e..6f376efc9fa81 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.json @@ -22,7 +22,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.proto.bin index 81e2c4d5fd54d..2f156c4336703 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_format_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.json index 798e79e6618f5..60242d45866aa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.json @@ -33,9 +33,11 @@ "literal": { "string": "FAILFAST" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.proto.bin index 8acd3b619b41e..2a3a14f8914a1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_csv.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.json index ddfa91abca05e..850ae645c0497 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.proto.bin index ad95d0f2b343d..5a1990c957290 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.json index ddfa91abca05e..850ae645c0497 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.proto.bin index ad95d0f2b343d..5a1990c957290 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_json_with_json_schema.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.json index 81d6608adb18f..bbe54e42d2fe8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.json @@ -18,7 +18,8 @@ "literal": { "long": "1" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.proto.bin index b1c34caaf62f0..5b92281e1f289 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_unixtime.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.json index 5d63fd829f302..ce7d64ebdc541 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.json @@ -22,7 +22,8 @@ "literal": { "string": "-08:00" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.proto.bin index 34bf9c64f3a97..c562454e01d11 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_utc_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.json index cfcd40a74b3a7..f137f14baac3a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.proto.bin index 1cc3a26c254fb..dc331f2b26074 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.json b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.json index cfcd40a74b3a7..f137f14baac3a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"a\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"b\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.proto.bin index 1cc3a26c254fb..dc331f2b26074 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_from_xml_with_json_schema.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_get.json b/sql/connect/common/src/test/resources/query-tests/queries/function_get.json index 7a2a89447c079..fd8f86b0f518c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_get.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_get.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_get.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_get.proto.bin index be40df955a407..9561814a9e2d2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_get.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_get.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.json b/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.json index 17adf9230a6eb..505bfcc07bf89 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.json @@ -22,7 +22,8 @@ "literal": { "string": "$.device_type" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.proto.bin index 08ad8f4f91bad..df7774ef54842 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_get_json_object.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.json b/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.json index ef33382022a53..8bb777bfc37b2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.json @@ -22,7 +22,8 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin index 15575e4f7cbb9..b7684864503db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_getbit.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.json b/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.json index bf5d50edec84f..92414ef27da5b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.json @@ -31,7 +31,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.proto.bin index 44d9d5f8cfb2d..a1ba220f29429 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_greatest.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hash.json b/sql/connect/common/src/test/resources/query-tests/queries/function_hash.json index 6ef504a006457..04363f49f72b8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_hash.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_hash.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hash.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_hash.proto.bin index 284700c4c5ea9..dea61a10eeb52 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_hash.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_hash.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hex.json b/sql/connect/common/src/test/resources/query-tests/queries/function_hex.json index af9d0dd298277..7cfc7a5647adf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_hex.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_hex.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hex.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_hex.proto.bin index 9d8c3b5e23584..52d2da3ea1e8b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_hex.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_hex.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.json b/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.json index 548b4977ddc50..2758e1a0638f6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.proto.bin index 81dbcd476ecbc..465557aa7f48b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_histogram_numeric.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hour.json b/sql/connect/common/src/test/resources/query-tests/queries/function_hour.json index 2621b9f81913c..e13d18f244903 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_hour.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_hour.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hour.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_hour.proto.bin index 6cdb50364c133..f57e2333ba855 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_hour.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_hour.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hours.json b/sql/connect/common/src/test/resources/query-tests/queries/function_hours.json index a72a8656362fd..d521c709a93d2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_hours.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_hours.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hours.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_hours.proto.bin index 6e8203b89e320..1c73b8d22fbb3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_hours.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_hours.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.json b/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.json index 2d0d6be0164bc..42fe4ac079293 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.proto.bin index 3ad07a2a1ee45..8dce8d0b02a00 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_hypot.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.json index 2a426fe6fff3a..577d45ddfe6e8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.proto.bin index c1307c2be8caa..9840001010b01 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ifnull.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.json index 46b1b87e03246..aa97c83dba131 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.proto.bin index b1c50e3aaf4b2..8dd2e803b2234 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.json index 6392912efe85d..f3fcfd7a86b4e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.json @@ -26,7 +26,8 @@ "literal": { "string": "/" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.proto.bin index de0d89f2c8cee..6a5714f5731c4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ilike_with_escape.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.json b/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.json index 896bb3d0209da..733efb38bac41 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.proto.bin index 72df35bd9b387..9129dbf467c28 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_initcap.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_inline.json b/sql/connect/common/src/test/resources/query-tests/queries/function_inline.json index 4abdac736d0fe..f63e0184608ee 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_inline.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_inline.json @@ -21,9 +21,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_inline.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_inline.proto.bin index 261e28e3acaa8..105edc74658df 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_inline.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_inline.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.json b/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.json index d74ee83eeff3e..8ad2752040c41 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.json @@ -21,9 +21,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.proto.bin index d757e5afe3050..fc847de167c69 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_inline_outer.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.json index 2b478579f377d..69f7aae01de2b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "input_file_block_length" + "functionName": "input_file_block_length", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.proto.bin index 55684ba7d1b02..6dd26fb322275 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.json b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.json index a85f58c3b9aaf..d05cfb4126397 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "input_file_block_start" + "functionName": "input_file_block_start", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.proto.bin index 6fa8027cc82e2..eed5dac21f88a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_block_start.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.json b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.json index 47f2e461eba46..7801d2afa72b1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "input_file_name" + "functionName": "input_file_name", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.proto.bin index c3c6414d5d881..b1eae731498e6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_input_file_name.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.json b/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.json index 7ae72f8f88e5c..2743ad7263a7f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.json @@ -21,9 +21,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.proto.bin index 4d3d2624609e7..2bf328f8a9db5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_is_variant_null.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.json b/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.json index f594918ed930a..12d3d19d7797a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.proto.bin index 1030abda5b8c2..8c2fad75be346 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_isnan.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.json b/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.json index 6ec209e7c24fc..eb011fe4664eb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.proto.bin index 1b37308d24753..5bd84491a2b94 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_isnotnull.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.json b/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.json index 7443fc97f42cf..11608a0de2195 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.proto.bin index 3d1fbd4dedfe7..05217e9e9085b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_isnull.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.json b/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.json index 196dd4869577f..828ccaf2c570e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.proto.bin index b5cd2ea0e929b..971395d4b3573 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_java_method.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.json index 36223a451e3f8..1789c0bef0840 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.proto.bin index 817c803d83033..79aefffa10332 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_json_array_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.json b/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.json index f8667a1012a08..9f9f60134485c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.proto.bin index 4be9477ec9185..023ce9ea65266 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_json_object_keys.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.json b/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.json index 32de63452c364..4f2b072db3e25 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.json @@ -30,7 +30,8 @@ "literal": { "string": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.proto.bin index e51be42b38d34..60b1685ece0f1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_json_tuple.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.json b/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.json index 7399d7a6da388..386c3c8205046 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.proto.bin index 848a4842e2462..30cd079db28d1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_kurtosis.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lag.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lag.json index dd1cba376f3c7..f0f38c35c0c5f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lag.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lag.json @@ -35,7 +35,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lag.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lag.proto.bin index 7fd85861fb8c8..3abeec8a65bfb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lag.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lag.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.json b/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.json index 2cb1635caf47e..cbd450c2859fc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.proto.bin index 1afb5c02ae347..925a919daffc0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_last_day.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.json index e78a456082cbd..18e55564d6ac0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.proto.bin index c04f8385995ee..0baa09d55bc20 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_ignore_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.json index cb509997e6533..d1388758fe8a6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.proto.bin index cee9838b70438..80bf3dfcc9abb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_last_value_with_respect_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.json index 6d1be02c78545..cb147d6998478 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.proto.bin index f6590582c6f5a..2477c1e58803d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_ignore_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.json b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.json index f26e5887ed527..dd68e3d189c03 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.json @@ -22,7 +22,8 @@ "literal": { "boolean": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.proto.bin index 69221737be671..a4a02664b5030 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_last_with_respect_nulls.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.json index a1610815b6c7d..a3a293bc7b1b8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.proto.bin index d5627abb0a5d0..e18e7e6781e76 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lcase.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lead.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lead.json index ef76586d381dd..aab3c54f48954 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lead.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lead.json @@ -32,7 +32,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lead.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lead.proto.bin index 9bcdcdb3617a9..f72eea8071743 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lead.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lead.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_least.json b/sql/connect/common/src/test/resources/query-tests/queries/function_least.json index 403531c9f6958..d0bcb0b07e404 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_least.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_least.json @@ -31,7 +31,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_least.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_least.proto.bin index c9ead802a9616..bcb27b7e02114 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_least.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_least.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_left.json b/sql/connect/common/src/test/resources/query-tests/queries/function_left.json index e629782ba6d5b..dbf9948ee3555 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_left.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_left.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_left.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_left.proto.bin index 497cf68194e88..c5b4ff7f56763 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_left.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_left.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_len.json b/sql/connect/common/src/test/resources/query-tests/queries/function_len.json index 884f875a961da..3b353abe2eac6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_len.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_len.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_len.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_len.proto.bin index 939a6c9c3360b..f787d98c0698e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_len.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_len.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_length.json index f2c3c69255897..cbb6cf10974ad 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_length.proto.bin index a14f94085b3b6..f4a53e9fa48d4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.json b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.json index 10caaf184fee5..878dc8f564869 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.json @@ -22,7 +22,8 @@ "literal": { "string": "bob" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.proto.bin index 75b48541b7663..0742bd0585665 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.json b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.json index 5cc30772e8e88..aceb63829aa8a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.json @@ -26,7 +26,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.proto.bin index 22e1a3328756e..2ba96d65869c6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_levenshtein_with_threshold.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_like.json b/sql/connect/common/src/test/resources/query-tests/queries/function_like.json index 3ce3431e50f8f..23b9578abf5b3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_like.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_like.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_like.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_like.proto.bin index d9a13f5c79bce..2f27591f3d68e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_like.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_like.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.json b/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.json index 0313398f0ad60..04e27ea1de9e1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.json @@ -26,7 +26,8 @@ "literal": { "string": "/" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.proto.bin index cc5fefe193fb9..0668e85576e04 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_like_with_escape.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ln.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ln.json index ababbc52d088d..4fe4ccd2aa34d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ln.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ln.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin index ecb87a1fc4102..3296765cb86cd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ln.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.json index 68281d2e6d9d1..5683069790afa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "localtimestamp" + "functionName": "localtimestamp", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.proto.bin index b1a9e70c7c802..e280db3af4ccd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_localtimestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_locate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_locate.json index 7939fdd2c7559..285374acfe075 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_locate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_locate.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_locate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_locate.proto.bin index cc7ced9957a52..e43b34bf0665f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_locate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_locate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.json b/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.json index 269f39701608a..87fa8050f4005 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.json @@ -26,7 +26,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.proto.bin index 162ab0108c132..b11ee8f8ae770 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_locate_with_pos.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log.json b/sql/connect/common/src/test/resources/query-tests/queries/function_log.json index ababbc52d088d..4fe4ccd2aa34d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_log.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_log.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_log.proto.bin index ecb87a1fc4102..3296765cb86cd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_log.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_log.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log10.json b/sql/connect/common/src/test/resources/query-tests/queries/function_log10.json index 13292d83c4727..bced949b9aaf1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_log10.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_log10.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log10.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_log10.proto.bin index 22d4655a6efbd..1a363b7043dc3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_log10.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_log10.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.json b/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.json index 4e9e6847c3c36..95942299457da 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.proto.bin index 9a72c377b0cc4..6097a554cee23 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_log1p.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log2.json b/sql/connect/common/src/test/resources/query-tests/queries/function_log2.json index ec29e154a0e1d..4fe3d22e03415 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_log2.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_log2.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log2.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_log2.proto.bin index 34e3780650540..39aab70e5ac3e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_log2.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_log2.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.json b/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.json index 6bc2a4ec3335a..3d6e05077dd6c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.proto.bin index 2e64e15ed5555..95f9352d73f93 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_log_with_base.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lower.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lower.json index f7fe5beba2c02..e2b4a5816e686 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lower.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lower.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lower.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lower.proto.bin index 7c736d93f7729..f9f4d930ebc23 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lower.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lower.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.json index b9f3e6700bfa4..7cdd2b75dadc6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.json @@ -26,7 +26,8 @@ "literal": { "string": "-" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.proto.bin index 470ab1cc44add..8576f2b0be3b7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.json b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.json index aeb39ba09ad20..60eaa163e632d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.json @@ -26,7 +26,8 @@ "literal": { "binary": "DAoPDg==" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.proto.bin index b4acebb394c7a..a8651574a6bbb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_lpad_binary.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.json index dd3b459520221..1c9ac2e57ffbb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.proto.bin index 162b6a7337bb9..ed81a583ccaac 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.json index 01dc81bdae7bc..f67f689d0d950 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.proto.bin index 0cd62226c9716..6a9d3bf7b00ef 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ltrim_with_pattern.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.json index a363298dd123a..0f78921e965e5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.json @@ -26,7 +26,8 @@ "literal": { "integer": 14 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.proto.bin index 0526825fccade..c47ea55dbe6cc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_date.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.json index 24cd85d7ae442..66e5da74d5e45 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "make_dt_interval" + "functionName": "make_dt_interval", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.proto.bin index 09c5a25a10cc2..00bedb03e9dc4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.json index fb65f1fcd9def..9fe3d9dedbc83 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.proto.bin index 11c67d98f9f49..70cd3d296df6a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.json index e08a9c3b083ee..f902694ef7774 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.proto.bin index bd16de042f61d..d8107489e1e78 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.json index a4b8c14538ae3..5a760c1b469fb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.proto.bin index 7595205c6bb0e..89a249a8535fc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.json index 20eaa7521d3d9..745012b755db3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.proto.bin index 6db1bc8b51bea..ba6a2b4e10f82 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_dt_interval_days_hours_mins_secs.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.json index cdbe616565287..eceeeddd4b15b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "make_interval" + "functionName": "make_interval", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.proto.bin index 8d4327eeff426..c569945e7b4c1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.json index e5afa5ec3349a..e49924238ced8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.proto.bin index 7be990a47aba6..84ed402ba145f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.json index 9de86b70c169e..a7f87a2015bc1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.proto.bin index 219cc5a023d45..a886b4d11afd3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.json index c387757c6f739..7ca71d529325b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.proto.bin index 7fb48227f69c7..af3f2d4b4519f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.json index 54274116ee951..d0741256492d1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.proto.bin index 58d62b76ac5f9..0d3355c516bdd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.json index a6e343532ec2c..aecddb3dcb53f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.json @@ -34,7 +34,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.proto.bin index 3133c2d497ea6..52b889c8ad81b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.json index d921fda962896..b5ed4c2e8d4e8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.proto.bin index cf2ad98b5c16c..7d06032ee8a45 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.json index 09f95bc933b10..372b13ce44621 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin index dca680fd90b61..033d88d328ed1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.json index a58259eefe742..7572d311648c0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.proto.bin index 3eac6534c6510..ecc490af0f3c4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_with_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.json index 5c87a856fc6c0..9e10c499ee9ee 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.proto.bin index b35f9fd474607..2baa0d4f269d4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ltz_without_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.json index 4cc4f1a11acdc..6241859ae9269 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.proto.bin index 5a6554443ceca..17dc37d99f63f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_ntz.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.json index a7a5ff132c083..a8d427ae58c0c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.proto.bin index 77c0d5961c804..660c4399e35f4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_with_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.json index 286ed33f82e10..8426f3dd45143 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.proto.bin index f91efead687a8..348befe79dbf8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_timestamp_without_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.json index dc1848be0cc42..f37ca015eed91 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "make_ym_interval" + "functionName": "make_ym_interval", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.proto.bin index eaffc7c237094..0c63f66caf9a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.json index d789064ad9b8e..81b70cf342205 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.proto.bin index 1938b7c53bdd1..7486df5d7530d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.json b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.json index aeffbbb4a1a14..1eb67c528682e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.proto.bin index f03f6ecce83de..5a85f5981b948 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_make_ym_interval_years_months.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map.json index ca9d3bf2bcc71..830445d3facf8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map.json @@ -30,7 +30,8 @@ "literal": { "string": "dummy" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map.proto.bin index 229a48b75131d..32b233fa939db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.json index f56f6cee20ab0..6e076de494412 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.json @@ -55,11 +55,14 @@ }, "name": ["b"] } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.proto.bin index 0a76d3a1193ea..6c04979af6da7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_concat.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.json index 56833f9651023..c25e508806149 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.json @@ -22,7 +22,8 @@ "literal": { "string": "xyz" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.proto.bin index e517479020e16..09bd7fa7c4796 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_contains_key.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.json index 0226506545010..310b3980931a1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.proto.bin index f1451d4ad7ba4..1050436839a0a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_entries.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.json index b50e77c0bf8e9..48c7871244f19 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.json @@ -31,7 +31,8 @@ "literal": { "string": "baz" } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -40,7 +41,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.proto.bin index 7f3d0c31fd6fe..a6143e55099ff 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_filter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.json index 1eb1f7d2ef066..0ce0f1f547174 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.json @@ -25,7 +25,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -38,9 +39,11 @@ "literal": { "string": "two" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.proto.bin index f5333b1c882bc..e85aba1a35ce7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_arrays.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.json index 1e48a1c2082df..e505d884d85f6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.json @@ -34,7 +34,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["x_1"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -43,9 +44,11 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.proto.bin index 0dd0d31350991..88d178e75c4fe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_from_entries.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.json index 5af013295cd9f..1e25cd90ac88c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.proto.bin index ee19968bacc2c..b45fde5acd6bd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_keys.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.json index 3c5eb651801dc..96ca6b6807963 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.proto.bin index 4cd7c488ada48..6708875252f51 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_values.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.json b/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.json index d13bd8dce75f3..9e913743009d1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.json @@ -53,7 +53,8 @@ } } } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -64,7 +65,8 @@ "nameParts": ["z_3"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.proto.bin index 2770b083e32ef..65dba7ed7bbd1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_map_zip_with.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mask.json index c0473466a3e1c..f4c6dad9b6385 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mask.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mask.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mask.proto.bin index 5e94c2675937d..6397cc4fe4adc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mask.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mask.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.json index 571d514e72ded..6dc93c69e10e8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.json @@ -22,7 +22,8 @@ "literal": { "string": "X" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.proto.bin index 0f6c4b579c4f5..699d662fcca48 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.json index ae527d70cf162..671a19a8900af 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.json @@ -26,7 +26,8 @@ "literal": { "string": "x" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.proto.bin index 5a6b4d7caa60e..c754da066d573 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.json index e7fee11d3169e..f12f44b3ecb38 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.json @@ -30,7 +30,8 @@ "literal": { "string": "n" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.proto.bin index f0a2e7cb643af..473f724d4126c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.json index d6076ae558bc7..1ddb661b636bb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.json @@ -34,7 +34,8 @@ "literal": { "string": "*" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.proto.bin index cb5f090361b20..bf641173435f3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mask_with_specific_upperChar_lowerChar_digitChar_otherChar.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_max.json b/sql/connect/common/src/test/resources/query-tests/queries/function_max.json index b23dd9d14c643..1514b34f8b462 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_max.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_max.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin index 788c9539b5767..d36b5d79ecbd9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.json b/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.json index da311e340cc50..1048a30325e5a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.proto.bin index 284c2453af8bd..1f1832962cb34 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_max_by.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_md5.json b/sql/connect/common/src/test/resources/query-tests/queries/function_md5.json index e8718594b0be3..d954d60a9c68a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_md5.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_md5.json @@ -26,7 +26,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_md5.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_md5.proto.bin index d3ec7c26a2ede..87ee03e940081 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_md5.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_md5.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_median.json b/sql/connect/common/src/test/resources/query-tests/queries/function_median.json index 7331454b9ecb0..a358a25d85705 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_median.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_median.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_median.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_median.proto.bin index 59533e5be5992..5a80a2f7cd44b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_median.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_median.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_min.json b/sql/connect/common/src/test/resources/query-tests/queries/function_min.json index 1b7266b6774e4..3fba2b795a224 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_min.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_min.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_min.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_min.proto.bin index b82f4c5309222..2ae3da3391fa5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_min.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_min.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.json b/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.json index d2478f5e81abe..6c9b99ad7d43d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.proto.bin index ddc642b95000c..da76415ec74a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_min_by.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_minute.json b/sql/connect/common/src/test/resources/query-tests/queries/function_minute.json index 7c749cdff82f5..c94a8703d38e9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_minute.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_minute.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_minute.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_minute.proto.bin index e81b7dad85331..b0b743773e902 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_minute.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_minute.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_mode.json index 8e8183e9e0883..a4f3b601ad47a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_mode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_mode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_mode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_mode.proto.bin index dca0953a387b1..e3dfc96922e62 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_mode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_mode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.json b/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.json index 0a14f1008976e..01ca4536c97f9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "monotonically_increasing_id" + "functionName": "monotonically_increasing_id", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.proto.bin index 724ce3ac6904c..2e86a0566afaf 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_monotonically_increasing_id.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_month.json b/sql/connect/common/src/test/resources/query-tests/queries/function_month.json index 7ea1e5d0375e9..6343ed28faa01 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_month.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_month.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_month.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_month.proto.bin index b97100a6fe2ec..1c09b5d1f26c2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_month.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_month.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.json b/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.json index c5ad3485252f1..7da09908cadaa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.proto.bin index 4518bb8d74253..958b9b34dff86 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_monthname.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months.json b/sql/connect/common/src/test/resources/query-tests/queries/function_months.json index 278bab76a6544..235893cf6cdf3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_months.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_months.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_months.proto.bin index fdcd96750dc9c..1a689d77c7019 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_months.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_months.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.json b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.json index 0fa772d26cd41..f02f9c3ea416a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.json @@ -16,13 +16,15 @@ "functionName": "months_between", "arguments": [{ "unresolvedFunction": { - "functionName": "current_date" + "functionName": "current_date", + "isInternal": false } }, { "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.proto.bin index 22ddc1813e0fb..34b5c49c83375 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.json b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.json index d11bfbd7f2426..f03709aece83c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.json @@ -16,7 +16,8 @@ "functionName": "months_between", "arguments": [{ "unresolvedFunction": { - "functionName": "current_date" + "functionName": "current_date", + "isInternal": false } }, { "unresolvedAttribute": { @@ -26,7 +27,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.proto.bin index bf9c545911ffd..a072570756411 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_months_between_with_roundoff.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.json index c4d92131ed06c..a23893ddb0fcf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.proto.bin index b595cfc282036..87c310823426b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_named_struct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.json b/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.json index 69daab270c2b9..151c3d830716f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.proto.bin index f314a73dcae65..edddf04956a74 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_nanvl.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_negate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_negate.json index e269fabe44be1..96f9359193fbd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_negate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_negate.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_negate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_negate.proto.bin index 9c56c111ceee6..f4d42e82e0c0d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_negate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_negate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_negative.json b/sql/connect/common/src/test/resources/query-tests/queries/function_negative.json index e269fabe44be1..96f9359193fbd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_negative.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_negative.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_negative.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_negative.proto.bin index 9c56c111ceee6..f4d42e82e0c0d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_negative.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_negative.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.json b/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.json index 486523dcad3ec..ad6f3bb22ff82 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.json @@ -22,7 +22,8 @@ "literal": { "string": "Mon" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.proto.bin index a97bd75f129db..8cece90ab671b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_next_day.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_now.json b/sql/connect/common/src/test/resources/query-tests/queries/function_now.json index 98556585c3e31..1ceb0bd1366ff 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_now.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_now.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "now" + "functionName": "now", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_now.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_now.proto.bin index a8fcd67fa1982..f9ab22b653c09 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_now.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_now.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.json b/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.json index 4c764a5d5603c..97f434a6d71ae 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.json @@ -28,7 +28,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.proto.bin index f87e1695f22e3..cd6eeac2e054e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_nth_value.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.json index 2346a788b64bd..595cfe02b8631 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.json @@ -20,7 +20,8 @@ "literal": { "integer": 4 } - }] + }], + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.proto.bin index d9ccd2e8a6007..fa7d6cac0bf17 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ntile.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.json b/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.json index 3892eb19fc52c..ac9f5620f9243 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.proto.bin index 9bbf5f4ccb8ac..0217381b686ff 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_nullif.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.json b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.json index 483448c26d114..1bf1f22bcad2b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.proto.bin index 21a9b37eb65ec..663f7c714883c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.json b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.json index 8db7f9ba6292c..408d6eba0a05b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.proto.bin index 8b7f90bf27552..627a671a085e2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_nvl2.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.json b/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.json index 7be9ac82662a4..bfd624216f70d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.proto.bin index 484ebbb6487b0..3f94747ea2595 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_octet_length.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.json b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.json index b580570f923a6..28cb276fe4cb1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.json @@ -26,7 +26,8 @@ "literal": { "integer": 4 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.proto.bin index 2110ae9c14610..43d75d4a07231 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.json b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.json index 99d5426c46fba..79f0be1011dab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.json @@ -30,7 +30,8 @@ "literal": { "string": "3" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.proto.bin index 9a09d28d84fde..9521c75b4a83e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_overlay_with_len.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.json b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.json index dfcf56c19223e..4a84cbe99b65d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.proto.bin index a7187fa2c1af0..cea247c862173 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_json.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.json b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.json index e03b86c21eb94..e193266998299 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.proto.bin index 56917289c1ec9..f15b2316d3c2e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.json b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.json index bd627911ef22d..862e3b5aa8d3a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.proto.bin index 231622cbd8a6b..543b9dd6f43dc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_parse_url_with_key.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.json b/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.json index d8778ec8cd81d..4770d3d81d6d5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.json @@ -15,7 +15,8 @@ "window": { "windowFunction": { "unresolvedFunction": { - "functionName": "percent_rank" + "functionName": "percent_rank", + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.proto.bin index d668f7e1504cb..2dcfbf3777ee9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_percent_rank.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.json b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.json index 6289464de2a37..16f5f1f2b4b7b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.json @@ -26,7 +26,8 @@ "literal": { "integer": 20 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.proto.bin index f44ec86888f6c..159c21c647729 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_approx.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json index f57804426643d..fb548fd0233e1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json @@ -26,7 +26,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin index 91d6279f9bd8c..13be09fc2901a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json index 44e2c98a4dc60..3af4be6aad5cd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json @@ -22,7 +22,8 @@ "literal": { "double": 0.3 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin index 45b807e5ffbd2..93156e49b4556 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pi.json b/sql/connect/common/src/test/resources/query-tests/queries/function_pi.json index 46474dfd8e369..d73ca1d6ca691 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_pi.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_pi.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "pi" + "functionName": "pi", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pi.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_pi.proto.bin index 14f018904bfb7..33fee15270257 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_pi.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_pi.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.json b/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.json index 1dc2cb54cbb67..fa6edbcda84f8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.proto.bin index a2bb94dbb5173..068b878b77ca5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_pmod.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.json index f8a9db37e62be..261ee78e20b43 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.proto.bin index fc50f5f4c85b7..9b22124951e80 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.json b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.json index 0e8cd4c1509e1..71d11fd72cd8e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.proto.bin index 19d700665e7f5..d771cb8c33739 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_posexplode_outer.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_position.json b/sql/connect/common/src/test/resources/query-tests/queries/function_position.json index 7b005e2bb8213..e71a363461f8d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_position.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_position.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_position.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_position.proto.bin index 34b7e301fe943..db2530e0625ba 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_position.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_position.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.json b/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.json index 2cd04992d1da8..0a1f3fc42a29a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.proto.bin index b34eaf80f8866..4f092eae4057a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_position_with_start.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_positive.json b/sql/connect/common/src/test/resources/query-tests/queries/function_positive.json index a8b3a2d6244bb..26f8ae17bbd19 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_positive.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_positive.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_positive.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_positive.proto.bin index 5507abce8caac..32e7859676cfe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_positive.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_positive.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pow.json b/sql/connect/common/src/test/resources/query-tests/queries/function_pow.json index 187636fb360c6..b9b24218fd99e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_pow.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_pow.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_pow.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_pow.proto.bin index 6e1d3b06fe87a..52a249ada18f6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_pow.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_pow.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_power.json b/sql/connect/common/src/test/resources/query-tests/queries/function_power.json index 187636fb360c6..b9b24218fd99e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_power.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_power.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_power.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_power.proto.bin index 6e1d3b06fe87a..52a249ada18f6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_power.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_power.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_printf.json b/sql/connect/common/src/test/resources/query-tests/queries/function_printf.json index 73ca595e8650b..c3ead5f1388b5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_printf.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_printf.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin index 3fb3862f44d91..b78d5046c98dc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_product.json b/sql/connect/common/src/test/resources/query-tests/queries/function_product.json index 1dfb7f81912d3..802f3e77e8cd1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_product.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_product.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_product.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_product.proto.bin index 8c3fbd31eb6b3..a7ff0061481b4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_product.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_product.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.json b/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.json index b95867e0be963..eb8e75f914318 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.proto.bin index fdc2d96fb08ca..d9ad4e20a3aed 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_quarter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_radians.json b/sql/connect/common/src/test/resources/query-tests/queries/function_radians.json index 837960dedc653..83f211272c123 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_radians.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_radians.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_radians.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_radians.proto.bin index 33a2521b22ac9..e371fb5947a5b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_radians.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_radians.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.json b/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.json index 5318466706bd8..6fd5cfe1194d3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.json @@ -18,7 +18,8 @@ "literal": { "string": "kaboom" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.proto.bin index 7fbd33b9869ca..678fc8cda7afb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_raise_error.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.json index 453ea54bd0ef3..67c1250a72ff8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.json @@ -18,7 +18,8 @@ "literal": { "long": "133" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.proto.bin index 566a49d641293..cb15624497821 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rand_with_seed.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.json b/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.json index ef84f05c3e193..f043d22159b20 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.json @@ -18,7 +18,8 @@ "literal": { "long": "133" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.proto.bin index b0064842bf308..6f601e62c262b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_randn_with_seed.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.json b/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.json index 11238a43ec1a3..fc0c945b825ae 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.json @@ -18,7 +18,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.proto.bin index aa4208afedb88..918de15afa6b5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_random_with_seed.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rank.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rank.json index 93c8dc38d668a..905af83a134f2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rank.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rank.json @@ -15,7 +15,8 @@ "window": { "windowFunction": { "unresolvedFunction": { - "functionName": "rank" + "functionName": "rank", + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rank.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rank.proto.bin index 3aef331fb1739..8d72dd7fddafa 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rank.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rank.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.json b/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.json index 4928145bda572..cad612fbc66d2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.json @@ -35,7 +35,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -55,7 +56,8 @@ "nameParts": ["x_3"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.proto.bin index 2532c111e3874..89868c4ea5ef6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_reduce.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.json b/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.json index 2b0fe7911150c..7ae607ccbeb92 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.proto.bin index 31c6c9bf13150..be0973b5020b9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_reflect.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.json index 005d7264969f2..890dc31aa5a19 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.json @@ -22,7 +22,8 @@ "literal": { "string": "[a-z]+b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.proto.bin index 0379829055998..c138f434647ff 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.json index 540f1821f50e4..23b90b66e6115 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.json @@ -22,7 +22,8 @@ "literal": { "string": "\\d+" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.proto.bin index 3afcfd8c21e7c..ec3970a0434ef 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.json index 5d9c7a5b4a5ab..e3b3650c16ba5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.json @@ -26,7 +26,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.proto.bin index 32ba8b6dcb5e9..9fa17177b5be4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.json index ebe2f581e3de2..04186ace547a0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.json @@ -26,7 +26,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.proto.bin index 2cf31e5f75f4f..12d2feefb602c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_with_regex_group_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.json index 84a2e378ed2e3..34459b011dcf3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.json @@ -22,7 +22,8 @@ "literal": { "string": "(\\d+)([a-z]+)" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.proto.bin index 529cae91ce595..3fbcb69e8ae54 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_extract_all_without_regex_group_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.json index cb44dda5ba2c2..69d171eca7f55 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.json @@ -26,7 +26,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.proto.bin index 55cc77eb3cd1f..32649c83ecc75 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_with_regex_group_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.json index eeab13abaa6da..e5ee8f177efdd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.json @@ -22,7 +22,8 @@ "literal": { "string": "\\d+(a|b|m)" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.proto.bin index 3aee655d92c65..791af5111d72d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_instr_without_regex_group_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.json index 289fb3d9b4eab..4856ead38ac16 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.json @@ -22,7 +22,8 @@ "literal": { "string": "[a-z]+b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.proto.bin index e7bb85bfa47d8..65e383f0e95f0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_like.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.json index 83dd7a8569fd4..b6a237881c400 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.json @@ -26,7 +26,8 @@ "literal": { "string": "XXX" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.proto.bin index b7d3fde25cf85..6d7dd2cb762f6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_replace.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.json index 067652959a94f..0e0c6ea3c4ff1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.json @@ -22,7 +22,8 @@ "literal": { "string": "\\d{2}(a|b|m)" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.proto.bin index 43b987c612cd9..5c629e755a99b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regexp_substr.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.json index 4fdc9b035d764..0c220a9401193 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.proto.bin index 5771d141728ad..e71110d6a3511 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgx.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.json index af225fdf5a895..a3ce82193c43a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.proto.bin index 0a6dcf0106ac7..8c9084a77a0fb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_avgy.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.json index 510fc78140a6e..cc51b8bd0a10e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.proto.bin index b1eff9f4d0329..325137df60e1b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.json index a8596615a2d7f..961fd09d0a7d3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.proto.bin index b9a1c0eff8943..49ee2215109cd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_intercept.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.json index 9f88c6ad41268..7d6d482dd2430 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.proto.bin index 0011348d3880a..aca6143f423a6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_r2.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.json index 9503b2c6feff3..7d93ecaf46afb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.proto.bin index 69c918a7861f2..ccfe35dbe4485 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_slope.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.json index fb243c9989ecf..705e3c357a7f7 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.proto.bin index df31a2e6851f9..9ebce4d26e382 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxx.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.json index 459deaa391e8d..4c35e57128935 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.proto.bin index db51c0bc32a79..8d7b682aef3a2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_sxy.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.json b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.json index 877fbc3aa7c51..624dfa2bf855d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.proto.bin index 6452b277a6e27..0bf9aa5bcf263 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_regr_syy.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_replace.json b/sql/connect/common/src/test/resources/query-tests/queries/function_replace.json index 2f6df6833f368..730207dc6e7d3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_replace.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_replace.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_replace.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_replace.proto.bin index 0564f7ed57583..a1a5792013595 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_replace.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_replace.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.json b/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.json index 2e91450552c19..4b4039e16e5b1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.proto.bin index 136a6b31821af..8aea298529272 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_replace_with_specified_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.json b/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.json index 93869adfbedca..3f028731c409e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "e" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.proto.bin index dd7f2d5de513d..f03f20dbba7f0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_reverse.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_right.json b/sql/connect/common/src/test/resources/query-tests/queries/function_right.json index 843f5be44a650..966e92157ed94 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_right.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_right.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_right.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_right.proto.bin index b8d0156c98132..d48b49b52e79e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_right.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_right.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rint.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rint.json index ea5bcebf81d72..af368a5694875 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rint.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rint.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rint.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rint.proto.bin index bd47adc8476fa..3f8c7c35ceec5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rint.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rint.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.json index fe8480a0800d1..ec6188eb31ac6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.json @@ -22,7 +22,8 @@ "literal": { "string": "[a-z]+b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.proto.bin index 79bbbe92c7fdb..028162c3ddcab 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rlike.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_round.json b/sql/connect/common/src/test/resources/query-tests/queries/function_round.json index 585a0befb224d..d42711c424c46 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_round.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_round.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_round.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_round.proto.bin index 8625ccb1a58f1..40e173d9df4a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_round.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_round.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.json index 3d5ac8afe3db3..9972a7e942c96 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.json @@ -15,7 +15,8 @@ "window": { "windowFunction": { "unresolvedFunction": { - "functionName": "row_number" + "functionName": "row_number", + "isInternal": false } }, "partitionSpec": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.proto.bin index 90b4fcb27d3f1..4368883ca2e36 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_row_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.json index d9b78a0cfd7a9..8c9aaf8242e81 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.json @@ -26,7 +26,8 @@ "literal": { "string": "-" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.proto.bin index d4c355afee0b7..8f945f6329135 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.json index 0daaf1636f13d..3e04a1bb094b0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.json @@ -26,7 +26,8 @@ "literal": { "binary": "CwoLDg==" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.proto.bin index c6f9f22146c61..8396bd5bc016e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rpad_binary.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.json index 5fe66e8e33596..39980dbd802d8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.proto.bin index 4320bf6ac397c..e11f621a033f6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.json b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.json index 0ac2401f9eacf..a41ad58bb73ba 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.proto.bin index 1332f5b330000..6755240b51672 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_rtrim_with_pattern.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.json b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.json index 6df6438a1a9ca..0e6f8f1425d6e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.json @@ -29,9 +29,11 @@ "literal": { "string": "|" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.proto.bin index 99475ddf30d11..6ee33f3a9e986 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_csv.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.json b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.json index 06110d326e1ef..5760bbfc038dc 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.json @@ -18,7 +18,8 @@ "literal": { "string": "[{\"col\":01}]" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.proto.bin index c4ca00e629262..f1934d780d8fd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.json b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.json index ab05ffa940c50..bca6b670d8b5d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.json @@ -29,9 +29,11 @@ "literal": { "string": "true" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.proto.bin index 482485501dd37..7b125c550aef4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_json_with_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.json b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.json index c4ea467bc1a24..c0ab18fd4b4b4 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.json @@ -21,9 +21,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.proto.bin index 0971460bf4112..ecea29dcd41ac 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.json index 19bf62f70b20f..c31f58aa3e320 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.json @@ -21,9 +21,11 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.proto.bin index 68c872ef0d4d2..a6a3f5bccd26b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_schema_of_variant_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sec.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sec.json index 1cab2239755ca..b54347f1488ab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sec.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sec.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sec.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sec.proto.bin index 8760f57a6d4f0..f1b4e4f830a0c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sec.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sec.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_second.json b/sql/connect/common/src/test/resources/query-tests/queries/function_second.json index c77a572b88aa0..5040147fa8b2b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_second.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_second.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_second.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_second.proto.bin index 193c46e917ba2..196f57b276cad 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_second.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_second.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.json index 412ac0272dd57..a2ace14172722 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.proto.bin index 4b62f22574d32..0ce754a8375e6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json index 869e074ccd604..ffb3065d74494 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.json @@ -22,7 +22,8 @@ "literal": { "string": "en" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin index 7514b380a1c82..bf003eecdcd40 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json index 991b42faddb76..5a6cab4cfd609 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.json @@ -26,7 +26,8 @@ "literal": { "string": "US" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin index 01c0136c6df16..ce9b087ef03bb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sentences_with_language_and_country.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.json index b8bd1b68c9a8f..e9bc5b437502d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.json @@ -22,7 +22,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.proto.bin index 36f1980f4ec2b..190d2a8225059 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sequence.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.json b/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.json index 07afa4a77c1b9..03ed1976a708b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "session_user" + "functionName": "session_user", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.proto.bin index 948e3eeed60ac..62904540ce677 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_session_user.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.json b/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.json index 92995656bd265..6178d6a73ab77 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.json @@ -22,7 +22,8 @@ "literal": { "string": "10 minutes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.proto.bin index 364ecdf2aaa28..b669a186b2e2b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_session_window.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sha.json index 57c5cb5bbd270..2385eb652cc15 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sha.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sha.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sha.proto.bin index e99760e49222d..66d9291a14101 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sha.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sha.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.json index ce5014ac2f7e6..b19270ee3fdcb 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.json @@ -26,7 +26,8 @@ } } } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.proto.bin index 3fdfdb2a072de..004ef664ee8d8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sha1.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.json index 5278d604e97b9..ed90b3b939ee6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.json @@ -30,7 +30,8 @@ "literal": { "integer": 512 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.proto.bin index 20a0ee1082ae2..bf16ad9677137 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sha2.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.json b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.json index 12decd300ab03..c02f85d5d56b1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.proto.bin index 94bfbc99fce2d..5ebdbbc9996ee 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftleft.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.json b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.json index c2295c4abaaa2..eabafb977393a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.proto.bin index 910d12f50d6a9..6a089c2ffa344 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftright.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.json b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.json index 875e26a5a5652..4b32899df264a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.json @@ -22,7 +22,8 @@ "literal": { "integer": 2 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.proto.bin index aba9c425dca96..d732f7244aa0e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_shiftrightunsigned.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sign.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sign.json index 34451969078b0..3491a453f6b68 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sign.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sign.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin index ff866c97303ed..35083f8b9a89a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_signum.json b/sql/connect/common/src/test/resources/query-tests/queries/function_signum.json index bcf6ad7eb174d..02ab0e364fd10 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_signum.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_signum.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_signum.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_signum.proto.bin index af52abfb7f25b..65c838e408540 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_signum.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_signum.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sin.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sin.json index cb5b0da073456..a6be1adb3249d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sin.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sin.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sin.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sin.proto.bin index a63f574fa59cb..1f746a3ab76f9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sin.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sin.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.json index e0f46b428611e..c84ac26b64222 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.proto.bin index 2f17ab02a6d94..545a0749d7973 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sinh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_size.json b/sql/connect/common/src/test/resources/query-tests/queries/function_size.json index 37c9cd1ac1ba7..97a996e23a790 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_size.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_size.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "f" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_size.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_size.proto.bin index a8ae600a3dd7a..4f80765b81564 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_size.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_size.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.json b/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.json index 4b14c8d5ca79c..4ed304e15c67e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.proto.bin index 889f96b2d2a39..73a8e1e4d1998 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_skewness.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_slice.json b/sql/connect/common/src/test/resources/query-tests/queries/function_slice.json index b0a63248784ea..c9229b9487c7d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_slice.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_slice.json @@ -26,7 +26,8 @@ "literal": { "integer": 5 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_slice.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_slice.proto.bin index 620a006f775d6..923214f8cf339 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_slice.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_slice.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_some.json b/sql/connect/common/src/test/resources/query-tests/queries/function_some.json index bd6e28468e357..7b440a8d7e17a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_some.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_some.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "flag" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_some.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_some.proto.bin index 0293719148506..43a3ab993a3a4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_some.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_some.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.json index b42bede5cd172..24ffc7c2d387e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.json @@ -22,7 +22,8 @@ "literal": { "boolean": true } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.proto.bin index 994048af2afc4..0414c104be556 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sort_array.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.json b/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.json index 851745b32ebe0..0e428e3e199d2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "spark_partition_id" + "functionName": "spark_partition_id", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.proto.bin index df99cd64e7203..843ca4273b7d8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_spark_partition_id.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split.json b/sql/connect/common/src/test/resources/query-tests/queries/function_split.json index 001d44dcaaf6e..a00e18d77628c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_split.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_split.json @@ -22,7 +22,8 @@ "literal": { "string": ";" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_split.proto.bin index cab0bde7b6da2..d0da01bba86dd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_split.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_split.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.json b/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.json index 81ced1555d3e4..a9c6a3ec9e2d0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.proto.bin index 2c1948f20dc22..fde88be6654b0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_split_part.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.json b/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.json index 98ef0e54e6211..5cdf413daf98c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.proto.bin index a87702f83d1bd..e5a712f11a74e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_split_using_columns.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.json b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.json index 45a7588838ff8..0e2f5ac83e77c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.json @@ -26,7 +26,8 @@ "literal": { "integer": 10 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.proto.bin index 497297fad8715..adc0a101eaf19 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.json b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.json index 138f9d70b2c85..73b7355de15e7 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.proto.bin index 04e24be40e9d8..a25da7d145181 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_split_with_limit_using_columns.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.json index f9a2b76520c13..6ca6327142558 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.proto.bin index e98e3bdfdb665..8b2ed11bd0e28 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sqrt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stack.json b/sql/connect/common/src/test/resources/query-tests/queries/function_stack.json index 14865c72df228..f714739b185cf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_stack.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_stack.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stack.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_stack.proto.bin index 5e5e12478d682..f578d0d2bb952 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_stack.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_stack.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.json b/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.json index ce2b0ac658c4a..4fb08d9de4760 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.proto.bin index 2f09e8095f5a0..f46d166a7c554 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_startswith.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_std.json b/sql/connect/common/src/test/resources/query-tests/queries/function_std.json index cbdb4ea9e5e83..5bb7ed27a5a74 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_std.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_std.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin index 7e34b0427c23b..26b1f4e167534 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.json b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.json index 1403817886ca0..d27469b26a7fe 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.proto.bin index 8d214eea8e74e..b24c06af4fd04 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.json b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.json index 35e3a08b219f8..038b6e1fbb70a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.proto.bin index b679f55014f97..e311b7d0311e8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_pop.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.json index 17cd0fd5e5976..3f7b829e4821f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.proto.bin index 9f22eba5e39aa..7f888c1c07d77 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_stddev_samp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.json b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.json index 2cfd095f8fe62..3c640499aba54 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.proto.bin index 9732a829513a8..7c125c71e5150 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.json b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.json index 228c939a43ef2..c3b52657efd2e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.json @@ -22,7 +22,8 @@ "literal": { "string": "," } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.proto.bin index 069c15db9af76..e65abe3472b91 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_and_keyValue_delimiter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.json b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.json index 7e02c7f13d2ec..2af5fcbb3fbf2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.json @@ -26,7 +26,8 @@ "literal": { "string": "\u003d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.proto.bin index 86a9d15b6512d..4e90cc32e37aa 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_str_to_map_with_pair_delimiter.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_struct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_struct.json index ba950215a2591..f88910dc3f494 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_struct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_struct.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_struct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_struct.proto.bin index 079c2be3c52e5..90e12eb597175 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_struct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_struct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substr.json b/sql/connect/common/src/test/resources/query-tests/queries/function_substr.json index ef6d225821c37..510d501b5c9b0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_substr.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_substr.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substr.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_substr.proto.bin index 934201c433381..6b0871916a8a2 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_substr.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_substr.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.json b/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.json index d8492899d69bc..b09ef5ed2723e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.proto.bin index 0fab03c025061..e5a3de2dc6c58 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_substr_with_len.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring.json b/sql/connect/common/src/test/resources/query-tests/queries/function_substring.json index 84a70cf1c0236..5590cd2660922 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_substring.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_substring.json @@ -26,7 +26,8 @@ "literal": { "integer": 5 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_substring.proto.bin index d302cd95c7434..bdb3a13e5d9c3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_substring.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_substring.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.json index dc81d925957cd..a5396bb478197 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.json @@ -26,7 +26,8 @@ "literal": { "integer": 5 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.proto.bin index 192bb2e300dc3..9ac474c32e1bd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.json b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.json index ba28b1c7f5700..8f01512673cf8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.proto.bin index f14b44ef5a501..838e1d9a8bb90 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_substring_using_columns.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sum.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sum.json index e9526a20b67fb..28b4ea5bbe856 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sum.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sum.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sum.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sum.proto.bin index 0e347bbc0a167..05c69d6f94029 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sum.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sum.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.json b/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.json index 4614cf99ad3a6..0de8f3d36c22f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.json @@ -19,7 +19,8 @@ "unparsedIdentifier": "a" } }], - "isDistinct": true + "isDistinct": true, + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.proto.bin index b4cf704391a4d..6c345201d8eea 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_sum_distinct.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_tan.json b/sql/connect/common/src/test/resources/query-tests/queries/function_tan.json index ead160a7e3ac2..38ca851765599 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_tan.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_tan.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_tan.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_tan.proto.bin index d674dc033b2cd..ba28964c9befb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_tan.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_tan.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.json b/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.json index bcd12c664427e..e9e3996bc5aaa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.proto.bin index 21c28c3ef88e6..d635c5020a53f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_tanh.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.json b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.json index 8fd71bb36d85e..c23a5c3bfa129 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.proto.bin index 5ab8ec531e073..142672a0929e8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_add.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.json b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.json index 635cbb45460e6..c779b0936dc63 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "t" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.proto.bin index 3a81fd8b318c0..c2053f46f6a55 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_diff.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.json b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.json index e43aa6d7115bd..985a23c536e1c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "x" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.proto.bin index c8ca8eedef3c0..f17fc48d4418b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_micros.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.json b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.json index afcdf42d7b3be..52389f7fe5fab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "x" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.proto.bin index bbe401c39f3d1..f63af6ecb1fb8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_millis.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.json b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.json index e6892d17708b3..526d22229facf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "x" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.proto.bin index 102afbdda9021..0c7647735eed5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_timestamp_seconds.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.json index 156c3a5b3ca65..fcf1d35f42169 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.proto.bin index a1da0e6e2eda1..2b02883e03a30 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.json index 8c78cc6f8b99f..325e92437d515 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.json @@ -22,7 +22,8 @@ "literal": { "string": "utf-8" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.proto.bin index 2f2364e5abab1..5be46a049a535 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_binary_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.json index 404a89a87ecb2..117955fa60c3a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.json @@ -22,7 +22,8 @@ "literal": { "string": "$99.99" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.proto.bin index 087e212c39f4e..6ed0c2cdde8a9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_char.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.json index 6b3856f5ac0af..cfbce992619b8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.json @@ -29,9 +29,11 @@ "literal": { "string": "|" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.proto.bin index a3017643a330a..318966eb2b58b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_csv.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.json index 8b9d50aa578b8..0da88a6158438 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "s" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.proto.bin index 59178487eef58..bfd79f65053c8 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.json index 48ae80d1e70ed..1f6250ec0656b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.json @@ -22,7 +22,8 @@ "literal": { "string": "yyyy-MM-dd" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.proto.bin index 2641d660ff69f..8118ff7ee4705 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_date_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.json index 7ceeb9d113cd3..0482c5cba9500 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.json @@ -29,9 +29,11 @@ "literal": { "string": "dd/MM/yyyy" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.proto.bin index c9461c1aa961c..f3d94b476135a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_json.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.json index abb71e80a769c..8df8436bf647d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.json @@ -22,7 +22,8 @@ "literal": { "string": "$99.99" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.proto.bin index 189c73553c5db..0d160ed3239c1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.json index 323c57e2ef58a..4e9013a1c0ff2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "s" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.proto.bin index ec6bd64f98187..1c8f18761c5c6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.json index 59a79f39eb612..c994b0968d099 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.proto.bin index 9cabae3e75657..45d6d1549d98a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.json index 08cb9c153f77f..6ff1dca87de6a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.proto.bin index 22fd3d07dfc43..6aebdbd4f8667 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ltz_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.json index 6808047ef2094..53678e814da88 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.proto.bin index 5cd4cfddbd164..9a0c00065da25 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.json index 03e38801bfa56..240c53dd5c31c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.proto.bin index 3a5d3dd970200..f38ad9460ff52 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_ntz_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.json index 30f34528319c7..1988d2fb5a863 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.json @@ -22,7 +22,8 @@ "literal": { "string": "yyyy-MM-dd HH:mm:ss.SSSS" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.proto.bin index 9c2d6d354ca73..939f9151de2a4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_timestamp_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.json index 15a42b814a629..6a1c5f3677e7f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.proto.bin index 1c70f303e6fc2..2887468cea205 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.json index d6f4280d4464e..ce072b6395620 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.proto.bin index 141ff1fa320d5..803ec8311f552 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_unix_timestamp_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.json index 015fbb5cf534a..fbd86a28a12f3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.json @@ -22,7 +22,8 @@ "literal": { "string": "-04:00" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.proto.bin index b2b65089604a2..bfaf5d2af8a9c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_utc_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.json b/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.json index 3694a68dc8f5c..732eb4b426dde 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.json @@ -22,7 +22,8 @@ "literal": { "string": "$99.99" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.proto.bin index 005c9ab064c9b..b912951a75519 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_to_varchar.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform.json b/sql/connect/common/src/test/resources/query-tests/queries/function_transform.json index 3ad6fe9435644..c8ec0608b13af 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_transform.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_transform.json @@ -31,14 +31,16 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "arguments": [{ "nameParts": ["x_1"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_transform.proto.bin index 266b093f7a99b..e4efe8865b124 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_transform.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_transform.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.json b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.json index 86349f460adaa..ed7487a58d3b8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.json @@ -40,7 +40,8 @@ } } } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -49,7 +50,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.proto.bin index 827b6f273ceea..e04eabb98195f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_keys.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.json b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.json index 02aeca229ce5d..6e76f8a0554f9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.json @@ -41,7 +41,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.proto.bin index b4a653ff77a5d..e24a6320c40b3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_values.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.json b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.json index df5e15b44fdd3..49dec8db7da73 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.json @@ -31,7 +31,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -40,7 +41,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.proto.bin index e502c18dcd9e8..30b1901f42f58 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_transform_with_index.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_translate.json b/sql/connect/common/src/test/resources/query-tests/queries/function_translate.json index 93d155c2857fb..ad5f98152258e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_translate.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_translate.json @@ -26,7 +26,8 @@ "literal": { "string": "bar" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_translate.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_translate.proto.bin index 1ce32c8d2843e..ec9c556cfef09 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_translate.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_translate.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trim.json b/sql/connect/common/src/test/resources/query-tests/queries/function_trim.json index d2700174bca3d..a7925c2c7b5d2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_trim.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_trim.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trim.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_trim.proto.bin index d5f4f21510fc6..0ea9051f33837 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_trim.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_trim.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.json b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.json index fc3281c921531..cb566a6b98dfe 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.proto.bin index 2136b55656212..4423ab1a02a37 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_trim_with_pattern.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.json b/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.json index 4c596cd863261..01e528241eedd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.json @@ -22,7 +22,8 @@ "literal": { "string": "mm" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.proto.bin index cdcee95af6344..acfd2bb94483b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_trunc.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.json index 80300b5b5778a..2a34feb64a6d5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.proto.bin index c1cb613b3943f..f9efaff46f3ec 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_add.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.json index 80e10f4786a81..fd2ba4bd66bdf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.proto.bin index c2a477e5320c7..cc94446a39c8d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.json index cc4ea4bfe5fb9..360742eaf0b69 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.proto.bin index 22919795e3e6a..b1fde39e2b632 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.json index 1f1fc777959a2..f02b625bd9e6a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.proto.bin index b16d49e2428a2..1774a4dfaba19 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.json index b7e7cd41bda8f..b0c434231378b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.json @@ -34,7 +34,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.proto.bin index d406961d5ccfc..6f75571c79ff3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_aes_decrypt_with_mode_padding_aad.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.json index 1216f4b5c635f..9ac56e71ad8b6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.proto.bin index 8ab7a5d19e380..1b378b28e9ca7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_avg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.json index d7d012756e62f..d7bd1d47d1b2a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.proto.bin index 05c8d4a193adb..3ba2e35029a78 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_divide.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.json index c2651e4ad7253..8fb878d1358f2 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.proto.bin index b86d5efd4096b..1fc1045fc085e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_array.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.json index c4e5bc2f415ee..e200f3fa9d278 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.proto.bin index 2f6c54f2fa5ec..1f8fdb10899a6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_element_at_map.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.json index a7a2348496040..4c3bafee572d1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.proto.bin index d459b6e8ec677..acb1952d621d7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.json index 14aaa41ee2cb5..4508e5610dc81 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.proto.bin index 5123b995417ba..f12d4affbff93 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.json index a6ac2f27e3dc5..672acdf6b2d3d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.proto.bin index cecfca97f7e20..555eb56ff0438 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.json index c9d4f1d4d2f1f..70782ac30ea49 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.proto.bin index 423172405c397..b9d6de2d00662 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.json index 7f2a42f01db45..fda1b3ea21a36 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.json @@ -34,7 +34,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.proto.bin index 71259b402aa51..3f477ddeab7db 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.json index 35ab05a90b3cd..b2db53de2c58e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.proto.bin index f8cf29d15aabf..72dd0ffd9d539 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.json index 2f9c1d019359b..5a1389ff2665e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin index d7343a059b53d..90bf07eb4e9e3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_interval_years_months_weeks_days_hours_mins_secs.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.json index 179f6e06988fc..36559a50d7aa4 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.proto.bin index d0c60ba1c7bf8..34dcc6bf46092 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_with_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.json index 29aa2096c2273..36e121b976dbf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.proto.bin index 9caf6f6ba5285..c0eaf743e46fe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ltz_without_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.json index 6b8d31d0c58e5..b131e0d07fdea 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.proto.bin index 7d7e2a8029def..a350a6861ed9a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_ntz.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.json index 79e11efc20d41..b2fe4db0dabab 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.json @@ -42,7 +42,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.proto.bin index 53b9839cf8c1f..8519bccb5a1a7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_with_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.json index 39ce728a38862..549071a8a964c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.json @@ -38,7 +38,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.proto.bin index 74918d42f89c6..2d8c06fb5a2a5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_make_timestamp_without_timezone.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.json index df22654c82031..0838efb2c1eb3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.proto.bin index 8912423235e0b..703d897792bcb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_multiply.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.json index 91177eb4a5857..fca2a95a83ab8 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.proto.bin index cc1f159cfd78c..a97f0801944be 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_json.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.json index b9603d5af2634..0b4e011421d6a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.proto.bin index 696c4ddde519c..080be30a46d23 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.json index 137ed4bd9bc80..d173704b73354 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.proto.bin index f4a13872e3c8f..b39bd2cb28cbb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_parse_url_with_key.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.json index de3fae90c2c4b..0d787f37493b6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.json @@ -26,7 +26,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.proto.bin index e38e0e5c06548..5971e17d1041b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_reflect.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.json index f3a5df24cce88..d1bae052a945c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.proto.bin index f0cb5f5027873..4951a8e3c5fd6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_subtract.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.json index 41e93d1fcf956..7961eb8eb5596 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.proto.bin index dce7d9df359c9..6e4cbe5a01090 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_sum.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.json index 9b57b6b26b562..164304ef4b0ae 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.json @@ -22,7 +22,8 @@ "literal": { "string": "format" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.proto.bin index 28b7059160757..3a1b90ca42f70 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.json index 2498ff9a7872f..6c676dc702a35 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.proto.bin index 682eb1821a3a1..4a7ed3da5738c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_binary_without_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.json index 44e894743dfc8..f206393079de7 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.json @@ -22,7 +22,8 @@ "literal": { "string": "99,999" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.proto.bin index c2eba8a19d5df..2eb9ff68df8fe 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.json index d00967823a33c..e0a532043e00c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.proto.bin index 4f0300d48a6fc..ff61c6147cd84 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.json index 4fdfc38ca539b..8589656d5ed52 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.proto.bin index 91a4156e305f6..72a6868870487 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_to_timestamp_without_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.json index d51704c8f62e2..e73de5e669362 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.proto.bin index 3e84921b12206..e15e5e2d902da 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_url_decode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.json b/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.json index 9a4a4e25f19e6..b8ce69c4bcf39 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.json @@ -21,7 +21,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }, { "literal": { @@ -31,7 +32,8 @@ "literal": { "string": "int" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.proto.bin index b16bbf4c7a4e9..82584c937aaa7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_try_variant_get.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.json b/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.json index 7a6fcfcbcf898..7f9808d9fd947 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.proto.bin index a042a6e8d7607..585d98f767904 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_typeof.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.json b/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.json index 7193142acdb6f..5580e31e26ffa 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.proto.bin index 3e17a01d4b1f5..8a2b70936e0f9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_ucase.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.json index 6af2a00ed160e..af85c7b64779c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.proto.bin index f37ceb91bf42b..f446e0ad73f45 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unbase64.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.json index 7c409d023f76a..1cea642cc9c68 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.proto.bin index fbac2821fdb07..757eca2dc04d5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unhex.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.json index 1a7ae09f46dad..2e617c4e6b8b1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.json @@ -25,9 +25,11 @@ "literal": { "string": "yyyy-MM-dd" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.proto.bin index 9c05e42bfad30..6ed08ec71d76a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_date.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.json index 07f5cd1d53dbd..f7bb6d9ba6264 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.json @@ -25,9 +25,11 @@ "literal": { "string": "yyyy-MM-dd HH:mm:ss.SSSS" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.proto.bin index c3f44d766f8b1..2574acb3c8d95 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_micros.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.json index aac02cc807aa0..4a7c077e88bc1 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.json @@ -25,9 +25,11 @@ "literal": { "string": "yyyy-MM-dd HH:mm:ss.SSSS" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.proto.bin index f0456e03e3fc1..1865aac8a7340 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_millis.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.json index 428cb26cd9c86..dc3fc3fae0c05 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.json @@ -25,9 +25,11 @@ "literal": { "string": "yyyy-MM-dd HH:mm:ss.SSSS" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.proto.bin index fdaf50e7322bb..2ca04f640cda4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_seconds.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.json index e590f7778f2ea..0780a83d74088 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.json @@ -16,9 +16,11 @@ "functionName": "unix_timestamp", "arguments": [{ "unresolvedFunction": { - "functionName": "current_timestamp" + "functionName": "current_timestamp", + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.proto.bin index cb3d967ae0123..6f8cf9115629f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.json b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.json index d2e087a5d8a24..93699abb33a7b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.json @@ -22,7 +22,8 @@ "literal": { "string": "yyyy-MM-dd HH:mm:ss.SSSS" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.proto.bin index ddfcdff63d11a..f838c9dd31912 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_unix_timestamp_with_format.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_upper.json b/sql/connect/common/src/test/resources/query-tests/queries/function_upper.json index 208ee9231a13c..36f1f0258ca82 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_upper.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_upper.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_upper.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_upper.proto.bin index 5ddbfce96e71b..a7d6be43571a0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_upper.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_upper.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.json index d4cdeeb6c48c5..2e8003e4e9a9f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.proto.bin index e347e73c3aef1..ff6fb3793e671 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_url_decode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.json b/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.json index 5d221e0fea6f4..9df8a4683ea65 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.proto.bin index 9313fb8249859..eb9e31e63697b 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_url_encode.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_user.json b/sql/connect/common/src/test/resources/query-tests/queries/function_user.json index aaf3de9ba034e..c3ab4ae4be94c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_user.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_user.json @@ -13,7 +13,8 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "user" + "functionName": "user", + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin index dbd64cae9f360..17b0d6a4ecc90 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_user.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.json b/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.json index 9c74ce4a984f8..d91c325ef41ba 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.proto.bin index 7ca6e8d3b811b..603e856366f10 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_var_pop.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.json b/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.json index 979313dd0510d..8132510e61129 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.proto.bin index 9bd042ad339e7..99a8d28ec0e72 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_var_samp.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_variance.json b/sql/connect/common/src/test/resources/query-tests/queries/function_variance.json index 90a97c3becf4d..4bfce573c50c5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_variance.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_variance.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_variance.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_variance.proto.bin index fd494fc496391..e3c0f8512c0c6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_variance.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_variance.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.json b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.json index ab0acd29d505b..d44a94f365b56 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.json @@ -21,7 +21,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }, { "literal": { @@ -31,7 +32,8 @@ "literal": { "string": "int" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.proto.bin index fe9b76bb97c4a..f1ac3c8c0ad63 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_variant_get.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.json b/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.json index b757700291752..82f37d343207e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.proto.bin index 1954103269eb2..cc47e1928103a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_weekday.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.json b/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.json index 3f46a98569e24..de4ce19a12b5e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.proto.bin index ec9b22522360e..e5c742732e76d 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_weekofyear.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_window.json b/sql/connect/common/src/test/resources/query-tests/queries/function_window.json index bdcb6a398800f..95bde679468e0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_window.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_window.json @@ -30,7 +30,8 @@ "literal": { "string": "0 second" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_window.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_window.proto.bin index 8cffcc1e9f673..303b9673c8ab5 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_window.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_window.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.json b/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.json index 4809ea21261c4..2107ac2f12ecd 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.json @@ -35,7 +35,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "wt" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.proto.bin index c143520df08ce..3de3a3e156a2c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_window_time.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.json index 3dea90a13653d..fd36b378137c6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b/text()" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.proto.bin index aabfc76f8a7e1..d07c5d50fc3a4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.json index 793d459ec165b..0749c2d422314 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.proto.bin index 544caab4ecc5b..e70d2e5c3edd6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_boolean.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.json index f88a06641b8f4..d1c77ae96a86b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.proto.bin index 9c4ea31712021..de580971683b3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_double.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.json index 94932891225d7..b13e12a2d7e9f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.proto.bin index 32dfbc00cfa44..4285deba56d7c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_float.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.json index 0dcef00ed20d4..2a55744cb38c9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.proto.bin index e6298b37dbe36..afe3b10e4cd86 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_int.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.json index c740d2bad4f5f..3d4d9267a6a50 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.proto.bin index d240600eabbae..7cb6efd6ab2d0 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_long.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.json index b164bb6a32ac7..cf1303b54d160 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.proto.bin index b967d3e55cc5f..c589c8ecc775a 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_number.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.json index 5d3a3e9983707..4aa5e3aae7fc9 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.proto.bin index 9ae27bd973853..f407b525cdfa7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_short.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.json index 26e4130ae2c4b..94fad0de2851f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.json @@ -22,7 +22,8 @@ "literal": { "string": "a/b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.proto.bin index 5384301238b1e..c31ae5065a513 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xpath_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.json b/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.json index c20739d09ff10..5000f3b164766 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "g" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.proto.bin index 414c76fc5ce7f..de84e70acef5e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_xxhash64.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_year.json b/sql/connect/common/src/test/resources/query-tests/queries/function_year.json index b8a4ee5a16525..9fadb5c411b9b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_year.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_year.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "d" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_year.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_year.proto.bin index 623bc9ac6d81f..91bbfdc180efc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_year.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_year.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_years.json b/sql/connect/common/src/test/resources/query-tests/queries/function_years.json index 2e87307320271..7b0ab3d287ece 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_years.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_years.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": true } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_years.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_years.proto.bin index 30c25423fd563..575f56951f017 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_years.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_years.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.json b/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.json index 660ca1931137e..66d53c39742ba 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.json @@ -35,7 +35,8 @@ "unresolvedNamedLambdaVariable": { "nameParts": ["y_2"] } - }] + }], + "isInternal": false } }, "arguments": [{ @@ -44,7 +45,8 @@ "nameParts": ["y_2"] }] } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.proto.bin index edbfe197af4dc..1bf478358f357 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/function_zip_with.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.json index b7b4c98518e6b..e36ad1de4960d 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -45,7 +47,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -55,7 +58,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -65,7 +69,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -75,7 +80,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -84,7 +90,8 @@ "unresolvedStar": { "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -94,7 +101,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.proto.bin index d7b1b94ed04a2..22eacb3a01b03 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.json index e61616786158e..cf42aabd68160 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.json @@ -24,7 +24,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -33,7 +34,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.proto.bin index d6daa1cc31f7d..a12bd0699df1f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_columns.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.json index 285c13f4bc8b3..e5261b8a18a09 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.json @@ -31,7 +31,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -41,7 +42,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.proto.bin index 674d506fa4a07..93bb39f16d1a6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_agg_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.json index 0ded46cf6cc7c..4110779d80c56 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.proto.bin index 444b0c3853f16..d43d816891aa6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_avg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.json index f92e22493e07b..04d6e91f9c7cf 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.json @@ -26,7 +26,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "name": ["count"] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.proto.bin index 5bb539195df9a..fd009c3a636ca 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_count.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.json index ed186ff713519..643f6c0676a30 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.proto.bin index 11cd163e91738..7bb503d75e92f 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_max.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.json index 0ded46cf6cc7c..4110779d80c56 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.proto.bin index 444b0c3853f16..d43d816891aa6 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_mean.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.json index 8c0ad283cb0a4..3d7546547d98f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.proto.bin index 2bc985a1fe9f3..cd0488b381612 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_min.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.json b/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.json index 788b964491c6a..61133d3fe4321 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "b", "planId": "0" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.proto.bin index e92041399cbca..d5f8bc8c47698 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupby_sum.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.json b/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.json index 6e84824ec7a3a..a81c24a9fa077 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.json @@ -25,7 +25,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unparsedIdentifier": "a", "planId": "0" } - }] + }], + "isInternal": false } }], "groupingSets": [{ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.proto.bin index ce0294096706e..48d51737b9a41 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/groupingSets.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.json b/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.json index 8ff81d95d2988..c1c1bcdf0dc87 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -39,7 +40,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -52,7 +54,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.proto.bin index d1dded43ddf99..41e1a426e25b9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/grouping_and_grouping_id.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.json index 52f66cf2dc6b9..914f304f56a2e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.proto.bin index 68b74817c3268..ba473ed2e2855 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.json index 52f66cf2dc6b9..914f304f56a2e 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.proto.bin index 68b74817c3268..ba473ed2e2855 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.json index fbd4ca05d9e99..590187f7fe63b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.json @@ -22,7 +22,8 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.proto.bin index bea4e1a642ab2..5b83337b80bee 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_columnName_lgConfigK_int.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.json index fbd4ca05d9e99..590187f7fe63b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.json @@ -22,7 +22,8 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.proto.bin index bea4e1a642ab2..5b83337b80bee 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.json index fbd4ca05d9e99..590187f7fe63b 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.json @@ -22,7 +22,8 @@ "literal": { "integer": 0 } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.proto.bin index bea4e1a642ab2..5b83337b80bee 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_sketch_agg_with_column_lgConfigK_int.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.json index 74b3e7c4a7410..216afd0f5975a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.proto.bin index e19b476247a24..309e80a86ee38 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.json index 74b3e7c4a7410..216afd0f5975a 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "bytes" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.proto.bin index e19b476247a24..309e80a86ee38 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.json index bb6413a94ced3..e733e086af5d6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.json @@ -22,7 +22,8 @@ "literal": { "boolean": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.proto.bin index 4f7f236583949..c71e656127200 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_columnName_allowDifferentLgConfigK_boolean.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.json index bb6413a94ced3..e733e086af5d6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.json @@ -22,7 +22,8 @@ "literal": { "boolean": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.proto.bin index 4f7f236583949..c71e656127200 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.json b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.json index bb6413a94ced3..e733e086af5d6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.json @@ -22,7 +22,8 @@ "literal": { "boolean": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.proto.bin index 4f7f236583949..c71e656127200 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/hll_union_agg_with_column_allowDifferentLgConfigK_boolean.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/join_condition.json b/sql/connect/common/src/test/resources/query-tests/queries/join_condition.json index 993cd98a7dd16..7151d0420f6b5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/join_condition.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/join_condition.json @@ -46,7 +46,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "r.id" } - }] + }], + "isInternal": false } }, "joinType": "JOIN_TYPE_LEFT_ANTI" diff --git a/sql/connect/common/src/test/resources/query-tests/queries/join_condition.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/join_condition.proto.bin index 1d11fe5e75bcc..4784998b49cca 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/join_condition.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/join_condition.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.json b/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.json index 527338c56ae60..9308d6babdb25 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.json @@ -46,7 +46,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "r.a" } - }] + }], + "isInternal": false } }, "joinType": "JOIN_TYPE_INNER" diff --git a/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.proto.bin index 5d3de55da9cf8..a49cc6ef30806 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/join_inner_condition.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/pivot.json b/sql/connect/common/src/test/resources/query-tests/queries/pivot.json index 2af86606b9fcb..f085d1a43b678 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/pivot.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/pivot.json @@ -24,7 +24,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }], "pivot": { diff --git a/sql/connect/common/src/test/resources/query-tests/queries/pivot.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/pivot.proto.bin index f545179e84968..73c88bf97535e 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/pivot.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/pivot.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.json b/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.json index aa043613795c4..9d5b22bce6e89 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.json @@ -24,7 +24,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }], "pivot": { diff --git a/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.proto.bin index 588b56f247e07..d722db0e17ea9 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/pivot_without_column_values.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.json b/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.json index 1102db18830bd..146904dc898e3 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.json @@ -30,7 +30,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "name": ["count"] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.proto.bin index 64dbb597c3650..8949050821a12 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/rollup_column.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.json b/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.json index 5082051031f81..6fe3659064e79 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.json @@ -32,7 +32,8 @@ "literal": { "integer": 1 } - }] + }], + "isInternal": false } }, "name": ["count"] diff --git a/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.proto.bin index 63fdead641dad..3843ae77a9bfc 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/rollup_string.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.json b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.json index 90ef62c5f415b..e3dcf84ae9c39 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.json @@ -28,11 +28,14 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.proto.bin index 2273a16d4e6a8..b72a7233c4c04 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_1-arg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.json b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.json index c9c6c75235694..d144dcf8b8af5 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.proto.bin index 37f3915cd8d18..18763400b4abb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_2-arg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.json b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.json index 23850dcb136ef..55b64d26d4904 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -48,7 +50,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.proto.bin index b3b56953a8586..d535835523de3 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_3-arg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.json b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.json index 2bbdb60794db5..da0adf605f977 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -48,7 +50,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -61,7 +64,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.proto.bin index bacccff22ae0a..50197b862ad14 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_4-arg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.json b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.json index 4f57c0ef82145..196a91b9fbb81 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.json @@ -22,7 +22,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -35,7 +36,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -48,7 +50,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -61,7 +64,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }, { "unresolvedFunction": { @@ -74,7 +78,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "b" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.proto.bin index 2c51e2088885f..e2ff25edd34cd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/select_typed_5-arg.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/toJSON.json b/sql/connect/common/src/test/resources/query-tests/queries/toJSON.json index 9a99a18853cf1..9faba08d9792c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/toJSON.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/toJSON.json @@ -21,9 +21,11 @@ "unresolvedStar": { "planId": "0" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/toJSON.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/toJSON.proto.bin index e930ee76aae97..0cd2c3d35c6b4 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/toJSON.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/toJSON.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.json b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.json index 6079e13bbfc93..b632fba4a0192 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.json @@ -22,7 +22,8 @@ "literal": { "string": "{\"type\": \"int\", \"name\": \"id\"}" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.proto.bin index 2843fbb67fecf..6c3907802968c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_with_schema.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.json b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.json index fa19d2120b94f..dd289a6abcc16 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.json @@ -18,7 +18,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.proto.bin index 4e7251125e4ce..59bce6aac25c7 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_avro_without_schema.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json index 921c1b800a089..e71bddc0b19a0 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.json @@ -22,7 +22,8 @@ "literal": { "string": "org.apache.spark.connect.proto.StorageLevel" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin index 5cc7c49882c03..f49e6d227ddcd 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json index 0843b469384e0..c6ccee6f35c3f 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.json @@ -26,7 +26,8 @@ "literal": { "binary": "CvwBCgxjb21tb24ucHJvdG8SDXNwYXJrLmNvbm5lY3QisAEKDFN0b3JhZ2VMZXZlbBIZCgh1c2VfZGlzaxgBIAEoCFIHdXNlRGlzaxIdCgp1c2VfbWVtb3J5GAIgASgIUgl1c2VNZW1vcnkSIAoMdXNlX29mZl9oZWFwGAMgASgIUgp1c2VPZmZIZWFwEiIKDGRlc2VyaWFsaXplZBgEIAEoCFIMZGVzZXJpYWxpemVkEiAKC3JlcGxpY2F0aW9uGAUgASgFUgtyZXBsaWNhdGlvbkIiCh5vcmcuYXBhY2hlLnNwYXJrLmNvbm5lY3QucHJvdG9QAWIGcHJvdG8z" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin index c3fe14aef47da..c30bc963ce0eb 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json index 76307b3141f7f..bc676b7aa5b1c 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.json @@ -37,9 +37,11 @@ "literal": { "string": "2" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin index a387611c1ad55..971d6b358711c 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_descFilePath_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json index 8787f0fc15d77..fa3d57a251cb6 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.json @@ -33,9 +33,11 @@ "literal": { "string": "2" } - }] + }], + "isInternal": false } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin index 9ef8348446ad4..7f955b1013fe1 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/to_protobuf_messageClassName_options.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/where_column.json b/sql/connect/common/src/test/resources/query-tests/queries/where_column.json index bef80a7e6ed5a..15cca60f2a407 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/where_column.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/where_column.json @@ -22,7 +22,8 @@ "literal": { "long": "1" } - }] + }], + "isInternal": false } } } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/where_column.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/where_column.proto.bin index e472ed0715b62..ef377b7044366 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/where_column.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/where_column.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.json b/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.json index 93d3b5297d9e1..3e7c112776655 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.json @@ -30,7 +30,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "a" } - }] + }], + "isInternal": false } }] } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.proto.bin index f212e97bc1c5a..2667912763248 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/width_bucket.proto.bin differ diff --git a/sql/connect/common/src/test/resources/query-tests/queries/window.json b/sql/connect/common/src/test/resources/query-tests/queries/window.json index 23fd5c1556ec5..ad9555f3ae898 100644 --- a/sql/connect/common/src/test/resources/query-tests/queries/window.json +++ b/sql/connect/common/src/test/resources/query-tests/queries/window.json @@ -20,7 +20,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "partitionSpec": [{ @@ -42,7 +43,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "partitionSpec": [{ @@ -64,7 +66,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "orderSpec": [{ @@ -94,7 +97,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "orderSpec": [{ @@ -124,7 +128,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "orderSpec": [{ @@ -163,7 +168,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } }, "orderSpec": [{ @@ -202,7 +208,8 @@ "unresolvedAttribute": { "unparsedIdentifier": "id" } - }] + }], + "isInternal": false } } } diff --git a/sql/connect/common/src/test/resources/query-tests/queries/window.proto.bin b/sql/connect/common/src/test/resources/query-tests/queries/window.proto.bin index a89c0d6a6a3f4..01616601af0ea 100644 Binary files a/sql/connect/common/src/test/resources/query-tests/queries/window.proto.bin and b/sql/connect/common/src/test/resources/query-tests/queries/window.proto.bin differ diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index d6ade1ac91264..c0b4384af8b6d 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -45,7 +45,7 @@ import org.apache.spark.internal.LogKeys.{DATAFRAME_ID, SESSION_ID} import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile, TaskResourceProfile, TaskResourceRequest} import org.apache.spark.sql.{Column, Dataset, Encoders, ForeachWriter, Observation, RelationalGroupedDataset, Row, SparkSession} import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier, QueryPlanningTracker} -import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LazyExpression, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedTableValuedFunction, UnresolvedTranspose} +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LazyExpression, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunction, UnresolvedTranspose} import org.apache.spark.sql.catalyst.encoders.{encoderFor, AgnosticEncoder, ExpressionEncoder, RowEncoder} import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.UnboundRowEncoder import org.apache.spark.sql.catalyst.expressions._ @@ -77,7 +77,7 @@ import org.apache.spark.sql.execution.stat.StatFunctions import org.apache.spark.sql.execution.streaming.GroupStateImpl.groupStateTimeoutFromString import org.apache.spark.sql.execution.streaming.StreamingQueryWrapper import org.apache.spark.sql.expressions.{Aggregator, ReduceAggregator, SparkUserDefinedFunction, UserDefinedAggregator, UserDefinedFunction} -import org.apache.spark.sql.internal.{CatalogImpl, MergeIntoWriterImpl, TypedAggUtils} +import org.apache.spark.sql.internal.{CatalogImpl, MergeIntoWriterImpl, TypedAggUtils, UserDefinedFunctionUtils} import org.apache.spark.sql.streaming.{GroupStateTimeout, OutputMode, StreamingQuery, StreamingQueryListener, StreamingQueryProgress, Trigger} import org.apache.spark.sql.types._ import org.apache.spark.sql.util.CaseInsensitiveStringMap @@ -566,10 +566,9 @@ class SparkConnectPlanner( } private def transformToDF(rel: proto.ToDF): LogicalPlan = { - Dataset - .ofRows(session, transformRelation(rel.getInput)) - .toDF(rel.getColumnNamesList.asScala.toSeq: _*) - .logicalPlan + UnresolvedSubqueryColumnAliases( + rel.getColumnNamesList.asScala.toSeq, + transformRelation(rel.getInput)) } private def transformMapPartitions(rel: proto.MapPartitions): LogicalPlan = { @@ -667,7 +666,8 @@ class SparkConnectPlanner( private def transformTypedGroupMap( rel: proto.GroupMap, commonUdf: proto.CommonInlineUserDefinedFunction): LogicalPlan = { - val udf = TypedScalaUdf(commonUdf) + val unpackedUdf = unpackUdf(commonUdf) + val udf = TypedScalaUdf(unpackedUdf, None) val ds = UntypedKeyValueGroupedDataset( rel.getInput, rel.getGroupingExpressionsList, @@ -697,6 +697,18 @@ class SparkConnectPlanner( InternalOutputModes(rel.getOutputMode) } + val stateSchema = DataTypeProtoConverter.toCatalystType(rel.getStateSchema) match { + case s: StructType => s + case other => + throw InvalidPlanInput( + s"Invalid state schema dataType $other for flatMapGroupsWithState") + } + val stateEncoder = TypedScalaUdf.encoderFor( + // the state agnostic encoder is the second element in the input encoders. + unpackedUdf.inputEncoders.tail.head, + "state", + Some(DataTypeUtils.toAttributes(stateSchema))) + val flatMapGroupsWithState = if (hasInitialState) { new FlatMapGroupsWithState( udf.function @@ -706,7 +718,7 @@ class SparkConnectPlanner( ds.groupingAttributes, ds.dataAttributes, udf.outputObjAttr, - initialDs.vEncoder.asInstanceOf[ExpressionEncoder[Any]], + stateEncoder.asInstanceOf[ExpressionEncoder[Any]], outputMode, rel.getIsMapGroupsWithState, timeoutConf, @@ -725,7 +737,7 @@ class SparkConnectPlanner( ds.groupingAttributes, ds.dataAttributes, udf.outputObjAttr, - initialDs.vEncoder.asInstanceOf[ExpressionEncoder[Any]], + stateEncoder.asInstanceOf[ExpressionEncoder[Any]], outputMode, rel.getIsMapGroupsWithState, timeoutConf, @@ -875,18 +887,20 @@ class SparkConnectPlanner( logicalPlan: LogicalPlan, groupingExprs: java.util.List[proto.Expression], sortOrder: Seq[SortOrder]): UntypedKeyValueGroupedDataset = { + val analyzed = session.sessionState.executePlan(logicalPlan).analyzed + assertPlan(groupingExprs.size() >= 1) val dummyFunc = TypedScalaUdf(groupingExprs.get(0), None) val groupExprs = groupingExprs.asScala.toSeq.drop(1).map(expr => transformExpression(expr)) val (qe, aliasedGroupings) = - RelationalGroupedDataset.handleGroupingExpression(logicalPlan, session, groupExprs) + RelationalGroupedDataset.handleGroupingExpression(analyzed, session, groupExprs) UntypedKeyValueGroupedDataset( dummyFunc.outEnc, dummyFunc.inEnc, qe.analyzed, - logicalPlan.output, + analyzed.output, aliasedGroupings, sortOrder) } @@ -895,20 +909,22 @@ class SparkConnectPlanner( logicalPlan: LogicalPlan, groupingExprs: java.util.List[proto.Expression], sortOrder: Seq[SortOrder]): UntypedKeyValueGroupedDataset = { + val analyzed = session.sessionState.executePlan(logicalPlan).analyzed + assertPlan(groupingExprs.size() == 1) - val groupFunc = TypedScalaUdf(groupingExprs.get(0), Some(logicalPlan.output)) + val groupFunc = TypedScalaUdf(groupingExprs.get(0), Some(analyzed.output)) val vEnc = groupFunc.inEnc val kEnc = groupFunc.outEnc - val withGroupingKey = AppendColumns(groupFunc.function, vEnc, kEnc, logicalPlan) + val withGroupingKey = AppendColumns(groupFunc.function, vEnc, kEnc, analyzed) // The input logical plan of KeyValueGroupedDataset need to be executed and analyzed - val analyzed = session.sessionState.executePlan(withGroupingKey).analyzed + val withGroupingKeyAnalyzed = session.sessionState.executePlan(withGroupingKey).analyzed UntypedKeyValueGroupedDataset( kEnc, vEnc, - analyzed, - logicalPlan.output, + withGroupingKeyAnalyzed, + analyzed.output, withGroupingKey.newColumns, sortOrder) } @@ -943,10 +959,7 @@ class SparkConnectPlanner( } } - def apply( - commonUdf: proto.CommonInlineUserDefinedFunction, - inputAttrs: Option[Seq[Attribute]] = None): TypedScalaUdf = { - val udf = unpackUdf(commonUdf) + def apply(udf: UdfPacket, inputAttrs: Option[Seq[Attribute]]): TypedScalaUdf = { // There might be more than one inputs, but we only interested in the first one. // Most typed API takes one UDF input. // For the few that takes more than one inputs, e.g. grouping function mapping UDFs, @@ -956,6 +969,13 @@ class SparkConnectPlanner( TypedScalaUdf(udf.function, udf.outputEncoder, inEnc, inputAttrs) } + def apply( + commonUdf: proto.CommonInlineUserDefinedFunction, + inputAttrs: Option[Seq[Attribute]] = None): TypedScalaUdf = { + val udf = unpackUdf(commonUdf) + apply(udf, inputAttrs) + } + def encoderFor( encoder: AgnosticEncoder[_], errorType: String, @@ -1457,8 +1477,9 @@ class SparkConnectPlanner( private def transformTypedFilter( fun: proto.CommonInlineUserDefinedFunction, child: LogicalPlan): TypedFilter = { - val udf = TypedScalaUdf(fun, Some(child.output)) - TypedFilter(udf.function, child)(udf.inEnc) + val analyzed = session.sessionState.executePlan(child).analyzed + val udf = TypedScalaUdf(fun, Some(analyzed.output)) + TypedFilter(udf.function, analyzed)(udf.inEnc) } private def transformProject(rel: proto.Project): LogicalPlan = { @@ -1633,14 +1654,18 @@ class SparkConnectPlanner( fun.getArgumentsList.asScala.map(transformExpression).toSeq, isDistinct = fun.getIsDistinct) } else { - // Spark Connect historically used the global namespace to lookup a couple of internal - // functions (e.g. product, collect_top_k, unwrap_udt, ...). In Spark 4 we moved these - // functions to a dedicated namespace, however in order to stay backwards compatible we still - // need to allow connect to use the global namespace. Here we check if a function is - // registered in the internal function registry, and we reroute the lookup to the internal - // registry. val name = fun.getFunctionName - val internal = FunctionRegistry.internal.functionExists(FunctionIdentifier(name)) + val internal = if (fun.hasIsInternal) { + fun.getIsInternal + } else { + // Spark Connect historically used the global namespace to look up a couple of internal + // functions (e.g. product, collect_top_k, unwrap_udt, ...). In Spark 4 we moved these + // functions to a dedicated namespace, however in order to stay backwards compatible we + // still need to allow Connect to use the global namespace. Here we check if a function is + // registered in the internal function registry, and we reroute the lookup to the internal + // registry. + FunctionRegistry.internal.functionExists(FunctionIdentifier(name)) + } UnresolvedFunction( name :: Nil, fun.getArgumentsList.asScala.map(transformExpression).toSeq, @@ -1722,34 +1747,36 @@ class SparkConnectPlanner( } /** - * Translates a Scala user-defined function from proto to the Catalyst expression. + * Translates a Scala user-defined function or aggregator from proto to the corresponding + * Catalyst expression. * * @param fun - * Proto representation of the Scala user-defined function. + * Proto representation of the Scala user-defined function or aggregator. * @return - * ScalaUDF. + * An expression, either a ScalaUDF or a ScalaAggregator. */ private def transformScalaUDF(fun: proto.CommonInlineUserDefinedFunction): Expression = { - val udf = fun.getScalarScalaUdf - val udfPacket = unpackUdf(fun) - if (udf.getAggregate) { - ScalaAggregator( - transformScalaFunction(fun).asInstanceOf[UserDefinedAggregator[Any, Any, Any]], - fun.getArgumentsList.asScala.map(transformExpression).toSeq) - .toAggregateExpression() - } else { - ScalaUDF( - function = udfPacket.function, - dataType = transformDataType(udf.getOutputType), - children = fun.getArgumentsList.asScala.map(transformExpression).toSeq, - inputEncoders = udfPacket.inputEncoders.map(e => Try(ExpressionEncoder(e)).toOption), - outputEncoder = Option(ExpressionEncoder(udfPacket.outputEncoder)), - udfName = Option(fun.getFunctionName), - nullable = udf.getNullable, - udfDeterministic = fun.getDeterministic) + val children = fun.getArgumentsList.asScala.map(transformExpression).toSeq + transformScalaFunction(fun) match { + case udf: SparkUserDefinedFunction => + UserDefinedFunctionUtils.toScalaUDF(udf, children) + case uda: UserDefinedAggregator[_, _, _] => + ScalaAggregator(uda, children).toAggregateExpression() + case other => + throw InvalidPlanInput( + s"Unsupported UserDefinedFunction implementation: ${other.getClass}") } } + /** + * Translates a Scala user-defined function or aggregator. from proto to a UserDefinedFunction. + * + * @param fun + * Proto representation of the Scala user-defined function or aggregator. + * @return + * A concrete UserDefinedFunction implementation, either a SparkUserDefinedFunction or a + * UserDefinedAggregator. + */ private def transformScalaFunction( fun: proto.CommonInlineUserDefinedFunction): UserDefinedFunction = { val udf = fun.getScalarScalaUdf diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala index 5dced7acfb0d2..36f3bcd1e6cd7 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala @@ -301,7 +301,7 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio // Clean up all artifacts. // Note: there can be concurrent AddArtifact calls still adding something. - artifactManager.cleanUpResources() + artifactManager.close() // Clean up running streaming queries. // Note: there can be concurrent streaming queries being started. diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala index c5e484e022bc4..06bc24b6ccae6 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectConfigHandler.scala @@ -73,11 +73,21 @@ class SparkConnectConfigHandler(responseObserver: StreamObserver[proto.ConfigRes private def handleSet( operation: proto.ConfigRequest.Set, conf: RuntimeConfig): proto.ConfigResponse.Builder = { + val silent = operation.hasSilent && operation.getSilent val builder = proto.ConfigResponse.newBuilder() operation.getPairsList.asScala.iterator.foreach { pair => val (key, value) = SparkConnectConfigHandler.toKeyValue(pair) - conf.set(key, value.orNull) - getWarning(key).foreach(builder.addWarnings) + try { + conf.set(key, value.orNull) + getWarning(key).foreach(builder.addWarnings) + } catch { + case e: Throwable => + if (silent) { + builder.addWarnings(s"Failed to set $key to $value due to ${e.getMessage}") + } else { + throw e + } + } } builder } diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectReleaseSessionHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectReleaseSessionHandler.scala index ec7a7f3bd242c..c36f07fc67f8f 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectReleaseSessionHandler.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectReleaseSessionHandler.scala @@ -37,7 +37,8 @@ class SparkConnectReleaseSessionHandler( val maybeSession = SparkConnectService.sessionManager.getIsolatedSessionIfPresent(key) maybeSession.foreach(f => responseBuilder.setServerSideSessionId(f.serverSessionId)) - SparkConnectService.sessionManager.closeSession(key) + val allowReconnect = v.getAllowReconnect + SparkConnectService.sessionManager.closeSession(key, allowReconnect) responseObserver.onNext(responseBuilder.build()) responseObserver.onCompleted() diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala index b0b74a36e187b..c59fd02a829ae 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SparkConnectSessionManager.scala @@ -134,7 +134,9 @@ class SparkConnectSessionManager extends Logging { } // Removes session from sessionStore and returns it. - private def removeSessionHolder(key: SessionKey): Option[SessionHolder] = { + private def removeSessionHolder( + key: SessionKey, + allowReconnect: Boolean = false): Option[SessionHolder] = { var sessionHolder: Option[SessionHolder] = None // The session holder should remain in the session store until it is added to the closed session @@ -144,9 +146,11 @@ class SparkConnectSessionManager extends Logging { sessionHolder = Option(sessionStore.get(key)) sessionHolder.foreach { s => - // Put into closedSessionsCache to prevent the same session from being recreated by - // getOrCreateIsolatedSession. - closedSessionsCache.put(s.key, s.getSessionHolderInfo) + if (!allowReconnect) { + // Put into closedSessionsCache to prevent the same session from being recreated by + // getOrCreateIsolatedSession when reconnection isn't allowed. + closedSessionsCache.put(s.key, s.getSessionHolderInfo) + } // Then, remove the session holder from the session store. sessionStore.remove(key) @@ -154,17 +158,21 @@ class SparkConnectSessionManager extends Logging { sessionHolder } - // Shut downs the session after removing. - private def shutdownSessionHolder(sessionHolder: SessionHolder): Unit = { + // Shuts down the session after removing. + private def shutdownSessionHolder( + sessionHolder: SessionHolder, + allowReconnect: Boolean = false): Unit = { sessionHolder.close() - // Update in closedSessionsCache: above it wasn't updated with closedTime etc. yet. - closedSessionsCache.put(sessionHolder.key, sessionHolder.getSessionHolderInfo) + if (!allowReconnect) { + // Update in closedSessionsCache: above it wasn't updated with closedTime etc. yet. + closedSessionsCache.put(sessionHolder.key, sessionHolder.getSessionHolderInfo) + } } - def closeSession(key: SessionKey): Unit = { - val sessionHolder = removeSessionHolder(key) + def closeSession(key: SessionKey, allowReconnect: Boolean = false): Unit = { + val sessionHolder = removeSessionHolder(key, allowReconnect) // Rest of the cleanup: the session cannot be accessed anymore by getOrCreateIsolatedSession. - sessionHolder.foreach(shutdownSessionHolder(_)) + sessionHolder.foreach(shutdownSessionHolder(_, allowReconnect)) } private[connect] def shutdown(): Unit = { diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkConnectServerTest.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkConnectServerTest.scala index b04c42a730785..3c857554dc756 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkConnectServerTest.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/SparkConnectServerTest.scala @@ -128,6 +128,17 @@ trait SparkConnectServerTest extends SharedSparkSession { req.build() } + protected def buildReleaseSessionRequest( + sessionId: String = defaultSessionId, + allowReconnect: Boolean = false) = { + proto.ReleaseSessionRequest + .newBuilder() + .setUserContext(userContext) + .setSessionId(sessionId) + .setAllowReconnect(allowReconnect) + .build() + } + protected def buildPlan(query: String) = { proto.Plan.newBuilder().setRoot(dsl.sql(query)).build() } diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala index 84d5fd68d4c73..aaeb5d9fe509a 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala @@ -26,9 +26,10 @@ import org.apache.spark.connect.proto import org.apache.spark.connect.proto.Expression.{Alias, ExpressionString, UnresolvedStar} import org.apache.spark.sql.{AnalysisException, Dataset, Row} import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation} import org.apache.spark.sql.catalyst.expressions.{AttributeReference, UnsafeProjection} import org.apache.spark.sql.catalyst.plans.logical +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.connect.SparkConnectTestUtils import org.apache.spark.sql.connect.common.InvalidPlanInput @@ -884,4 +885,36 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest { intercept[AnalysisException](Dataset.ofRows(spark, logical)) } + + test("Internal functions") { + def getProjectRelationWithFn(name: String, isInternal: Option[Boolean]): proto.Relation = { + val fn = proto.Expression.UnresolvedFunction.newBuilder.setFunctionName(name) + isInternal.foreach(fn.setIsInternal) + val proj = proto.Project.newBuilder + .setInput(readRel) + .addExpressions(proto.Expression.newBuilder.setUnresolvedFunction(fn)) + proto.Relation.newBuilder.setProject(proj).build() + } + + def getUnresolvedFunction(plan: LogicalPlan): UnresolvedFunction = + plan.expressions.head.asInstanceOf[UnresolvedAlias].child.asInstanceOf[UnresolvedFunction] + + // "bloom_filter_agg" is an internal function. + val plan1 = transform(getProjectRelationWithFn("bloom_filter_agg", isInternal = None)) + val fn1 = getUnresolvedFunction(plan1) + assert(fn1.nameParts.head == "bloom_filter_agg") + assert(fn1.isInternal) + + // "abcde" is not an internal function. + val plan2 = transform(getProjectRelationWithFn("abcde", isInternal = None)) + val fn2 = getUnresolvedFunction(plan2) + assert(fn2.nameParts.head == "abcde") + assert(!fn2.isInternal) + + // "abcde" is not an internal function but we could set it to be internal. + val plan3 = transform(getProjectRelationWithFn("abcde", isInternal = Some(true))) + val fn3 = getUnresolvedFunction(plan3) + assert(fn3.nameParts.head == "abcde") + assert(fn3.isInternal) + } } diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectServiceE2ESuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectServiceE2ESuite.scala index f86298a8b5b98..f24560259a883 100644 --- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectServiceE2ESuite.scala +++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/service/SparkConnectServiceE2ESuite.scala @@ -245,4 +245,26 @@ class SparkConnectServiceE2ESuite extends SparkConnectServerTest { assert(queryError.getMessage.contains("INVALID_HANDLE.SESSION_CHANGED")) } } + + test("Client is allowed to reconnect to released session if allow_reconnect is set") { + withRawBlockingStub { stub => + val sessionId = UUID.randomUUID.toString() + val iter = + stub.executePlan( + buildExecutePlanRequest( + buildPlan("select * from range(1000000)"), + sessionId = sessionId)) + iter.hasNext // guarantees the request was received by server. + + stub.releaseSession(buildReleaseSessionRequest(sessionId, allowReconnect = true)) + + val iter2 = + stub.executePlan( + buildExecutePlanRequest( + buildPlan("select * from range(1000000)"), + sessionId = sessionId)) + // guarantees the request was received by server. No exception should be thrown on reuse + iter2.hasNext + } + } } diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk21-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk21-results.txt index c15fb78f2f165..4b3fc3f4d4a1f 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk21-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-jdk21-results.txt @@ -2,153 +2,153 @@ Rebasing dates/timestamps in Parquet datasource ================================================================================================ -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 11193 11193 0 8.9 111.9 1.0X -before 1582, noop 7070 7070 0 14.1 70.7 1.6X -after 1582, rebase EXCEPTION 19836 19836 0 5.0 198.4 0.6X -after 1582, rebase LEGACY 19368 19368 0 5.2 193.7 0.6X -after 1582, rebase CORRECTED 19627 19627 0 5.1 196.3 0.6X -before 1582, rebase LEGACY 16301 16301 0 6.1 163.0 0.7X -before 1582, rebase CORRECTED 15612 15612 0 6.4 156.1 0.7X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1582, noop 10901 10901 0 9.2 109.0 1.0X +before 1582, noop 6649 6649 0 15.0 66.5 1.6X +after 1582, rebase EXCEPTION 20020 20020 0 5.0 200.2 0.5X +after 1582, rebase LEGACY 20257 20257 0 4.9 202.6 0.5X +after 1582, rebase CORRECTED 20050 20050 0 5.0 200.5 0.5X +before 1582, rebase LEGACY 16178 16178 0 6.2 161.8 0.7X +before 1582, rebase CORRECTED 16322 16322 0 6.1 163.2 0.7X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase EXCEPTION 11601 11657 95 8.6 116.0 1.0X -after 1582, vec off, rebase LEGACY 11671 11751 72 8.6 116.7 1.0X -after 1582, vec off, rebase CORRECTED 11593 11632 67 8.6 115.9 1.0X -after 1582, vec on, rebase EXCEPTION 2394 2427 33 41.8 23.9 4.8X -after 1582, vec on, rebase LEGACY 2466 2489 38 40.6 24.7 4.7X -after 1582, vec on, rebase CORRECTED 2487 2500 18 40.2 24.9 4.7X -before 1582, vec off, rebase LEGACY 11937 11951 14 8.4 119.4 1.0X -before 1582, vec off, rebase CORRECTED 11542 11600 60 8.7 115.4 1.0X -before 1582, vec on, rebase LEGACY 2708 2729 30 36.9 27.1 4.3X -before 1582, vec on, rebase CORRECTED 2436 2445 8 41.1 24.4 4.8X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1582, vec off, rebase EXCEPTION 11905 11978 106 8.4 119.1 1.0X +after 1582, vec off, rebase LEGACY 12069 12083 17 8.3 120.7 1.0X +after 1582, vec off, rebase CORRECTED 11835 11861 40 8.4 118.4 1.0X +after 1582, vec on, rebase EXCEPTION 2453 2492 38 40.8 24.5 4.9X +after 1582, vec on, rebase LEGACY 2471 2494 22 40.5 24.7 4.8X +after 1582, vec on, rebase CORRECTED 2386 2442 59 41.9 23.9 5.0X +before 1582, vec off, rebase LEGACY 12049 12076 27 8.3 120.5 1.0X +before 1582, vec off, rebase CORRECTED 11948 11999 48 8.4 119.5 1.0X +before 1582, vec on, rebase LEGACY 2787 2810 24 35.9 27.9 4.3X +before 1582, vec on, rebase CORRECTED 2443 2457 14 40.9 24.4 4.9X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2403 2403 0 41.6 24.0 1.0X -before 1900, noop 2443 2443 0 40.9 24.4 1.0X -after 1900, rebase EXCEPTION 12805 12805 0 7.8 128.1 0.2X -after 1900, rebase LEGACY 12529 12529 0 8.0 125.3 0.2X -after 1900, rebase CORRECTED 12474 12474 0 8.0 124.7 0.2X -before 1900, rebase LEGACY 14628 14628 0 6.8 146.3 0.2X -before 1900, rebase CORRECTED 12601 12601 0 7.9 126.0 0.2X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2429 2429 0 41.2 24.3 1.0X +before 1900, noop 2334 2334 0 42.8 23.3 1.0X +after 1900, rebase EXCEPTION 14358 14358 0 7.0 143.6 0.2X +after 1900, rebase LEGACY 14210 14210 0 7.0 142.1 0.2X +after 1900, rebase CORRECTED 14052 14052 0 7.1 140.5 0.2X +before 1900, rebase LEGACY 16255 16255 0 6.2 162.5 0.1X +before 1900, rebase CORRECTED 14010 14010 0 7.1 140.1 0.2X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 15545 15658 99 6.4 155.4 1.0X -after 1900, vec off, rebase LEGACY 15927 15945 25 6.3 159.3 1.0X -after 1900, vec off, rebase CORRECTED 15558 15620 54 6.4 155.6 1.0X -after 1900, vec on, rebase EXCEPTION 4050 4074 34 24.7 40.5 3.8X -after 1900, vec on, rebase LEGACY 4024 4059 32 24.9 40.2 3.9X -after 1900, vec on, rebase CORRECTED 4062 4074 17 24.6 40.6 3.8X -before 1900, vec off, rebase LEGACY 18219 18234 22 5.5 182.2 0.9X -before 1900, vec off, rebase CORRECTED 15584 15633 45 6.4 155.8 1.0X -before 1900, vec on, rebase LEGACY 6080 6106 23 16.4 60.8 2.6X -before 1900, vec on, rebase CORRECTED 4045 4057 14 24.7 40.4 3.8X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1900, vec off, rebase EXCEPTION 16891 16906 13 5.9 168.9 1.0X +after 1900, vec off, rebase LEGACY 16737 16761 21 6.0 167.4 1.0X +after 1900, vec off, rebase CORRECTED 16555 16614 54 6.0 165.5 1.0X +after 1900, vec on, rebase EXCEPTION 4046 4094 54 24.7 40.5 4.2X +after 1900, vec on, rebase LEGACY 4057 4067 11 24.6 40.6 4.2X +after 1900, vec on, rebase CORRECTED 4063 4068 6 24.6 40.6 4.2X +before 1900, vec off, rebase LEGACY 18775 18838 56 5.3 187.7 0.9X +before 1900, vec off, rebase CORRECTED 16681 16713 36 6.0 166.8 1.0X +before 1900, vec on, rebase LEGACY 6105 6129 32 16.4 61.0 2.8X +before 1900, vec on, rebase CORRECTED 4046 4067 27 24.7 40.5 4.2X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2378 2378 0 42.0 23.8 1.0X -before 1900, noop 2426 2426 0 41.2 24.3 1.0X -after 1900, rebase EXCEPTION 14475 14475 0 6.9 144.8 0.2X -after 1900, rebase LEGACY 13685 13685 0 7.3 136.8 0.2X -after 1900, rebase CORRECTED 13448 13448 0 7.4 134.5 0.2X -before 1900, rebase LEGACY 15085 15085 0 6.6 150.8 0.2X -before 1900, rebase CORRECTED 13668 13668 0 7.3 136.7 0.2X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2415 2415 0 41.4 24.1 1.0X +before 1900, noop 2438 2438 0 41.0 24.4 1.0X +after 1900, rebase EXCEPTION 13703 13703 0 7.3 137.0 0.2X +after 1900, rebase LEGACY 13404 13404 0 7.5 134.0 0.2X +after 1900, rebase CORRECTED 13538 13538 0 7.4 135.4 0.2X +before 1900, rebase LEGACY 15245 15245 0 6.6 152.5 0.2X +before 1900, rebase CORRECTED 13242 13242 0 7.6 132.4 0.2X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 14595 14621 23 6.9 146.0 1.0X -after 1900, vec off, rebase LEGACY 14689 14699 12 6.8 146.9 1.0X -after 1900, vec off, rebase CORRECTED 14626 14648 25 6.8 146.3 1.0X -after 1900, vec on, rebase EXCEPTION 3732 3745 14 26.8 37.3 3.9X -after 1900, vec on, rebase LEGACY 3753 3771 29 26.6 37.5 3.9X -after 1900, vec on, rebase CORRECTED 3714 3734 23 26.9 37.1 3.9X -before 1900, vec off, rebase LEGACY 17073 17151 107 5.9 170.7 0.9X -before 1900, vec off, rebase CORRECTED 14575 14613 33 6.9 145.8 1.0X -before 1900, vec on, rebase LEGACY 5581 5602 34 17.9 55.8 2.6X -before 1900, vec on, rebase CORRECTED 3680 3698 30 27.2 36.8 4.0X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1900, vec off, rebase EXCEPTION 14618 14680 54 6.8 146.2 1.0X +after 1900, vec off, rebase LEGACY 14578 14613 35 6.9 145.8 1.0X +after 1900, vec off, rebase CORRECTED 14621 14691 61 6.8 146.2 1.0X +after 1900, vec on, rebase EXCEPTION 3727 3738 16 26.8 37.3 3.9X +after 1900, vec on, rebase LEGACY 3715 3738 22 26.9 37.1 3.9X +after 1900, vec on, rebase CORRECTED 3705 3727 21 27.0 37.1 3.9X +before 1900, vec off, rebase LEGACY 16907 16943 32 5.9 169.1 0.9X +before 1900, vec off, rebase CORRECTED 14704 14711 6 6.8 147.0 1.0X +before 1900, vec on, rebase LEGACY 5588 5610 30 17.9 55.9 2.6X +before 1900, vec on, rebase CORRECTED 3730 3744 15 26.8 37.3 3.9X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2418 2418 0 41.4 24.2 1.0X -before 1900, noop 2413 2413 0 41.5 24.1 1.0X -after 1900, rebase EXCEPTION 11749 11749 0 8.5 117.5 0.2X -after 1900, rebase LEGACY 11757 11757 0 8.5 117.6 0.2X -after 1900, rebase CORRECTED 12081 12081 0 8.3 120.8 0.2X -before 1900, rebase LEGACY 13503 13503 0 7.4 135.0 0.2X -before 1900, rebase CORRECTED 11649 11649 0 8.6 116.5 0.2X - -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2391 2391 0 41.8 23.9 1.0X +before 1900, noop 2431 2431 0 41.1 24.3 1.0X +after 1900, rebase EXCEPTION 11856 11856 0 8.4 118.6 0.2X +after 1900, rebase LEGACY 11077 11077 0 9.0 110.8 0.2X +after 1900, rebase CORRECTED 11292 11292 0 8.9 112.9 0.2X +before 1900, rebase LEGACY 13101 13101 0 7.6 131.0 0.2X +before 1900, rebase CORRECTED 11333 11333 0 8.8 113.3 0.2X + +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 14322 14376 53 7.0 143.2 1.0X -after 1900, vec off, rebase LEGACY 14453 14478 22 6.9 144.5 1.0X -after 1900, vec off, rebase CORRECTED 14429 14523 94 6.9 144.3 1.0X -after 1900, vec on, rebase EXCEPTION 4876 4914 42 20.5 48.8 2.9X -after 1900, vec on, rebase LEGACY 4418 4465 41 22.6 44.2 3.2X -after 1900, vec on, rebase CORRECTED 4876 4909 50 20.5 48.8 2.9X -before 1900, vec off, rebase LEGACY 17196 17238 45 5.8 172.0 0.8X -before 1900, vec off, rebase CORRECTED 14462 14509 57 6.9 144.6 1.0X -before 1900, vec on, rebase LEGACY 6120 6135 16 16.3 61.2 2.3X -before 1900, vec on, rebase CORRECTED 4887 4929 42 20.5 48.9 2.9X +after 1900, vec off, rebase EXCEPTION 14732 14828 88 6.8 147.3 1.0X +after 1900, vec off, rebase LEGACY 15011 15061 58 6.7 150.1 1.0X +after 1900, vec off, rebase CORRECTED 14924 14978 86 6.7 149.2 1.0X +after 1900, vec on, rebase EXCEPTION 4935 4964 37 20.3 49.4 3.0X +after 1900, vec on, rebase LEGACY 4488 4509 19 22.3 44.9 3.3X +after 1900, vec on, rebase CORRECTED 4896 4934 33 20.4 49.0 3.0X +before 1900, vec off, rebase LEGACY 16917 17029 151 5.9 169.2 0.9X +before 1900, vec off, rebase CORRECTED 14924 14953 35 6.7 149.2 1.0X +before 1900, vec on, rebase LEGACY 6010 6023 12 16.6 60.1 2.5X +before 1900, vec on, rebase CORRECTED 4916 4939 25 20.3 49.2 3.0X ================================================================================================ Rebasing dates/timestamps in ORC datasource ================================================================================================ -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 12614 12614 0 7.9 126.1 1.0X -before 1582, noop 6620 6620 0 15.1 66.2 1.9X -after 1582 17066 17066 0 5.9 170.7 0.7X -before 1582 10573 10573 0 9.5 105.7 1.2X +after 1582, noop 12628 12628 0 7.9 126.3 1.0X +before 1582, noop 6710 6710 0 14.9 67.1 1.9X +after 1582 16956 16956 0 5.9 169.6 0.7X +before 1582 10840 10840 0 9.2 108.4 1.2X -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 8422 8523 131 11.9 84.2 1.0X -after 1582, vec on 2386 2401 20 41.9 23.9 3.5X -before 1582, vec off 8447 8474 42 11.8 84.5 1.0X -before 1582, vec on 2526 2542 24 39.6 25.3 3.3X +after 1582, vec off 8521 8619 138 11.7 85.2 1.0X +after 1582, vec on 2414 2421 10 41.4 24.1 3.5X +before 1582, vec off 8621 8693 89 11.6 86.2 1.0X +before 1582, vec on 2555 2562 8 39.1 25.5 3.3X -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2336 2336 0 42.8 23.4 1.0X -before 1900, noop 2309 2309 0 43.3 23.1 1.0X -after 1900 9646 9646 0 10.4 96.5 0.2X -before 1900 12150 12150 0 8.2 121.5 0.2X +after 1900, noop 2290 2290 0 43.7 22.9 1.0X +before 1900, noop 2313 2313 0 43.2 23.1 1.0X +after 1900 10097 10097 0 9.9 101.0 0.2X +before 1900 12223 12223 0 8.2 122.2 0.2X -OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 21.0.5+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 9781 9845 86 10.2 97.8 1.0X -after 1900, vec on 3778 3792 13 26.5 37.8 2.6X -before 1900, vec off 11757 11781 21 8.5 117.6 0.8X -before 1900, vec on 5490 5511 21 18.2 54.9 1.8X +after 1900, vec off 9908 9927 29 10.1 99.1 1.0X +after 1900, vec on 3868 3881 18 25.9 38.7 2.6X +before 1900, vec off 11663 11671 7 8.6 116.6 0.8X +before 1900, vec on 5396 5408 10 18.5 54.0 1.8X diff --git a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt index 249b478e772a8..980897bc96b9b 100644 --- a/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt +++ b/sql/core/benchmarks/DateTimeRebaseBenchmark-results.txt @@ -2,153 +2,153 @@ Rebasing dates/timestamps in Parquet datasource ================================================================================================ -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save DATE to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 11170 11170 0 9.0 111.7 1.0X -before 1582, noop 6505 6505 0 15.4 65.0 1.7X -after 1582, rebase EXCEPTION 19873 19873 0 5.0 198.7 0.6X -after 1582, rebase LEGACY 19726 19726 0 5.1 197.3 0.6X -after 1582, rebase CORRECTED 19931 19931 0 5.0 199.3 0.6X -before 1582, rebase LEGACY 15590 15590 0 6.4 155.9 0.7X -before 1582, rebase CORRECTED 15523 15523 0 6.4 155.2 0.7X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1582, noop 13229 13229 0 7.6 132.3 1.0X +before 1582, noop 7736 7736 0 12.9 77.4 1.7X +after 1582, rebase EXCEPTION 21925 21925 0 4.6 219.2 0.6X +after 1582, rebase LEGACY 21714 21714 0 4.6 217.1 0.6X +after 1582, rebase CORRECTED 21864 21864 0 4.6 218.6 0.6X +before 1582, rebase LEGACY 16338 16338 0 6.1 163.4 0.8X +before 1582, rebase CORRECTED 16312 16312 0 6.1 163.1 0.8X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load DATE from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off, rebase EXCEPTION 11571 11596 22 8.6 115.7 1.0X -after 1582, vec off, rebase LEGACY 11892 11909 27 8.4 118.9 1.0X -after 1582, vec off, rebase CORRECTED 11681 11724 47 8.6 116.8 1.0X -after 1582, vec on, rebase EXCEPTION 2516 2530 13 39.7 25.2 4.6X -after 1582, vec on, rebase LEGACY 2555 2563 8 39.1 25.5 4.5X -after 1582, vec on, rebase CORRECTED 2487 2503 22 40.2 24.9 4.7X -before 1582, vec off, rebase LEGACY 11947 11996 69 8.4 119.5 1.0X -before 1582, vec off, rebase CORRECTED 11792 11821 41 8.5 117.9 1.0X -before 1582, vec on, rebase LEGACY 2826 2856 25 35.4 28.3 4.1X -before 1582, vec on, rebase CORRECTED 2465 2489 21 40.6 24.6 4.7X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1582, vec off, rebase EXCEPTION 11106 11144 39 9.0 111.1 1.0X +after 1582, vec off, rebase LEGACY 11476 11510 49 8.7 114.8 1.0X +after 1582, vec off, rebase CORRECTED 11373 11409 62 8.8 113.7 1.0X +after 1582, vec on, rebase EXCEPTION 2522 2548 31 39.7 25.2 4.4X +after 1582, vec on, rebase LEGACY 2628 2642 15 38.1 26.3 4.2X +after 1582, vec on, rebase CORRECTED 2490 2547 61 40.2 24.9 4.5X +before 1582, vec off, rebase LEGACY 11671 11743 88 8.6 116.7 1.0X +before 1582, vec off, rebase CORRECTED 11350 11379 41 8.8 113.5 1.0X +before 1582, vec on, rebase LEGACY 2855 2857 2 35.0 28.6 3.9X +before 1582, vec on, rebase CORRECTED 2559 2570 16 39.1 25.6 4.3X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_INT96 to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2340 2340 0 42.7 23.4 1.0X -before 1900, noop 2284 2284 0 43.8 22.8 1.0X -after 1900, rebase EXCEPTION 13230 13230 0 7.6 132.3 0.2X -after 1900, rebase LEGACY 13238 13238 0 7.6 132.4 0.2X -after 1900, rebase CORRECTED 13264 13264 0 7.5 132.6 0.2X -before 1900, rebase LEGACY 15216 15216 0 6.6 152.2 0.2X -before 1900, rebase CORRECTED 13382 13382 0 7.5 133.8 0.2X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2239 2239 0 44.7 22.4 1.0X +before 1900, noop 2229 2229 0 44.9 22.3 1.0X +after 1900, rebase EXCEPTION 13150 13150 0 7.6 131.5 0.2X +after 1900, rebase LEGACY 13004 13004 0 7.7 130.0 0.2X +after 1900, rebase CORRECTED 13216 13216 0 7.6 132.2 0.2X +before 1900, rebase LEGACY 14965 14965 0 6.7 149.6 0.1X +before 1900, rebase CORRECTED 13218 13218 0 7.6 132.2 0.2X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_INT96 from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 16170 16195 22 6.2 161.7 1.0X -after 1900, vec off, rebase LEGACY 16720 16755 31 6.0 167.2 1.0X -after 1900, vec off, rebase CORRECTED 16152 16213 54 6.2 161.5 1.0X -after 1900, vec on, rebase EXCEPTION 4090 4101 18 24.4 40.9 4.0X -after 1900, vec on, rebase LEGACY 4114 4144 33 24.3 41.1 3.9X -after 1900, vec on, rebase CORRECTED 4158 4191 28 24.0 41.6 3.9X -before 1900, vec off, rebase LEGACY 18554 18584 31 5.4 185.5 0.9X -before 1900, vec off, rebase CORRECTED 16192 16267 84 6.2 161.9 1.0X -before 1900, vec on, rebase LEGACY 6256 6271 22 16.0 62.6 2.6X -before 1900, vec on, rebase CORRECTED 4074 4104 27 24.5 40.7 4.0X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1900, vec off, rebase EXCEPTION 17738 17763 26 5.6 177.4 1.0X +after 1900, vec off, rebase LEGACY 16932 16966 49 5.9 169.3 1.0X +after 1900, vec off, rebase CORRECTED 16738 16751 12 6.0 167.4 1.1X +after 1900, vec on, rebase EXCEPTION 4046 4072 34 24.7 40.5 4.4X +after 1900, vec on, rebase LEGACY 4081 4090 11 24.5 40.8 4.3X +after 1900, vec on, rebase CORRECTED 4015 4046 32 24.9 40.1 4.4X +before 1900, vec off, rebase LEGACY 19319 19369 43 5.2 193.2 0.9X +before 1900, vec off, rebase CORRECTED 16460 16479 19 6.1 164.6 1.1X +before 1900, vec on, rebase LEGACY 5956 5960 5 16.8 59.6 3.0X +before 1900, vec on, rebase CORRECTED 4036 4086 83 24.8 40.4 4.4X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_MICROS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2390 2390 0 41.8 23.9 1.0X -before 1900, noop 2291 2291 0 43.6 22.9 1.0X -after 1900, rebase EXCEPTION 12537 12537 0 8.0 125.4 0.2X -after 1900, rebase LEGACY 12047 12047 0 8.3 120.5 0.2X -after 1900, rebase CORRECTED 12151 12151 0 8.2 121.5 0.2X -before 1900, rebase LEGACY 13960 13960 0 7.2 139.6 0.2X -before 1900, rebase CORRECTED 11985 11985 0 8.3 119.9 0.2X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2262 2262 0 44.2 22.6 1.0X +before 1900, noop 2253 2253 0 44.4 22.5 1.0X +after 1900, rebase EXCEPTION 11901 11901 0 8.4 119.0 0.2X +after 1900, rebase LEGACY 11564 11564 0 8.6 115.6 0.2X +after 1900, rebase CORRECTED 11804 11804 0 8.5 118.0 0.2X +before 1900, rebase LEGACY 13264 13264 0 7.5 132.6 0.2X +before 1900, rebase CORRECTED 11816 11816 0 8.5 118.2 0.2X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_MICROS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 14977 15018 36 6.7 149.8 1.0X -after 1900, vec off, rebase LEGACY 14924 14960 33 6.7 149.2 1.0X -after 1900, vec off, rebase CORRECTED 14965 14994 26 6.7 149.7 1.0X -after 1900, vec on, rebase EXCEPTION 3810 3819 8 26.2 38.1 3.9X -after 1900, vec on, rebase LEGACY 3829 3835 8 26.1 38.3 3.9X -after 1900, vec on, rebase CORRECTED 3785 3837 47 26.4 37.9 4.0X -before 1900, vec off, rebase LEGACY 17323 17343 19 5.8 173.2 0.9X -before 1900, vec off, rebase CORRECTED 14933 14962 26 6.7 149.3 1.0X -before 1900, vec on, rebase LEGACY 5763 5783 17 17.4 57.6 2.6X -before 1900, vec on, rebase CORRECTED 3798 3817 32 26.3 38.0 3.9X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1900, vec off, rebase EXCEPTION 15095 15151 58 6.6 150.9 1.0X +after 1900, vec off, rebase LEGACY 15243 15292 47 6.6 152.4 1.0X +after 1900, vec off, rebase CORRECTED 15200 15211 11 6.6 152.0 1.0X +after 1900, vec on, rebase EXCEPTION 3758 3772 14 26.6 37.6 4.0X +after 1900, vec on, rebase LEGACY 3775 3811 39 26.5 37.8 4.0X +after 1900, vec on, rebase CORRECTED 3784 3796 16 26.4 37.8 4.0X +before 1900, vec off, rebase LEGACY 17548 17583 41 5.7 175.5 0.9X +before 1900, vec off, rebase CORRECTED 15155 15166 10 6.6 151.6 1.0X +before 1900, vec on, rebase LEGACY 5640 5665 43 17.7 56.4 2.7X +before 1900, vec on, rebase CORRECTED 3803 3856 64 26.3 38.0 4.0X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP_MILLIS to parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2309 2309 0 43.3 23.1 1.0X -before 1900, noop 2358 2358 0 42.4 23.6 1.0X -after 1900, rebase EXCEPTION 11266 11266 0 8.9 112.7 0.2X -after 1900, rebase LEGACY 11582 11582 0 8.6 115.8 0.2X -after 1900, rebase CORRECTED 11555 11555 0 8.7 115.5 0.2X -before 1900, rebase LEGACY 13600 13600 0 7.4 136.0 0.2X -before 1900, rebase CORRECTED 12113 12113 0 8.3 121.1 0.2X - -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +after 1900, noop 2253 2253 0 44.4 22.5 1.0X +before 1900, noop 2253 2253 0 44.4 22.5 1.0X +after 1900, rebase EXCEPTION 11471 11471 0 8.7 114.7 0.2X +after 1900, rebase LEGACY 11173 11173 0 8.9 111.7 0.2X +after 1900, rebase CORRECTED 11497 11497 0 8.7 115.0 0.2X +before 1900, rebase LEGACY 13368 13368 0 7.5 133.7 0.2X +before 1900, rebase CORRECTED 11570 11570 0 8.6 115.7 0.2X + +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP_MILLIS from parquet: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off, rebase EXCEPTION 15076 15125 46 6.6 150.8 1.0X -after 1900, vec off, rebase LEGACY 15480 15491 16 6.5 154.8 1.0X -after 1900, vec off, rebase CORRECTED 15171 15189 17 6.6 151.7 1.0X -after 1900, vec on, rebase EXCEPTION 3976 4001 28 25.2 39.8 3.8X -after 1900, vec on, rebase LEGACY 4582 4609 46 21.8 45.8 3.3X -after 1900, vec on, rebase CORRECTED 3934 3953 29 25.4 39.3 3.8X -before 1900, vec off, rebase LEGACY 17602 17644 37 5.7 176.0 0.9X -before 1900, vec off, rebase CORRECTED 15201 15238 34 6.6 152.0 1.0X -before 1900, vec on, rebase LEGACY 6306 6311 6 15.9 63.1 2.4X -before 1900, vec on, rebase CORRECTED 3926 3961 50 25.5 39.3 3.8X +after 1900, vec off, rebase EXCEPTION 15315 15363 82 6.5 153.1 1.0X +after 1900, vec off, rebase LEGACY 15462 15506 70 6.5 154.6 1.0X +after 1900, vec off, rebase CORRECTED 15394 15426 38 6.5 153.9 1.0X +after 1900, vec on, rebase EXCEPTION 3910 3941 28 25.6 39.1 3.9X +after 1900, vec on, rebase LEGACY 4503 4516 22 22.2 45.0 3.4X +after 1900, vec on, rebase CORRECTED 3915 3919 5 25.5 39.1 3.9X +before 1900, vec off, rebase LEGACY 17595 17636 46 5.7 176.0 0.9X +before 1900, vec off, rebase CORRECTED 15344 15354 17 6.5 153.4 1.0X +before 1900, vec on, rebase LEGACY 6188 6202 23 16.2 61.9 2.5X +before 1900, vec on, rebase CORRECTED 3890 3899 8 25.7 38.9 3.9X ================================================================================================ Rebasing dates/timestamps in ORC datasource ================================================================================================ -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save DATE to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, noop 11208 11208 0 8.9 112.1 1.0X -before 1582, noop 6567 6567 0 15.2 65.7 1.7X -after 1582 15130 15130 0 6.6 151.3 0.7X -before 1582 10992 10992 0 9.1 109.9 1.0X +after 1582, noop 13342 13342 0 7.5 133.4 1.0X +before 1582, noop 7875 7875 0 12.7 78.7 1.7X +after 1582 16884 16884 0 5.9 168.8 0.8X +before 1582 11948 11948 0 8.4 119.5 1.1X -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load DATE from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1582, vec off 8770 8777 8 11.4 87.7 1.0X -after 1582, vec on 2445 2478 31 40.9 24.5 3.6X -before 1582, vec off 8820 8896 106 11.3 88.2 1.0X -before 1582, vec on 2580 2615 37 38.8 25.8 3.4X +after 1582, vec off 8890 8914 29 11.2 88.9 1.0X +after 1582, vec on 2459 2499 64 40.7 24.6 3.6X +before 1582, vec off 8453 8462 8 11.8 84.5 1.1X +before 1582, vec on 2618 2636 16 38.2 26.2 3.4X -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Save TIMESTAMP to ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, noop 2221 2221 0 45.0 22.2 1.0X -before 1900, noop 2218 2218 0 45.1 22.2 1.0X -after 1900 9916 9916 0 10.1 99.2 0.2X -before 1900 12130 12130 0 8.2 121.3 0.2X +after 1900, noop 2181 2181 0 45.9 21.8 1.0X +before 1900, noop 2161 2161 0 46.3 21.6 1.0X +after 1900 10953 10953 0 9.1 109.5 0.2X +before 1900 12293 12293 0 8.1 122.9 0.2X -OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure +OpenJDK 64-Bit Server VM 17.0.13+11-LTS on Linux 6.5.0-1025-azure AMD EPYC 7763 64-Core Processor Load TIMESTAMP from ORC: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -after 1900, vec off 10569 11038 802 9.5 105.7 1.0X -after 1900, vec on 4361 4415 55 22.9 43.6 2.4X -before 1900, vec off 12223 12227 7 8.2 122.2 0.9X -before 1900, vec on 6103 6136 30 16.4 61.0 1.7X +after 1900, vec off 10398 10414 25 9.6 104.0 1.0X +after 1900, vec on 4165 4173 7 24.0 41.7 2.5X +before 1900, vec off 12138 12211 112 8.2 121.4 0.9X +before 1900, vec on 5781 5784 4 17.3 57.8 1.8X diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index b9ae0e5b91318..e4e782a50e3d4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -92,6 +92,23 @@ private[sql] object Dataset { dataset } + def apply[T]( + sparkSession: SparkSession, + logicalPlan: LogicalPlan, + encoderGenerator: () => Encoder[T]): Dataset[T] = { + val dataset = new Dataset(sparkSession, logicalPlan, encoderGenerator) + // Eagerly bind the encoder so we verify that the encoder matches the underlying + // schema. The user will get an error if this is not the case. + // optimization: it is guaranteed that [[InternalRow]] can be converted to [[Row]] so + // do not do this check in that case. this check can be expensive since it requires running + // the whole [[Analyzer]] to resolve the deserializer + if (!dataset.queryExecution.isLazyAnalysis + && dataset.encoder.clsTag.runtimeClass != classOf[Row]) { + dataset.resolvedEnc + } + dataset + } + def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame = sparkSession.withActive { val qe = sparkSession.sessionState.executePlan(logicalPlan) @@ -241,8 +258,13 @@ class Dataset[T] private[sql]( this(queryExecution, () => encoder) } + def this( + sparkSession: SparkSession, logicalPlan: LogicalPlan, encoderGenerator: () => Encoder[T]) = { + this(sparkSession.sessionState.executePlan(logicalPlan), encoderGenerator) + } + def this(sparkSession: SparkSession, logicalPlan: LogicalPlan, encoder: Encoder[T]) = { - this(sparkSession.sessionState.executePlan(logicalPlan), encoder) + this(sparkSession, logicalPlan, () => encoder) } def this(sqlContext: SQLContext, logicalPlan: LogicalPlan, encoder: Encoder[T]) = { @@ -508,16 +530,8 @@ class Dataset[T] private[sql]( /** @inheritdoc */ @scala.annotation.varargs - def toDF(colNames: String*): DataFrame = { - require(schema.size == colNames.size, - "The number of columns doesn't match.\n" + - s"Old column names (${schema.size}): " + schema.fields.map(_.name).mkString(", ") + "\n" + - s"New column names (${colNames.size}): " + colNames.mkString(", ")) - - val newCols = logicalPlan.output.zip(colNames).map { case (oldAttribute, newName) => - Column(oldAttribute).as(newName) - } - select(newCols : _*) + def toDF(colNames: String*): DataFrame = withPlan { + UnresolvedSubqueryColumnAliases(colNames, logicalPlan) } /** @inheritdoc */ @@ -854,7 +868,7 @@ class Dataset[T] private[sql]( } /** @inheritdoc */ - def as(alias: String): Dataset[T] = withTypedPlan { + def as(alias: String): Dataset[T] = withSameTypedPlan { SubqueryAlias(alias, logicalPlan) } @@ -909,7 +923,7 @@ class Dataset[T] private[sql]( } /** @inheritdoc */ - def filter(condition: Column): Dataset[T] = withTypedPlan { + def filter(condition: Column): Dataset[T] = withSameTypedPlan { Filter(condition.expr, logicalPlan) } @@ -1026,6 +1040,29 @@ class Dataset[T] private[sql]( ) } + /** + * Converts the DataFrame into a `TableArg` object, which can be used as a table argument + * in a user-defined table function (UDTF). + * + * After obtaining a `TableArg` from a DataFrame using this method, you can specify + * partitioning and ordering for the table argument by calling methods such as `partitionBy`, + * `orderBy`, and `withSinglePartition` on the `TableArg` instance. + * - partitionBy(*cols): Partitions the data based on the specified columns. + * This method cannot be called after withSinglePartition() has been called. + * - orderBy(*cols): Orders the data within partitions based on the specified columns. + * - withSinglePartition(): Indicates that the data should be treated as a single partition. + * This method cannot be called after partitionBy() has been called. + * + * @group untypedrel + * @since 4.0.0 + */ + def asTable(): TableArg = { + new TableArg( + FunctionTableSubqueryArgumentExpression(plan = logicalPlan), + sparkSession + ) + } + /** @inheritdoc */ def scalar(): Column = { Column(ExpressionColumnNode(ScalarSubqueryExpr(logicalPlan))) @@ -1038,7 +1075,7 @@ class Dataset[T] private[sql]( /** @inheritdoc */ @scala.annotation.varargs - def observe(name: String, expr: Column, exprs: Column*): Dataset[T] = withTypedPlan { + def observe(name: String, expr: Column, exprs: Column*): Dataset[T] = withSameTypedPlan { CollectMetrics(name, (expr +: exprs).map(_.named), logicalPlan, id) } @@ -1050,12 +1087,12 @@ class Dataset[T] private[sql]( } /** @inheritdoc */ - def limit(n: Int): Dataset[T] = withTypedPlan { + def limit(n: Int): Dataset[T] = withSameTypedPlan { Limit(Literal(n), logicalPlan) } /** @inheritdoc */ - def offset(n: Int): Dataset[T] = withTypedPlan { + def offset(n: Int): Dataset[T] = withSameTypedPlan { Offset(Literal(n), logicalPlan) } @@ -1142,7 +1179,7 @@ class Dataset[T] private[sql]( /** @inheritdoc */ def sample(withReplacement: Boolean, fraction: Double, seed: Long): Dataset[T] = { - withTypedPlan { + withSameTypedPlan { Sample(0.0, fraction, withReplacement, seed, logicalPlan) } } @@ -1340,7 +1377,7 @@ class Dataset[T] private[sql]( def dropDuplicates(): Dataset[T] = dropDuplicates(this.columns) /** @inheritdoc */ - def dropDuplicates(colNames: Seq[String]): Dataset[T] = withTypedPlan { + def dropDuplicates(colNames: Seq[String]): Dataset[T] = withSameTypedPlan { val groupCols = groupColsFromDropDuplicates(colNames) Deduplicate(groupCols, logicalPlan) } @@ -1351,7 +1388,7 @@ class Dataset[T] private[sql]( } /** @inheritdoc */ - def dropDuplicatesWithinWatermark(colNames: Seq[String]): Dataset[T] = withTypedPlan { + def dropDuplicatesWithinWatermark(colNames: Seq[String]): Dataset[T] = withSameTypedPlan { val groupCols = groupColsFromDropDuplicates(colNames) // UnsupportedOperationChecker will fail the query if this is called with batch Dataset. DeduplicateWithinWatermark(groupCols, logicalPlan) @@ -1511,7 +1548,7 @@ class Dataset[T] private[sql]( } /** @inheritdoc */ - def repartition(numPartitions: Int): Dataset[T] = withTypedPlan { + def repartition(numPartitions: Int): Dataset[T] = withSameTypedPlan { Repartition(numPartitions, shuffle = true, logicalPlan) } @@ -1526,7 +1563,7 @@ class Dataset[T] private[sql]( s"""Invalid partitionExprs specified: $sortOrders |For range partitioning use repartitionByRange(...) instead. """.stripMargin) - withTypedPlan { + withSameTypedPlan { RepartitionByExpression(partitionExprs.map(_.expr), logicalPlan, numPartitions) } } @@ -1539,13 +1576,13 @@ class Dataset[T] private[sql]( case expr: SortOrder => expr case expr: Expression => SortOrder(expr, Ascending) }) - withTypedPlan { + withSameTypedPlan { RepartitionByExpression(sortOrder, logicalPlan, numPartitions) } } /** @inheritdoc */ - def coalesce(numPartitions: Int): Dataset[T] = withTypedPlan { + def coalesce(numPartitions: Int): Dataset[T] = withSameTypedPlan { Repartition(numPartitions, shuffle = false, logicalPlan) } @@ -2240,7 +2277,7 @@ class Dataset[T] private[sql]( SortOrder(expr, Ascending) } } - withTypedPlan { + withSameTypedPlan { Sort(sortOrder, global = global, logicalPlan) } } @@ -2255,6 +2292,11 @@ class Dataset[T] private[sql]( Dataset(sparkSession, logicalPlan) } + /** A convenient function to wrap a logical plan and produce a Dataset. */ + @inline private def withSameTypedPlan(logicalPlan: LogicalPlan): Dataset[T] = { + Dataset(sparkSession, logicalPlan, encoderGenerator) + } + /** A convenient function to wrap a set based logical plan and produce a Dataset. */ @inline private def withSetOperator[U : Encoder](logicalPlan: LogicalPlan): Dataset[U] = { if (isUnTyped) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala b/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala new file mode 100644 index 0000000000000..133775c0b666c --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/TableArg.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, FunctionTableSubqueryArgumentExpression, SortOrder} + +class TableArg( + private[sql] val expression: FunctionTableSubqueryArgumentExpression, + sparkSession: SparkSession) + extends TableValuedFunctionArgument { + import sparkSession.toRichColumn + + private def isPartitioned: Boolean = + expression.partitionByExpressions.nonEmpty || expression.withSinglePartition + + @scala.annotation.varargs + def partitionBy(cols: Column*): TableArg = { + if (isPartitioned) { + throw new IllegalArgumentException( + "Cannot call partitionBy() after partitionBy() or withSinglePartition() has been called." + ) + } + val partitionByExpressions = cols.map(_.expr) + new TableArg( + expression.copy( + partitionByExpressions = partitionByExpressions), + sparkSession) + } + + @scala.annotation.varargs + def orderBy(cols: Column*): TableArg = { + if (!isPartitioned) { + throw new IllegalArgumentException( + "Please call partitionBy() or withSinglePartition() before orderBy()." + ) + } + val orderByExpressions = cols.map { col => + col.expr match { + case sortOrder: SortOrder => sortOrder + case expr: Expression => SortOrder(expr, Ascending) + } + } + new TableArg( + expression.copy(orderByExpressions = orderByExpressions), + sparkSession) + } + + def withSinglePartition(): TableArg = { + if (isPartitioned) { + throw new IllegalArgumentException( + "Cannot call withSinglePartition() after partitionBy() or " + + "withSinglePartition() has been called." + ) + } + new TableArg( + expression.copy(withSinglePartition = true), + sparkSession) + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala index e33fe38b160af..49fe494903cdc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/api/python/PythonSQLUtils.scala @@ -148,6 +148,29 @@ private[sql] object PythonSQLUtils extends Logging { DataType.fromJson(json).asInstanceOf[StructType].toDDL } + def ddlToJson(ddl: String): String = { + val dataType = try { + // DDL format, "fieldname datatype, fieldname datatype". + StructType.fromDDL(ddl) + } catch { + case e: Throwable => + try { + // For backwards compatibility, "integer", "struct" and etc. + parseDataType(ddl) + } catch { + case _: Throwable => + try { + // For backwards compatibility, "fieldname: datatype, fieldname: datatype" case. + parseDataType(s"struct<${ddl.trim}>") + } catch { + case _: Throwable => + throw e + } + } + } + dataType.json + } + def unresolvedNamedLambdaVariable(name: String): Column = Column(internal.UnresolvedNamedLambdaVariable.apply(name)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala index 65f361cbae3f2..6394cef9fc760 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala @@ -17,7 +17,8 @@ package org.apache.spark.sql.artifact -import java.io.File +import java.io.{File, IOException} +import java.lang.ref.Cleaner import java.net.{URI, URL, URLClassLoader} import java.nio.ByteBuffer import java.nio.file.{CopyOption, Files, Path, Paths, StandardCopyOption} @@ -30,8 +31,8 @@ import scala.reflect.ClassTag import org.apache.commons.io.{FilenameUtils, FileUtils} import org.apache.hadoop.fs.{LocalFileSystem, Path => FSPath} -import org.apache.spark.{JobArtifactSet, JobArtifactState, SparkEnv, SparkException, SparkUnsupportedOperationException} -import org.apache.spark.internal.Logging +import org.apache.spark.{JobArtifactSet, JobArtifactState, SparkContext, SparkEnv, SparkException, SparkUnsupportedOperationException} +import org.apache.spark.internal.{Logging, LogKeys, MDC} import org.apache.spark.internal.config.{CONNECT_SCALA_UDF_STUB_PREFIXES, EXECUTOR_USER_CLASS_PATH_FIRST} import org.apache.spark.sql.{Artifact, SparkSession} import org.apache.spark.sql.internal.SQLConf @@ -51,7 +52,7 @@ import org.apache.spark.util.{ChildFirstURLClassLoader, StubClassLoader, Utils} * * @param session The object used to hold the Spark Connect session state. */ -class ArtifactManager(session: SparkSession) extends Logging { +class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging { import ArtifactManager._ // The base directory where all artifacts are stored. @@ -373,34 +374,28 @@ class ArtifactManager(session: SparkSession) extends Logging { newArtifactManager } + private val cleanUpStateForGlobalResources = ArtifactStateForCleanup( + session.sessionUUID, + session.sparkContext, + state, + artifactPath) + // Ensure that no reference to `this` is captured/help by the cleanup lambda + private def getCleanable: Cleaner.Cleanable = cleaner.register( + this, + () => ArtifactManager.cleanUpGlobalResources(cleanUpStateForGlobalResources) + ) + private var cleanable = getCleanable + /** * Cleans up all resources specific to this `session`. */ - private[sql] def cleanUpResources(): Unit = { + private def cleanUpResources(): Unit = { logDebug( s"Cleaning up resources for session with sessionUUID ${session.sessionUUID}") - // Clean up added files - val fileserver = SparkEnv.get.rpcEnv.fileServer - val sparkContext = session.sparkContext - if (state != null) { - val shouldUpdateEnv = sparkContext.addedFiles.contains(state.uuid) || - sparkContext.addedArchives.contains(state.uuid) || - sparkContext.addedJars.contains(state.uuid) - if (shouldUpdateEnv) { - sparkContext.addedFiles.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeFile)) - sparkContext.addedArchives.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeFile)) - sparkContext.addedJars.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeJar)) - sparkContext.postEnvironmentUpdate() - } - } - - // Clean up cached relations - val blockManager = sparkContext.env.blockManager - blockManager.removeCache(session.sessionUUID) - - // Clean up artifacts folder - FileUtils.deleteDirectory(artifactPath.toFile) + // Clean up global resources via the Cleaner process. + // Note that this will only be run once per instance. + cleanable.clean() // Clean up internal trackers jarsList.clear() @@ -412,6 +407,17 @@ class ArtifactManager(session: SparkSession) extends Logging { cachedClassLoader = None } + override def close(): Unit = { + cleanUpResources() + } + + private[sql] def cleanUpResourcesForTesting(): Unit = { + cleanUpResources() + // Tests reuse the same instance so we need to re-register the cleanable otherwise, it is run + // only once per instance. + cleanable = getCleanable + } + def uploadArtifactToFs( remoteRelativePath: Path, serverLocalStagingPath: Path): Unit = { @@ -481,4 +487,51 @@ object ArtifactManager extends Logging { throw SparkException.internalError(s"Block $fromId not found in the block manager.") } } + + // Shared cleaner instance + private val cleaner: Cleaner = Cleaner.create() + + /** + * Helper method to clean up global resources (i.e. resources associated with the calling + * instance but held externally in sparkContext, blockManager, disk etc.) + */ + private def cleanUpGlobalResources(cleanupState: ArtifactStateForCleanup): Unit = { + // Clean up added files + val (sparkSessionUUID, sparkContext, state, artifactPath) = ( + cleanupState.sparkSessionUUID, + cleanupState.sparkContext, + cleanupState.jobArtifactState, + cleanupState.artifactPath) + val fileServer = SparkEnv.get.rpcEnv.fileServer + if (state != null) { + val shouldUpdateEnv = sparkContext.addedFiles.contains(state.uuid) || + sparkContext.addedArchives.contains(state.uuid) || + sparkContext.addedJars.contains(state.uuid) + if (shouldUpdateEnv) { + sparkContext.addedFiles.remove(state.uuid).foreach(_.keys.foreach(fileServer.removeFile)) + sparkContext.addedArchives.remove(state.uuid).foreach(_.keys.foreach(fileServer.removeFile)) + sparkContext.addedJars.remove(state.uuid).foreach(_.keys.foreach(fileServer.removeJar)) + sparkContext.postEnvironmentUpdate() + } + } + + // Clean up cached relations + val blockManager = sparkContext.env.blockManager + blockManager.removeCache(sparkSessionUUID) + + // Clean up artifacts folder + try { + FileUtils.deleteDirectory(artifactPath.toFile) + } catch { + case e: IOException => + logWarning(log"Failed to delete directory ${MDC(LogKeys.PATH, artifactPath.toFile)}: " + + log"${MDC(LogKeys.EXCEPTION, e.getMessage)}", e) + } + } } + +private[artifact] case class ArtifactStateForCleanup( + sparkSessionUUID: String, + sparkContext: SparkContext, + jobArtifactState: JobArtifactState, + artifactPath: Path) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index fa28a2cb9ead6..6a388a7849f75 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -497,6 +497,27 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) case CreateFunction(ResolvedIdentifier(catalog, _), _, _, _, _) => throw QueryCompilationErrors.missingCatalogAbilityError(catalog, "CREATE FUNCTION") + + case c @ CreateUserDefinedFunction( + ResolvedIdentifierInSessionCatalog(ident), _, _, _, _, _, _, _, _, _, _, _) => + CreateUserDefinedFunctionCommand( + FunctionIdentifier(ident.table, ident.database, ident.catalog), + c.inputParamText, + c.returnTypeText, + c.exprText, + c.queryText, + c.comment, + c.isDeterministic, + c.containsSQL, + c.language, + c.isTableFunc, + isTemp = false, + c.ignoreIfExists, + c.replace) + + case CreateUserDefinedFunction( + ResolvedIdentifier(catalog, _), _, _, _, _, _, _, _, _, _, _, _) => + throw QueryCompilationErrors.missingCatalogAbilityError(catalog, "CREATE FUNCTION") } private def constructV1TableCmd( @@ -726,7 +747,7 @@ class ResolveSessionCatalog(val catalogManager: CatalogManager) private def supportsV1Command(catalog: CatalogPlugin): Boolean = { isSessionCatalog(catalog) && ( - SQLConf.get.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).isEmpty || + SQLConf.get.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION) == "builtin" || catalog.isInstanceOf[CatalogExtension]) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/SQLFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/SQLFunction.scala index 8ae0341e5646c..c0bd4ac80f5ed 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/SQLFunction.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/SQLFunction.scala @@ -17,9 +17,16 @@ package org.apache.spark.sql.catalyst.catalog +import scala.collection.mutable + +import org.json4s.JsonAST.{JArray, JString} +import org.json4s.jackson.JsonMethods.{compact, render} + import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.catalog.UserDefinedFunction._ +import org.apache.spark.sql.catalyst.expressions.{Expression, ScalarSubquery} import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation, Project} import org.apache.spark.sql.types.{DataType, StructType} /** @@ -56,10 +63,48 @@ case class SQLFunction( assert((isTableFunc && returnType.isRight) || (!isTableFunc && returnType.isLeft)) override val language: RoutineLanguage = LanguageSQL + + /** + * Optionally get the function body as an expression or query using the given parser. + */ + def getExpressionAndQuery( + parser: ParserInterface, + isTableFunc: Boolean): (Option[Expression], Option[LogicalPlan]) = { + // The RETURN clause of the CREATE FUNCTION statement looks like this in the parser: + // RETURN (query | expression) + // If the 'query' matches and parses as a SELECT clause of one item with no FROM clause, and + // this is a scalar function, we skip a level of subquery expression wrapping by using the + // referenced expression directly. + val parsedExpression = exprText.map(parser.parseExpression) + val parsedQuery = queryText.map(parser.parsePlan) + (parsedExpression, parsedQuery) match { + case (None, Some(Project(expr :: Nil, _: OneRowRelation))) + if !isTableFunc => + (Some(expr), None) + case (Some(ScalarSubquery(Project(expr :: Nil, _: OneRowRelation), _, _, _, _, _, _)), None) + if !isTableFunc => + (Some(expr), None) + case (_, _) => + (parsedExpression, parsedQuery) + } + } } object SQLFunction { + private val SQL_FUNCTION_PREFIX = "sqlFunction." + + private val FUNCTION_CATALOG_AND_NAMESPACE = "catalogAndNamespace.numParts" + private val FUNCTION_CATALOG_AND_NAMESPACE_PART_PREFIX = "catalogAndNamespace.part." + + private val FUNCTION_REFERRED_TEMP_VIEW_NAMES = "referredTempViewNames" + private val FUNCTION_REFERRED_TEMP_FUNCTION_NAMES = "referredTempFunctionsNames" + private val FUNCTION_REFERRED_TEMP_VARIABLE_NAMES = "referredTempVariableNames" + + def parseDefault(text: String, parser: ParserInterface): Expression = { + parser.parseExpression(text) + } + /** * This method returns an optional DataType indicating, when present, either the return type for * scalar user-defined functions, or a StructType indicating the names and types of the columns in @@ -92,4 +137,43 @@ object SQLFunction { } } } + + def isSQLFunction(className: String): Boolean = className == SQL_FUNCTION_PREFIX + + /** + * Convert the current catalog and namespace to properties. + */ + def catalogAndNamespaceToProps( + currentCatalog: String, + currentNamespace: Seq[String]): Map[String, String] = { + val props = new mutable.HashMap[String, String] + val parts = currentCatalog +: currentNamespace + if (parts.nonEmpty) { + props.put(FUNCTION_CATALOG_AND_NAMESPACE, parts.length.toString) + parts.zipWithIndex.foreach { case (name, index) => + props.put(s"$FUNCTION_CATALOG_AND_NAMESPACE_PART_PREFIX$index", name) + } + } + props.toMap + } + + /** + * Convert the temporary object names to properties. + */ + def referredTempNamesToProps( + viewNames: Seq[Seq[String]], + functionsNames: Seq[String], + variableNames: Seq[Seq[String]]): Map[String, String] = { + val viewNamesJson = + JArray(viewNames.map(nameParts => JArray(nameParts.map(JString).toList)).toList) + val functionsNamesJson = JArray(functionsNames.map(JString).toList) + val variableNamesJson = + JArray(variableNames.map(nameParts => JArray(nameParts.map(JString).toList)).toList) + + val props = new mutable.HashMap[String, String] + props.put(FUNCTION_REFERRED_TEMP_VIEW_NAMES, compact(render(viewNamesJson))) + props.put(FUNCTION_REFERRED_TEMP_FUNCTION_NAMES, compact(render(functionsNamesJson))) + props.put(FUNCTION_REFERRED_TEMP_VARIABLE_NAMES, compact(render(variableNamesJson))) + props.toMap + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunction.scala index 1473f19cb71bd..6567062841de5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunction.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/catalog/UserDefinedFunction.scala @@ -56,6 +56,8 @@ trait UserDefinedFunction { } object UserDefinedFunction { + val SQL_CONFIG_PREFIX = "sqlConfig." + def parseTableSchema(text: String, parser: ParserInterface): StructType = { val parsed = parser.parseTableSchema(text) CharVarcharUtils.failIfHasCharVarchar(parsed).asInstanceOf[StructType] diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SQLFunctionNode.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SQLFunctionNode.scala new file mode 100644 index 0000000000000..0a3274af33b5b --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SQLFunctionNode.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.plans.logical + +import org.apache.spark.sql.catalyst.catalog.SQLFunction +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.trees.TreePattern.FUNCTION_TABLE_RELATION_ARGUMENT_EXPRESSION +import org.apache.spark.sql.errors.DataTypeErrors.toSQLId +import org.apache.spark.sql.errors.QueryCompilationErrors + +/** + * A container for holding a SQL function query plan and its function identifier. + * + * @param function: the SQL function that this node represents. + * @param child: the SQL function body. + */ +case class SQLFunctionNode( + function: SQLFunction, + child: LogicalPlan) extends UnaryNode { + override def output: Seq[Attribute] = child.output + override def stringArgs: Iterator[Any] = Iterator(function.name, child) + override protected def withNewChildInternal(newChild: LogicalPlan): SQLFunctionNode = + copy(child = newChild) + + // Throw a reasonable error message when trying to call a SQL UDF with TABLE argument(s). + if (child.containsPattern(FUNCTION_TABLE_RELATION_ARGUMENT_EXPRESSION)) { + throw QueryCompilationErrors + .tableValuedArgumentsNotYetImplementedForSqlFunctions("call", toSQLId(function.name.funcName)) + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala index b0ce2bb4293e1..23ae5ee7b9be4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/util/V2ExpressionBuilder.scala @@ -305,6 +305,8 @@ class V2ExpressionBuilder(e: Expression, isPredicate: Boolean = false) extends L case _: Md5 => generateExpressionWithName("MD5", expr, isPredicate) case _: Sha1 => generateExpressionWithName("SHA1", expr, isPredicate) case _: Sha2 => generateExpressionWithName("SHA2", expr, isPredicate) + case _: StringLPad => generateExpressionWithName("LPAD", expr, isPredicate) + case _: StringRPad => generateExpressionWithName("RPAD", expr, isPredicate) // TODO supports other expressions case ApplyFunctionExpression(function, children) => val childrenExpressions = children.flatMap(generateExpression(_)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/InsertSortForLimitAndOffset.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/InsertSortForLimitAndOffset.scala index 6c7a9206a8e39..fa63e04d91b00 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/InsertSortForLimitAndOffset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/InsertSortForLimitAndOffset.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.execution +import scala.annotation.tailrec + import org.apache.spark.sql.catalyst.expressions.SortOrder import org.apache.spark.sql.catalyst.plans.physical.SinglePartition import org.apache.spark.sql.catalyst.rules.Rule @@ -49,6 +51,7 @@ object InsertSortForLimitAndOffset extends Rule[SparkPlan] { } object SinglePartitionShuffleWithGlobalOrdering { + @tailrec def unapply(plan: SparkPlan): Option[Seq[SortOrder]] = plan match { case ShuffleExchangeExec(SinglePartition, SparkPlanWithGlobalOrdering(ordering), _, _) => Some(ordering) @@ -61,6 +64,7 @@ object InsertSortForLimitAndOffset extends Rule[SparkPlan] { // Note: this is not implementing a generalized notion of "global order preservation", but just // tackles the regular ORDER BY semantics with optional LIMIT (top-K). object SparkPlanWithGlobalOrdering { + @tailrec def unapply(plan: SparkPlan): Option[Seq[SortOrder]] = plan match { case p: SortExec if p.global => Some(p.sortOrder) case p: LocalLimitExec => unapply(p.child) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 8d5ddb2d85c4e..744ab03d5d037 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -723,8 +723,19 @@ class SparkSqlAstBuilder extends AstBuilder { withIdentClause(ctx.identifierReference(), functionIdentifier => { if (ctx.TEMPORARY == null) { - // TODO: support creating persistent UDFs. - operationNotAllowed(s"creating persistent SQL functions is not supported", ctx) + CreateUserDefinedFunction( + UnresolvedIdentifier(functionIdentifier), + inputParamText, + returnTypeText, + exprText, + queryText, + comment, + deterministic, + containsSQL, + language, + isTableFunc, + ctx.EXISTS != null, + ctx.REPLACE != null) } else { // Disallow to define a temporary function with `IF NOT EXISTS` if (ctx.EXISTS != null) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala index d2aaa93fcca06..25598a12af222 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionCommand.scala @@ -17,9 +17,19 @@ package org.apache.spark.sql.execution.command -import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.SparkException +import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.FunctionIdentifier -import org.apache.spark.sql.catalyst.catalog.SQLFunction +import org.apache.spark.sql.catalyst.analysis.{Analyzer, UnresolvedAlias, UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation} +import org.apache.spark.sql.catalyst.catalog.{SessionCatalog, SQLFunction, UserDefinedFunctionErrors} +import org.apache.spark.sql.catalyst.expressions.{Alias, Cast, Generator, LateralSubquery, Literal, ScalarSubquery, SubqueryExpression, WindowExpression} +import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression +import org.apache.spark.sql.catalyst.plans.Inner +import org.apache.spark.sql.catalyst.plans.logical.{LateralJoin, LogicalPlan, OneRowRelation, Project, SQLFunctionNode, UnresolvedWith} +import org.apache.spark.sql.catalyst.trees.TreePattern.UNRESOLVED_ATTRIBUTE +import org.apache.spark.sql.errors.QueryCompilationErrors +import org.apache.spark.sql.execution.command.CreateUserDefinedFunctionCommand._ +import org.apache.spark.sql.types.{DataType, StructField, StructType} /** * The DDL command that creates a SQL function. @@ -52,10 +62,13 @@ case class CreateSQLFunctionCommand( replace: Boolean) extends CreateUserDefinedFunctionCommand { - override def run(sparkSession: SparkSession): Seq[Row] = { - import SQLFunction._ + import SQLFunction._ + override def run(sparkSession: SparkSession): Seq[Row] = { val parser = sparkSession.sessionState.sqlParser + val analyzer = sparkSession.sessionState.analyzer + val catalog = sparkSession.sessionState.catalog + val conf = sparkSession.sessionState.conf val inputParam = inputParamText.map(parser.parseTableSchema) val returnType = parseReturnTypeText(returnTypeText, isTableFunc, parser) @@ -72,8 +85,313 @@ case class CreateSQLFunctionCommand( isTableFunc, Map.empty) - // TODO: Implement the rest of the method. + val newFunction = { + val (expression, query) = function.getExpressionAndQuery(parser, isTableFunc) + assert(query.nonEmpty || expression.nonEmpty) + + // Check if the function can be replaced. + if (replace && catalog.functionExists(name)) { + checkFunctionSignatures(catalog, name) + } + + // Build function input. + val inputPlan = if (inputParam.isDefined) { + val param = inputParam.get + checkParameterNotNull(param, inputParamText.get) + checkParameterNameDuplication(param, conf, name) + checkDefaultsTrailing(param, name) + + // Qualify the input parameters with the function name so that attributes referencing + // the function input parameters can be resolved correctly. + val qualifier = Seq(name.funcName) + val input = param.map(p => Alias( + { + val defaultExpr = p.getDefault() + if (defaultExpr.isEmpty) { + Literal.create(null, p.dataType) + } else { + val defaultPlan = parseDefault(defaultExpr.get, parser) + if (SubqueryExpression.hasSubquery(defaultPlan)) { + throw new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.NOT_A_VALID_DEFAULT_EXPRESSION", + messageParameters = + Map("functionName" -> name.funcName, "parameterName" -> p.name)) + } else if (defaultPlan.containsPattern(UNRESOLVED_ATTRIBUTE)) { + // TODO(SPARK-50698): use parsed expression instead of expression string. + defaultPlan.collect { + case a: UnresolvedAttribute => + throw QueryCompilationErrors.unresolvedAttributeError( + "UNRESOLVED_COLUMN", a.sql, Seq.empty, a.origin) + } + } + Cast(defaultPlan, p.dataType) + } + }, p.name)(qualifier = qualifier)) + Project(input, OneRowRelation()) + } else { + OneRowRelation() + } + + // Build the function body and check if the function body can be analyzed successfully. + val (unresolvedPlan, analyzedPlan, inferredReturnType) = if (!isTableFunc) { + // Build SQL scalar function plan. + val outputExpr = if (query.isDefined) ScalarSubquery(query.get) else expression.get + val plan: LogicalPlan = returnType.map { t => + val retType: DataType = t match { + case Left(t) => t + case _ => throw SparkException.internalError( + "Unexpected return type for a scalar SQL UDF.") + } + val outputCast = Seq(Alias(Cast(outputExpr, retType), name.funcName)()) + Project(outputCast, inputPlan) + }.getOrElse { + // If no explicit RETURNS clause is present, infer the result type from the function body. + val outputAlias = Seq(Alias(outputExpr, name.funcName)()) + Project(outputAlias, inputPlan) + } + + // Check the function body can be analyzed correctly. + val analyzed = analyzer.execute(plan) + val (resolved, resolvedReturnType) = analyzed match { + case p @ Project(expr :: Nil, _) if expr.resolved => + (p, Left(expr.dataType)) + case other => + (other, function.returnType) + } + + // Check if the SQL function body contains aggregate/window functions. + // This check needs to be performed before checkAnalysis to provide better error messages. + checkAggOrWindowOrGeneratorExpr(resolved) + + // Check if the SQL function body can be analyzed. + checkFunctionBodyAnalysis(analyzer, function, resolved) + + (plan, resolved, resolvedReturnType) + } else { + // Build SQL table function plan. + if (query.isEmpty) { + throw UserDefinedFunctionErrors.bodyIsNotAQueryForSqlTableUdf(name.funcName) + } + + // Construct a lateral join to analyze the function body. + val plan = LateralJoin(inputPlan, LateralSubquery(query.get), Inner, None) + val analyzed = analyzer.execute(plan) + val newPlan = analyzed match { + case Project(_, j: LateralJoin) => j + case j: LateralJoin => j + case _ => throw SparkException.internalError("Unexpected plan returned when " + + s"creating a SQL TVF: ${analyzed.getClass.getSimpleName}.") + } + val maybeResolved = newPlan.asInstanceOf[LateralJoin].right.plan + + // Check if the function body can be analyzed. + checkFunctionBodyAnalysis(analyzer, function, maybeResolved) + + // Get the function's return schema. + val returnParam: StructType = returnType.map { + case Right(t) => t + case Left(_) => throw SparkException.internalError( + "Unexpected return schema for a SQL table function.") + }.getOrElse { + // If no explicit RETURNS clause is present, infer the result type from the function body. + // To detect this, we search for instances of the UnresolvedAlias expression. Examples: + // CREATE TABLE t USING PARQUET AS VALUES (0, 1), (1, 2) AS tab(c1, c2); + // SELECT c1 FROM t --> UnresolvedAttribute: 'c1 + // SELECT c1 + 1 FROM t --> UnresolvedAlias: unresolvedalias(('c1 + 1), None) + // SELECT c1 + 1 AS a FROM t --> Alias: ('c1 + 1) AS a#2 + query.get match { + case Project(projectList, _) if projectList.exists(_.isInstanceOf[UnresolvedAlias]) => + throw UserDefinedFunctionErrors.missingColumnNamesForSqlTableUdf(name.funcName) + case _ => + StructType(analyzed.asInstanceOf[LateralJoin].right.plan.output.map { col => + StructField(col.name, col.dataType) + }) + } + } + + // Check the return columns cannot have NOT NULL specified. + checkParameterNotNull(returnParam, returnTypeText) + + // Check duplicated return column names. + checkReturnsColumnDuplication(returnParam, conf, name) + + // Check if the actual output size equals to the number of return parameters. + val outputSize = maybeResolved.output.size + if (outputSize != returnParam.size) { + throw new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.RETURN_COLUMN_COUNT_MISMATCH", + messageParameters = Map( + "outputSize" -> s"$outputSize", + "returnParamSize" -> s"${returnParam.size}", + "name" -> s"$name" + ) + ) + } + + (plan, analyzed, Right(returnParam)) + } + + // A permanent function is not allowed to reference temporary objects. + // This should be called after `qe.assertAnalyzed()` (i.e., `plan` can be resolved) + verifyTemporaryObjectsNotExists(catalog, isTemp, name, unresolvedPlan, analyzedPlan) + + // Generate function properties. + val properties = generateFunctionProperties(sparkSession, unresolvedPlan, analyzedPlan) + + // Derive determinism of the SQL function. + val deterministic = analyzedPlan.deterministic + + function.copy( + // Assign the return type, inferring from the function body if needed. + returnType = inferredReturnType, + deterministic = Some(function.deterministic.getOrElse(deterministic)), + properties = properties + ) + } + + // TODO: create/register sql functions in catalog Seq.empty } + + /** + * Check if the function body can be analyzed. + */ + private def checkFunctionBodyAnalysis( + analyzer: Analyzer, + function: SQLFunction, + body: LogicalPlan): Unit = { + analyzer.checkAnalysis(SQLFunctionNode(function, body)) + } + + /** Check whether the new function is replacing an existing SQL function. */ + private def checkFunctionSignatures(catalog: SessionCatalog, name: FunctionIdentifier): Unit = { + val info = catalog.lookupFunctionInfo(name) + if (!isSQLFunction(info.getClassName)) { + throw new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.CANNOT_REPLACE_NON_SQL_UDF_WITH_SQL_UDF", + messageParameters = Map("name" -> s"$name") + ) + } + } + + /** + * Collect all temporary views and functions and return the identifiers separately + * This func traverses the unresolved plan `child`. Below are the reasons: + * 1) Analyzer replaces unresolved temporary views by a SubqueryAlias with the corresponding + * logical plan. After replacement, it is impossible to detect whether the SubqueryAlias is + * added/generated from a temporary view. + * 2) The temp functions are represented by multiple classes. Most are inaccessible from this + * package (e.g., HiveGenericUDF). + * 3) Temporary SQL functions, once resolved, cannot be identified as temp functions. + */ + private def collectTemporaryObjectsInUnresolvedPlan( + catalog: SessionCatalog, + child: LogicalPlan): (Seq[Seq[String]], Seq[String]) = { + import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ + def collectTempViews(child: LogicalPlan): Seq[Seq[String]] = { + child.flatMap { + case UnresolvedRelation(nameParts, _, _) if catalog.isTempView(nameParts) => + Seq(nameParts) + case w: UnresolvedWith if !w.resolved => w.innerChildren.flatMap(collectTempViews) + case plan if !plan.resolved => plan.expressions.flatMap(_.flatMap { + case e: SubqueryExpression => collectTempViews(e.plan) + case _ => Seq.empty + }) + case _ => Seq.empty + }.distinct + } + + def collectTempFunctions(child: LogicalPlan): Seq[String] = { + child.flatMap { + case w: UnresolvedWith if !w.resolved => w.innerChildren.flatMap(collectTempFunctions) + case plan if !plan.resolved => + plan.expressions.flatMap(_.flatMap { + case e: SubqueryExpression => collectTempFunctions(e.plan) + case e: UnresolvedFunction + if catalog.isTemporaryFunction(e.nameParts.asFunctionIdentifier) => + Seq(e.nameParts.asFunctionIdentifier.funcName) + case _ => Seq.empty + }) + case _ => Seq.empty + }.distinct + } + (collectTempViews(child), collectTempFunctions(child)) + } + + /** + * Permanent functions are not allowed to reference temp objects, including temp functions + * and temp views. + */ + private def verifyTemporaryObjectsNotExists( + catalog: SessionCatalog, + isTemporary: Boolean, + name: FunctionIdentifier, + child: LogicalPlan, + analyzed: LogicalPlan): Unit = { + import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ + if (!isTemporary) { + val (tempViews, tempFunctions) = collectTemporaryObjectsInUnresolvedPlan(catalog, child) + tempViews.foreach { nameParts => + throw UserDefinedFunctionErrors.invalidTempViewReference( + routineName = name.asMultipart, tempViewName = nameParts) + } + tempFunctions.foreach { funcName => + throw UserDefinedFunctionErrors.invalidTempFuncReference( + routineName = name.asMultipart, tempFuncName = funcName) + } + val tempVars = ViewHelper.collectTemporaryVariables(analyzed) + tempVars.foreach { varName => + throw UserDefinedFunctionErrors.invalidTempVarReference( + routineName = name.asMultipart, varName = varName) + } + } + } + + /** + * Check if the SQL function body contains aggregate/window/generate functions. + * Note subqueries inside the SQL function body can contain aggregate/window/generate functions. + */ + private def checkAggOrWindowOrGeneratorExpr(plan: LogicalPlan): Unit = { + if (plan.resolved) { + plan.transformAllExpressions { + case e if e.isInstanceOf[WindowExpression] || e.isInstanceOf[Generator] || + e.isInstanceOf[AggregateExpression] => + throw new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.CANNOT_CONTAIN_COMPLEX_FUNCTIONS", + messageParameters = Map("queryText" -> s"${exprText.orElse(queryText).get}") + ) + } + } + } + + /** + * Generate the function properties, including: + * 1. the SQL configs when creating the function. + * 2. the catalog and database name when creating the function. This will be used to provide + * context during nested function resolution. + * 3. referred temporary object names if the function is a temp function. + */ + private def generateFunctionProperties( + session: SparkSession, + plan: LogicalPlan, + analyzed: LogicalPlan): Map[String, String] = { + val catalog = session.sessionState.catalog + val conf = session.sessionState.conf + val manager = session.sessionState.catalogManager + + // Only collect temporary object names when the function is a temp function. + val (tempViews, tempFunctions) = if (isTemp) { + collectTemporaryObjectsInUnresolvedPlan(catalog, plan) + } else { + (Nil, Nil) + } + val tempVars = ViewHelper.collectTemporaryVariables(analyzed) + + sqlConfigsToProps(conf) ++ + catalogAndNamespaceToProps( + manager.currentCatalog.name, + manager.currentNamespace.toIndexedSeq) ++ + referredTempNamesToProps(tempViews, tempFunctions, tempVars) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala index bebb0f5cf6c38..1ee3c8a4c388f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/CreateUserDefinedFunctionCommand.scala @@ -17,9 +17,15 @@ package org.apache.spark.sql.execution.command +import java.util.Locale + +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.catalog.{LanguageSQL, RoutineLanguage, UserDefinedFunctionErrors} +import org.apache.spark.sql.catalyst.catalog.UserDefinedFunction._ import org.apache.spark.sql.catalyst.plans.logical.IgnoreCachedData +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StructType /** * The base class for CreateUserDefinedFunctionCommand @@ -74,4 +80,108 @@ object CreateUserDefinedFunctionCommand { throw UserDefinedFunctionErrors.unsupportedUserDefinedFunction(other) } } + + /** + * Convert SQL configs to properties by prefixing all configs with a key. + * When converting a function to [[org.apache.spark.sql.catalyst.catalog.CatalogFunction]] or + * [[org.apache.spark.sql.catalyst.expressions.ExpressionInfo]], all SQL configs and other + * function properties (such as the function parameters and the function return type) + * are saved together in a property map. + */ + def sqlConfigsToProps(conf: SQLConf): Map[String, String] = { + val modifiedConfs = ViewHelper.getModifiedConf(conf) + modifiedConfs.map { case (key, value) => s"$SQL_CONFIG_PREFIX$key" -> value } + } + + /** + * Check whether the function parameters contain duplicated column names. + * It takes the function input parameter struct as input and verifies that there is no duplicates + * in the parameter column names. + * If any duplicates are found, it throws an exception with helpful information for users to + * fix the wrong function parameters. + * + * Perform this check while registering the function to fail early. + * This check does not need to run the function itself. + */ + def checkParameterNameDuplication( + param: StructType, + conf: SQLConf, + name: FunctionIdentifier): Unit = { + val names = if (conf.caseSensitiveAnalysis) { + param.fields.map(_.name) + } else { + param.fields.map(_.name.toLowerCase(Locale.ROOT)) + } + if (names.distinct.length != names.length) { + val duplicateColumns = names.groupBy(identity).collect { + case (x, ys) if ys.length > 1 => s"`$x`" + } + throw UserDefinedFunctionErrors.duplicateParameterNames( + routineName = name.funcName, + names = duplicateColumns.toSeq.sorted.mkString(", ")) + } + } + + /** + * Check whether the function has duplicate column names in the RETURNS clause. + */ + def checkReturnsColumnDuplication( + columns: StructType, + conf: SQLConf, + name: FunctionIdentifier): Unit = { + val names = if (conf.caseSensitiveAnalysis) { + columns.fields.map(_.name) + } else { + columns.fields.map(_.name.toLowerCase(Locale.ROOT)) + } + if (names.distinct.length != names.length) { + val duplicateColumns = names.groupBy(identity).collect { + case (x, ys) if ys.length > 1 => s"`$x`" + } + throw UserDefinedFunctionErrors.duplicateReturnsColumns( + routineName = name.funcName, + columns = duplicateColumns.toSeq.sorted.mkString(", ")) + } + } + + /** + * Check whether the function parameters contain non trailing defaults. + * For languages that support default values for input parameters, + * this check ensures once a default value is given to a parameter, + * all subsequent parameters must also have a default value. It throws error if otherwise. + * + * Perform this check on function input parameters while registering the function to fail early. + * This check does not need to run the function itself. + */ + def checkDefaultsTrailing(param: StructType, name: FunctionIdentifier): Unit = { + var defaultFound = false + var previousParamName = ""; + param.fields.foreach { field => + if (field.getDefault().isEmpty && defaultFound) { + throw new AnalysisException( + errorClass = "USER_DEFINED_FUNCTIONS.NOT_A_VALID_DEFAULT_PARAMETER_POSITION", + messageParameters = Map( + "functionName" -> name.funcName, + "parameterName" -> previousParamName, + "nextParameterName" -> field.name)) + } + defaultFound |= field.getDefault().isDefined + previousParamName = field.name + } + } + + /** + * Check whether the function input or return columns (for TABLE Return type) have NOT NULL + * specified. Throw exception if NOT NULL is found. + * + * Perform this check on function input and return parameters while registering the function + * to fail early. This check does not need to run the function itself. + */ + def checkParameterNotNull(param: StructType, input: String): Unit = { + param.fields.foreach { field => + if (!field.nullable) { + throw UserDefinedFunctionErrors.cannotSpecifyNotNullOnFunctionParameters(input) + } + } + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala index d5a72fd6c441a..f654c846c8a57 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala @@ -464,12 +464,19 @@ object ViewHelper extends SQLConfHelper with Logging { } /** - * Convert the view SQL configs to `properties`. + * Get all configurations that are modifiable and should be captured. */ - private def sqlConfigsToProps(conf: SQLConf): Map[String, String] = { - val modifiedConfs = conf.getAllConfs.filter { case (k, _) => + def getModifiedConf(conf: SQLConf): Map[String, String] = { + conf.getAllConfs.filter { case (k, _) => conf.isModifiable(k) && shouldCaptureConfig(k) } + } + + /** + * Convert the view SQL configs to `properties`. + */ + private def sqlConfigsToProps(conf: SQLConf): Map[String, String] = { + val modifiedConfs = getModifiedConf(conf) // Some configs have dynamic default values, such as SESSION_LOCAL_TIMEZONE whose // default value relies on the JVM system timezone. We need to always capture them to // to make sure we apply the same configs when reading the view. @@ -690,7 +697,7 @@ object ViewHelper extends SQLConfHelper with Logging { /** * Collect all temporary SQL variables and return the identifiers separately. */ - private def collectTemporaryVariables(child: LogicalPlan): Seq[Seq[String]] = { + def collectTemporaryVariables(child: LogicalPlan): Seq[Seq[String]] = { def collectTempVars(child: LogicalPlan): Seq[Seq[String]] = { child.flatMap { plan => plan.expressions.flatMap(_.flatMap { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScan.scala index 83d219c28983b..33ba4f772a13a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScan.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PushVariantIntoScan.scala @@ -21,7 +21,7 @@ import scala.collection.mutable.HashMap import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.expressions.variant.{VariantGet, VariantPathParser} +import org.apache.spark.sql.catalyst.expressions.variant._ import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, Subquery} import org.apache.spark.sql.catalyst.rules.Rule @@ -54,7 +54,7 @@ case class VariantMetadata( .build() ).build() - def parsedPath(): Array[VariantPathParser.PathSegment] = { + def parsedPath(): Array[VariantPathSegment] = { VariantPathParser.parse(path).getOrElse { val name = if (failOnError) "variant_get" else "try_variant_get" throw QueryExecutionErrors.invalidVariantGetPath(path, name) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala index 64c2a3126ca9a..daeb8e88a924b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaConverter.scala @@ -756,6 +756,14 @@ class SparkToParquetSchemaConverter( .addField(convertField(StructField("metadata", BinaryType, nullable = false))) .named(field.name) + case s: StructType if SparkShreddingUtils.isVariantShreddingStruct(s) => + // Variant struct takes a Variant and writes to Parquet as a shredded schema. + val group = Types.buildGroup(repetition) + s.fields.foreach { f => + group.addField(convertField(f)) + } + group.named(field.name) + case StructType(fields) => fields.foldLeft(Types.buildGroup(repetition)) { (builder, field) => builder.addField(convertField(field)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala index 3e111252bc6fe..663182d8d1820 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetUtils.scala @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.datasources.{AggregatePushDownUtils, Outpu import org.apache.spark.sql.execution.datasources.v2.V2ColumnUtils import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} import org.apache.spark.sql.internal.SQLConf.PARQUET_AGGREGATE_PUSHDOWN_ENABLED -import org.apache.spark.sql.types.{ArrayType, AtomicType, DataType, MapType, StructField, StructType, UserDefinedType} +import org.apache.spark.sql.types.{ArrayType, AtomicType, DataType, MapType, StructField, StructType, UserDefinedType, VariantType} import org.apache.spark.util.ArrayImplicits._ object ParquetUtils extends Logging { @@ -420,6 +420,22 @@ object ParquetUtils extends Logging { statistics.getNumNulls; } + // Replaces each VariantType in the schema with the corresponding type in the shredding schema. + // Used for testing, where we force a single shredding schema for all Variant fields. + // Does not touch Variant fields nested in arrays, maps, or UDTs. + private def replaceVariantTypes(schema: StructType, shreddingSchema: StructType): StructType = { + val newFields = schema.fields.zip(shreddingSchema.fields).map { + case (field, shreddingField) => + field.dataType match { + case s: StructType => + field.copy(dataType = replaceVariantTypes(s, shreddingSchema)) + case VariantType => field.copy(dataType = shreddingSchema) + case _ => field + } + } + StructType(newFields) + } + def prepareWrite( sqlConf: SQLConf, job: Job, @@ -454,8 +470,23 @@ object ParquetUtils extends Logging { ParquetOutputFormat.setWriteSupportClass(job, classOf[ParquetWriteSupport]) + val shreddingSchema = if (sqlConf.getConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED) && + !sqlConf.getConf(SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST).isEmpty) { + // Convert the schema to a shredding schema, and replace it anywhere that there is a + // VariantType in the original schema. + val simpleShreddingSchema = DataType.fromDDL( + sqlConf.getConf(SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST) + ) + val oneShreddingSchema = SparkShreddingUtils.variantShreddingSchema(simpleShreddingSchema) + val schemaWithMetadata = SparkShreddingUtils.addWriteShreddingMetadata(oneShreddingSchema) + Some(replaceVariantTypes(dataSchema, schemaWithMetadata)) + } else { + None + } + // This metadata is useful for keeping UDTs like Vector/Matrix. ParquetWriteSupport.setSchema(dataSchema, conf) + shreddingSchema.foreach(ParquetWriteSupport.setShreddingSchema(_, conf)) // Sets flags for `ParquetWriteSupport`, which converts Catalyst schema to Parquet // schema and writes actual rows to Parquet files. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala index 89a1cd5d4375a..35eb57a2e4fb2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.execution.datasources.DataSourceUtils import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf} import org.apache.spark.sql.types._ +import org.apache.spark.types.variant.Variant /** * A Parquet [[WriteSupport]] implementation that writes Catalyst [[InternalRow]]s as Parquet @@ -59,6 +60,10 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { // Schema of the `InternalRow`s to be written private var schema: StructType = _ + // Schema of the `InternalRow`s to be written, with VariantType replaced with its shredding + // schema, if appropriate. + private var shreddedSchema: StructType = _ + // `ValueWriter`s for all fields of the schema private var rootFieldWriters: Array[ValueWriter] = _ @@ -95,7 +100,16 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { override def init(configuration: Configuration): WriteContext = { val schemaString = configuration.get(ParquetWriteSupport.SPARK_ROW_SCHEMA) + val shreddedSchemaString = configuration.get(ParquetWriteSupport.SPARK_VARIANT_SHREDDING_SCHEMA) this.schema = StructType.fromString(schemaString) + // If shreddingSchemaString is provided, we use that everywhere in the writer, except for + // setting the spark schema in the Parquet metadata. If it isn't provided, it means that there + // are no shredded Variant columns, so it is identical to this.schema. + this.shreddedSchema = if (shreddedSchemaString == null) { + this.schema + } else { + StructType.fromString(shreddedSchemaString) + } this.writeLegacyParquetFormat = { // `SQLConf.PARQUET_WRITE_LEGACY_FORMAT` should always be explicitly set in ParquetRelation assert(configuration.get(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key) != null) @@ -108,9 +122,9 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { SQLConf.ParquetOutputTimestampType.withName(configuration.get(key)) } - this.rootFieldWriters = schema.map(_.dataType).map(makeWriter).toArray[ValueWriter] + this.rootFieldWriters = shreddedSchema.map(_.dataType).map(makeWriter).toArray[ValueWriter] - val messageType = new SparkToParquetSchemaConverter(configuration).convert(schema) + val messageType = new SparkToParquetSchemaConverter(configuration).convert(shreddedSchema) val metadata = Map( SPARK_VERSION_METADATA_KEY -> SPARK_VERSION_SHORT, ParquetReadSupport.SPARK_METADATA_KEY -> schemaString @@ -132,13 +146,23 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { } } - logDebug( - s"""Initialized Parquet WriteSupport with Catalyst schema: - |${schema.prettyJson} - |and corresponding Parquet message type: - |$messageType - """.stripMargin) - + if (shreddedSchemaString == null) { + logDebug( + s"""Initialized Parquet WriteSupport with Catalyst schema: + |${schema.prettyJson} + |and corresponding Parquet message type: + |$messageType + """.stripMargin) + } else { + logDebug( + s"""Initialized Parquet WriteSupport with Catalyst schema: + |${schema.prettyJson} + |and shredding schema: + |${shreddedSchema.prettyJson} + |and corresponding Parquet message type: + |$messageType + """.stripMargin) + } new WriteContext(messageType, metadata.asJava) } @@ -148,7 +172,7 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { override def write(row: InternalRow): Unit = { consumeMessage { - writeFields(row, schema, rootFieldWriters) + writeFields(row, shreddedSchema, rootFieldWriters) } } @@ -250,6 +274,17 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { } } + case s: StructType if SparkShreddingUtils.isVariantShreddingStruct(s) => + val fieldWriters = s.map(_.dataType).map(makeWriter).toArray[ValueWriter] + val variantShreddingSchema = SparkShreddingUtils.buildVariantSchema(s) + (row: SpecializedGetters, ordinal: Int) => + val v = row.getVariant(ordinal) + val variant = new Variant(v.getValue, v.getMetadata) + val shreddedValues = SparkShreddingUtils.castShredded(variant, variantShreddingSchema) + consumeGroup { + writeFields(shreddedValues, s, fieldWriters) + } + case t: StructType => val fieldWriters = t.map(_.dataType).map(makeWriter).toArray[ValueWriter] (row: SpecializedGetters, ordinal: Int) => @@ -499,6 +534,10 @@ class ParquetWriteSupport extends WriteSupport[InternalRow] with Logging { object ParquetWriteSupport { val SPARK_ROW_SCHEMA: String = "org.apache.spark.sql.parquet.row.attributes" + // A version of `SPARK_ROW_SCHEMA`, where one or more Variant attributes have been replace with a + // shredded struct schema. + val SPARK_VARIANT_SHREDDING_SCHEMA: String = + "org.apache.spark.sql.parquet.variant.shredding.attributes" def setSchema(schema: StructType, configuration: Configuration): Unit = { configuration.set(SPARK_ROW_SCHEMA, schema.json) @@ -506,4 +545,8 @@ object ParquetWriteSupport { ParquetOutputFormat.WRITER_VERSION, ParquetProperties.WriterVersion.PARQUET_1_0.toString) } + + def setShreddingSchema(shreddingSchema: StructType, configuration: Configuration): Unit = { + configuration.set(SPARK_VARIANT_SHREDDING_SCHEMA, shreddingSchema.json) + } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala index 34c167aea363a..ffb6704061e66 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala @@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.expressions.variant._ -import org.apache.spark.sql.catalyst.expressions.variant.VariantPathParser.PathSegment import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData} import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} import org.apache.spark.sql.execution.RowToColumnConverter @@ -56,9 +55,9 @@ case class SparkShreddedRow(row: SpecializedGetters) extends ShreddingUtils.Shre override def numElements(): Int = row.asInstanceOf[ArrayData].numElements() } -// The search result of a `PathSegment` in a `VariantSchema`. +// The search result of a `VariantPathSegment` in a `VariantSchema`. case class SchemaPathSegment( - rawPath: PathSegment, + rawPath: VariantPathSegment, // Whether this path segment is an object or array extraction. isObject: Boolean, // `schema.typedIdx`, if the path exists in the schema (for object extraction, the schema @@ -448,6 +447,8 @@ case object SparkShreddingUtils { val TypedValueFieldName = "typed_value"; val MetadataFieldName = "metadata"; + val VARIANT_WRITE_SHREDDING_KEY: String = "__VARIANT_WRITE_SHREDDING_KEY" + def buildVariantSchema(schema: DataType): VariantSchema = { schema match { case s: StructType => buildVariantSchema(s, topLevel = true) @@ -512,6 +513,27 @@ case object SparkShreddingUtils { } } + /** + * Given a schema that represents a valid shredding schema (e.g. constructed by + * SparkShreddingUtils.variantShreddingSchema), add metadata to the top-level fields to mark it + * as a shredding schema for writers. + */ + def addWriteShreddingMetadata(schema: StructType): StructType = { + val newFields = schema.fields.map { f => + f.copy(metadata = new + MetadataBuilder() + .withMetadata(f.metadata) + .putNull(VARIANT_WRITE_SHREDDING_KEY).build()) + } + StructType(newFields) + } + + // Check if the struct is marked with metadata set by addWriteShreddingMetadata - i.e. it + // represents a Variant converted to a shredding schema for writing. + def isVariantShreddingStruct(s: StructType): Boolean = { + s.fields.length > 0 && s.fields.forall(_.metadata.contains(VARIANT_WRITE_SHREDDING_KEY)) + } + /* * Given a Spark schema that represents a valid shredding schema (e.g. constructed by * SparkShreddingUtils.variantShreddingSchema), return the corresponding VariantSchema. @@ -691,11 +713,11 @@ case object SparkShreddingUtils { // found at a certain level of the file type, then `typedIdx` will be -1 starting from // this position, and the final `schema` will be null. for (i <- rawPath.indices) { - val isObject = rawPath(i).isLeft + val isObject = rawPath(i).isInstanceOf[ObjectExtraction] var typedIdx = -1 var extractionIdx = -1 rawPath(i) match { - case scala.util.Left(key) if schema != null && schema.objectSchema != null => + case ObjectExtraction(key) if schema != null && schema.objectSchema != null => val fieldIdx = schema.objectSchemaMap.get(key) if (fieldIdx != null) { typedIdx = schema.typedIdx @@ -704,7 +726,7 @@ case object SparkShreddingUtils { } else { schema = null } - case scala.util.Right(index) if schema != null && schema.arraySchema != null => + case ArrayExtraction(index) if schema != null && schema.arraySchema != null => typedIdx = schema.typedIdx extractionIdx = index schema = schema.arraySchema @@ -747,8 +769,8 @@ case object SparkShreddingUtils { var v = new Variant(row.getBinary(variantIdx), topLevelMetadata) while (pathIdx < pathLen) { v = pathList(pathIdx).rawPath match { - case scala.util.Left(key) if v.getType == Type.OBJECT => v.getFieldByKey(key) - case scala.util.Right(index) if v.getType == Type.ARRAY => v.getElementAtIndex(index) + case ObjectExtraction(key) if v.getType == Type.OBJECT => v.getFieldByKey(key) + case ArrayExtraction(index) if v.getType == Type.ARRAY => v.getElementAtIndex(index) case _ => null } if (v == null) return null diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala index e9927cdcc7a33..0a533645648e6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalog.scala @@ -83,7 +83,7 @@ class V2SessionCatalog(catalog: SessionCatalog) } private def hasCustomSessionCatalog: Boolean = { - catalog.conf.contains(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION.key) + catalog.conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION) != "builtin" } override def loadTable(ident: Identifier): Table = { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasPythonRunner.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasPythonRunner.scala index 79e199f598b0c..f415ae2543d34 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasPythonRunner.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasPythonRunner.scala @@ -275,7 +275,7 @@ class TransformWithStateInPandasPythonPreInitRunner( override def stop(): Unit = { super.stop() closeServerSocketChannelSilently(stateServerSocket) - daemonThread.stop() + daemonThread.interrupt() } private def startStateServer(): Unit = { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasStateServer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasStateServer.scala index a48d0bfd15034..e37e4266b46b8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasStateServer.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/TransformWithStateInPandasStateServer.scala @@ -159,6 +159,11 @@ class TransformWithStateInPandasStateServer( logWarning(log"No more data to read from the socket") statefulProcessorHandle.setHandleState(StatefulProcessorHandleState.CLOSED) return + case _: InterruptedException => + logInfo(log"Thread interrupted, shutting down state server") + Thread.currentThread().interrupt() + statefulProcessorHandle.setHandleState(StatefulProcessorHandleState.CLOSED) + return case e: Exception => logError(log"Error reading message: ${MDC(LogKeys.ERROR, e.getMessage)}", e) sendResponse(1, e.getMessage) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala index 388ede5d062e5..575e3d4072b8c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala @@ -24,7 +24,7 @@ import scala.collection.mutable.ArrayBuffer import net.razorvine.pickle.Pickler import org.apache.spark.api.python.{PythonEvalType, PythonFunction, PythonWorkerUtils, SpecialLengths} -import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} +import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession, TableArg, TableValuedFunctionArgument} import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Descending, Expression, FunctionTableSubqueryArgumentExpression, NamedArgumentExpression, NullsFirst, NullsLast, PythonUDAF, PythonUDF, PythonUDTF, PythonUDTFAnalyzeResult, PythonUDTFSelectedExpression, SortOrder, UnresolvedPolymorphicPythonUDTF} import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.plans.logical.{Generate, LogicalPlan, NamedParametersSupport, OneRowRelation} @@ -160,8 +160,16 @@ case class UserDefinedPythonTableFunction( } /** Returns a [[DataFrame]] that will evaluate to calling this UDTF with the given input. */ - def apply(session: SparkSession, exprs: Column*): DataFrame = { - val udtf = builder(exprs.map(session.expression), session.sessionState.sqlParser) + def apply(session: SparkSession, exprs: TableValuedFunctionArgument*): DataFrame = { + val parser = session.sessionState.sqlParser + val expressions = exprs.map { + case col: Column => session.expression(col) + case tableArg: TableArg => tableArg.expression + case other => throw new IllegalArgumentException( + s"Unsupported argument type: ${other.getClass.getName}" + ) + } + val udtf = builder(expressions, parser) Dataset.ofRows(session, udtf) } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/DataFrameWriterImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/DataFrameWriterImpl.scala index 0069062e63078..5a96db5e34bbd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/DataFrameWriterImpl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/DataFrameWriterImpl.scala @@ -383,6 +383,11 @@ final class DataFrameWriterImpl[T] private[sql](ds: Dataset[T]) extends DataFram } } + private def hasCustomSessionCatalog: Boolean = { + df.sparkSession.sessionState.conf + .getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION) != "builtin" + } + /** * Saves the content of the `DataFrame` as the specified table. * @@ -426,8 +431,7 @@ final class DataFrameWriterImpl[T] private[sql](ds: Dataset[T]) extends DataFram import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ val session = df.sparkSession - val canUseV2 = lookupV2Provider().isDefined || (df.sparkSession.sessionState.conf.getConf( - SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).isDefined && + val canUseV2 = lookupV2Provider().isDefined || (hasCustomSessionCatalog && !df.sparkSession.sessionState.catalogManager.catalog(CatalogManager.SESSION_CATALOG_NAME) .isInstanceOf[CatalogExtension]) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala index 2d50d37e2cb83..ee47491b803bb 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala @@ -790,6 +790,7 @@ class ForStatementExec( case ForState.VariableCleanup => dropVariablesExec.getTreeIterator.hasNext }) + @scala.annotation.tailrec override def next(): CompoundStatementExec = state match { case ForState.VariableAssignment => diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-command.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-command.sql.out index 926e3cd003046..c12076b85b1df 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-command.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-command.sql.out @@ -4,7 +4,7 @@ CREATE TABLE cte_tbl USING csv AS WITH s AS (SELECT 42 AS col) SELECT * FROM s -- !query analysis CreateDataSourceTableAsSelectCommand `spark_catalog`.`default`.`cte_tbl`, ErrorIfExists, [col] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias s : +- Project [42 AS col#x] : +- OneRowRelation @@ -26,7 +26,7 @@ CREATE TEMPORARY VIEW cte_view AS WITH s AS (SELECT 42 AS col) SELECT * FROM s -- !query analysis CreateViewCommand `cte_view`, WITH s AS (SELECT 42 AS col) SELECT * FROM s, false, false, LocalTempView, UNSUPPORTED, true +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias s : +- Project [42 AS col#x] : +- OneRowRelation @@ -43,7 +43,7 @@ Project [col#x] +- View (`cte_view`, [col#x]) +- Project [cast(col#x as int) AS col#x] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias s : +- Project [42 AS col#x] : +- OneRowRelation @@ -58,7 +58,7 @@ INSERT INTO cte_tbl SELECT * FROM S -- !query analysis InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/cte_tbl, false, CSV, [path=file:[not included in comparison]/{warehouse_dir}/cte_tbl], Append, `spark_catalog`.`default`.`cte_tbl`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/cte_tbl), [col] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias s : +- Project [43 AS col#x] : +- OneRowRelation @@ -80,7 +80,7 @@ INSERT INTO cte_tbl WITH s AS (SELECT 44 AS col) SELECT * FROM s -- !query analysis InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/cte_tbl, false, CSV, [path=file:[not included in comparison]/{warehouse_dir}/cte_tbl], Append, `spark_catalog`.`default`.`cte_tbl`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/cte_tbl), [col] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias s : +- Project [44 AS col#x] : +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nested.sql.out index 0365c4dcd8e09..1b968d0431f4a 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nested.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nested.sql.out @@ -7,11 +7,11 @@ WITH t as ( SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x] : +- SubqueryAlias t2 @@ -30,7 +30,7 @@ SELECT max(c) FROM ( Aggregate [max(c#x) AS max(c)#x] +- SubqueryAlias __auto_generated_subquery_name +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] @@ -48,7 +48,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [1 AS 1#x] : : +- OneRowRelation @@ -66,7 +66,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [id#xL] AS scalarsubquery(id)#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias unreferenced : : +- Project [outer(id#xL)] : : +- OneRowRelation @@ -83,7 +83,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [id#xL] AS scalarsubquery(id)#xL] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias unreferenced : : +- Project [1 AS 1#x] : : +- OneRowRelation @@ -128,15 +128,15 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [2#x] : +- SubqueryAlias t @@ -160,18 +160,18 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : : +- Aggregate [max(c#x) AS max(c)#x] : : +- SubqueryAlias __auto_generated_subquery_name : : +- WithCTE -: : :- CTERelationDef xxxx, false, false +: : :- CTERelationDef xxxx, false : : : +- SubqueryAlias t : : : +- Project [2#x AS c#x] : : : +- Project [2 AS 2#x] @@ -199,24 +199,24 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [3 AS 3#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [3#x] : +- SubqueryAlias t : +- CTERelationRef xxxx, true, [3#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [3#x] : +- SubqueryAlias t2 @@ -234,12 +234,12 @@ SELECT max(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] @@ -261,12 +261,12 @@ SELECT sum(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] @@ -291,17 +291,17 @@ SELECT sum(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [3#x AS c#x] : +- Project [3 AS 3#x] @@ -323,13 +323,13 @@ SELECT ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- OneRowRelation +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE - : :- CTERelationDef xxxx, false, false + : :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [2 AS 2#x] : : +- OneRowRelation @@ -349,14 +349,14 @@ SELECT ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- OneRowRelation +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : : +- WithCTE - : : :- CTERelationDef xxxx, false, false + : : :- CTERelationDef xxxx, false : : : +- SubqueryAlias t : : : +- Project [2 AS 2#x] : : : +- OneRowRelation @@ -378,19 +378,19 @@ SELECT ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- OneRowRelation +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE - : :- CTERelationDef xxxx, false, false + : :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [2 AS 2#x] : : +- OneRowRelation : +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : : +- WithCTE - : : :- CTERelationDef xxxx, false, false + : : :- CTERelationDef xxxx, false : : : +- SubqueryAlias t : : : +- Project [3 AS 3#x] : : : +- OneRowRelation @@ -410,7 +410,7 @@ WHERE c IN ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] @@ -418,7 +418,7 @@ WithCTE +- Project [c#x] +- Filter c#x IN (list#x []) : +- WithCTE - : :- CTERelationDef xxxx, false, false + : :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [2#x AS c#x] : : +- Project [2 AS 2#x] @@ -440,16 +440,16 @@ WITH SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x] : +- SubqueryAlias t2 : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [2 AS 2#x] : +- OneRowRelation @@ -468,15 +468,15 @@ WITH SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias abc : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias aBc : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x] : +- SubqueryAlias aBC @@ -494,13 +494,13 @@ SELECT ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias abc : +- Project [1 AS 1#x] : +- OneRowRelation +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE - : :- CTERelationDef xxxx, false, false + : :- CTERelationDef xxxx, false : : +- SubqueryAlias aBc : : +- Project [2 AS 2#x] : : +- OneRowRelation @@ -522,16 +522,16 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t1 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t3 : +- Project [1#x] : +- SubqueryAlias t1 : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1#x] : +- SubqueryAlias t3 @@ -553,11 +553,11 @@ SELECT * FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_outer : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner : +- Project [1#x] : +- SubqueryAlias cte_outer @@ -586,16 +586,16 @@ SELECT * FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_outer : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner_inner : +- Project [1#x] : +- SubqueryAlias cte_outer : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner : +- Project [1#x] : +- SubqueryAlias __auto_generated_subquery_name diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nonlegacy.sql.out index 53dcd46361dc1..fcfe64097069b 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nonlegacy.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/cte-nonlegacy.sql.out @@ -7,11 +7,11 @@ WITH t as ( SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x] : +- SubqueryAlias t2 @@ -30,7 +30,7 @@ SELECT max(c) FROM ( Aggregate [max(c#x) AS max(c)#x] +- SubqueryAlias __auto_generated_subquery_name +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] @@ -48,7 +48,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [1 AS 1#x] : : +- OneRowRelation @@ -66,7 +66,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [id#xL] AS scalarsubquery(id)#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias unreferenced : : +- Project [outer(id#xL)] : : +- OneRowRelation @@ -83,7 +83,7 @@ SELECT ( -- !query analysis Project [scalar-subquery#x [id#xL] AS scalarsubquery(id)#xL] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias unreferenced : : +- Project [1 AS 1#x] : : +- OneRowRelation @@ -153,18 +153,18 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [scalar-subquery#x [] AS scalarsubquery()#x] : : +- Aggregate [max(c#x) AS max(c)#x] : : +- SubqueryAlias __auto_generated_subquery_name : : +- WithCTE -: : :- CTERelationDef xxxx, false, false +: : :- CTERelationDef xxxx, false : : : +- SubqueryAlias t : : : +- Project [2#x AS c#x] : : : +- Project [2 AS 2#x] @@ -211,12 +211,12 @@ SELECT max(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] @@ -238,12 +238,12 @@ SELECT sum(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] @@ -268,17 +268,17 @@ SELECT sum(c) FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS c#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [2#x AS c#x] : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [3#x AS c#x] : +- Project [3 AS 3#x] @@ -384,16 +384,16 @@ WITH SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x] : +- SubqueryAlias t2 : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [2 AS 2#x] : +- OneRowRelation @@ -454,16 +454,16 @@ WITH SELECT * FROM t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t1 : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t3 : +- Project [1#x] : +- SubqueryAlias t1 : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [1#x] : +- SubqueryAlias t3 @@ -485,11 +485,11 @@ SELECT * FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_outer : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner : +- Project [1#x] : +- SubqueryAlias cte_outer @@ -518,16 +518,16 @@ SELECT * FROM ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_outer : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner_inner : +- Project [1#x] : +- SubqueryAlias cte_outer : +- CTERelationRef xxxx, true, [1#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte_inner : +- Project [1#x] : +- SubqueryAlias __auto_generated_subquery_name diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/cte.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/cte.sql.out index 9a221f6a699cd..cdd3698ce9af4 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/cte.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/cte.sql.out @@ -17,6 +17,19 @@ CreateViewCommand `t2`, select * from values 0, 1 as t(id), false, false, LocalT +- LocalRelation [id#x] +-- !query +create temporary view t3 as select * from t +-- !query analysis +CreateViewCommand `t3`, select * from t, false, false, LocalTempView, UNSUPPORTED, true + +- Project [id#x] + +- SubqueryAlias t + +- View (`t`, [id#x]) + +- Project [cast(id#x as int) AS id#x] + +- Project [id#x] + +- SubqueryAlias t + +- LocalRelation [id#x] + + -- !query WITH s AS (SELECT 1 FROM s) SELECT * FROM s -- !query analysis @@ -62,7 +75,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException WITH t AS (SELECT 1 FROM t) SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS 1#x] : +- SubqueryAlias t @@ -76,6 +89,27 @@ WithCTE +- CTERelationRef xxxx, true, [1#x], false, false +-- !query +WITH t AS (SELECT 1) SELECT * FROM t3 +-- !query analysis +WithCTE +:- CTERelationDef xxxx, false +: +- SubqueryAlias t +: +- Project [1 AS 1#x] +: +- OneRowRelation ++- Project [id#x] + +- SubqueryAlias t3 + +- View (`t3`, [id#x]) + +- Project [cast(id#x as int) AS id#x] + +- Project [id#x] + +- SubqueryAlias t + +- View (`t`, [id#x]) + +- Project [cast(id#x as int) AS id#x] + +- Project [id#x] + +- SubqueryAlias t + +- LocalRelation [id#x] + + -- !query WITH s1 AS (SELECT 1 FROM s2), s2 AS (SELECT 1 FROM s1) SELECT * FROM s1, s2 -- !query analysis @@ -100,7 +134,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException WITH t1 AS (SELECT * FROM t2), t2 AS (SELECT 2 FROM t1) SELECT * FROM t1 cross join t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t1 : +- Project [id#x] : +- SubqueryAlias t2 @@ -109,7 +143,7 @@ WithCTE : +- Project [id#x] : +- SubqueryAlias t : +- LocalRelation [id#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [2 AS 2#x] : +- SubqueryAlias t1 @@ -134,7 +168,7 @@ FROM CTE1 t1 CROSS JOIN CTE1 t2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias CTE1 : +- Project [id#x AS id#x] : +- Join Cross @@ -168,7 +202,7 @@ WITH t(x) AS (SELECT 1) SELECT * FROM t WHERE x = 1 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS x#x] : +- Project [1 AS 1#x] @@ -184,7 +218,7 @@ WITH t(x, y) AS (SELECT 1, 2) SELECT * FROM t WHERE x = 1 AND y = 2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS x#x, 2#x AS y#x] : +- Project [1 AS 1#x, 2 AS 2#x] @@ -200,7 +234,7 @@ WITH t(x, x) AS (SELECT 1, 2) SELECT * FROM t -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1#x AS x#x, 2#x AS x#x] : +- Project [1 AS 1#x, 2 AS 2#x] @@ -300,48 +334,48 @@ WITH w1(c1) AS SELECT * FROM w1 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w8 : +- Project [1#x AS c8#x] : +- Project [1 AS 1#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w7 : +- Project [c8#x AS c7#x] : +- Project [c8#x] : +- SubqueryAlias w8 : +- CTERelationRef xxxx, true, [c8#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w6 : +- Project [c7#x AS c6#x] : +- Project [c7#x] : +- SubqueryAlias w7 : +- CTERelationRef xxxx, true, [c7#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w5 : +- Project [c6#x AS c5#x] : +- Project [c6#x] : +- SubqueryAlias w6 : +- CTERelationRef xxxx, true, [c6#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w4 : +- Project [c5#x AS c4#x] : +- Project [c5#x] : +- SubqueryAlias w5 : +- CTERelationRef xxxx, true, [c5#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w3 : +- Project [c4#x AS c3#x] : +- Project [c4#x] : +- SubqueryAlias w4 : +- CTERelationRef xxxx, true, [c4#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w2 : +- Project [c3#x AS c2#x] : +- Project [c3#x] : +- SubqueryAlias w3 : +- CTERelationRef xxxx, true, [c3#x], false, false -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias w1 : +- Project [c2#x AS c1#x] : +- Project [c2#x] @@ -379,7 +413,7 @@ WITH same_name AS (SELECT 42) SELECT * FROM same_name, (SELECT 10) AS same_name -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias same_name : +- Project [42 AS 42#x] : +- OneRowRelation @@ -419,7 +453,7 @@ WITH q AS (SELECT 'foo' AS x) SELECT x, typeof(x) FROM q -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q : +- Project [foo AS x#x] : +- OneRowRelation @@ -479,7 +513,7 @@ SELECT * FROM Project [y#x] +- SubqueryAlias __auto_generated_subquery_name +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias q : +- Project [1 AS x#x] : +- OneRowRelation @@ -493,7 +527,7 @@ select (with q as (select 1 x) select * from q) -- !query analysis Project [scalar-subquery#x [] AS scalarsubquery()#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias q : : +- Project [1 AS x#x] : : +- OneRowRelation @@ -508,7 +542,7 @@ select 1 in (with q as (select 1) select * from q) -- !query analysis Project [1 IN (list#x []) AS (1 IN (listquery()))#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias q : : +- Project [1 AS 1#x] : : +- OneRowRelation @@ -549,11 +583,11 @@ from T1 z -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias T1 : +- Project [1 AS a#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias T1 : +- Project [2 AS b#x] : +- OneRowRelation @@ -580,15 +614,15 @@ from (WITH TtTt as (select 3 c) select * from ttTT, `tttT_2`) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias TTtt : +- Project [1 AS a#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias tTTt_2 : +- Project [2 AS a#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias TtTt : +- Project [3 AS c#x] : +- OneRowRelation @@ -609,7 +643,7 @@ from (select 1 x, 2 y) T -- !query analysis Project [scalar-subquery#x [x#x] AS scalarsubquery(x)#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias q : : +- Project [outer(x#x)] : : +- OneRowRelation @@ -628,7 +662,7 @@ from (select 1 x, 2 y) T -- !query analysis Project [scalar-subquery#x [x#x && y#x] AS scalarsubquery(x, y)#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias q : : +- Project [3 AS z#x] : : +- OneRowRelation @@ -646,11 +680,11 @@ select * from (with q2 as (select * from q1) select * from q2) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q1 : +- Project [1 AS x#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q2 : +- Project [x#x] : +- SubqueryAlias q1 @@ -668,11 +702,11 @@ select * from (with q1 as (select x+1 from q1) select * from q1) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q1 : +- Project [1 AS x#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q1 : +- Project [(x#x + 1) AS (x + 1)#x] : +- SubqueryAlias q1 @@ -709,7 +743,7 @@ with cte1 as (Select id as j from t) select * from cte1 where j = (select max(j) from cte1 as cte2) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [id#x AS j#x] : +- SubqueryAlias t @@ -778,3 +812,9 @@ DropTempViewCommand t DROP VIEW IF EXISTS t2 -- !query analysis DropTempViewCommand t2 + + +-- !query +DROP VIEW IF EXISTS t3 +-- !query analysis +DropTempViewCommand t3 diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/double-quoted-identifiers-enabled.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/double-quoted-identifiers-enabled.sql.out index 35713167bc38d..f2b1d91d7e447 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/double-quoted-identifiers-enabled.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/double-quoted-identifiers-enabled.sql.out @@ -411,7 +411,7 @@ CREATE TEMPORARY VIEW "myview"("c1") AS -- !query analysis CreateViewCommand `myview`, [(c1,None)], WITH "v"("a") AS (SELECT 1) SELECT "a" FROM "v", false, false, LocalTempView, UNSUPPORTED, true +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias v : +- Project [1#x AS a#x] : +- Project [1 AS 1#x] @@ -431,7 +431,7 @@ Project [a1#x AS a2#x] +- View (`myview`, [c1#x]) +- Project [cast(a#x as int) AS c1#x] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias v : +- Project [1#x AS a#x] : +- Project [1 AS 1#x] diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out index 9384f17fa74c1..87d537f949813 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/identifier-clause.sql.out @@ -1012,11 +1012,11 @@ WITH S(c1, c2) AS (VALUES(1, 2), (2, 3)), SELECT IDENTIFIER(agg)(IDENTIFIER(col)) FROM IDENTIFIER(tab) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias S : +- Project [col1#x AS c1#x, col2#x AS c2#x] : +- LocalRelation [col1#x, col2#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias T : +- Project [col1#x AS c1#x, col2#x AS c2#x] : +- LocalRelation [col1#x, col2#x] @@ -1031,11 +1031,11 @@ WITH S(c1, c2) AS (VALUES(1, 2), (2, 3)), SELECT IDENTIFIER('max')(IDENTIFIER('c1')) FROM IDENTIFIER('T') -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias S : +- Project [col1#x AS c1#x, col2#x AS c2#x] : +- LocalRelation [col1#x, col2#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias T : +- Project [col1#x AS c1#x, col2#x AS c2#x] : +- LocalRelation [col1#x, col2#x] @@ -1049,7 +1049,7 @@ WITH ABC(c1, c2) AS (VALUES(1, 2), (2, 3)) SELECT IDENTIFIER('max')(IDENTIFIER('c1')) FROM IDENTIFIER('A' || 'BC') -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias ABC : +- Project [col1#x AS c1#x, col2#x AS c2#x] : +- LocalRelation [col1#x, col2#x] diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/join-lateral.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/join-lateral.sql.out index c93109e428362..390fcf5e30152 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/join-lateral.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/join-lateral.sql.out @@ -1358,14 +1358,14 @@ WITH cte1 AS ( SELECT * FROM cte2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [c1#x] : +- SubqueryAlias spark_catalog.default.t1 : +- View (`spark_catalog`.`default`.`t1`, [c1#x, c2#x]) : +- Project [cast(col1#x as int) AS c1#x, cast(col2#x as int) AS c2#x] : +- LocalRelation [col1#x, col2#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte2 : +- Project [c1#x, c2#x] : +- LateralJoin lateral-subquery#x [c1#x], Inner diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/non-excludable-rule.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/non-excludable-rule.sql.out index 7f1ca3dda902d..4a717488e0172 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/non-excludable-rule.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/non-excludable-rule.sql.out @@ -36,7 +36,7 @@ WITH tmp AS ( SELECT id FROM range(3) WHERE id > (SELECT max(id) FROM tmp) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias tmp : +- Intersect false : :- Project [id#xL] diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out index 441b3627bc928..2b2d69db348d3 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/window_part3.sql.out @@ -89,7 +89,7 @@ FROM cte WINDOW w AS (ORDER BY x rows between 1 preceding and 1 following) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte : +- Project [id#xL AS x#xL] : +- Project [id#xL] @@ -111,7 +111,7 @@ FROM cte WINDOW w AS (ORDER BY x range between 1 preceding and 1 following) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte : +- Project [id#xL AS x#xL] : +- Project [id#xL] @@ -134,7 +134,7 @@ FROM cte WINDOW w AS (ORDER BY x rows between 1 preceding and 1 following) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte : +- Project [1#xL AS x#xL] : +- Union false, false @@ -167,7 +167,7 @@ FROM cte WINDOW w AS (ORDER BY x range between 1 preceding and 1 following) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte : +- Project [1#xL AS x#xL] : +- Union false, false diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out index b92e70e91a4ba..1219562068dd9 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/with.sql.out @@ -4,7 +4,7 @@ WITH q1(x,y) AS (SELECT 1,2) SELECT * FROM q1, q1 AS q2 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias q1 : +- Project [1#x AS x#x, 2#x AS y#x] : +- Project [1 AS 1#x, 2 AS 2#x] @@ -185,7 +185,7 @@ InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_d with cte(foo) as ( select 42 ) select * from ((select foo from cte)) q -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte : +- Project [42#x AS foo#x] : +- Project [42 AS 42#x] @@ -207,11 +207,11 @@ WITH outermost(x) AS ( SELECT * FROM outermost ORDER BY 1 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias innermost : +- Project [2 AS 2#x] : +- OneRowRelation -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias outermost : +- Project [1#x AS x#x] : +- Distinct @@ -412,7 +412,7 @@ org.apache.spark.sql.catalyst.parser.ParseException with ordinality as (select 1 as x) select * from ordinality -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias ordinality : +- Project [1 AS x#x] : +- OneRowRelation @@ -453,7 +453,7 @@ with test as (select 42) insert into test select * from test InsertIntoHadoopFsRelationCommand file:[not included in comparison]/{warehouse_dir}/test, false, Parquet, [path=file:[not included in comparison]/{warehouse_dir}/test], Append, `spark_catalog`.`default`.`test`, org.apache.spark.sql.execution.datasources.InMemoryFileIndex(file:[not included in comparison]/{warehouse_dir}/test), [i] +- Project [cast(42#x as int) AS i#x] +- WithCTE - :- CTERelationDef xxxx, false, false + :- CTERelationDef xxxx, false : +- SubqueryAlias test : +- Project [42 AS 42#x] : +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-session-variables.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-session-variables.sql.out index da4542fe9bb1c..77e7174167eca 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/sql-session-variables.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/sql-session-variables.sql.out @@ -2044,7 +2044,7 @@ Project [1 AS 1#x] WITH v1 AS (SELECT var1 AS c1) SELECT c1 AS `1` FROM v1 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias v1 : +- Project [variablereference(system.session.var1=1) AS c1#x] : +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/exists-subquery/exists-cte.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/exists-subquery/exists-cte.sql.out index f210bc49030fc..abaf6a2432251 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/exists-subquery/exists-cte.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/exists-subquery/exists-cte.sql.out @@ -101,7 +101,7 @@ WHERE a.bonus_amt > 30 WHERE a.emp_name = b.emp_name) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias bonus_cte : +- Project [emp_name#x, bonus_amt#x] : +- Filter exists#x [emp_name#x] @@ -162,7 +162,7 @@ WHERE EXISTS (SELECT * WHERE bonus.emp_name = a.emp_name) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias emp_cte : +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] : +- Filter ((id#x >= 100) AND (id#x <= 300)) @@ -172,7 +172,7 @@ WithCTE : +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] : +- SubqueryAlias EMP : +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias dept_cte : +- Project [dept_id#x, dept_name#x, state#x] : +- Filter (dept_id#x = 10) @@ -225,7 +225,7 @@ WHERE e.dept_id = d.dept_id WHERE e.emp_name = a.emp_name) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias emp_cte : +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] : +- Filter ((id#x >= 100) AND (id#x <= 300)) @@ -235,7 +235,7 @@ WithCTE : +- Project [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] : +- SubqueryAlias EMP : +- LocalRelation [id#x, emp_name#x, hiredate#x, salary#x, dept_id#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias dept_cte : +- Project [dept_id#x, dept_name#x, state#x] : +- Filter (dept_id#x = 10) @@ -299,7 +299,7 @@ WHERE EXISTS (SELECT dept_id, GROUP BY emp_name -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias empdept : +- Project [id#x, salary#x, emp_name#x, dept_id#x] : +- Filter id#x IN (100,200) @@ -352,7 +352,7 @@ WHERE NOT EXISTS (SELECT dept_id, GROUP BY emp_name -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias empdept : +- Project [id#x, salary#x, emp_name#x, dept_id#x] : +- Filter id#x IN (100,200) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-multiple-columns.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-multiple-columns.sql.out index 85e28de736a0c..230ffc005e90d 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-multiple-columns.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-multiple-columns.sql.out @@ -307,7 +307,7 @@ FROM ( on cte1.t1b = cte2.t1b) s -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter named_struct(t1b, t1b#x, t1d, t1d#xL) IN (list#x [t1c#x]) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-with-cte.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-with-cte.sql.out index 03a4c3ffd4fcb..199b876fb9a86 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-with-cte.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/in-subquery/in-with-cte.sql.out @@ -123,7 +123,7 @@ WHERE t1b IN (SELECT cte1.t1b WHERE cte1.t1b > 0) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter (t1a#x = val1a) @@ -177,7 +177,7 @@ GROUP BY t1a, t1b, t1c HAVING t1c IS NOT NULL -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- SubqueryAlias t1 @@ -250,7 +250,7 @@ WHERE t1c IN ON cte1.t1d > cte6.t1d) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x, t1c#x, t1d#xL, t1e#x] : +- SubqueryAlias t1 @@ -316,7 +316,7 @@ FROM (SELECT * ON cte1.t1b = cte4.t1b) s -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter (t1b#x IN (list#x []) AND (t1a#x = val1b)) @@ -393,7 +393,7 @@ WHERE t1b IN GROUP BY t1b -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x, t1h#x] : +- Filter t1a#x IN (list#x [t1b#x]) @@ -453,7 +453,7 @@ FROM ( ) s -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x, t1c#x] : +- Filter (t1b#x IN (list#x [t1c#x]) AND (t1a#x = val1b)) @@ -515,7 +515,7 @@ FROM (SELECT cte1.t1a, GROUP BY s.t1b -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter t1b#x IN (list#x [t1c#x]) @@ -569,7 +569,7 @@ WHERE s.t1b IN ON t1.t1a = cte1.t1a) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter t1b#x IN (list#x [t1c#x]) @@ -627,7 +627,7 @@ WHERE t1b NOT IN (SELECT cte1.t1b t1c > 10 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x] : +- Filter (t1a#x = val1d) @@ -683,7 +683,7 @@ WHERE t1b NOT IN ORDER BY t1c DESC -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias cte1 : +- Project [t1a#x, t1b#x, t1c#x, t1d#xL, t1h#x] : +- Filter NOT t1d#xL IN (list#x []) diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/scalar-subquery/scalar-subquery-select.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/scalar-subquery/scalar-subquery-select.sql.out index 3b1ffa94c17f2..2a3a87e5cab81 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/scalar-subquery/scalar-subquery-select.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/subquery/scalar-subquery/scalar-subquery-select.sql.out @@ -617,7 +617,7 @@ SELECT c1, (WITH t AS (SELECT 1 AS a) SELECT a + c1 FROM t) FROM t1 -- !query analysis Project [c1#x, scalar-subquery#x [c1#x] AS scalarsubquery(c1)#x] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [1 AS a#x] : : +- OneRowRelation @@ -636,7 +636,7 @@ SELECT c1, (WITH t AS (SELECT * FROM t2 WHERE c1 = t1.c1) SELECT SUM(c2) FROM t) -- !query analysis Project [c1#x, scalar-subquery#x [c1#x] AS scalarsubquery(c1)#xL] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [c1#x, c2#x] : : +- Filter (c1#x = outer(c1#x)) @@ -664,7 +664,7 @@ SELECT c1, ( -- !query analysis Project [c1#x, scalar-subquery#x [c1#x] AS scalarsubquery(c1)#xL] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t3 : : +- Project [(c1#x + 1) AS c1#x, (c2#x + 1) AS c2#x] : : +- SubqueryAlias t2 @@ -672,7 +672,7 @@ Project [c1#x, scalar-subquery#x [c1#x] AS scalarsubquery(c1)#xL] : : +- Project [cast(c1#x as int) AS c1#x, cast(c2#x as int) AS c2#x] : : +- SubqueryAlias t2 : : +- LocalRelation [c1#x, c2#x] -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t4 : : +- Project [c1#x, c2#x] : : +- Filter (outer(c1#x) = c1#x) @@ -697,7 +697,7 @@ SELECT c1, ( -- !query analysis Project [c1#x, scalar-subquery#x [c1#x] AS scalarsubquery(c1)#xL] : +- WithCTE -: :- CTERelationDef xxxx, false, false +: :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [c1#x, c2#x] : : +- SubqueryAlias t2 @@ -732,7 +732,7 @@ SELECT * FROM t1 WHERE c1 > ( ) -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias v : +- Project [c1#x, c2#x] : +- SubqueryAlias t2 @@ -743,7 +743,7 @@ WithCTE +- Project [c1#x, c2#x] +- Filter (cast(c1#x as bigint) > scalar-subquery#x [c1#x]) : +- WithCTE - : :- CTERelationDef xxxx, false, false + : :- CTERelationDef xxxx, false : : +- SubqueryAlias t : : +- Project [c1#x, c2#x] : : +- SubqueryAlias t2 @@ -771,7 +771,7 @@ WITH t AS (SELECT 1 AS a) SELECT c1, (SELECT a FROM t WHERE a = c1) FROM t1 -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t : +- Project [1 AS a#x] : +- OneRowRelation @@ -1019,7 +1019,7 @@ SELECT (SELECT sum(1) FROM T WHERE a = col OR upper(col)= 'Y') FROM (SELECT null as col) as foo -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias T : +- Project [1 AS a#x] : +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/transform.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/transform.sql.out index 04e019fdbaa41..d5376913f9ae4 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/transform.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/transform.sql.out @@ -874,7 +874,7 @@ WITH temp AS ( SELECT t1.b FROM temp t1 JOIN temp t2 ON t1.b = t2.b -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias temp : +- ScriptTransformation cat, [b#x], ScriptInputOutputSchema(List(),List(),None,None,List(),List(),None,None,false) : +- Project [a#x] diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/using-join.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/using-join.sql.out index 89f988fe2b616..d26c5ba4430da 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/using-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/using-join.sql.out @@ -817,12 +817,12 @@ FROM t1 FULL OUTER JOIN t2 USING (key) WHERE t1.key NOT LIKE 'bb.%' -- !query analysis WithCTE -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t1 : +- Project [key#x] : +- SubqueryAlias t : +- LocalRelation [key#x] -:- CTERelationDef xxxx, false, false +:- CTERelationDef xxxx, false : +- SubqueryAlias t2 : +- Project [key#x] : +- SubqueryAlias t diff --git a/sql/core/src/test/resources/sql-tests/inputs/cte.sql b/sql/core/src/test/resources/sql-tests/inputs/cte.sql index 67a94ce61617d..1e17529d545bf 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/cte.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/cte.sql @@ -1,5 +1,6 @@ create temporary view t as select * from values 0, 1, 2 as t(id); create temporary view t2 as select * from values 0, 1 as t(id); +create temporary view t3 as select * from t; -- WITH clause should not fall into infinite loop by referencing self WITH s AS (SELECT 1 FROM s) SELECT * FROM s; @@ -10,6 +11,9 @@ SELECT * FROM r; -- WITH clause should reference the base table WITH t AS (SELECT 1 FROM t) SELECT * FROM t; +-- Table `t` referenced by a view should take precedence over the top CTE `t` +WITH t AS (SELECT 1) SELECT * FROM t3; + -- WITH clause should not allow cross reference WITH s1 AS (SELECT 1 FROM s2), s2 AS (SELECT 1 FROM s1) SELECT * FROM s1, s2; @@ -175,3 +179,4 @@ with cte as (select * from cte) select * from cte; -- Clean up DROP VIEW IF EXISTS t; DROP VIEW IF EXISTS t2; +DROP VIEW IF EXISTS t3; diff --git a/sql/core/src/test/resources/sql-tests/results/cte.sql.out b/sql/core/src/test/resources/sql-tests/results/cte.sql.out index 754a8832ef6c5..4367ae1d5f2d0 100644 --- a/sql/core/src/test/resources/sql-tests/results/cte.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/cte.sql.out @@ -15,6 +15,14 @@ struct<> +-- !query +create temporary view t3 as select * from t +-- !query schema +struct<> +-- !query output + + + -- !query WITH s AS (SELECT 1 FROM s) SELECT * FROM s -- !query schema @@ -70,6 +78,16 @@ struct<1:int> 1 +-- !query +WITH t AS (SELECT 1) SELECT * FROM t3 +-- !query schema +struct +-- !query output +0 +1 +2 + + -- !query WITH s1 AS (SELECT 1 FROM s2), s2 AS (SELECT 1 FROM s1) SELECT * FROM s1, s2 -- !query schema @@ -580,3 +598,11 @@ DROP VIEW IF EXISTS t2 struct<> -- !query output + + +-- !query +DROP VIEW IF EXISTS t3 +-- !query schema +struct<> +-- !query output + diff --git a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out index 7d96a3e98c832..f9c9df3f9bf5f 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out @@ -252,6 +252,7 @@ REAL false RECORDREADER false RECORDWRITER false RECOVER false +RECURSIVE true REDUCE false REFERENCES true REFRESH false @@ -432,6 +433,7 @@ ORDER OUTER OVERLAPS PRIMARY +RECURSIVE REFERENCES RIGHT SELECT diff --git a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out index 6cbfe519a76f6..67e5e4170d789 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out @@ -252,6 +252,7 @@ REAL false RECORDREADER false RECORDWRITER false RECOVER false +RECURSIVE false REDUCE false REFERENCES false REFRESH false diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out index 6cbfe519a76f6..67e5e4170d789 100644 --- a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out @@ -252,6 +252,7 @@ REAL false RECORDREADER false RECORDWRITER false RECOVER false +RECURSIVE false REDUCE false REFERENCES false REFRESH false diff --git a/sql/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala index b253c4a70bbf9..e1b0676831549 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/AggregateHashMapSuite.scala @@ -21,7 +21,9 @@ import org.scalatest.BeforeAndAfter import org.apache.spark.SparkConf import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.tags.SlowSQLTest +@SlowSQLTest class SingleLevelAggregateHashMapSuite extends DataFrameAggregateSuite with BeforeAndAfter { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.CODEGEN_FALLBACK.key, "false") @@ -37,6 +39,7 @@ class SingleLevelAggregateHashMapSuite extends DataFrameAggregateSuite with Befo } } +@SlowSQLTest class TwoLevelAggregateHashMapSuite extends DataFrameAggregateSuite with BeforeAndAfter { override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.CODEGEN_FALLBACK.key, "false") @@ -52,6 +55,7 @@ class TwoLevelAggregateHashMapSuite extends DataFrameAggregateSuite with BeforeA } } +@SlowSQLTest class TwoLevelAggregateHashMapWithVectorizedMapSuite extends DataFrameAggregateSuite with BeforeAndAfter { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala index ad80dc65926bc..141d6b219f2a7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala @@ -39,10 +39,12 @@ import org.apache.spark.sql.test.SQLTestData.DecimalData import org.apache.spark.sql.types._ import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, MINUTE, SECOND} import org.apache.spark.sql.types.YearMonthIntervalType.{MONTH, YEAR} +import org.apache.spark.tags.SlowSQLTest import org.apache.spark.unsafe.types.CalendarInterval case class Fact(date: Int, hour: Int, minute: Int, room_name: String, temp: Double) +@SlowSQLTest class DataFrameAggregateSuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlanHelper { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala index f94cf89276ec4..fdfb909d9ba73 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSubquerySuite.scala @@ -418,6 +418,30 @@ class DataFrameSubquerySuite extends QueryTest with SharedSparkSession { } } + test("lateral join with star expansion") { + withView("t1", "t2") { + val t1 = table1() + val t2 = table2() + + checkAnswer( + t1.lateralJoin(spark.range(1).select().select($"*")), + sql("SELECT * FROM t1, LATERAL (SELECT *)") + ) + checkAnswer( + t1.lateralJoin(t2.select($"*")), + sql("SELECT * FROM t1, LATERAL (SELECT * FROM t2)") + ) + checkAnswer( + t1.lateralJoin(t2.select($"t1.*".outer(), $"t2.*")), + sql("SELECT * FROM t1, LATERAL (SELECT t1.*, t2.* FROM t2)") + ) + checkAnswer( + t1.lateralJoin(t2.alias("t1").select($"t1.*")), + sql("SELECT * FROM t1, LATERAL (SELECT t1.* FROM t2 AS t1)") + ) + } + } + test("lateral join with different join types") { withView("t1") { val t1 = table1() @@ -444,6 +468,18 @@ class DataFrameSubquerySuite extends QueryTest with SharedSparkSession { } } + test("lateral join with subquery alias") { + withView("t1") { + val t1 = table1() + + checkAnswer( + t1.lateralJoin(spark.range(1).select($"c1".outer(), $"c2".outer()).toDF("a", "b").as("s")) + .select("a", "b"), + sql("SELECT a, b FROM t1, LATERAL (SELECT c1, c2) s(a, b)") + ) + } + } + test("lateral join with correlated equality / non-equality predicates") { withView("t1", "t2") { val t1 = table1() @@ -516,8 +552,8 @@ class DataFrameSubquerySuite extends QueryTest with SharedSparkSession { checkAnswer( t1.lateralJoin( - t2.where($"t1.c1".outer() === $"t2.c1").select($"c2"), "left" - ).join(t1.as("t3"), $"t2.c2" === $"t3.c2", "left"), + t2.where($"t1.c1".outer() === $"t2.c1").select($"c2").as("s"), "left" + ).join(t1.as("t3"), $"s.c2" === $"t3.c2", "left"), sql( """ |SELECT * FROM t1 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala index 4f2cd275ffdfd..637e0cf964fe5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameTableValuedFunctionsSuite.scala @@ -60,11 +60,11 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi val t3 = spark.table("t3") checkAnswer( - t1.lateralJoin(spark.tvf.explode(array($"c1".outer(), $"c2".outer()))), + t1.lateralJoin(spark.tvf.explode(array($"c1".outer(), $"c2".outer())).toDF("c3").as("t2")), sql("SELECT * FROM t1, LATERAL EXPLODE(ARRAY(c1, c2)) t2(c3)") ) checkAnswer( - t3.lateralJoin(spark.tvf.explode($"c2".outer())), + t3.lateralJoin(spark.tvf.explode($"c2".outer()).toDF("v").as("t2")), sql("SELECT * FROM t3, LATERAL EXPLODE(c2) t2(v)") ) checkAnswer( @@ -112,11 +112,12 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi val t3 = spark.table("t3") checkAnswer( - t1.lateralJoin(spark.tvf.explode_outer(array($"c1".outer(), $"c2".outer()))), + t1.lateralJoin( + spark.tvf.explode_outer(array($"c1".outer(), $"c2".outer())).toDF("c3").as("t2")), sql("SELECT * FROM t1, LATERAL EXPLODE_OUTER(ARRAY(c1, c2)) t2(c3)") ) checkAnswer( - t3.lateralJoin(spark.tvf.explode_outer($"c2".outer())), + t3.lateralJoin(spark.tvf.explode_outer($"c2".outer()).toDF("v").as("t2")), sql("SELECT * FROM t3, LATERAL EXPLODE_OUTER(c2) t2(v)") ) checkAnswer( @@ -164,8 +165,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi ) checkAnswer( arrayStruct.lateralJoin( - spark.tvf.inline($"arr".outer()), - $"id" === $"col1", + spark.tvf.inline($"arr".outer()).toDF("k", "v").as("t"), + $"id" === $"k", "left" ), sql("SELECT * FROM array_struct LEFT JOIN LATERAL INLINE(arr) t(k, v) ON id = k") @@ -210,8 +211,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi ) checkAnswer( arrayStruct.lateralJoin( - spark.tvf.inline_outer($"arr".outer()), - $"id" === $"col1", + spark.tvf.inline_outer($"arr".outer()).toDF("k", "v").as("t"), + $"id" === $"k", "left" ), sql("SELECT * FROM array_struct LEFT JOIN LATERAL INLINE_OUTER(arr) t(k, v) ON id = k") @@ -249,8 +250,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi jsonTable.as("t1").lateralJoin( spark.tvf.json_tuple( $"t1.jstring".outer(), - lit("f1"), lit("f2"), lit("f3"), lit("f4"), lit("f5")) - ).select($"key", $"c0", $"c1", $"c2", $"c3", $"c4"), + lit("f1"), lit("f2"), lit("f3"), lit("f4"), lit("f5")).as("t2") + ).select($"t1.key", $"t2.*"), sql("SELECT t1.key, t2.* FROM json_table t1, " + "LATERAL json_tuple(t1.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') t2") ) @@ -258,9 +259,9 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi jsonTable.as("t1").lateralJoin( spark.tvf.json_tuple( $"jstring".outer(), - lit("f1"), lit("f2"), lit("f3"), lit("f4"), lit("f5")) - ).where($"c0".isNotNull) - .select($"key", $"c0", $"c1", $"c2", $"c3", $"c4"), + lit("f1"), lit("f2"), lit("f3"), lit("f4"), lit("f5")).as("t2") + ).where($"t2.c0".isNotNull) + .select($"t1.key", $"t2.*"), sql("SELECT t1.key, t2.* FROM json_table t1, " + "LATERAL json_tuple(t1.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') t2 " + "WHERE t2.c0 IS NOT NULL") @@ -388,21 +389,21 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi checkAnswer( t1.lateralJoin( - spark.tvf.stack(lit(2), lit("Key"), $"c1".outer(), lit("Value"), $"c2".outer()) - ).select($"col0", $"col1"), + spark.tvf.stack(lit(2), lit("Key"), $"c1".outer(), lit("Value"), $"c2".outer()).as("t") + ).select($"t.*"), sql("SELECT t.* FROM t1, LATERAL stack(2, 'Key', c1, 'Value', c2) t") ) checkAnswer( t1.lateralJoin( - spark.tvf.stack(lit(1), $"c1".outer(), $"c2".outer()) - ).select($"col0".as("x"), $"col1".as("y")), + spark.tvf.stack(lit(1), $"c1".outer(), $"c2".outer()).toDF("x", "y").as("t") + ).select($"t.*"), sql("SELECT t.* FROM t1 JOIN LATERAL stack(1, c1, c2) t(x, y)") ) checkAnswer( t1.join(t3, $"t1.c1" === $"t3.c1") .lateralJoin( - spark.tvf.stack(lit(1), $"t1.c2".outer(), $"t3.c2".outer()) - ).select($"col0", $"col1"), + spark.tvf.stack(lit(1), $"t1.c2".outer(), $"t3.c2".outer()).as("t") + ).select($"t.*"), sql("SELECT t.* FROM t1 JOIN t3 ON t1.c1 = t3.c1 JOIN LATERAL stack(1, t1.c2, t3.c2) t") ) } @@ -466,8 +467,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi checkAnswer( variantTable.as("t1").lateralJoin( - spark.tvf.variant_explode($"v".outer()) - ).select($"id", $"pos", $"key", $"value"), + spark.tvf.variant_explode($"v".outer()).as("t") + ).select($"t1.id", $"t.*"), sql("SELECT t1.id, t.* FROM variant_table AS t1, LATERAL variant_explode(v) AS t") ) } @@ -519,8 +520,8 @@ class DataFrameTableValuedFunctionsSuite extends QueryTest with SharedSparkSessi checkAnswer( variantTable.as("t1").lateralJoin( - spark.tvf.variant_explode_outer($"v".outer()) - ).select($"id", $"pos", $"key", $"value"), + spark.tvf.variant_explode_outer($"v".outer()).as("t") + ).select($"t1.id", $"t.*"), sql("SELECT t1.id, t.* FROM variant_table AS t1, LATERAL variant_explode_outer(v) AS t") ) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala index 46a24acb475c4..43e6111fc99ca 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSCollationQueryTestSuite.scala @@ -67,6 +67,7 @@ class TPCDSCollationQueryTestSuite extends QueryTest with TPCDSBase with SQLQuer // To make output results deterministic override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.SHUFFLE_PARTITIONS.key, "1") + .remove("spark.hadoop.fs.file.impl") protected override def createSparkSession: TestSparkSession = { new TestSparkSession(new SparkContext("local[1]", this.getClass.getSimpleName, sparkConf)) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala index ffd15eb46a48e..e8b36d8b130cf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala @@ -33,8 +33,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSBase { // Disable read-side char padding so that the generated code is less than 8000. super.sparkConf.set(SQLConf.READ_SIDE_CHAR_PADDING, false) - // q72 is skipped due to GitHub Actions' memory limit. - tpcdsQueries.filterNot(sys.env.contains("GITHUB_ACTIONS") && _ == "q72").foreach { name => + tpcdsQueries.foreach { name => val queryString = resourceToString(s"tpcds/$name.sql", classLoader = Thread.currentThread().getContextClassLoader) test(name) { @@ -44,8 +43,7 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSBase { } } - // q72 is skipped due to GitHub Actions' memory limit. - tpcdsQueriesV2_7_0.filterNot(sys.env.contains("GITHUB_ACTIONS") && _ == "q72").foreach { name => + tpcdsQueriesV2_7_0.foreach { name => val queryString = resourceToString(s"tpcds-v2.7.0/$name.sql", classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index bde6155529872..c1246a167b8cc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -62,6 +62,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp // To make output results deterministic override protected def sparkConf: SparkConf = super.sparkConf .set(SQLConf.SHUFFLE_PARTITIONS.key, "1") + .remove("spark.hadoop.fs.file.impl") protected override def createSparkSession: TestSparkSession = { new TestSparkSession(new SparkContext("local[1]", this.getClass.getSimpleName, sparkConf)) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala index f09e896fb39a3..a24982aea1585 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala @@ -48,7 +48,7 @@ class ArtifactManagerSuite extends SharedSparkSession { private def sessionUUID: String = spark.sessionUUID override def afterEach(): Unit = { - artifactManager.cleanUpResources() + artifactManager.cleanUpResourcesForTesting() super.afterEach() } @@ -209,7 +209,7 @@ class ArtifactManagerSuite extends SharedSparkSession { assert(expectedPath.toFile.exists()) // Remove resources - artifactManager.cleanUpResources() + artifactManager.cleanUpResourcesForTesting() assert(blockManager.getLocalBytes(blockId).isEmpty) assert(!expectedPath.toFile.exists()) @@ -294,7 +294,7 @@ class ArtifactManagerSuite extends SharedSparkSession { val sessionDirectory = artifactManager.artifactPath.toFile assert(sessionDirectory.exists()) - artifactManager.cleanUpResources() + artifactManager.cleanUpResourcesForTesting() assert(!sessionDirectory.exists()) assert(ArtifactManager.artifactRootDirectory.toFile.exists()) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 87d0a1ff4e7bf..8d255e9efda54 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -278,8 +278,8 @@ class DataSourceV2SQLSuiteV1Filter test("CreateTable: without USING clause") { withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key -> "false") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") val testCatalog = catalog("testcat").asTableCatalog sql("CREATE TABLE testcat.t1 (id int)") @@ -785,8 +785,8 @@ class DataSourceV2SQLSuiteV1Filter } test("CreateTableAsSelect: v2 session catalog can load v1 source table") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data") df.createOrReplaceTempView("source") @@ -846,8 +846,8 @@ class DataSourceV2SQLSuiteV1Filter // TODO: ignored by SPARK-31707, restore the test after create table syntax unification ignore("CreateTableAsSelect: without USING clause") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") val testCatalog = catalog("testcat").asTableCatalog sql("CREATE TABLE testcat.t1 AS SELECT 1 i") @@ -1086,11 +1086,11 @@ class DataSourceV2SQLSuiteV1Filter Seq(true, false).foreach { useV1Table => val format = if (useV1Table) "json" else v2Format if (useV1Table) { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") } else { spark.conf.set( - V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName) + V2_SESSION_CATALOG_IMPLEMENTATION, classOf[InMemoryTableSessionCatalog].getName) } withTable("t") { @@ -1815,8 +1815,8 @@ class DataSourceV2SQLSuiteV1Filter } test("SPARK-46972: asymmetrical replacement for char/varchar in V2SessionCatalog.createTable") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") withTable("t") { sql(s"CREATE TABLE t(c char(1), v varchar(2)) USING $v2Source") } @@ -2533,8 +2533,8 @@ class DataSourceV2SQLSuiteV1Filter } test("SPARK-30001: session catalog name can be specified in SQL statements") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") withTable("t") { sql("CREATE TABLE t USING json AS SELECT 1 AS i") @@ -2598,8 +2598,8 @@ class DataSourceV2SQLSuiteV1Filter } test("SPARK-30094: current namespace is used during table resolution") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") withTable("spark_catalog.default.t", "testcat.ns.t") { sql("CREATE TABLE t USING parquet AS SELECT 1") @@ -2613,8 +2613,8 @@ class DataSourceV2SQLSuiteV1Filter } test("SPARK-30284: CREATE VIEW should track the current catalog and namespace") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") val sessionCatalogName = CatalogManager.SESSION_CATALOG_NAME sql("CREATE NAMESPACE testcat.ns1.ns2") @@ -2651,8 +2651,8 @@ class DataSourceV2SQLSuiteV1Filter } test("COMMENT ON NAMESPACE") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") // Session catalog is used. sql("CREATE NAMESPACE ns") checkNamespaceComment("ns", "minor revision") @@ -2685,8 +2685,8 @@ class DataSourceV2SQLSuiteV1Filter } test("COMMENT ON TABLE") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") // Session catalog is used. withTable("t") { sql("CREATE TABLE t(k int) USING json") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala index fd022580db42b..26f64ceb33fe3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala @@ -100,8 +100,8 @@ trait DeleteFromTests extends DatasourceV2SQLBase { } test("DeleteFrom: DELETE is only supported with v2 tables") { - // unset this config to use the default v2 session catalog. - spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key) + // use the default v2 session catalog. + spark.conf.set(V2_SESSION_CATALOG_IMPLEMENTATION, "builtin") val v1Table = "tbl" withTable(v1Table) { sql(s"CREATE TABLE $v1Table" + diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala index 0078c3f9f65de..31b002a1e245d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExternalAppendOnlyUnsafeRowArrayBenchmark.scala @@ -32,10 +32,10 @@ import org.apache.spark.util.collection.unsafe.sort.UnsafeExternalSorter * {{{ * 1. without sbt: * bin/spark-submit --class --jars - * 2. build/sbt build/sbt ";project sql;set javaOptions - * in Test += \"-Dspark.memory.debugFill=false\";Test/runMain " - * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt ";project sql;set javaOptions - * in Test += \"-Dspark.memory.debugFill=false\";Test/runMain " + * 2. build/sbt build/sbt ";project sql; + * set Test / javaOptions += \"-Dspark.memory.debugFill=false\";Test/runMain " + * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt ";project sql; + * set Test / javaOptions += \"-Dspark.memory.debugFill=false\";Test/runMain " * Results will be written to * "benchmarks/ExternalAppendOnlyUnsafeRowArrayBenchmark-results.txt". * }}} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala index d298b98aaa8da..ad28fd5176d99 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala @@ -749,7 +749,6 @@ class AdaptiveQueryExecSuite // so retry several times here to avoid unit test failure. eventually(timeout(15.seconds), interval(500.milliseconds)) { withSQLConf( - SQLConf.ARTIFACTS_SESSION_ISOLATION_ALWAYS_APPLY_CLASSLOADER.key -> "true", SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", SQLConf.NON_EMPTY_PARTITION_RATIO_FOR_BROADCAST_JOIN.key -> "0.5") { // `testData` is small enough to be broadcast but has empty partition ratio over the config. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionParserSuite.scala new file mode 100644 index 0000000000000..75b42c6440719 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/CreateSQLFunctionParserSuite.scala @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedIdentifier} +import org.apache.spark.sql.catalyst.catalog.LanguageSQL +import org.apache.spark.sql.catalyst.plans.logical.CreateUserDefinedFunction +import org.apache.spark.sql.execution.SparkSqlParser + +class CreateSQLFunctionParserSuite extends AnalysisTest { + private lazy val parser = new SparkSqlParser() + + private def intercept(sqlCommand: String, messages: String*): Unit = + interceptParseException(parser.parsePlan)(sqlCommand, messages: _*)() + + private def checkParseError( + sqlCommand: String, + errorClass: String, + parameters: Map[String, String], + queryContext: Array[ExpectedContext] = Array.empty): Unit = + assertParseErrorClass(parser.parsePlan, sqlCommand, errorClass, parameters, queryContext) + + // scalastyle:off argcount + private def createSQLFunction( + nameParts: Seq[String], + inputParamText: Option[String] = None, + returnTypeText: String = "INT", + exprText: Option[String] = None, + queryText: Option[String] = None, + comment: Option[String] = None, + isDeterministic: Option[Boolean] = None, + containsSQL: Option[Boolean] = None, + isTableFunc: Boolean = false, + ignoreIfExists: Boolean = false, + replace: Boolean = false): CreateUserDefinedFunction = { + // scalastyle:on argcount + CreateUserDefinedFunction( + UnresolvedIdentifier(nameParts), + inputParamText = inputParamText, + returnTypeText = returnTypeText, + exprText = exprText, + queryText = queryText, + comment = comment, + isDeterministic = isDeterministic, + containsSQL = containsSQL, + language = LanguageSQL, + isTableFunc = isTableFunc, + ignoreIfExists = ignoreIfExists, + replace = replace) + } + + // scalastyle:off argcount + private def createSQLFunctionCommand( + name: String, + inputParamText: Option[String] = None, + returnTypeText: String = "INT", + exprText: Option[String] = None, + queryText: Option[String] = None, + comment: Option[String] = None, + isDeterministic: Option[Boolean] = None, + containsSQL: Option[Boolean] = None, + isTableFunc: Boolean = false, + ignoreIfExists: Boolean = false, + replace: Boolean = false): CreateSQLFunctionCommand = { + // scalastyle:on argcount + CreateSQLFunctionCommand( + FunctionIdentifier(name), + inputParamText = inputParamText, + returnTypeText = returnTypeText, + exprText = exprText, + queryText = queryText, + comment = comment, + isDeterministic = isDeterministic, + containsSQL = containsSQL, + isTableFunc = isTableFunc, + isTemp = true, + ignoreIfExists = ignoreIfExists, + replace = replace) + } + + test("create temporary SQL functions") { + comparePlans( + parser.parsePlan("CREATE TEMPORARY FUNCTION a() RETURNS INT RETURN 1"), + createSQLFunctionCommand("a", exprText = Some("1"))) + + comparePlans( + parser.parsePlan( + "CREATE TEMPORARY FUNCTION a(x INT) RETURNS TABLE (a INT) RETURN SELECT x"), + createSQLFunctionCommand( + name = "a", + inputParamText = Some("x INT"), + returnTypeText = "a INT", + queryText = Some("SELECT x"), + isTableFunc = true)) + + comparePlans( + parser.parsePlan("CREATE OR REPLACE TEMPORARY FUNCTION a() RETURNS INT RETURN 1"), + createSQLFunctionCommand("a", exprText = Some("1"), replace = true)) + + checkParseError( + "CREATE TEMPORARY FUNCTION a.b() RETURNS INT RETURN 1", + errorClass = "INVALID_SQL_SYNTAX.CREATE_TEMP_FUNC_WITH_DATABASE", + parameters = Map("database" -> "`a`"), + queryContext = Array( + ExpectedContext("CREATE TEMPORARY FUNCTION a.b() RETURNS INT RETURN 1", 0, 51) + ) + ) + + checkParseError( + "CREATE TEMPORARY FUNCTION a.b.c() RETURNS INT RETURN 1", + errorClass = "INVALID_SQL_SYNTAX.MULTI_PART_NAME", + parameters = Map( + "statement" -> "CREATE TEMPORARY FUNCTION", + "name" -> "`a`.`b`.`c`"), + queryContext = Array( + ExpectedContext("CREATE TEMPORARY FUNCTION a.b.c() RETURNS INT RETURN 1", 0, 53) + ) + ) + + checkParseError( + "CREATE TEMPORARY FUNCTION IF NOT EXISTS a() RETURNS INT RETURN 1", + errorClass = "INVALID_SQL_SYNTAX.CREATE_TEMP_FUNC_WITH_IF_NOT_EXISTS", + parameters = Map.empty, + queryContext = Array( + ExpectedContext("CREATE TEMPORARY FUNCTION IF NOT EXISTS a() RETURNS INT RETURN 1", 0, 63) + ) + ) + } + + test("create persistent SQL functions") { + comparePlans( + parser.parsePlan("CREATE FUNCTION a() RETURNS INT RETURN 1"), + createSQLFunction(Seq("a"), exprText = Some("1"))) + + comparePlans( + parser.parsePlan("CREATE FUNCTION a.b(x INT) RETURNS INT RETURN x"), + createSQLFunction(Seq("a", "b"), Some("x INT"), exprText = Some("x"))) + + comparePlans(parser.parsePlan( + "CREATE FUNCTION a.b.c(x INT) RETURNS TABLE (a INT) RETURN SELECT x"), + createSQLFunction(Seq("a", "b", "c"), Some("x INT"), returnTypeText = "a INT", None, + Some("SELECT x"), isTableFunc = true)) + + comparePlans(parser.parsePlan("CREATE FUNCTION IF NOT EXISTS a() RETURNS INT RETURN 1"), + createSQLFunction(Seq("a"), exprText = Some("1"), ignoreIfExists = true) + ) + + comparePlans(parser.parsePlan("CREATE OR REPLACE FUNCTION a() RETURNS INT RETURN 1"), + createSQLFunction(Seq("a"), exprText = Some("1"), replace = true)) + + comparePlans( + parser.parsePlan( + """ + |CREATE FUNCTION a(x INT COMMENT 'x') RETURNS INT + |LANGUAGE SQL DETERMINISTIC CONTAINS SQL + |COMMENT 'function' + |RETURN x + |""".stripMargin), + createSQLFunction(Seq("a"), inputParamText = Some("x INT COMMENT 'x'"), + exprText = Some("x"), isDeterministic = Some(true), containsSQL = Some(true), + comment = Some("function")) + ) + + intercept("CREATE OR REPLACE FUNCTION IF NOT EXISTS a() RETURNS INT RETURN 1", + "Cannot create a routine with both IF NOT EXISTS and REPLACE specified") + } + + test("create SQL functions with unsupported routine characteristics") { + intercept("CREATE FUNCTION foo() RETURNS INT LANGUAGE blah RETURN 1", + "Operation not allowed: Unsupported language for user defined functions: blah") + + intercept("CREATE FUNCTION foo() RETURNS INT SPECIFIC foo1 RETURN 1", + "Operation not allowed: SQL function with SPECIFIC name is not supported") + + intercept("CREATE FUNCTION foo() RETURNS INT NO SQL RETURN 1", + "Operation not allowed: SQL function with NO SQL is not supported") + + intercept("CREATE FUNCTION foo() RETURNS INT NO SQL CONTAINS SQL RETURN 1", + "Found duplicate clauses: SQL DATA ACCESS") + + intercept("CREATE FUNCTION foo() RETURNS INT RETURNS NULL ON NULL INPUT RETURN 1", + "Operation not allowed: SQL function with RETURNS NULL ON NULL INPUT is not supported") + + intercept("CREATE FUNCTION foo() RETURNS INT SQL SECURITY INVOKER RETURN 1", + "Operation not allowed: SQL function with SQL SECURITY INVOKER is not supported") + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index d38708ab3745c..3dea8593b428d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.command import org.apache.spark.SparkThrowable -import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, GlobalTempView, LocalTempView, SchemaCompensation, UnresolvedAttribute, UnresolvedFunctionName, UnresolvedIdentifier} import org.apache.spark.sql.catalyst.catalog.{ArchiveResource, FileResource, FunctionResource, JarResource} import org.apache.spark.sql.catalyst.dsl.expressions._ @@ -37,9 +36,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { super.parseException(parser.parsePlan)(sqlText) } - private def intercept(sqlCommand: String, messages: String*): Unit = - interceptParseException(parser.parsePlan)(sqlCommand, messages: _*)() - private def compareTransformQuery(sql: String, expected: LogicalPlan): Unit = { val plan = parser.parsePlan(sql).asInstanceOf[ScriptTransformation].copy(ioschema = null) comparePlans(plan, expected, checkAnalysis = false) @@ -827,44 +823,4 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { parser.parsePlan("SHOW CATALOGS LIKE 'defau*'"), ShowCatalogsCommand(Some("defau*"))) } - - test("Create SQL functions") { - comparePlans( - parser.parsePlan("CREATE TEMP FUNCTION foo() RETURNS INT RETURN 1"), - CreateSQLFunctionCommand( - FunctionIdentifier("foo"), - inputParamText = None, - returnTypeText = "INT", - exprText = Some("1"), - queryText = None, - comment = None, - isDeterministic = None, - containsSQL = None, - isTableFunc = false, - isTemp = true, - ignoreIfExists = false, - replace = false)) - intercept("CREATE FUNCTION foo() RETURNS INT RETURN 1", - "Operation not allowed: creating persistent SQL functions is not supported") - } - - test("create SQL functions with unsupported routine characteristics") { - intercept("CREATE FUNCTION foo() RETURNS INT LANGUAGE blah RETURN 1", - "Operation not allowed: Unsupported language for user defined functions: blah") - - intercept("CREATE FUNCTION foo() RETURNS INT SPECIFIC foo1 RETURN 1", - "Operation not allowed: SQL function with SPECIFIC name is not supported") - - intercept("CREATE FUNCTION foo() RETURNS INT NO SQL RETURN 1", - "Operation not allowed: SQL function with NO SQL is not supported") - - intercept("CREATE FUNCTION foo() RETURNS INT NO SQL CONTAINS SQL RETURN 1", - "Found duplicate clauses: SQL DATA ACCESS") - - intercept("CREATE FUNCTION foo() RETURNS INT RETURNS NULL ON NULL INPUT RETURN 1", - "Operation not allowed: SQL function with RETURNS NULL ON NULL INPUT is not supported") - - intercept("CREATE FUNCTION foo() RETURNS INT SQL SECURITY INVOKER RETURN 1", - "Operation not allowed: SQL function with SQL SECURITY INVOKER is not supported") - } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala new file mode 100644 index 0000000000000..8bb5a4b1d0bc5 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVariantShreddingSuite.scala @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources.parquet + +import java.io.File + +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.unsafe.types.VariantVal + +/** + * Test shredding Variant values in the Parquet reader/writer. + */ +class ParquetVariantShreddingSuite extends QueryTest with ParquetTest with SharedSparkSession { + + private def testWithTempDir(name: String)(block: File => Unit): Unit = test(name) { + withTempDir { dir => + block(dir) + } + } + + testWithTempDir("write shredded variant basic") { dir => + val schema = "a int, b string, c decimal(15, 1)" + val df = spark.sql( + """ + | select case + | when id = 0 then parse_json('{"a": 1, "b": "2", "c": 3.3, "d": 4.4}') + | when id = 1 then parse_json('{"a": [1,2,3], "b": "hello", "c": {"x": 0}}') + | when id = 2 then parse_json('{"A": 1, "c": 1.23}') + | end v from range(3) + |""".stripMargin) + val fullSchema = "v struct, b struct," + + "c struct>>" + withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString, + SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) { + df.write.mode("overwrite").parquet(dir.getAbsolutePath) + + + // Verify that we can read the full variant. The exact binary layout can change before and + // after shredding, so just check that the JSON representation matches. + checkAnswer( + spark.read.parquet(dir.getAbsolutePath).selectExpr("to_json(v)"), + df.selectExpr("to_json(v)").collect() + ) + + // Verify that it was shredded to the expected fields. + + val shreddedDf = spark.read.schema(fullSchema).parquet(dir.getAbsolutePath) + // Metadata should be unchanaged. + checkAnswer(shreddedDf.selectExpr("v.metadata"), + df.collect().map(v => Row(v.get(0).asInstanceOf[VariantVal].getMetadata)) + ) + + // Check typed values. + // Second row is not an integer, and third is A, not a + checkAnswer( + shreddedDf.selectExpr("v.typed_value.a.typed_value"), + Seq(Row(1), Row(null), Row(null))) + // b is missing from third row. + checkAnswer( + shreddedDf.selectExpr("v.typed_value.b.typed_value"), + Seq(Row("2"), Row("hello"), Row(null))) + // Second row is an object, third is the wrong scale. (Note: we may eventually allow the + // latter, in which case this test should be updated.) + checkAnswer( + shreddedDf.selectExpr("v.typed_value.c.typed_value"), + Seq(Row(3.3), Row(null), Row(null))) + + // Untyped values are more awkward to check, so for now just check their nullness. We + // can do more thorough checking once the reader is ready. + checkAnswer( + shreddedDf.selectExpr("v.value is null"), + // First row has "d" and third has "A". + Seq(Row(false), Row(true), Row(false))) + checkAnswer( + shreddedDf.selectExpr("v.typed_value.a.value is null"), + // First row is fully shredded, third is missing. + Seq(Row(true), Row(false), Row(true))) + checkAnswer( + shreddedDf.selectExpr("v.typed_value.b.value is null"), + // b is always fully shredded or missing. + Seq(Row(true), Row(true), Row(true))) + checkAnswer( + shreddedDf.selectExpr("v.typed_value.c.value is null"), + Seq(Row(true), Row(false), Row(false))) + // The a/b/c levels are not null, even if the field is missing. + checkAnswer( + shreddedDf.selectExpr( + "v.typed_value.a is null or v.typed_value.b is null or v.typed_value.c is null"), + Seq(Row(false), Row(false), Row(false))) + } + } + + testWithTempDir("write shredded variant array") { dir => + val schema = "array" + val df = spark.sql( + """ + | select case + | when id = 0 then parse_json('[1, "2", 3.5, null, 5]') + | when id = 1 then parse_json('{"a": [1, 2, 3]}') + | when id = 2 then parse_json('1') + | when id = 3 then parse_json('null') + | end v from range(4) + |""".stripMargin) + val fullSchema = "v struct>>" + withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString, + SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) { + df.write.mode("overwrite").parquet(dir.getAbsolutePath) + + // Verify that we can read the full variant. + checkAnswer( + spark.read.parquet(dir.getAbsolutePath).selectExpr("to_json(v)"), + df.selectExpr("to_json(v)").collect() + ) + + // Verify that it was shredded to the expected fields. + + val shreddedDf = spark.read.schema(fullSchema).parquet(dir.getAbsolutePath) + // Metadata should be unchanaged. + checkAnswer(shreddedDf.selectExpr("v.metadata"), + df.collect().map(v => Row(v.get(0).asInstanceOf[VariantVal].getMetadata)) + ) + + // Check typed values. + checkAnswer( + shreddedDf.selectExpr("v.typed_value.typed_value"), + Seq(Row(Array(1, null, null, null, 5)), Row(null), Row(null), Row(null))) + + // All the other array elements should have non-null value. + checkAnswer( + shreddedDf.selectExpr("transform(v.typed_value.value, v -> v is null)"), + Seq(Row(Array(true, false, false, false, true)), Row(null), Row(null), Row(null))) + + // The non-arrays should have non-null top-level value. + checkAnswer( + shreddedDf.selectExpr("v.value is null"), + Seq(Row(true), Row(false), Row(false), Row(false))) + } + } + + testWithTempDir("write no shredding schema") { dir => + // Check that we can write and read normally when shredding is enabled if + // we don't provide a shredding schema. + withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString) { + val df = spark.sql( + """ + | select parse_json('{"a": ' || id || ', "b": 2}') as v, + | array(parse_json('{"c": 3}'), 123::variant) as a + | from range(1, 3, 1, 1) + |""".stripMargin) + df.write.mode("overwrite").parquet(dir.getAbsolutePath) + checkAnswer( + spark.read.parquet(dir.getAbsolutePath), df.collect() + ) + } + } + + testWithTempDir("arrays and maps ignore shredding schema") { dir => + // Check that we don't try to shred array or map elements, even if a shredding schema + // is specified. + val schema = "a int" + val df = spark.sql( + """ select v, array(v) as arr, map('myKey', v) as m from + | (select parse_json('{"a":' || id || '}') v from range(3)) + |""".stripMargin) + val fullSchema = "v struct>>, " + + "arr array>, " + + "m map>" + withSQLConf(SQLConf.VARIANT_WRITE_SHREDDING_ENABLED.key -> true.toString, + SQLConf.VARIANT_FORCE_SHREDDING_SCHEMA_FOR_TEST.key -> schema) { + df.write.mode("overwrite").parquet(dir.getAbsolutePath) + + // Verify that we can read the full variant. + checkAnswer( + spark.read.parquet(dir.getAbsolutePath).selectExpr("to_json(v)"), + df.selectExpr("to_json(v)").collect() + ) + + // Verify that it was shredded to the expected fields. + + val shreddedDf = spark.read.schema(fullSchema).parquet(dir.getAbsolutePath) + // Metadata should be unchanaged. + checkAnswer(shreddedDf.selectExpr("v.metadata"), + df.selectExpr("v").collect().map(v => Row(v.get(0).asInstanceOf[VariantVal].getMetadata)) + ) + checkAnswer(shreddedDf.selectExpr("arr[0].metadata"), + df.selectExpr("arr[0]").collect().map(v => + Row(v.get(0).asInstanceOf[VariantVal].getMetadata)) + ) + checkAnswer(shreddedDf.selectExpr("m['myKey'].metadata"), + df.selectExpr("m['myKey']").collect().map( + v => Row(v.get(0).asInstanceOf[VariantVal].getMetadata)) + ) + + // v should be fully shredded, but the array and map should not be. + checkAnswer( + shreddedDf.selectExpr( + "v.value is null"), + Seq(Row(true), Row(true), Row(true))) + checkAnswer( + shreddedDf.selectExpr( + "arr[0].value is null"), + Seq(Row(false), Row(false), Row(false))) + checkAnswer( + shreddedDf.selectExpr( + "m['myKey'].value is null"), + Seq(Row(false), Row(false), Row(false))) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceTransformWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceTransformWithStateSuite.scala index b402344a73177..fe224608fffd8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceTransformWithStateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/state/StateDataSourceTransformWithStateSuite.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.functions.{col, explode, timestamp_seconds} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.{InputMapRow, ListState, MapInputEvent, MapOutputEvent, MapStateTTLProcessor, MaxEventTimeStatefulProcessor, OutputMode, RunningCountStatefulProcessor, RunningCountStatefulProcessorWithProcTimeTimerUpdates, StatefulProcessor, StateStoreMetricsTest, TestMapStateProcessor, TimeMode, TimerValues, TransformWithStateSuiteUtils, Trigger, TTLConfig, ValueState} import org.apache.spark.sql.streaming.util.StreamManualClock +import org.apache.spark.tags.SlowSQLTest import org.apache.spark.util.Utils /** Stateful processor of single value state var with non-primitive type */ @@ -125,6 +126,7 @@ class SessionGroupsStatefulProcessorWithTTL extends /** * Test suite to verify integration of state data source reader with the transformWithState operator */ +@SlowSQLTest class StateDataSourceTransformWithStateSuite extends StateStoreMetricsTest with AlsoTestWithRocksDBFeatures with AlsoTestWithEncodingTypes { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala index fe910c21cb0c6..560292b263ba3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/xml/XmlSuite.scala @@ -1930,7 +1930,7 @@ class XmlSuite } checkXmlOptionErrorMessage(Map.empty, - "[XML_ROW_TAG_MISSING] `rowTag` option is required for reading files in XML format.", + "[XML_ROW_TAG_MISSING] `rowTag` option is required for reading/writing files in XML format.", QueryCompilationErrors.xmlRowTagRequiredError(XmlOptions.ROW_TAG).getCause) checkXmlOptionErrorMessage(Map("rowTag" -> ""), "'rowTag' option should not be an empty string.") @@ -1951,6 +1951,20 @@ class XmlSuite .xml(spark.createDataset(Seq(xmlString))) } + test("SPARK-50688: rowTag requirement for write") { + withTempDir { dir => + dir.delete() + val e = intercept[AnalysisException] { + spark.range(1).write.xml(dir.getCanonicalPath) + } + checkError( + exception = e, + condition = "XML_ROW_TAG_MISSING", + parameters = Map("rowTag" -> "`rowTag`") + ) + } + } + test("Primitive field casting") { val ts = Seq("2002-05-30 21:46:54", "2002-05-30T21:46:54", "2002-05-30T21:46:54.1234", "2002-05-30T21:46:54Z", "2002-05-30T21:46:54.1234Z", "2002-05-30T21:46:54-06:00", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateSuite.scala index 601699c8320fb..97dad5fe78a19 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/TransformWithStateSuite.scala @@ -38,6 +38,7 @@ import org.apache.spark.sql.functions.timestamp_seconds import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.util.StreamManualClock import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructType} +import org.apache.spark.tags.SlowSQLTest object TransformWithStateSuiteUtils { val NUM_SHUFFLE_PARTITIONS = 5 @@ -441,6 +442,7 @@ class SleepingTimerProcessor extends StatefulProcessor[String, String, String] { /** * Class that adds tests for transformWithState stateful streaming operator */ +@SlowSQLTest class TransformWithStateSuite extends StateStoreMetricsTest with AlsoTestWithRocksDBFeatures with AlsoTestWithEncodingTypes { diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala index 2acf25640ef78..da0ddd3a156f7 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala @@ -214,7 +214,7 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer { val sessionHandle = client.openSession(user, "") val infoValue = client.getInfo(sessionHandle, GetInfoType.CLI_ODBC_KEYWORDS) // scalastyle:off line.size.limit - assert(infoValue.getStringValue == "ADD,AFTER,AGGREGATE,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,AT,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONSTRAINT,CONTAINS,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFINED,DEFINER,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,END,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GLOBAL,GRANT,GROUP,GROUPING,HAVING,HOUR,HOURS,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MERGE,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NO,NONE,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,REDUCE,REFERENCES,REFRESH,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,START,STATISTICS,STORED,STRATIFY,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,X,YEAR,YEARS,ZONE") + assert(infoValue.getStringValue == "ADD,AFTER,AGGREGATE,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,AT,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONSTRAINT,CONTAINS,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFINED,DEFINER,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,END,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GLOBAL,GRANT,GROUP,GROUPING,HAVING,HOUR,HOURS,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MERGE,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NO,NONE,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,RECURSIVE,REDUCE,REFERENCES,REFRESH,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,START,STATISTICS,STORED,STRATIFY,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,X,YEAR,YEARS,ZONE") // scalastyle:on line.size.limit } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index c41370c96241a..5431066c30a9f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -72,7 +72,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd override def afterEach(): Unit = { try { - spark.artifactManager.cleanUpResources() + spark.artifactManager.cleanUpResourcesForTesting() } finally { super.afterEach() } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala index 770e1da94a1c7..7a0599cda2fe7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHiveSingleton.scala @@ -42,7 +42,7 @@ trait TestHiveSingleton extends SparkFunSuite with BeforeAndAfterAll { protected override def afterEach(): Unit = { try { - spark.artifactManager.cleanUpResources() + spark.artifactManager.cleanUpResourcesForTesting() } finally { super.afterEach() }