From 456e759421a172a2c4c4569f90b6831eef3382a4 Mon Sep 17 00:00:00 2001 From: Mihailo Timotic Date: Wed, 9 Oct 2024 16:06:41 +0200 Subject: [PATCH] Add type alias to map --- .../3221d272-3ff0-420e-a4bd-d91eb9b00a29/benchmarks | 1 + core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/src | 1 + .../tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/target | 1 + .../3270e079-5639-4351-a967-3083f2a973fe/benchmarks | 1 + core/tasks/3270e079-5639-4351-a967-3083f2a973fe/src | 1 + .../tasks/3270e079-5639-4351-a967-3083f2a973fe/target | 1 + .../3b2b9eee-4487-4964-b116-46a1cf04b6c9/benchmarks | 1 + core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/src | 1 + .../tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/target | 1 + .../5123f8c2-1e56-4cb9-a3d7-1761fde853a8/benchmarks | 1 + core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/src | 1 + .../tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/target | 1 + .../7b995caf-4467-48d7-8214-a2dfeaa2a8bb/benchmarks | 1 + core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/src | 1 + .../tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/target | 1 + .../ded36344-5710-4c12-8b60-e188d9269fc7/benchmarks | 1 + core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/src | 1 + .../tasks/ded36344-5710-4c12-8b60-e188d9269fc7/target | 1 + .../sql/catalyst/analysis/DeduplicateRelations.scala | 11 +++++++---- 19 files changed, 25 insertions(+), 4 deletions(-) create mode 120000 core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/benchmarks create mode 120000 core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/src create mode 120000 core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/target create mode 120000 core/tasks/3270e079-5639-4351-a967-3083f2a973fe/benchmarks create mode 120000 core/tasks/3270e079-5639-4351-a967-3083f2a973fe/src create mode 120000 core/tasks/3270e079-5639-4351-a967-3083f2a973fe/target create mode 120000 core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/benchmarks create mode 120000 core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/src create mode 120000 core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/target create mode 120000 core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/benchmarks create mode 120000 core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/src create mode 120000 core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/target create mode 120000 core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/benchmarks create mode 120000 core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/src create mode 120000 core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/target create mode 120000 core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/benchmarks create mode 120000 core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/src create mode 120000 core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/target diff --git a/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/benchmarks b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/src b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/target b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/3221d272-3ff0-420e-a4bd-d91eb9b00a29/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/benchmarks b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/src b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/target b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/3270e079-5639-4351-a967-3083f2a973fe/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/benchmarks b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/src b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/target b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/3b2b9eee-4487-4964-b116-46a1cf04b6c9/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/benchmarks b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/src b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/target b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/5123f8c2-1e56-4cb9-a3d7-1761fde853a8/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/benchmarks b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/src b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/target b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/7b995caf-4467-48d7-8214-a2dfeaa2a8bb/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/benchmarks b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/benchmarks new file mode 120000 index 0000000000000..40e1acf441fa9 --- /dev/null +++ b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/benchmarks @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./benchmarks \ No newline at end of file diff --git a/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/src b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/src new file mode 120000 index 0000000000000..b5f58663ed01c --- /dev/null +++ b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/src @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./src \ No newline at end of file diff --git a/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/target b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/target new file mode 120000 index 0000000000000..23c8fb6053e83 --- /dev/null +++ b/core/tasks/ded36344-5710-4c12-8b60-e188d9269fc7/target @@ -0,0 +1 @@ +/Users/mihailo.timotic/spark/core/./target \ No newline at end of file diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala index acc2f2b11d6ec..8181078c519fc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DeduplicateRelations.scala @@ -25,6 +25,9 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreePattern._ object DeduplicateRelations extends Rule[LogicalPlan] { + + type ExprIdMap = mutable.HashMap[Class[_], mutable.HashSet[Long]] + override def apply(plan: LogicalPlan): LogicalPlan = { val newPlan = renewDuplicatedRelations(mutable.HashMap.empty, plan)._1 @@ -75,7 +78,7 @@ object DeduplicateRelations extends Rule[LogicalPlan] { } private def existDuplicatedExprId( - existingRelations: mutable.HashMap[Class[_], mutable.HashSet[Long]], + existingRelations: ExprIdMap, planClass: Class[_], exprIds: Seq[Long]): Boolean = { val attrSet = existingRelations.getOrElse(planClass, mutable.HashSet.empty) exprIds.exists(attrSet.contains) @@ -89,7 +92,7 @@ object DeduplicateRelations extends Rule[LogicalPlan] { * whether the plan is changed or not) */ private def renewDuplicatedRelations( - existingRelations: mutable.HashMap[Class[_], mutable.HashSet[Long]], + existingRelations: ExprIdMap, plan: LogicalPlan): (LogicalPlan, Boolean) = plan match { case p: LogicalPlan if p.isStreaming => (plan, false) @@ -192,7 +195,7 @@ object DeduplicateRelations extends Rule[LogicalPlan] { } private def deduplicate( - existingRelations: mutable.HashMap[Class[_], mutable.HashSet[Long]], + existingRelations: ExprIdMap, plan: LogicalPlan): (LogicalPlan, Boolean) = { var planChanged = false val newPlan = if (plan.children.nonEmpty) { @@ -276,7 +279,7 @@ object DeduplicateRelations extends Rule[LogicalPlan] { } private def deduplicateAndRenew[T <: LogicalPlan]( - existingRelations: mutable.HashMap[Class[_], mutable.HashSet[Long]], plan: T, + existingRelations: ExprIdMap, plan: T, getExprIds: T => Seq[Long], copyNewPlan: T => T): (LogicalPlan, Boolean) = { var (newPlan, planChanged) = deduplicate(existingRelations, plan)