Skip to content

Commit

Permalink
Resolve scalastyle errors.
Browse files Browse the repository at this point in the history
  • Loading branch information
ptkool committed Apr 20, 2017
1 parent 3f1e6a1 commit 3986247
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 29 deletions.
2 changes: 1 addition & 1 deletion python/pyspark/sql/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ def nanvl(col1, col2):

@since(2.2)
def no_collapse(df):
"""Marks a DataFrame as small enough for use in broadcast joins."""
"""Marks a DataFrame as non-collapsible."""

sc = SparkContext._active_spark_context
return DataFrame(sc._jvm.functions.no_collapse(df._jdf), df.sql_ctx)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -537,10 +537,5 @@ class PlanParserSuite extends PlanTest {
comparePlans(
parsePlan("SELECT /*+ MAPJOIN(t) */ a from t where true group by a order by a"),
Hint("MAPJOIN", Seq("t"), table("t").where(Literal(true)).groupBy('a)('a)).orderBy('a.asc))

comparePlans(
parsePlan("SELECT a FROM (SELECT /*+ NO_COLLAPSE */ * FROM t) t1"),
SubqueryAlias("t1", Hint("NO_COLLAPSE", Seq.empty, table("t").select(star())))
.select('a))
}
}
47 changes: 24 additions & 23 deletions sql/core/src/main/scala/org/apache/spark/sql/functions.scala
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@ package org.apache.spark.sql

import scala.collection.JavaConverters._
import scala.language.implicitConversions
import scala.reflect.runtime.universe.{TypeTag, typeTag}
import scala.reflect.runtime.universe.{typeTag, TypeTag}
import scala.util.Try
import scala.util.control.NonFatal

import org.apache.spark.annotation.{Experimental, InterfaceStability}
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.catalyst.analysis.{Star, UnresolvedFunction}
Expand Down Expand Up @@ -1006,33 +1007,33 @@ object functions {
def map(cols: Column*): Column = withExpr { CreateMap(cols.map(_.expr)) }

/**
* Marks a DataFrame as small enough for use in broadcast joins.
*
* The following example marks the right DataFrame for broadcast hash join using `joinKey`.
* {{{
* // left and right are DataFrames
* left.join(broadcast(right), "joinKey")
* }}}
*
* @group normal_funcs
* @since 1.5.0
*/
* Marks a DataFrame as small enough for use in broadcast joins.
*
* The following example marks the right DataFrame for broadcast hash join using `joinKey`.
* {{{
* // left and right are DataFrames
* left.join(broadcast(right), "joinKey")
* }}}
*
* @group normal_funcs
* @since 1.5.0
*/
def broadcast[T](df: Dataset[T]): Dataset[T] = {
Dataset[T](df.sparkSession, BroadcastHint(df.logicalPlan))(df.exprEnc)
}

/**
* Marks a DataFrame as small enough for use in broadcast joins.
*
* The following example marks the right DataFrame for broadcast hash join using `joinKey`.
* {{{
* // left and right are DataFrames
* left.join(broadcast(right), "joinKey")
* }}}
*
* @group normal_funcs
* @since 1.5.0
*/
* Marks a DataFrame as non-collapsible.
*
* For example:
* {{{
* df1 = no_collapse(df.select((df.col("qty") * lit(10).alias("c1")))
* df2 = df1.select(col("c1") + lit(1)), col("c1") + lit(2)))
* }}}
*
* @group normal_funcs
* @since 2.2.0
*/
def no_collapse[T](df: Dataset[T]): Dataset[T] = {
Dataset[T](df.sparkSession, NoCollapseHint(df.logicalPlan))(df.exprEnc)
}
Expand Down

0 comments on commit 3986247

Please sign in to comment.