+ * Within each long[] buffer, position {@code 2 * i} holds a pointer pointer to the record at
+ * index {@code i}, while position {@code 2 * i + 1} in the array holds an 8-byte key prefix.
+ */
+final class UnsafeSortDataFormat extends SortDataFormat
+ * Map(
+ * "tachyon" -> Set("true"),
+ * "zone": -> Set("us-east-1a", "us-east-1b")
+ * )
+ * }}}
+ *
+ * Mesos documentation: http://mesos.apache.org/documentation/attributes-resources/
+ * https://github.com/apache/mesos/blob/master/src/common/values.cpp
+ * https://github.com/apache/mesos/blob/master/src/common/attributes.cpp
+ *
+ * @param constraintsVal constaints string consisting of ';' separated key-value pairs (separated
+ * by ':')
+ * @return Map of constraints to match resources offers.
+ */
+ def parseConstraintString(constraintsVal: String): Map[String, Set[String]] = {
+ /*
+ Based on mesos docs:
+ attributes : attribute ( ";" attribute )*
+ attribute : labelString ":" ( labelString | "," )+
+ labelString : [a-zA-Z0-9_/.-]
+ */
+ val splitter = Splitter.on(';').trimResults().withKeyValueSeparator(':')
+ // kv splitter
+ if (constraintsVal.isEmpty) {
+ Map()
+ } else {
+ try {
+ Map() ++ mapAsScalaMap(splitter.split(constraintsVal)).map {
+ case (k, v) =>
+ if (v == null || v.isEmpty) {
+ (k, Set[String]())
+ } else {
+ (k, v.split(',').toSet)
+ }
+ }
+ } catch {
+ case NonFatal(e) =>
+ throw new IllegalArgumentException(s"Bad constraint string: $constraintsVal", e)
+ }
+ }
+ }
+
+ // These defaults copied from YARN
+ private val MEMORY_OVERHEAD_FRACTION = 0.10
+ private val MEMORY_OVERHEAD_MINIMUM = 384
+
+ /**
+ * Return the amount of memory to allocate to each executor, taking into account
+ * container overheads.
+ * @param sc SparkContext to use to get `spark.mesos.executor.memoryOverhead` value
+ * @return memory requirement as (0.1 *
@@ -346,7 +346,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
// machine and to send back the result (but not the time to fetch the task result,
// if it needed to be fetched from the block manager on the worker).
val schedulerDelays = validTasks.map { case TaskUIData(info, metrics, _) =>
- getSchedulerDelay(info, metrics.get).toDouble
+ getSchedulerDelay(info, metrics.get, currentTime).toDouble
}
val schedulerDelayTitle = Scheduler Delay
@@ -544,7 +544,7 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val serializationTimeProportion = toProportion(serializationTime)
val deserializationTime = metricsOpt.map(_.executorDeserializeTime).getOrElse(0L)
val deserializationTimeProportion = toProportion(deserializationTime)
- val gettingResultTime = getGettingResultTime(taskUIData.taskInfo)
+ val gettingResultTime = getGettingResultTime(taskUIData.taskInfo, currentTime)
val gettingResultTimeProportion = toProportion(gettingResultTime)
val schedulerDelay = totalExecutionTime -
(executorComputingTime + shuffleReadTime + shuffleWriteTime +
@@ -685,11 +685,11 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
else metrics.map(_.executorRunTime).getOrElse(1L)
val formatDuration = if (info.status == "RUNNING") UIUtils.formatDuration(duration)
else metrics.map(m => UIUtils.formatDuration(m.executorRunTime)).getOrElse("")
- val schedulerDelay = metrics.map(getSchedulerDelay(info, _)).getOrElse(0L)
+ val schedulerDelay = metrics.map(getSchedulerDelay(info, _, currentTime)).getOrElse(0L)
val gcTime = metrics.map(_.jvmGCTime).getOrElse(0L)
val taskDeserializationTime = metrics.map(_.executorDeserializeTime).getOrElse(0L)
val serializationTime = metrics.map(_.resultSerializationTime).getOrElse(0L)
- val gettingResultTime = getGettingResultTime(info)
+ val gettingResultTime = getGettingResultTime(info, currentTime)
val maybeAccumulators = info.accumulables
val accumulatorsReadable = maybeAccumulators.map{acc => s"${acc.name}: ${acc.update.get}"}
@@ -852,32 +852,31 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
{errorSummary}{details}
}
- private def getGettingResultTime(info: TaskInfo): Long = {
- if (info.gettingResultTime > 0) {
- if (info.finishTime > 0) {
+ private def getGettingResultTime(info: TaskInfo, currentTime: Long): Long = {
+ if (info.gettingResult) {
+ if (info.finished) {
info.finishTime - info.gettingResultTime
} else {
// The task is still fetching the result.
- System.currentTimeMillis - info.gettingResultTime
+ currentTime - info.gettingResultTime
}
} else {
0L
}
}
- private def getSchedulerDelay(info: TaskInfo, metrics: TaskMetrics): Long = {
- val totalExecutionTime =
- if (info.gettingResult) {
- info.gettingResultTime - info.launchTime
- } else if (info.finished) {
- info.finishTime - info.launchTime
- } else {
- 0
- }
- val executorOverhead = (metrics.executorDeserializeTime +
- metrics.resultSerializationTime)
- math.max(
- 0,
- totalExecutionTime - metrics.executorRunTime - executorOverhead - getGettingResultTime(info))
+ private def getSchedulerDelay(info: TaskInfo, metrics: TaskMetrics, currentTime: Long): Long = {
+ if (info.finished) {
+ val totalExecutionTime = info.finishTime - info.launchTime
+ val executorOverhead = (metrics.executorDeserializeTime +
+ metrics.resultSerializationTime)
+ math.max(
+ 0,
+ totalExecutionTime - metrics.executorRunTime - executorOverhead -
+ getGettingResultTime(info, currentTime))
+ } else {
+ // The task is still running and the metrics like executorRunTime are not available.
+ 0L
+ }
}
}
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
index 07db783c572cf..04f584621e71e 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StoragePage.scala
@@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
-import org.apache.spark.storage.RDDInfo
+import org.apache.spark.storage._
import org.apache.spark.ui.{UIUtils, WebUIPage}
import org.apache.spark.util.Utils
@@ -30,13 +30,25 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
private val listener = parent.listener
def render(request: HttpServletRequest): Seq[Node] = {
- val rdds = listener.rddInfoList
- val content = UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))
+ val content = rddTable(listener.rddInfoList) ++
+ receiverBlockTables(listener.allExecutorStreamBlockStatus.sortBy(_.executorId))
UIUtils.headerSparkPage("Storage", content, parent)
}
+ private[storage] def rddTable(rdds: Seq[RDDInfo]): Seq[Node] = {
+ if (rdds.isEmpty) {
+ // Don't show the rdd table if there is no RDD persisted.
+ Nil
+ } else {
+ RDDs
+ {UIUtils.listingTable(rddHeader, rddRow, rdds, id = Some("storage-by-rdd-table"))}
+ {rdd.storageLevel.description}
- {rdd.numCachedPartitions}
+ {rdd.numCachedPartitions.toString}
{"%.0f%%".format(rdd.numCachedPartitions * 100.0 / rdd.numPartitions)}
{Utils.bytesToString(rdd.memSize)}
{Utils.bytesToString(rdd.externalBlockStoreSize)}
@@ -64,4 +76,130 @@ private[ui] class StoragePage(parent: StorageTab) extends WebUIPage("") {
// scalastyle:on
}
+
+ private[storage] def receiverBlockTables(statuses: Seq[ExecutorStreamBlockStatus]): Seq[Node] = {
+ if (statuses.map(_.numStreamBlocks).sum == 0) {
+ // Don't show the tables if there is no stream block
+ Nil
+ } else {
+ val blocks = statuses.flatMap(_.blocks).groupBy(_.blockId).toSeq.sortBy(_._1.toString)
+
+ Receiver Blocks
+ {executorMetricsTable(statuses)}
+ {streamBlockTable(blocks)}
+ Aggregated Block Metrics by Executor
+ {UIUtils.listingTable(executorMetricsTableHeader, executorMetricsTableRow, statuses,
+ id = Some("storage-by-executor-stream-blocks"))}
+
+
+ }
+
+ private def streamBlockTable(blocks: Seq[(BlockId, Seq[BlockUIData])]): Seq[Node] = {
+ if (blocks.isEmpty) {
+ Nil
+ } else {
+
+ {status.executorId}
+
+
+ {status.location}
+
+
+ {Utils.bytesToString(status.totalMemSize)}
+
+
+ {Utils.bytesToString(status.totalExternalBlockStoreSize)}
+
+
+ {Utils.bytesToString(status.totalDiskSize)}
+
+
+ {status.numStreamBlocks.toString}
+
+ Blocks
+ {UIUtils.listingTable(
+ streamBlockTableHeader,
+ streamBlockTableRow,
+ blocks,
+ id = Some("storage-by-block-table"),
+ sortable = false)}
+
+ {
+ if (firstSubrow) {
+
+ }
+
+ private[storage] def streamBlockStorageLevelDescriptionAndSize(
+ block: BlockUIData): (String, Long) = {
+ if (block.storageLevel.useDisk) {
+ ("Disk", block.diskSize)
+ } else if (block.storageLevel.useMemory && block.storageLevel.deserialized) {
+ ("Memory", block.memSize)
+ } else if (block.storageLevel.useMemory && !block.storageLevel.deserialized) {
+ ("Memory Serialized", block.memSize)
+ } else if (block.storageLevel.useOffHeap) {
+ ("External", block.externalBlockStoreSize)
+ } else {
+ throw new IllegalStateException(s"Invalid Storage Level: ${block.storageLevel}")
+ }
+ }
+
}
diff --git a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
index 0351749700962..22e2993b3b5bd 100644
--- a/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/storage/StorageTab.scala
@@ -39,7 +39,8 @@ private[ui] class StorageTab(parent: SparkUI) extends SparkUITab(parent, "storag
* This class is thread-safe (unlike JobProgressListener)
*/
@DeveloperApi
-class StorageListener(storageStatusListener: StorageStatusListener) extends SparkListener {
+class StorageListener(storageStatusListener: StorageStatusListener) extends BlockStatusListener {
+
private[ui] val _rddInfoMap = mutable.Map[Int, RDDInfo]() // exposed for testing
def storageStatusList: Seq[StorageStatus] = storageStatusListener.storageStatusList
diff --git a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
index 305de4c75539d..43626b4ef4880 100644
--- a/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
+++ b/core/src/main/scala/org/apache/spark/util/ClosureCleaner.scala
@@ -448,10 +448,12 @@ private class InnerClosureFinder(output: Set[Class[_]]) extends ClassVisitor(ASM
if (op == INVOKESPECIAL && name == "
+ {block.blockId.toString}
+
+
+ {replication.toString}
+
+ }
+ }
+ {block.location}
+ {storageLevel}
+ {Utils.bytesToString(size)}
+
Initial size of Kryo's serialization buffer. Note that there will be one buffer
per core on each worker. This buffer will grow up to
-
spark.kryoserializer.buffer.max.mb if needed.
+ spark.kryoserializer.buffer.max if needed.
@@ -1007,9 +1007,9 @@ Apart from these, the following properties are also available, and may be useful
spark.rpc.numRetries3
+
Number of times to retry before an RPC task gives up.
An RPC task will run at most times of this number.
-
@@ -1029,8 +1029,8 @@ Apart from these, the following properties are also available, and may be useful
@@ -1206,7 +1206,7 @@ Apart from these, the following properties are also available, and may be useful
spark.rpc.lookupTimeout120s
- Duration for an RPC remote endpoint lookup operation to wait before timing out.
+ Duration for an RPC remote endpoint lookup operation to wait before timing out.
spark.dynamicAllocation.cachedExecutorIdleTimeout2 * executorIdleTimeout
+ infinity
If dynamic allocation is enabled and an executor which has cached data blocks has been idle for more than this duration,
the executor will be removed. For more details, see this
@@ -1222,7 +1222,7 @@ Apart from these, the following properties are also available, and may be useful
+spark.dynamicAllocation.maxExecutorsInteger.MAX_VALUE
+ infinity
Upper bound for the number of executors if dynamic allocation is enabled.
diff --git a/docs/ml-features.md b/docs/ml-features.md
index f88c0248c1a8a..54068debe2159 100644
--- a/docs/ml-features.md
+++ b/docs/ml-features.md
@@ -288,6 +288,94 @@ for words_label in wordsDataFrame.select("words", "label").take(3):
+## $n$-gram
+
+An [n-gram](https://en.wikipedia.org/wiki/N-gram) is a sequence of $n$ tokens (typically words) for some integer $n$. The `NGram` class can be used to transform input features into $n$-grams.
+
+`NGram` takes as input a sequence of strings (e.g. the output of a [Tokenizer](ml-features.html#tokenizer). The parameter `n` is used to determine the number of terms in each $n$-gram. The output will consist of a sequence of $n$-grams where each $n$-gram is represented by a space-delimited string of $n$ consecutive words. If the input sequence contains fewer than `n` strings, no output is produced.
+
+
+
# Troubleshooting and Debugging
diff --git a/docs/sparkr.md b/docs/sparkr.md
index 095ea4308cfeb..4385a4eeacd5c 100644
--- a/docs/sparkr.md
+++ b/docs/sparkr.md
@@ -68,7 +68,7 @@ you can specify the packages with the `packages` argument.
spark.mesos.constraintsAttribute based constraints to be matched against when accepting resource offers.
+
+ Attribute based constraints on mesos resource offers. By default, all resource offers will be accepted. Refer to Mesos Attributes & Resources for more information on attributes.
+
+
+
+
* Use this class to start Spark applications programmatically. The class uses a builder pattern * to allow clients to configure the Spark application and launch it as a child process. + *
*/ public class SparkLauncher { diff --git a/launcher/src/main/java/org/apache/spark/launcher/package-info.java b/launcher/src/main/java/org/apache/spark/launcher/package-info.java index 7ed756f4b8591..7c97dba511b28 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/package-info.java +++ b/launcher/src/main/java/org/apache/spark/launcher/package-info.java @@ -17,13 +17,17 @@ /** * Library for launching Spark applications. - * + * + ** This library allows applications to launch Spark programmatically. There's only one entry * point to the library - the {@link org.apache.spark.launcher.SparkLauncher} class. - *
+ * + * + ** To launch a Spark application, just instantiate a {@link org.apache.spark.launcher.SparkLauncher} * and configure the application to run. For example: - * + *
+ * *
* {@code
* import org.apache.spark.launcher.SparkLauncher;
diff --git a/make-distribution.sh b/make-distribution.sh
index 9f063da3a16c0..cac7032bb2e87 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -219,6 +219,7 @@ cp -r "$SPARK_HOME/ec2" "$DISTDIR"
if [ -d "$SPARK_HOME"/R/lib/SparkR ]; then
mkdir -p "$DISTDIR"/R/lib
cp -r "$SPARK_HOME/R/lib/SparkR" "$DISTDIR"/R/lib
+ cp "$SPARK_HOME/R/lib/sparkr.zip" "$DISTDIR"/R/lib
fi
# Download and copy in tachyon, if requested
diff --git a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
index a1f3851d804ff..aef2c019d2871 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala
@@ -95,6 +95,8 @@ class Pipeline(override val uid: String) extends Estimator[PipelineModel] {
/** @group setParam */
def setStages(value: Array[PipelineStage]): this.type = { set(stages, value); this }
+ // Below, we clone stages so that modifications to the list of stages will not change
+ // the Param value in the Pipeline.
/** @group getParam */
def getStages: Array[PipelineStage] = $(stages).clone()
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index 2e6eedd45ab07..8fc9199fb4602 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.classification
import scala.collection.mutable
-import breeze.linalg.{DenseVector => BDV, norm => brzNorm}
+import breeze.linalg.{DenseVector => BDV}
import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, OWLQN => BreezeOWLQN}
import org.apache.spark.{Logging, SparkException}
@@ -41,7 +41,7 @@ import org.apache.spark.storage.StorageLevel
*/
private[classification] trait LogisticRegressionParams extends ProbabilisticClassifierParams
with HasRegParam with HasElasticNetParam with HasMaxIter with HasFitIntercept with HasTol
- with HasThreshold
+ with HasThreshold with HasStandardization
/**
* :: Experimental ::
@@ -98,6 +98,18 @@ class LogisticRegression(override val uid: String)
def setFitIntercept(value: Boolean): this.type = set(fitIntercept, value)
setDefault(fitIntercept -> true)
+ /**
+ * Whether to standardize the training features before fitting the model.
+ * The coefficients of models will be always returned on the original scale,
+ * so it will be transparent for users. Note that when no regularization,
+ * with or without standardization, the models should be always converged to
+ * the same solution.
+ * Default is true.
+ * @group setParam
+ * */
+ def setStandardization(value: Boolean): this.type = set(standardization, value)
+ setDefault(standardization -> true)
+
/** @group setParam */
def setThreshold(value: Double): this.type = set(threshold, value)
setDefault(threshold -> 0.5)
@@ -116,7 +128,7 @@ class LogisticRegression(override val uid: String)
case ((summarizer: MultivariateOnlineSummarizer, labelSummarizer: MultiClassSummarizer),
(label: Double, features: Vector)) =>
(summarizer.add(features), labelSummarizer.add(label))
- },
+ },
combOp = (c1, c2) => (c1, c2) match {
case ((summarizer1: MultivariateOnlineSummarizer,
classSummarizer1: MultiClassSummarizer), (summarizer2: MultivariateOnlineSummarizer,
@@ -149,15 +161,28 @@ class LogisticRegression(override val uid: String)
val regParamL1 = $(elasticNetParam) * $(regParam)
val regParamL2 = (1.0 - $(elasticNetParam)) * $(regParam)
- val costFun = new LogisticCostFun(instances, numClasses, $(fitIntercept),
+ val costFun = new LogisticCostFun(instances, numClasses, $(fitIntercept), $(standardization),
featuresStd, featuresMean, regParamL2)
val optimizer = if ($(elasticNetParam) == 0.0 || $(regParam) == 0.0) {
new BreezeLBFGS[BDV[Double]]($(maxIter), 10, $(tol))
} else {
- // Remove the L1 penalization on the intercept
def regParamL1Fun = (index: Int) => {
- if (index == numFeatures) 0.0 else regParamL1
+ // Remove the L1 penalization on the intercept
+ if (index == numFeatures) {
+ 0.0
+ } else {
+ if ($(standardization)) {
+ regParamL1
+ } else {
+ // If `standardization` is false, we still standardize the data
+ // to improve the rate of convergence; as a result, we have to
+ // perform this reverse standardization by penalizing each component
+ // differently to get effectively the same objective function when
+ // the training dataset is not standardized.
+ if (featuresStd(index) != 0.0) regParamL1 / featuresStd(index) else 0.0
+ }
+ }
}
new BreezeOWLQN[Int, BDV[Double]]($(maxIter), 10, regParamL1Fun, $(tol))
}
@@ -166,18 +191,18 @@ class LogisticRegression(override val uid: String)
Vectors.zeros(if ($(fitIntercept)) numFeatures + 1 else numFeatures)
if ($(fitIntercept)) {
- /**
- * For binary logistic regression, when we initialize the weights as zeros,
- * it will converge faster if we initialize the intercept such that
- * it follows the distribution of the labels.
- *
- * {{{
- * P(0) = 1 / (1 + \exp(b)), and
- * P(1) = \exp(b) / (1 + \exp(b))
- * }}}, hence
- * {{{
- * b = \log{P(1) / P(0)} = \log{count_1 / count_0}
- * }}}
+ /*
+ For binary logistic regression, when we initialize the weights as zeros,
+ it will converge faster if we initialize the intercept such that
+ it follows the distribution of the labels.
+
+ {{{
+ P(0) = 1 / (1 + \exp(b)), and
+ P(1) = \exp(b) / (1 + \exp(b))
+ }}}, hence
+ {{{
+ b = \log{P(1) / P(0)} = \log{count_1 / count_0}
+ }}}
*/
initialWeightsWithIntercept.toArray(numFeatures)
= math.log(histogram(1).toDouble / histogram(0).toDouble)
@@ -186,39 +211,48 @@ class LogisticRegression(override val uid: String)
val states = optimizer.iterations(new CachedDiffFunction(costFun),
initialWeightsWithIntercept.toBreeze.toDenseVector)
- var state = states.next()
- val lossHistory = mutable.ArrayBuilder.make[Double]
+ val (weights, intercept, objectiveHistory) = {
+ /*
+ Note that in Logistic Regression, the objective history (loss + regularization)
+ is log-likelihood which is invariance under feature standardization. As a result,
+ the objective history from optimizer is the same as the one in the original space.
+ */
+ val arrayBuilder = mutable.ArrayBuilder.make[Double]
+ var state: optimizer.State = null
+ while (states.hasNext) {
+ state = states.next()
+ arrayBuilder += state.adjustedValue
+ }
- while (states.hasNext) {
- lossHistory += state.value
- state = states.next()
- }
- lossHistory += state.value
+ if (state == null) {
+ val msg = s"${optimizer.getClass.getName} failed."
+ logError(msg)
+ throw new SparkException(msg)
+ }
- // The weights are trained in the scaled space; we're converting them back to
- // the original space.
- val weightsWithIntercept = {
+ /*
+ The weights are trained in the scaled space; we're converting them back to
+ the original space.
+ Note that the intercept in scaled space and original space is the same;
+ as a result, no scaling is needed.
+ */
val rawWeights = state.x.toArray.clone()
var i = 0
- // Note that the intercept in scaled space and original space is the same;
- // as a result, no scaling is needed.
while (i < numFeatures) {
rawWeights(i) *= { if (featuresStd(i) != 0.0) 1.0 / featuresStd(i) else 0.0 }
i += 1
}
- Vectors.dense(rawWeights)
+
+ if ($(fitIntercept)) {
+ (Vectors.dense(rawWeights.dropRight(1)).compressed, rawWeights.last, arrayBuilder.result())
+ } else {
+ (Vectors.dense(rawWeights).compressed, 0.0, arrayBuilder.result())
+ }
}
if (handlePersistence) instances.unpersist()
- val (weights, intercept) = if ($(fitIntercept)) {
- (Vectors.dense(weightsWithIntercept.toArray.slice(0, weightsWithIntercept.size - 1)),
- weightsWithIntercept(weightsWithIntercept.size - 1))
- } else {
- (weightsWithIntercept, 0.0)
- }
-
- new LogisticRegressionModel(uid, weights.compressed, intercept)
+ copyValues(new LogisticRegressionModel(uid, weights, intercept))
}
override def copy(extra: ParamMap): LogisticRegression = defaultCopy(extra)
@@ -423,16 +457,12 @@ private class LogisticAggregator(
require(dim == data.size, s"Dimensions mismatch when adding new sample." +
s" Expecting $dim but got ${data.size}.")
- val dataSize = data.size
-
val localWeightsArray = weightsArray
val localGradientSumArray = gradientSumArray
numClasses match {
case 2 =>
- /**
- * For Binary Logistic Regression.
- */
+ // For Binary Logistic Regression.
val margin = - {
var sum = 0.0
data.foreachActive { (index, value) =>
@@ -518,11 +548,13 @@ private class LogisticCostFun(
data: RDD[(Double, Vector)],
numClasses: Int,
fitIntercept: Boolean,
+ standardization: Boolean,
featuresStd: Array[Double],
featuresMean: Array[Double],
regParamL2: Double) extends DiffFunction[BDV[Double]] {
override def calculate(weights: BDV[Double]): (Double, BDV[Double]) = {
+ val numFeatures = featuresStd.length
val w = Vectors.fromBreeze(weights)
val logisticAggregator = data.treeAggregate(new LogisticAggregator(w, numClasses, fitIntercept,
@@ -534,27 +566,43 @@ private class LogisticCostFun(
case (aggregator1, aggregator2) => aggregator1.merge(aggregator2)
})
- // regVal is the sum of weight squares for L2 regularization
- val norm = if (regParamL2 == 0.0) {
- 0.0
- } else if (fitIntercept) {
- brzNorm(Vectors.dense(weights.toArray.slice(0, weights.size -1)).toBreeze, 2.0)
- } else {
- brzNorm(weights, 2.0)
- }
- val regVal = 0.5 * regParamL2 * norm * norm
-
- val loss = logisticAggregator.loss + regVal
- val gradient = logisticAggregator.gradient
+ val totalGradientArray = logisticAggregator.gradient.toArray
- if (fitIntercept) {
- val wArray = w.toArray.clone()
- wArray(wArray.length - 1) = 0.0
- axpy(regParamL2, Vectors.dense(wArray), gradient)
+ // regVal is the sum of weight squares excluding intercept for L2 regularization.
+ val regVal = if (regParamL2 == 0.0) {
+ 0.0
} else {
- axpy(regParamL2, w, gradient)
+ var sum = 0.0
+ w.foreachActive { (index, value) =>
+ // If `fitIntercept` is true, the last term which is intercept doesn't
+ // contribute to the regularization.
+ if (index != numFeatures) {
+ // The following code will compute the loss of the regularization; also
+ // the gradient of the regularization, and add back to totalGradientArray.
+ sum += {
+ if (standardization) {
+ totalGradientArray(index) += regParamL2 * value
+ value * value
+ } else {
+ if (featuresStd(index) != 0.0) {
+ // If `standardization` is false, we still standardize the data
+ // to improve the rate of convergence; as a result, we have to
+ // perform this reverse standardization by penalizing each component
+ // differently to get effectively the same objective function when
+ // the training dataset is not standardized.
+ val temp = value / (featuresStd(index) * featuresStd(index))
+ totalGradientArray(index) += regParamL2 * temp
+ value * temp
+ } else {
+ 0.0
+ }
+ }
+ }
+ }
+ }
+ 0.5 * regParamL2 * sum
}
- (loss, gradient.toBreeze.asInstanceOf[BDV[Double]])
+ (logisticAggregator.loss + regVal, new BDV(totalGradientArray))
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizerModel.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizerModel.scala
new file mode 100644
index 0000000000000..6b77de89a0330
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizerModel.scala
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.feature
+
+import scala.collection.mutable
+
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.ml.UnaryTransformer
+import org.apache.spark.ml.param.{ParamMap, ParamValidators, IntParam}
+import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.linalg.{Vectors, VectorUDT, Vector}
+import org.apache.spark.sql.types.{StringType, ArrayType, DataType}
+
+/**
+ * :: Experimental ::
+ * Converts a text document to a sparse vector of token counts.
+ * @param vocabulary An Array over terms. Only the terms in the vocabulary will be counted.
+ */
+@Experimental
+class CountVectorizerModel (override val uid: String, val vocabulary: Array[String])
+ extends UnaryTransformer[Seq[String], Vector, CountVectorizerModel] {
+
+ def this(vocabulary: Array[String]) =
+ this(Identifiable.randomUID("cntVec"), vocabulary)
+
+ /**
+ * Corpus-specific filter to ignore scarce words in a document. For each document, terms with
+ * frequency (count) less than the given threshold are ignored.
+ * Default: 1
+ * @group param
+ */
+ val minTermFreq: IntParam = new IntParam(this, "minTermFreq",
+ "minimum frequency (count) filter used to neglect scarce words (>= 1). For each document, " +
+ "terms with frequency less than the given threshold are ignored.", ParamValidators.gtEq(1))
+
+ /** @group setParam */
+ def setMinTermFreq(value: Int): this.type = set(minTermFreq, value)
+
+ /** @group getParam */
+ def getMinTermFreq: Int = $(minTermFreq)
+
+ setDefault(minTermFreq -> 1)
+
+ override protected def createTransformFunc: Seq[String] => Vector = {
+ val dict = vocabulary.zipWithIndex.toMap
+ document =>
+ val termCounts = mutable.HashMap.empty[Int, Double]
+ document.foreach { term =>
+ dict.get(term) match {
+ case Some(index) => termCounts.put(index, termCounts.getOrElse(index, 0.0) + 1.0)
+ case None => // ignore terms not in the vocabulary
+ }
+ }
+ Vectors.sparse(dict.size, termCounts.filter(_._2 >= $(minTermFreq)).toSeq)
+ }
+
+ override protected def validateInputType(inputType: DataType): Unit = {
+ require(inputType.sameType(ArrayType(StringType)),
+ s"Input type must be ArrayType(StringType) but got $inputType.")
+ }
+
+ override protected def outputDataType: DataType = new VectorUDT()
+
+ override def copy(extra: ParamMap): CountVectorizerModel = {
+ val copied = new CountVectorizerModel(uid, vocabulary)
+ copyValues(copied, extra)
+ }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
index ca3c1cfb56b7f..72b545e5db3e4 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
@@ -106,6 +106,12 @@ class StandardScalerModel private[ml] (
scaler: feature.StandardScalerModel)
extends Model[StandardScalerModel] with StandardScalerParams {
+ /** Standard deviation of the StandardScalerModel */
+ val std: Vector = scaler.std
+
+ /** Mean of the StandardScalerModel */
+ val mean: Vector = scaler.mean
+
/** @group setParam */
def setInputCol(value: String): this.type = set(inputCol, value)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
index 50c0d855066f8..d034d7ec6b60e 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/params.scala
@@ -341,9 +341,7 @@ trait Params extends Identifiable with Serializable {
* those are checked during schema validation.
*/
def validateParams(): Unit = {
- params.filter(isDefined).foreach { param =>
- param.asInstanceOf[Param[Any]].validate($(param))
- }
+ // Do nothing by default. Override to handle Param interactions.
}
/**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
index b0a6af171c01f..f7ae1de522e01 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala
@@ -54,8 +54,7 @@ private[shared] object SharedParamsCodeGen {
isValid = "ParamValidators.gtEq(1)"),
ParamDesc[Boolean]("fitIntercept", "whether to fit an intercept term", Some("true")),
ParamDesc[Boolean]("standardization", "whether to standardize the training features" +
- " prior to fitting the model sequence. Note that the coefficients of models are" +
- " always returned on the original scale.", Some("true")),
+ " before fitting the model.", Some("true")),
ParamDesc[Long]("seed", "random seed", Some("this.getClass.getName.hashCode.toLong")),
ParamDesc[Double]("elasticNetParam", "the ElasticNet mixing parameter, in range [0, 1]." +
" For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.",
@@ -135,7 +134,7 @@ private[shared] object SharedParamsCodeGen {
s"""
|/**
- | * (private[ml]) Trait for shared param $name$defaultValueDoc.
+ | * Trait for shared param $name$defaultValueDoc.
| */
|private[ml] trait Has$Name extends Params {
|
@@ -174,7 +173,6 @@ private[shared] object SharedParamsCodeGen {
|package org.apache.spark.ml.param.shared
|
|import org.apache.spark.ml.param._
- |import org.apache.spark.util.Utils
|
|// DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
|
diff --git a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
index bbe08939b6d75..65e48e4ee5083 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala
@@ -18,14 +18,13 @@
package org.apache.spark.ml.param.shared
import org.apache.spark.ml.param._
-import org.apache.spark.util.Utils
// DO NOT MODIFY THIS FILE! It was generated by SharedParamsCodeGen.
// scalastyle:off
/**
- * (private[ml]) Trait for shared param regParam.
+ * Trait for shared param regParam.
*/
private[ml] trait HasRegParam extends Params {
@@ -40,7 +39,7 @@ private[ml] trait HasRegParam extends Params {
}
/**
- * (private[ml]) Trait for shared param maxIter.
+ * Trait for shared param maxIter.
*/
private[ml] trait HasMaxIter extends Params {
@@ -55,7 +54,7 @@ private[ml] trait HasMaxIter extends Params {
}
/**
- * (private[ml]) Trait for shared param featuresCol (default: "features").
+ * Trait for shared param featuresCol (default: "features").
*/
private[ml] trait HasFeaturesCol extends Params {
@@ -72,7 +71,7 @@ private[ml] trait HasFeaturesCol extends Params {
}
/**
- * (private[ml]) Trait for shared param labelCol (default: "label").
+ * Trait for shared param labelCol (default: "label").
*/
private[ml] trait HasLabelCol extends Params {
@@ -89,7 +88,7 @@ private[ml] trait HasLabelCol extends Params {
}
/**
- * (private[ml]) Trait for shared param predictionCol (default: "prediction").
+ * Trait for shared param predictionCol (default: "prediction").
*/
private[ml] trait HasPredictionCol extends Params {
@@ -106,7 +105,7 @@ private[ml] trait HasPredictionCol extends Params {
}
/**
- * (private[ml]) Trait for shared param rawPredictionCol (default: "rawPrediction").
+ * Trait for shared param rawPredictionCol (default: "rawPrediction").
*/
private[ml] trait HasRawPredictionCol extends Params {
@@ -123,7 +122,7 @@ private[ml] trait HasRawPredictionCol extends Params {
}
/**
- * (private[ml]) Trait for shared param probabilityCol (default: "probability").
+ * Trait for shared param probabilityCol (default: "probability").
*/
private[ml] trait HasProbabilityCol extends Params {
@@ -140,7 +139,7 @@ private[ml] trait HasProbabilityCol extends Params {
}
/**
- * (private[ml]) Trait for shared param threshold.
+ * Trait for shared param threshold.
*/
private[ml] trait HasThreshold extends Params {
@@ -155,7 +154,7 @@ private[ml] trait HasThreshold extends Params {
}
/**
- * (private[ml]) Trait for shared param inputCol.
+ * Trait for shared param inputCol.
*/
private[ml] trait HasInputCol extends Params {
@@ -170,7 +169,7 @@ private[ml] trait HasInputCol extends Params {
}
/**
- * (private[ml]) Trait for shared param inputCols.
+ * Trait for shared param inputCols.
*/
private[ml] trait HasInputCols extends Params {
@@ -185,7 +184,7 @@ private[ml] trait HasInputCols extends Params {
}
/**
- * (private[ml]) Trait for shared param outputCol (default: uid + "__output").
+ * Trait for shared param outputCol (default: uid + "__output").
*/
private[ml] trait HasOutputCol extends Params {
@@ -202,7 +201,7 @@ private[ml] trait HasOutputCol extends Params {
}
/**
- * (private[ml]) Trait for shared param checkpointInterval.
+ * Trait for shared param checkpointInterval.
*/
private[ml] trait HasCheckpointInterval extends Params {
@@ -217,7 +216,7 @@ private[ml] trait HasCheckpointInterval extends Params {
}
/**
- * (private[ml]) Trait for shared param fitIntercept (default: true).
+ * Trait for shared param fitIntercept (default: true).
*/
private[ml] trait HasFitIntercept extends Params {
@@ -234,15 +233,15 @@ private[ml] trait HasFitIntercept extends Params {
}
/**
- * (private[ml]) Trait for shared param standardization (default: true).
+ * Trait for shared param standardization (default: true).
*/
private[ml] trait HasStandardization extends Params {
/**
- * Param for whether to standardize the training features prior to fitting the model sequence. Note that the coefficients of models are always returned on the original scale..
+ * Param for whether to standardize the training features before fitting the model..
* @group param
*/
- final val standardization: BooleanParam = new BooleanParam(this, "standardization", "whether to standardize the training features prior to fitting the model sequence. Note that the coefficients of models are always returned on the original scale.")
+ final val standardization: BooleanParam = new BooleanParam(this, "standardization", "whether to standardize the training features before fitting the model.")
setDefault(standardization, true)
@@ -251,7 +250,7 @@ private[ml] trait HasStandardization extends Params {
}
/**
- * (private[ml]) Trait for shared param seed (default: this.getClass.getName.hashCode.toLong).
+ * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong).
*/
private[ml] trait HasSeed extends Params {
@@ -268,7 +267,7 @@ private[ml] trait HasSeed extends Params {
}
/**
- * (private[ml]) Trait for shared param elasticNetParam.
+ * Trait for shared param elasticNetParam.
*/
private[ml] trait HasElasticNetParam extends Params {
@@ -283,7 +282,7 @@ private[ml] trait HasElasticNetParam extends Params {
}
/**
- * (private[ml]) Trait for shared param tol.
+ * Trait for shared param tol.
*/
private[ml] trait HasTol extends Params {
@@ -298,7 +297,7 @@ private[ml] trait HasTol extends Params {
}
/**
- * (private[ml]) Trait for shared param stepSize.
+ * Trait for shared param stepSize.
*/
private[ml] trait HasStepSize extends Params {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
index 1b1d7299fb496..8fc986056657d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala
@@ -22,18 +22,20 @@ import scala.collection.mutable
import breeze.linalg.{DenseVector => BDV, norm => brzNorm}
import breeze.optimize.{CachedDiffFunction, DiffFunction, LBFGS => BreezeLBFGS, OWLQN => BreezeOWLQN}
-import org.apache.spark.Logging
+import org.apache.spark.{Logging, SparkException}
import org.apache.spark.annotation.Experimental
import org.apache.spark.ml.PredictorParams
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.param.shared._
import org.apache.spark.ml.util.Identifiable
+import org.apache.spark.mllib.evaluation.RegressionMetrics
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.mllib.linalg.BLAS._
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.stat.MultivariateOnlineSummarizer
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.DataFrame
+import org.apache.spark.sql.{DataFrame, Row}
+import org.apache.spark.sql.functions.{col, udf}
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.StatCounter
@@ -132,7 +134,6 @@ class LinearRegression(override val uid: String)
val numFeatures = summarizer.mean.size
val yMean = statCounter.mean
val yStd = math.sqrt(statCounter.variance)
- // look at glmnet5.m L761 maaaybe that has info
// If the yStd is zero, then the intercept is yMean with zero weights;
// as a result, training is not needed.
@@ -140,7 +141,16 @@ class LinearRegression(override val uid: String)
logWarning(s"The standard deviation of the label is zero, so the weights will be zeros " +
s"and the intercept will be the mean of the label; as a result, training is not needed.")
if (handlePersistence) instances.unpersist()
- return new LinearRegressionModel(uid, Vectors.sparse(numFeatures, Seq()), yMean)
+ val weights = Vectors.sparse(numFeatures, Seq())
+ val intercept = yMean
+
+ val model = new LinearRegressionModel(uid, weights, intercept)
+ val trainingSummary = new LinearRegressionTrainingSummary(
+ model.transform(dataset).select($(predictionCol), $(labelCol)),
+ $(predictionCol),
+ $(labelCol),
+ Array(0D))
+ return copyValues(model.setSummary(trainingSummary))
}
val featuresMean = summarizer.mean.toArray
@@ -162,21 +172,33 @@ class LinearRegression(override val uid: String)
}
val initialWeights = Vectors.zeros(numFeatures)
- val states =
- optimizer.iterations(new CachedDiffFunction(costFun), initialWeights.toBreeze.toDenseVector)
-
- var state = states.next()
- val lossHistory = mutable.ArrayBuilder.make[Double]
-
- while (states.hasNext) {
- lossHistory += state.value
- state = states.next()
- }
- lossHistory += state.value
+ val states = optimizer.iterations(new CachedDiffFunction(costFun),
+ initialWeights.toBreeze.toDenseVector)
+
+ val (weights, objectiveHistory) = {
+ /*
+ Note that in Linear Regression, the objective history (loss + regularization) returned
+ from optimizer is computed in the scaled space given by the following formula.
+ {{{
+ L = 1/2n||\sum_i w_i(x_i - \bar{x_i}) / \hat{x_i} - (y - \bar{y}) / \hat{y}||^2 + regTerms
+ }}}
+ */
+ val arrayBuilder = mutable.ArrayBuilder.make[Double]
+ var state: optimizer.State = null
+ while (states.hasNext) {
+ state = states.next()
+ arrayBuilder += state.adjustedValue
+ }
+ if (state == null) {
+ val msg = s"${optimizer.getClass.getName} failed."
+ logError(msg)
+ throw new SparkException(msg)
+ }
- // The weights are trained in the scaled space; we're converting them back to
- // the original space.
- val weights = {
+ /*
+ The weights are trained in the scaled space; we're converting them back to
+ the original space.
+ */
val rawWeights = state.x.toArray.clone()
var i = 0
val len = rawWeights.length
@@ -184,17 +206,26 @@ class LinearRegression(override val uid: String)
rawWeights(i) *= { if (featuresStd(i) != 0.0) yStd / featuresStd(i) else 0.0 }
i += 1
}
- Vectors.dense(rawWeights)
+
+ (Vectors.dense(rawWeights).compressed, arrayBuilder.result())
}
- // The intercept in R's GLMNET is computed using closed form after the coefficients are
- // converged. See the following discussion for detail.
- // http://stats.stackexchange.com/questions/13617/how-is-the-intercept-computed-in-glmnet
+ /*
+ The intercept in R's GLMNET is computed using closed form after the coefficients are
+ converged. See the following discussion for detail.
+ http://stats.stackexchange.com/questions/13617/how-is-the-intercept-computed-in-glmnet
+ */
val intercept = if ($(fitIntercept)) yMean - dot(weights, Vectors.dense(featuresMean)) else 0.0
+
if (handlePersistence) instances.unpersist()
- // TODO: Converts to sparse format based on the storage, but may base on the scoring speed.
- copyValues(new LinearRegressionModel(uid, weights.compressed, intercept))
+ val model = copyValues(new LinearRegressionModel(uid, weights, intercept))
+ val trainingSummary = new LinearRegressionTrainingSummary(
+ model.transform(dataset).select($(predictionCol), $(labelCol)),
+ $(predictionCol),
+ $(labelCol),
+ objectiveHistory)
+ model.setSummary(trainingSummary)
}
override def copy(extra: ParamMap): LinearRegression = defaultCopy(extra)
@@ -212,13 +243,124 @@ class LinearRegressionModel private[ml] (
extends RegressionModel[Vector, LinearRegressionModel]
with LinearRegressionParams {
+ private var trainingSummary: Option[LinearRegressionTrainingSummary] = None
+
+ /**
+ * Gets summary (e.g. residuals, mse, r-squared ) of model on training set. An exception is
+ * thrown if `trainingSummary == None`.
+ */
+ def summary: LinearRegressionTrainingSummary = trainingSummary match {
+ case Some(summ) => summ
+ case None =>
+ throw new SparkException(
+ "No training summary available for this LinearRegressionModel",
+ new NullPointerException())
+ }
+
+ private[regression] def setSummary(summary: LinearRegressionTrainingSummary): this.type = {
+ this.trainingSummary = Some(summary)
+ this
+ }
+
+ /** Indicates whether a training summary exists for this model instance. */
+ def hasSummary: Boolean = trainingSummary.isDefined
+
+ /**
+ * Evaluates the model on a testset.
+ * @param dataset Test dataset to evaluate model on.
+ */
+ // TODO: decide on a good name before exposing to public API
+ private[regression] def evaluate(dataset: DataFrame): LinearRegressionSummary = {
+ val t = udf { features: Vector => predict(features) }
+ val predictionAndObservations = dataset
+ .select(col($(labelCol)), t(col($(featuresCol))).as($(predictionCol)))
+
+ new LinearRegressionSummary(predictionAndObservations, $(predictionCol), $(labelCol))
+ }
+
override protected def predict(features: Vector): Double = {
dot(features, weights) + intercept
}
override def copy(extra: ParamMap): LinearRegressionModel = {
- copyValues(new LinearRegressionModel(uid, weights, intercept), extra)
+ val newModel = copyValues(new LinearRegressionModel(uid, weights, intercept))
+ if (trainingSummary.isDefined) newModel.setSummary(trainingSummary.get)
+ newModel
+ }
+}
+
+/**
+ * :: Experimental ::
+ * Linear regression training results.
+ * @param predictions predictions outputted by the model's `transform` method.
+ * @param objectiveHistory objective function (scaled loss + regularization) at each iteration.
+ */
+@Experimental
+class LinearRegressionTrainingSummary private[regression] (
+ predictions: DataFrame,
+ predictionCol: String,
+ labelCol: String,
+ val objectiveHistory: Array[Double])
+ extends LinearRegressionSummary(predictions, predictionCol, labelCol) {
+
+ /** Number of training iterations until termination */
+ val totalIterations = objectiveHistory.length
+
+}
+
+/**
+ * :: Experimental ::
+ * Linear regression results evaluated on a dataset.
+ * @param predictions predictions outputted by the model's `transform` method.
+ */
+@Experimental
+class LinearRegressionSummary private[regression] (
+ @transient val predictions: DataFrame,
+ val predictionCol: String,
+ val labelCol: String) extends Serializable {
+
+ @transient private val metrics = new RegressionMetrics(
+ predictions
+ .select(predictionCol, labelCol)
+ .map { case Row(pred: Double, label: Double) => (pred, label) } )
+
+ /**
+ * Returns the explained variance regression score.
+ * explainedVariance = 1 - variance(y - \hat{y}) / variance(y)
+ * Reference: [[http://en.wikipedia.org/wiki/Explained_variation]]
+ */
+ val explainedVariance: Double = metrics.explainedVariance
+
+ /**
+ * Returns the mean absolute error, which is a risk function corresponding to the
+ * expected value of the absolute error loss or l1-norm loss.
+ */
+ val meanAbsoluteError: Double = metrics.meanAbsoluteError
+
+ /**
+ * Returns the mean squared error, which is a risk function corresponding to the
+ * expected value of the squared error loss or quadratic loss.
+ */
+ val meanSquaredError: Double = metrics.meanSquaredError
+
+ /**
+ * Returns the root mean squared error, which is defined as the square root of
+ * the mean squared error.
+ */
+ val rootMeanSquaredError: Double = metrics.rootMeanSquaredError
+
+ /**
+ * Returns R^2^, the coefficient of determination.
+ * Reference: [[http://en.wikipedia.org/wiki/Coefficient_of_determination]]
+ */
+ val r2: Double = metrics.r2
+
+ /** Residuals (predicted value - label value) */
+ @transient lazy val residuals: DataFrame = {
+ val t = udf { (pred: Double, label: Double) => pred - label}
+ predictions.select(t(col(predictionCol), col(labelCol)).as("residuals"))
}
+
}
/**
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 1929f9d02156e..22873909c33fa 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -17,6 +17,7 @@
package org.apache.spark.ml.tree
+import org.apache.spark.mllib.linalg.{Vectors, Vector}
/**
* Abstraction for Decision Tree models.
@@ -70,6 +71,10 @@ private[ml] trait TreeEnsembleModel {
/** Weights for each tree, zippable with [[trees]] */
def treeWeights: Array[Double]
+ /** Weights used by the python wrappers. */
+ // Note: An array cannot be returned directly due to serialization problems.
+ private[spark] def javaTreeWeights: Vector = Vectors.dense(treeWeights)
+
/** Summary of the model */
override def toString: String = {
// Implementing classes should generally override this method to be more descriptive.
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
index f51ee36d0dfcb..9e379d7d74b2f 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
@@ -93,26 +93,70 @@ class NaiveBayesModel private[mllib] (
override def predict(testData: Vector): Double = {
modelType match {
case Multinomial =>
- val prob = thetaMatrix.multiply(testData)
- BLAS.axpy(1.0, piVector, prob)
- labels(prob.argmax)
+ labels(multinomialCalculation(testData).argmax)
case Bernoulli =>
- testData.foreachActive { (index, value) =>
- if (value != 0.0 && value != 1.0) {
- throw new SparkException(
- s"Bernoulli naive Bayes requires 0 or 1 feature values but found $testData.")
- }
- }
- val prob = thetaMinusNegTheta.get.multiply(testData)
- BLAS.axpy(1.0, piVector, prob)
- BLAS.axpy(1.0, negThetaSum.get, prob)
- labels(prob.argmax)
- case _ =>
- // This should never happen.
- throw new UnknownError(s"Invalid modelType: $modelType.")
+ labels(bernoulliCalculation(testData).argmax)
+ }
+ }
+
+ /**
+ * Predict values for the given data set using the model trained.
+ *
+ * @param testData RDD representing data points to be predicted
+ * @return an RDD[Vector] where each entry contains the predicted posterior class probabilities,
+ * in the same order as class labels
+ */
+ def predictProbabilities(testData: RDD[Vector]): RDD[Vector] = {
+ val bcModel = testData.context.broadcast(this)
+ testData.mapPartitions { iter =>
+ val model = bcModel.value
+ iter.map(model.predictProbabilities)
}
}
+ /**
+ * Predict posterior class probabilities for a single data point using the model trained.
+ *
+ * @param testData array representing a single data point
+ * @return predicted posterior class probabilities from the trained model,
+ * in the same order as class labels
+ */
+ def predictProbabilities(testData: Vector): Vector = {
+ modelType match {
+ case Multinomial =>
+ posteriorProbabilities(multinomialCalculation(testData))
+ case Bernoulli =>
+ posteriorProbabilities(bernoulliCalculation(testData))
+ }
+ }
+
+ private def multinomialCalculation(testData: Vector) = {
+ val prob = thetaMatrix.multiply(testData)
+ BLAS.axpy(1.0, piVector, prob)
+ prob
+ }
+
+ private def bernoulliCalculation(testData: Vector) = {
+ testData.foreachActive((_, value) =>
+ if (value != 0.0 && value != 1.0) {
+ throw new SparkException(
+ s"Bernoulli naive Bayes requires 0 or 1 feature values but found $testData.")
+ }
+ )
+ val prob = thetaMinusNegTheta.get.multiply(testData)
+ BLAS.axpy(1.0, piVector, prob)
+ BLAS.axpy(1.0, negThetaSum.get, prob)
+ prob
+ }
+
+ private def posteriorProbabilities(logProb: DenseVector) = {
+ val logProbArray = logProb.toArray
+ val maxLog = logProbArray.max
+ val scaledProbs = logProbArray.map(lp => math.exp(lp - maxLog))
+ val probSum = scaledProbs.sum
+ new DenseVector(scaledProbs.map(_ / probSum))
+ }
+
override def save(sc: SparkContext, path: String): Unit = {
val data = NaiveBayesModel.SaveLoadV2_0.Data(labels, pi, theta, modelType)
NaiveBayesModel.SaveLoadV2_0.save(sc, path, data)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
index fc509d2ba1470..e459367333d26 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixture.scala
@@ -140,6 +140,10 @@ class GaussianMixture private (
// Get length of the input vectors
val d = breezeData.first().length
+ // Heuristic to distribute the computation of the [[MultivariateGaussian]]s, approximately when
+ // d > 25 except for when k is very small
+ val distributeGaussians = ((k - 1.0) / k) * d > 25
+
// Determine initial weights and corresponding Gaussians.
// If the user supplied an initial GMM, we use those values, otherwise
// we start with uniform weights, a random mean from the data, and
@@ -171,14 +175,25 @@ class GaussianMixture private (
// Create new distributions based on the partial assignments
// (often referred to as the "M" step in literature)
val sumWeights = sums.weights.sum
- var i = 0
- while (i < k) {
- val mu = sums.means(i) / sums.weights(i)
- BLAS.syr(-sums.weights(i), Vectors.fromBreeze(mu),
- Matrices.fromBreeze(sums.sigmas(i)).asInstanceOf[DenseMatrix])
- weights(i) = sums.weights(i) / sumWeights
- gaussians(i) = new MultivariateGaussian(mu, sums.sigmas(i) / sums.weights(i))
- i = i + 1
+
+ if (distributeGaussians) {
+ val numPartitions = math.min(k, 1024)
+ val tuples =
+ Seq.tabulate(k)(i => (sums.means(i), sums.sigmas(i), sums.weights(i)))
+ val (ws, gs) = sc.parallelize(tuples, numPartitions).map { case (mean, sigma, weight) =>
+ updateWeightsAndGaussians(mean, sigma, weight, sumWeights)
+ }.collect.unzip
+ Array.copy(ws, 0, weights, 0, ws.length)
+ Array.copy(gs, 0, gaussians, 0, gs.length)
+ } else {
+ var i = 0
+ while (i < k) {
+ val (weight, gaussian) =
+ updateWeightsAndGaussians(sums.means(i), sums.sigmas(i), sums.weights(i), sumWeights)
+ weights(i) = weight
+ gaussians(i) = gaussian
+ i = i + 1
+ }
}
llhp = llh // current becomes previous
@@ -192,6 +207,19 @@ class GaussianMixture private (
/** Java-friendly version of [[run()]] */
def run(data: JavaRDD[Vector]): GaussianMixtureModel = run(data.rdd)
+ private def updateWeightsAndGaussians(
+ mean: BDV[Double],
+ sigma: BreezeMatrix[Double],
+ weight: Double,
+ sumWeights: Double): (Double, MultivariateGaussian) = {
+ val mu = (mean /= weight)
+ BLAS.syr(-weight, Vectors.fromBreeze(mu),
+ Matrices.fromBreeze(sigma).asInstanceOf[DenseMatrix])
+ val newWeight = weight / sumWeights
+ val newGaussian = new MultivariateGaussian(mu, sigma / weight)
+ (newWeight, newGaussian)
+ }
+
/** Average of dense breeze vectors */
private def vectorMean(x: IndexedSeq[BV[Double]]): BDV[Double] = {
val v = BDV.zeros[Double](x(0).length)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
new file mode 100644
index 0000000000000..72d0ea0c12e1e
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/AssociationRules.scala
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.mllib.fpm
+
+import scala.reflect.ClassTag
+
+import org.apache.spark.Logging
+import org.apache.spark.annotation.Experimental
+import org.apache.spark.api.java.JavaRDD
+import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
+import org.apache.spark.mllib.fpm.AssociationRules.Rule
+import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset
+import org.apache.spark.rdd.RDD
+
+/**
+ * :: Experimental ::
+ *
+ * Generates association rules from a [[RDD[FreqItemset[Item]]]. This method only generates
+ * association rules which have a single item as the consequent.
+ *
+ * @since 1.5.0
+ */
+@Experimental
+class AssociationRules private[fpm] (
+ private var minConfidence: Double) extends Logging with Serializable {
+
+ /**
+ * Constructs a default instance with default parameters {minConfidence = 0.8}.
+ *
+ * @since 1.5.0
+ */
+ def this() = this(0.8)
+
+ /**
+ * Sets the minimal confidence (default: `0.8`).
+ *
+ * @since 1.5.0
+ */
+ def setMinConfidence(minConfidence: Double): this.type = {
+ require(minConfidence >= 0.0 && minConfidence <= 1.0)
+ this.minConfidence = minConfidence
+ this
+ }
+
+ /**
+ * Computes the association rules with confidence above [[minConfidence]].
+ * @param freqItemsets frequent itemset model obtained from [[FPGrowth]]
+ * @return a [[Set[Rule[Item]]] containing the assocation rules.
+ *
+ * @since 1.5.0
+ */
+ def run[Item: ClassTag](freqItemsets: RDD[FreqItemset[Item]]): RDD[Rule[Item]] = {
+ // For candidate rule X => Y, generate (X, (Y, freq(X union Y)))
+ val candidates = freqItemsets.flatMap { itemset =>
+ val items = itemset.items
+ items.flatMap { item =>
+ items.partition(_ == item) match {
+ case (consequent, antecedent) if !antecedent.isEmpty =>
+ Some((antecedent.toSeq, (consequent.toSeq, itemset.freq)))
+ case _ => None
+ }
+ }
+ }
+
+ // Join to get (X, ((Y, freq(X union Y)), freq(X))), generate rules, and filter by confidence
+ candidates.join(freqItemsets.map(x => (x.items.toSeq, x.freq)))
+ .map { case (antecendent, ((consequent, freqUnion), freqAntecedent)) =>
+ new Rule(antecendent.toArray, consequent.toArray, freqUnion, freqAntecedent)
+ }.filter(_.confidence >= minConfidence)
+ }
+
+ def run[Item](freqItemsets: JavaRDD[FreqItemset[Item]]): JavaRDD[Rule[Item]] = {
+ val tag = fakeClassTag[Item]
+ run(freqItemsets.rdd)(tag)
+ }
+}
+
+object AssociationRules {
+
+ /**
+ * :: Experimental ::
+ *
+ * An association rule between sets of items.
+ * @param antecedent hypotheses of the rule
+ * @param consequent conclusion of the rule
+ * @tparam Item item type
+ *
+ * @since 1.5.0
+ */
+ @Experimental
+ class Rule[Item] private[fpm] (
+ val antecedent: Array[Item],
+ val consequent: Array[Item],
+ freqUnion: Double,
+ freqAntecedent: Double) extends Serializable {
+
+ def confidence: Double = freqUnion.toDouble / freqAntecedent
+
+ require(antecedent.toSet.intersect(consequent.toSet).isEmpty, {
+ val sharedItems = antecedent.toSet.intersect(consequent.toSet)
+ s"A valid association rule must have disjoint antecedent and " +
+ s"consequent but ${sharedItems} is present in both."
+ })
+ }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
index abac08022ea47..e2370a52f4930 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
@@ -28,7 +28,7 @@ import org.apache.spark.{HashPartitioner, Logging, Partitioner, SparkException}
import org.apache.spark.annotation.Experimental
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.api.java.JavaSparkContext.fakeClassTag
-import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset
+import org.apache.spark.mllib.fpm.FPGrowth._
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
@@ -36,11 +36,23 @@ import org.apache.spark.storage.StorageLevel
* :: Experimental ::
*
* Model trained by [[FPGrowth]], which holds frequent itemsets.
- * @param freqItemsets frequent itemsets, which is an RDD of [[FreqItemset]]
+ * @param freqItemsets frequent itemset, which is an RDD of [[FreqItemset]]
* @tparam Item item type
+ *
+ * @since 1.3.0
*/
@Experimental
-class FPGrowthModel[Item: ClassTag](val freqItemsets: RDD[FreqItemset[Item]]) extends Serializable
+class FPGrowthModel[Item: ClassTag](val freqItemsets: RDD[FreqItemset[Item]]) extends Serializable {
+ /**
+ * Generates association rules for the [[Item]]s in [[freqItemsets]].
+ * @param confidence minimal confidence of the rules produced
+ * @since 1.5.0
+ */
+ def generateAssociationRules(confidence: Double): RDD[AssociationRules.Rule[Item]] = {
+ val associationRules = new AssociationRules(confidence)
+ associationRules.run(freqItemsets)
+ }
+}
/**
* :: Experimental ::
@@ -58,21 +70,26 @@ class FPGrowthModel[Item: ClassTag](val freqItemsets: RDD[FreqItemset[Item]]) ex
*
* @see [[http://en.wikipedia.org/wiki/Association_rule_learning Association rule learning
* (Wikipedia)]]
+ *
+ * @since 1.3.0
*/
@Experimental
class FPGrowth private (
private var minSupport: Double,
- private var numPartitions: Int,
- private var ordered: Boolean) extends Logging with Serializable {
+ private var numPartitions: Int) extends Logging with Serializable {
/**
* Constructs a default instance with default parameters {minSupport: `0.3`, numPartitions: same
- * as the input data, ordered: `false`}.
+ * as the input data}.
+ *
+ * @since 1.3.0
*/
- def this() = this(0.3, -1, false)
+ def this() = this(0.3, -1)
/**
* Sets the minimal support level (default: `0.3`).
+ *
+ * @since 1.3.0
*/
def setMinSupport(minSupport: Double): this.type = {
this.minSupport = minSupport
@@ -81,25 +98,20 @@ class FPGrowth private (
/**
* Sets the number of partitions used by parallel FP-growth (default: same as input data).
+ *
+ * @since 1.3.0
*/
def setNumPartitions(numPartitions: Int): this.type = {
this.numPartitions = numPartitions
this
}
- /**
- * Indicates whether to mine itemsets (unordered) or sequences (ordered) (default: false, mine
- * itemsets).
- */
- def setOrdered(ordered: Boolean): this.type = {
- this.ordered = ordered
- this
- }
-
/**
* Computes an FP-Growth model that contains frequent itemsets.
* @param data input data set, each element contains a transaction
* @return an [[FPGrowthModel]]
+ *
+ * @since 1.3.0
*/
def run[Item: ClassTag](data: RDD[Array[Item]]): FPGrowthModel[Item] = {
if (data.getStorageLevel == StorageLevel.NONE) {
@@ -165,7 +177,7 @@ class FPGrowth private (
.flatMap { case (part, tree) =>
tree.extract(minCount, x => partitioner.getPartition(x) == part)
}.map { case (ranks, count) =>
- new FreqItemset(ranks.map(i => freqItems(i)).reverse.toArray, count, ordered)
+ new FreqItemset(ranks.map(i => freqItems(i)).toArray, count)
}
}
@@ -181,12 +193,9 @@ class FPGrowth private (
itemToRank: Map[Item, Int],
partitioner: Partitioner): mutable.Map[Int, Array[Int]] = {
val output = mutable.Map.empty[Int, Array[Int]]
- // Filter the basket by frequent items pattern
+ // Filter the basket by frequent items pattern and sort their ranks.
val filtered = transaction.flatMap(itemToRank.get)
- if (!this.ordered) {
- ju.Arrays.sort(filtered)
- }
- // Generate conditional transactions
+ ju.Arrays.sort(filtered)
val n = filtered.length
var i = n - 1
while (i >= 0) {
@@ -203,6 +212,8 @@ class FPGrowth private (
/**
* :: Experimental ::
+ *
+ * @since 1.3.0
*/
@Experimental
object FPGrowth {
@@ -211,21 +222,16 @@ object FPGrowth {
* Frequent itemset.
* @param items items in this itemset. Java users should call [[FreqItemset#javaItems]] instead.
* @param freq frequency
- * @param ordered indicates if items represents an itemset (false) or sequence (true)
* @tparam Item item type
+ *
+ * @since 1.3.0
*/
- class FreqItemset[Item](val items: Array[Item], val freq: Long, val ordered: Boolean)
- extends Serializable {
-
- /**
- * Auxillary constructor, assumes unordered by default.
- */
- def this(items: Array[Item], freq: Long) {
- this(items, freq, false)
- }
+ class FreqItemset[Item](val items: Array[Item], val freq: Long) extends Serializable {
/**
* Returns items in a Java List.
+ *
+ * @since 1.3.0
*/
def javaItems: java.util.List[Item] = {
items.toList.asJava
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 75e7004464af9..0df07663405a3 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -24,9 +24,9 @@ import scala.collection.mutable.{ArrayBuilder => MArrayBuilder, HashSet => MHash
import breeze.linalg.{CSCMatrix => BSM, DenseMatrix => BDM, Matrix => BM}
import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.types._
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.types._
/**
* Trait for a local matrix.
@@ -147,7 +147,7 @@ private[spark] class MatrixUDT extends UserDefinedType[Matrix] {
))
}
- override def serialize(obj: Any): Row = {
+ override def serialize(obj: Any): InternalRow = {
val row = new GenericMutableRow(7)
obj match {
case sm: SparseMatrix =>
@@ -173,9 +173,7 @@ private[spark] class MatrixUDT extends UserDefinedType[Matrix] {
override def deserialize(datum: Any): Matrix = {
datum match {
- // TODO: something wrong with UDT serialization, should never happen.
- case m: Matrix => m
- case row: Row =>
+ case row: InternalRow =>
require(row.length == 7,
s"MatrixUDT.deserialize given row with length ${row.length} but requires length == 7")
val tpe = row.getByte(0)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
index c9c27425d2877..e048b01d92462 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
@@ -28,7 +28,7 @@ import breeze.linalg.{DenseVector => BDV, SparseVector => BSV, Vector => BV}
import org.apache.spark.SparkException
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.mllib.util.NumericParser
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
import org.apache.spark.sql.types._
@@ -175,7 +175,7 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true)))
}
- override def serialize(obj: Any): Row = {
+ override def serialize(obj: Any): InternalRow = {
obj match {
case SparseVector(size, indices, values) =>
val row = new GenericMutableRow(4)
@@ -191,17 +191,12 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
row.setNullAt(2)
row.update(3, values.toSeq)
row
- // TODO: There are bugs in UDT serialization because we don't have a clear separation between
- // TODO: internal SQL types and language specific types (including UDT). UDT serialize and
- // TODO: deserialize may get called twice. See SPARK-7186.
- case row: Row =>
- row
}
}
override def deserialize(datum: Any): Vector = {
datum match {
- case row: Row =>
+ case row: InternalRow =>
require(row.length == 4,
s"VectorUDT.deserialize given row with length ${row.length} but requires length == 4")
val tpe = row.getByte(0)
@@ -215,11 +210,6 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
val values = row.getAs[Iterable[Double]](3).toArray
new DenseVector(values)
}
- // TODO: There are bugs in UDT serialization because we don't have a clear separation between
- // TODO: internal SQL types and language specific types (including UDT). UDT serialize and
- // TODO: deserialize may get called twice. See SPARK-7186.
- case v: Vector =>
- v
}
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/Statistics.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/Statistics.scala
index 900007ec6bc74..90332028cfb3a 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/Statistics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/Statistics.scala
@@ -17,13 +17,16 @@
package org.apache.spark.mllib.stat
+import scala.annotation.varargs
+
import org.apache.spark.annotation.Experimental
import org.apache.spark.api.java.JavaRDD
import org.apache.spark.mllib.linalg.distributed.RowMatrix
import org.apache.spark.mllib.linalg.{Matrix, Vector}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.stat.correlation.Correlations
-import org.apache.spark.mllib.stat.test.{ChiSqTest, ChiSqTestResult}
+import org.apache.spark.mllib.stat.test.{ChiSqTest, ChiSqTestResult, KolmogorovSmirnovTest,
+ KolmogorovSmirnovTestResult}
import org.apache.spark.rdd.RDD
/**
@@ -158,4 +161,39 @@ object Statistics {
def chiSqTest(data: RDD[LabeledPoint]): Array[ChiSqTestResult] = {
ChiSqTest.chiSquaredFeatures(data)
}
+
+ /**
+ * Conduct the two-sided Kolmogorov-Smirnov (KS) test for data sampled from a
+ * continuous distribution. By comparing the largest difference between the empirical cumulative
+ * distribution of the sample data and the theoretical distribution we can provide a test for the
+ * the null hypothesis that the sample data comes from that theoretical distribution.
+ * For more information on KS Test:
+ * @see [[https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test]]
+ *
+ * @param data an `RDD[Double]` containing the sample of data to test
+ * @param cdf a `Double => Double` function to calculate the theoretical CDF at a given value
+ * @return [[org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult]] object containing test
+ * statistic, p-value, and null hypothesis.
+ */
+ def kolmogorovSmirnovTest(data: RDD[Double], cdf: Double => Double)
+ : KolmogorovSmirnovTestResult = {
+ KolmogorovSmirnovTest.testOneSample(data, cdf)
+ }
+
+ /**
+ * Convenience function to conduct a one-sample, two-sided Kolmogorov-Smirnov test for probability
+ * distribution equality. Currently supports the normal distribution, taking as parameters
+ * the mean and standard deviation.
+ * (distName = "norm")
+ * @param data an `RDD[Double]` containing the sample of data to test
+ * @param distName a `String` name for a theoretical distribution
+ * @param params `Double*` specifying the parameters to be used for the theoretical distribution
+ * @return [[org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult]] object containing test
+ * statistic, p-value, and null hypothesis.
+ */
+ @varargs
+ def kolmogorovSmirnovTest(data: RDD[Double], distName: String, params: Double*)
+ : KolmogorovSmirnovTestResult = {
+ KolmogorovSmirnovTest.testOneSample(data, distName, params: _*)
+ }
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
new file mode 100644
index 0000000000000..d89b0059d83f3
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/KolmogorovSmirnovTest.scala
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.mllib.stat.test
+
+import scala.annotation.varargs
+
+import org.apache.commons.math3.distribution.{NormalDistribution, RealDistribution}
+import org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest
+
+import org.apache.spark.Logging
+import org.apache.spark.rdd.RDD
+
+/**
+ * Conduct the two-sided Kolmogorov Smirnov (KS) test for data sampled from a
+ * continuous distribution. By comparing the largest difference between the empirical cumulative
+ * distribution of the sample data and the theoretical distribution we can provide a test for the
+ * the null hypothesis that the sample data comes from that theoretical distribution.
+ * For more information on KS Test:
+ * @see [[https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test]]
+ *
+ * Implementation note: We seek to implement the KS test with a minimal number of distributed
+ * passes. We sort the RDD, and then perform the following operations on a per-partition basis:
+ * calculate an empirical cumulative distribution value for each observation, and a theoretical
+ * cumulative distribution value. We know the latter to be correct, while the former will be off by
+ * a constant (how large the constant is depends on how many values precede it in other partitions).
+ * However, given that this constant simply shifts the empirical CDF upwards, but doesn't
+ * change its shape, and furthermore, that constant is the same within a given partition, we can
+ * pick 2 values in each partition that can potentially resolve to the largest global distance.
+ * Namely, we pick the minimum distance and the maximum distance. Additionally, we keep track of how
+ * many elements are in each partition. Once these three values have been returned for every
+ * partition, we can collect and operate locally. Locally, we can now adjust each distance by the
+ * appropriate constant (the cumulative sum of number of elements in the prior partitions divided by
+ * thedata set size). Finally, we take the maximum absolute value, and this is the statistic.
+ */
+private[stat] object KolmogorovSmirnovTest extends Logging {
+
+ // Null hypothesis for the type of KS test to be included in the result.
+ object NullHypothesis extends Enumeration {
+ type NullHypothesis = Value
+ val OneSampleTwoSided = Value("Sample follows theoretical distribution")
+ }
+
+ /**
+ * Runs a KS test for 1 set of sample data, comparing it to a theoretical distribution
+ * @param data `RDD[Double]` data on which to run test
+ * @param cdf `Double => Double` function to calculate the theoretical CDF
+ * @return [[org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult]] summarizing the test
+ * results (p-value, statistic, and null hypothesis)
+ */
+ def testOneSample(data: RDD[Double], cdf: Double => Double): KolmogorovSmirnovTestResult = {
+ val n = data.count().toDouble
+ val localData = data.sortBy(x => x).mapPartitions { part =>
+ val partDiffs = oneSampleDifferences(part, n, cdf) // local distances
+ searchOneSampleCandidates(partDiffs) // candidates: local extrema
+ }.collect()
+ val ksStat = searchOneSampleStatistic(localData, n) // result: global extreme
+ evalOneSampleP(ksStat, n.toLong)
+ }
+
+ /**
+ * Runs a KS test for 1 set of sample data, comparing it to a theoretical distribution
+ * @param data `RDD[Double]` data on which to run test
+ * @param distObj `RealDistribution` a theoretical distribution
+ * @return [[org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult]] summarizing the test
+ * results (p-value, statistic, and null hypothesis)
+ */
+ def testOneSample(data: RDD[Double], distObj: RealDistribution): KolmogorovSmirnovTestResult = {
+ val cdf = (x: Double) => distObj.cumulativeProbability(x)
+ testOneSample(data, cdf)
+ }
+
+ /**
+ * Calculate unadjusted distances between the empirical CDF and the theoretical CDF in a
+ * partition
+ * @param partData `Iterator[Double]` 1 partition of a sorted RDD
+ * @param n `Double` the total size of the RDD
+ * @param cdf `Double => Double` a function the calculates the theoretical CDF of a value
+ * @return `Iterator[(Double, Double)] `Unadjusted (ie. off by a constant) potential extrema
+ * in a partition. The first element corresponds to the (empirical CDF - 1/N) - CDF,
+ * the second element corresponds to empirical CDF - CDF. We can then search the resulting
+ * iterator for the minimum of the first and the maximum of the second element, and provide
+ * this as a partition's candidate extrema
+ */
+ private def oneSampleDifferences(partData: Iterator[Double], n: Double, cdf: Double => Double)
+ : Iterator[(Double, Double)] = {
+ // zip data with index (within that partition)
+ // calculate local (unadjusted) empirical CDF and subtract CDF
+ partData.zipWithIndex.map { case (v, ix) =>
+ // dp and dl are later adjusted by constant, when global info is available
+ val dp = (ix + 1) / n
+ val dl = ix / n
+ val cdfVal = cdf(v)
+ (dl - cdfVal, dp - cdfVal)
+ }
+ }
+
+ /**
+ * Search the unadjusted differences in a partition and return the
+ * two extrema (furthest below and furthest above CDF), along with a count of elements in that
+ * partition
+ * @param partDiffs `Iterator[(Double, Double)]` the unadjusted differences between empirical CDF
+ * and CDFin a partition, which come as a tuple of
+ * (empirical CDF - 1/N - CDF, empirical CDF - CDF)
+ * @return `Iterator[(Double, Double, Double)]` the local extrema and a count of elements
+ */
+ private def searchOneSampleCandidates(partDiffs: Iterator[(Double, Double)])
+ : Iterator[(Double, Double, Double)] = {
+ val initAcc = (Double.MaxValue, Double.MinValue, 0.0)
+ val pResults = partDiffs.foldLeft(initAcc) { case ((pMin, pMax, pCt), (dl, dp)) =>
+ (math.min(pMin, dl), math.max(pMax, dp), pCt + 1)
+ }
+ val results = if (pResults == initAcc) Array[(Double, Double, Double)]() else Array(pResults)
+ results.iterator
+ }
+
+ /**
+ * Find the global maximum distance between empirical CDF and CDF (i.e. the KS statistic) after
+ * adjusting local extrema estimates from individual partitions with the amount of elements in
+ * preceding partitions
+ * @param localData `Array[(Double, Double, Double)]` A local array containing the collected
+ * results of `searchOneSampleCandidates` across all partitions
+ * @param n `Double`The size of the RDD
+ * @return The one-sample Kolmogorov Smirnov Statistic
+ */
+ private def searchOneSampleStatistic(localData: Array[(Double, Double, Double)], n: Double)
+ : Double = {
+ val initAcc = (Double.MinValue, 0.0)
+ // adjust differences based on the number of elements preceding it, which should provide
+ // the correct distance between empirical CDF and CDF
+ val results = localData.foldLeft(initAcc) { case ((prevMax, prevCt), (minCand, maxCand, ct)) =>
+ val adjConst = prevCt / n
+ val dist1 = math.abs(minCand + adjConst)
+ val dist2 = math.abs(maxCand + adjConst)
+ val maxVal = Array(prevMax, dist1, dist2).max
+ (maxVal, prevCt + ct)
+ }
+ results._1
+ }
+
+ /**
+ * A convenience function that allows running the KS test for 1 set of sample data against
+ * a named distribution
+ * @param data the sample data that we wish to evaluate
+ * @param distName the name of the theoretical distribution
+ * @param params Variable length parameter for distribution's parameters
+ * @return [[org.apache.spark.mllib.stat.test.KolmogorovSmirnovTestResult]] summarizing the
+ * test results (p-value, statistic, and null hypothesis)
+ */
+ @varargs
+ def testOneSample(data: RDD[Double], distName: String, params: Double*)
+ : KolmogorovSmirnovTestResult = {
+ val distObj =
+ distName match {
+ case "norm" => {
+ if (params.nonEmpty) {
+ // parameters are passed, then can only be 2
+ require(params.length == 2, "Normal distribution requires mean and standard " +
+ "deviation as parameters")
+ new NormalDistribution(params(0), params(1))
+ } else {
+ // if no parameters passed in initializes to standard normal
+ logInfo("No parameters specified for normal distribution," +
+ "initialized to standard normal (i.e. N(0, 1))")
+ new NormalDistribution(0, 1)
+ }
+ }
+ case _ => throw new UnsupportedOperationException(s"$distName not yet supported through" +
+ s" convenience method. Current options are:['norm'].")
+ }
+
+ testOneSample(data, distObj)
+ }
+
+ private def evalOneSampleP(ksStat: Double, n: Long): KolmogorovSmirnovTestResult = {
+ val pval = 1 - new KolmogorovSmirnovTest().cdf(ksStat, n.toInt)
+ new KolmogorovSmirnovTestResult(pval, ksStat, NullHypothesis.OneSampleTwoSided.toString)
+ }
+}
+
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/TestResult.scala b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/TestResult.scala
index 4784f9e947908..f44be13706695 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/stat/test/TestResult.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/stat/test/TestResult.scala
@@ -90,3 +90,20 @@ class ChiSqTestResult private[stat] (override val pValue: Double,
super.toString
}
}
+
+/**
+ * :: Experimental ::
+ * Object containing the test results for the Kolmogorov-Smirnov test.
+ */
+@Experimental
+class KolmogorovSmirnovTestResult private[stat] (
+ override val pValue: Double,
+ override val statistic: Double,
+ override val nullHypothesis: String) extends TestResult[Int] {
+
+ override val degreesOfFreedom = 0
+
+ override def toString: String = {
+ "Kolmogorov-Smirnov test summary:\n" + super.toString
+ }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
index 6eaebaf7dba9f..e6bcff48b022c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/KMeansDataGenerator.scala
@@ -64,8 +64,10 @@ object KMeansDataGenerator {
def main(args: Array[String]) {
if (args.length < 6) {
+ // scalastyle:off println
println("Usage: KMeansGenerator " +
" []")
+ // scalastyle:on println
System.exit(1)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
index b4e33c98ba7e5..87eeb5db05d26 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LinearDataGenerator.scala
@@ -153,8 +153,10 @@ object LinearDataGenerator {
def main(args: Array[String]) {
if (args.length < 2) {
+ // scalastyle:off println
println("Usage: LinearDataGenerator " +
" [num_examples] [num_features] [num_partitions]")
+ // scalastyle:on println
System.exit(1)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
index 9d802678c4a77..c09cbe69bb971 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/LogisticRegressionDataGenerator.scala
@@ -64,8 +64,10 @@ object LogisticRegressionDataGenerator {
def main(args: Array[String]) {
if (args.length != 5) {
+ // scalastyle:off println
println("Usage: LogisticRegressionGenerator " +
" ")
+ // scalastyle:on println
System.exit(1)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
index bd73a866c8a82..16f430599a515 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
@@ -55,8 +55,10 @@ import org.apache.spark.rdd.RDD
object MFDataGenerator {
def main(args: Array[String]) {
if (args.length < 2) {
+ // scalastyle:off println
println("Usage: MFDataGenerator " +
" [m] [n] [rank] [trainSampFact] [noise] [sigma] [test] [testSampFact]")
+ // scalastyle:on println
System.exit(1)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
index a8e30cc9d730c..ad20b7694a779 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/SVMDataGenerator.scala
@@ -37,8 +37,10 @@ object SVMDataGenerator {
def main(args: Array[String]) {
if (args.length < 2) {
+ // scalastyle:off println
println("Usage: SVMGenerator " +
" [num_examples] [num_features] [num_partitions]")
+ // scalastyle:on println
System.exit(1)
}
diff --git a/mllib/src/test/java/org/apache/spark/ml/feature/JavaPCASuite.java b/mllib/src/test/java/org/apache/spark/ml/feature/JavaPCASuite.java
new file mode 100644
index 0000000000000..5cf43fec6f29e
--- /dev/null
+++ b/mllib/src/test/java/org/apache/spark/ml/feature/JavaPCASuite.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.feature;
+
+import java.io.Serializable;
+import java.util.List;
+
+import scala.Tuple2;
+
+import com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.mllib.linalg.distributed.RowMatrix;
+import org.apache.spark.mllib.linalg.Matrix;
+import org.apache.spark.mllib.linalg.Vector;
+import org.apache.spark.mllib.linalg.Vectors;
+import org.apache.spark.sql.DataFrame;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+
+public class JavaPCASuite implements Serializable {
+ private transient JavaSparkContext jsc;
+ private transient SQLContext sqlContext;
+
+ @Before
+ public void setUp() {
+ jsc = new JavaSparkContext("local", "JavaPCASuite");
+ sqlContext = new SQLContext(jsc);
+ }
+
+ @After
+ public void tearDown() {
+ jsc.stop();
+ jsc = null;
+ }
+
+ public static class VectorPair implements Serializable {
+ private Vector features = Vectors.dense(0.0);
+ private Vector expected = Vectors.dense(0.0);
+
+ public void setFeatures(Vector features) {
+ this.features = features;
+ }
+
+ public Vector getFeatures() {
+ return this.features;
+ }
+
+ public void setExpected(Vector expected) {
+ this.expected = expected;
+ }
+
+ public Vector getExpected() {
+ return this.expected;
+ }
+ }
+
+ @Test
+ public void testPCA() {
+ List points = Lists.newArrayList(
+ Vectors.sparse(5, new int[]{1, 3}, new double[]{1.0, 7.0}),
+ Vectors.dense(2.0, 0.0, 3.0, 4.0, 5.0),
+ Vectors.dense(4.0, 0.0, 0.0, 6.0, 7.0)
+ );
+ JavaRDD dataRDD = jsc.parallelize(points, 2);
+
+ RowMatrix mat = new RowMatrix(dataRDD.rdd());
+ Matrix pc = mat.computePrincipalComponents(3);
+ JavaRDD expected = mat.multiply(pc).rows().toJavaRDD();
+
+ JavaRDD featuresExpected = dataRDD.zip(expected).map(
+ new Function, VectorPair>() {
+ public VectorPair call(Tuple2 pair) {
+ VectorPair featuresExpected = new VectorPair();
+ featuresExpected.setFeatures(pair._1());
+ featuresExpected.setExpected(pair._2());
+ return featuresExpected;
+ }
+ }
+ );
+
+ DataFrame df = sqlContext.createDataFrame(featuresExpected, VectorPair.class);
+ PCAModel pca = new PCA()
+ .setInputCol("features")
+ .setOutputCol("pca_features")
+ .setK(3)
+ .fit(df);
+ List result = pca.transform(df).select("pca_features", "expected").toJavaRDD().collect();
+ for (Row r : result) {
+ Assert.assertEquals(r.get(1), r.get(0));
+ }
+ }
+}
diff --git a/mllib/src/test/java/org/apache/spark/mllib/clustering/JavaLDASuite.java b/mllib/src/test/java/org/apache/spark/mllib/clustering/JavaLDASuite.java
index 581c033f08ebe..b48f190f599a2 100644
--- a/mllib/src/test/java/org/apache/spark/mllib/clustering/JavaLDASuite.java
+++ b/mllib/src/test/java/org/apache/spark/mllib/clustering/JavaLDASuite.java
@@ -28,12 +28,13 @@
import org.junit.Before;
import org.junit.Test;
+import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.linalg.Matrix;
import org.apache.spark.mllib.linalg.Vector;
-
+import org.apache.spark.mllib.linalg.Vectors;
public class JavaLDASuite implements Serializable {
private transient JavaSparkContext sc;
@@ -110,7 +111,15 @@ public void distributedLDAModel() {
// Check: topic distributions
JavaPairRDD topicDistributions = model.javaTopicDistributions();
- assertEquals(topicDistributions.count(), corpus.count());
+ // SPARK-5562. since the topicDistribution returns the distribution of the non empty docs
+ // over topics. Compare it against nonEmptyCorpus instead of corpus
+ JavaPairRDD nonEmptyCorpus = corpus.filter(
+ new Function, Boolean>() {
+ public Boolean call(Tuple2 tuple2) {
+ return Vectors.norm(tuple2._2(), 1.0) != 0.0;
+ }
+ });
+ assertEquals(topicDistributions.count(), nonEmptyCorpus.count());
}
@Test
diff --git a/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaAssociationRulesSuite.java b/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaAssociationRulesSuite.java
new file mode 100644
index 0000000000000..b3815ae6039c0
--- /dev/null
+++ b/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaAssociationRulesSuite.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.mllib.fpm;
+
+import java.io.Serializable;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import com.google.common.collect.Lists;
+
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset;
+
+
+public class JavaAssociationRulesSuite implements Serializable {
+ private transient JavaSparkContext sc;
+
+ @Before
+ public void setUp() {
+ sc = new JavaSparkContext("local", "JavaFPGrowth");
+ }
+
+ @After
+ public void tearDown() {
+ sc.stop();
+ sc = null;
+ }
+
+ @Test
+ public void runAssociationRules() {
+
+ @SuppressWarnings("unchecked")
+ JavaRDD> freqItemsets = sc.parallelize(Lists.newArrayList(
+ new FreqItemset(new String[] {"a"}, 15L),
+ new FreqItemset(new String[] {"b"}, 35L),
+ new FreqItemset(new String[] {"a", "b"}, 18L)
+ ));
+
+ JavaRDD> results = (new AssociationRules()).run(freqItemsets);
+ }
+}
+
diff --git a/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaFPGrowthSuite.java b/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaFPGrowthSuite.java
index bd0edf2b9ea62..9ce2c52dca8b6 100644
--- a/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaFPGrowthSuite.java
+++ b/mllib/src/test/java/org/apache/spark/mllib/fpm/JavaFPGrowthSuite.java
@@ -29,7 +29,6 @@
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.mllib.fpm.FPGrowth.FreqItemset;
public class JavaFPGrowthSuite implements Serializable {
private transient JavaSparkContext sc;
@@ -62,10 +61,10 @@ public void runFPGrowth() {
.setNumPartitions(2)
.run(rdd);
- List> freqItemsets = model.freqItemsets().toJavaRDD().collect();
+ List> freqItemsets = model.freqItemsets().toJavaRDD().collect();
assertEquals(18, freqItemsets.size());
- for (FreqItemset itemset: freqItemsets) {
+ for (FPGrowth.FreqItemset itemset: freqItemsets) {
// Test return types.
List items = itemset.javaItems();
long freq = itemset.freq();
diff --git a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index ba8fbee84197c..b7dd44753896a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -77,6 +77,7 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
assert(lr.getRawPredictionCol === "rawPrediction")
assert(lr.getProbabilityCol === "probability")
assert(lr.getFitIntercept)
+ assert(lr.getStandardization)
val model = lr.fit(dataset)
model.transform(dataset)
.select("label", "probability", "prediction", "rawPrediction")
@@ -208,8 +209,11 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
}
test("binary logistic regression with intercept without regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(true)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(true).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(true).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -230,18 +234,22 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V5 -0.7996864
*/
val interceptR = 2.8366423
- val weightsR = Array(-0.5895848, 0.8931147, -0.3925051, -0.7996864)
+ val weightsR = Vectors.dense(-0.5895848, 0.8931147, -0.3925051, -0.7996864)
+
+ assert(model1.intercept ~== interceptR relTol 1E-3)
+ assert(model1.weights ~= weightsR relTol 1E-3)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-3)
+ // Without regularization, with or without standardization will converge to the same solution.
+ assert(model2.intercept ~== interceptR relTol 1E-3)
+ assert(model2.weights ~= weightsR relTol 1E-3)
}
test("binary logistic regression without intercept without regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(false)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(false).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(false).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -263,19 +271,24 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V5 -0.7407946
*/
val interceptR = 0.0
- val weightsR = Array(-0.3534996, 1.2964482, -0.3571741, -0.7407946)
+ val weightsR = Vectors.dense(-0.3534996, 1.2964482, -0.3571741, -0.7407946)
+
+ assert(model1.intercept ~== interceptR relTol 1E-3)
+ assert(model1.weights ~= weightsR relTol 1E-2)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-2)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-2)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-3)
+ // Without regularization, with or without standardization should converge to the same solution.
+ assert(model2.intercept ~== interceptR relTol 1E-3)
+ assert(model2.weights ~= weightsR relTol 1E-2)
}
test("binary logistic regression with intercept with L1 regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(true)
- .setElasticNetParam(1.0).setRegParam(0.12)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(1.0).setRegParam(0.12).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(1.0).setRegParam(0.12).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -295,20 +308,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.04325749
data.V5 -0.02481551
*/
- val interceptR = -0.05627428
- val weightsR = Array(0.0, 0.0, -0.04325749, -0.02481551)
-
- assert(model.intercept ~== interceptR relTol 1E-2)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-2)
- assert(model.weights(3) ~== weightsR(3) relTol 2E-2)
+ val interceptR1 = -0.05627428
+ val weightsR1 = Vectors.dense(0.0, 0.0, -0.04325749, -0.02481551)
+
+ assert(model1.intercept ~== interceptR1 relTol 1E-2)
+ assert(model1.weights ~= weightsR1 absTol 2E-2)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
+
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 1, lambda = 0.12,
+ standardize=FALSE))
+ weights
+
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) 0.3722152
+ data.V2 .
+ data.V3 .
+ data.V4 -0.1665453
+ data.V5 .
+ */
+ val interceptR2 = 0.3722152
+ val weightsR2 = Vectors.dense(0.0, 0.0, -0.1665453, 0.0)
+
+ assert(model2.intercept ~== interceptR2 relTol 1E-2)
+ assert(model2.weights ~= weightsR2 absTol 1E-3)
}
test("binary logistic regression without intercept with L1 regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(false)
- .setElasticNetParam(1.0).setRegParam(0.12)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(1.0).setRegParam(0.12).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(1.0).setRegParam(0.12).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -329,20 +368,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.05189203
data.V5 -0.03891782
*/
- val interceptR = 0.0
- val weightsR = Array(0.0, 0.0, -0.05189203, -0.03891782)
+ val interceptR1 = 0.0
+ val weightsR1 = Vectors.dense(0.0, 0.0, -0.05189203, -0.03891782)
+
+ assert(model1.intercept ~== interceptR1 relTol 1E-3)
+ assert(model1.weights ~= weightsR1 absTol 1E-3)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-2)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-2)
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 1, lambda = 0.12,
+ intercept=FALSE, standardize=FALSE))
+ weights
+
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) .
+ data.V2 .
+ data.V3 .
+ data.V4 -0.08420782
+ data.V5 .
+ */
+ val interceptR2 = 0.0
+ val weightsR2 = Vectors.dense(0.0, 0.0, -0.08420782, 0.0)
+
+ assert(model2.intercept ~== interceptR2 absTol 1E-3)
+ assert(model2.weights ~= weightsR2 absTol 1E-3)
}
test("binary logistic regression with intercept with L2 regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(true)
- .setElasticNetParam(0.0).setRegParam(1.37)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(0.0).setRegParam(1.37).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(0.0).setRegParam(1.37).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -362,20 +427,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.04865309
data.V5 -0.10062872
*/
- val interceptR = 0.15021751
- val weightsR = Array(-0.07251837, 0.10724191, -0.04865309, -0.10062872)
-
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-3)
+ val interceptR1 = 0.15021751
+ val weightsR1 = Vectors.dense(-0.07251837, 0.10724191, -0.04865309, -0.10062872)
+
+ assert(model1.intercept ~== interceptR1 relTol 1E-3)
+ assert(model1.weights ~= weightsR1 relTol 1E-3)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
+
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 0, lambda = 1.37,
+ standardize=FALSE))
+ weights
+
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) 0.48657516
+ data.V2 -0.05155371
+ data.V3 0.02301057
+ data.V4 -0.11482896
+ data.V5 -0.06266838
+ */
+ val interceptR2 = 0.48657516
+ val weightsR2 = Vectors.dense(-0.05155371, 0.02301057, -0.11482896, -0.06266838)
+
+ assert(model2.intercept ~== interceptR2 relTol 1E-3)
+ assert(model2.weights ~= weightsR2 relTol 1E-3)
}
test("binary logistic regression without intercept with L2 regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(false)
- .setElasticNetParam(0.0).setRegParam(1.37)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(0.0).setRegParam(1.37).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(0.0).setRegParam(1.37).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -396,20 +487,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.04708770
data.V5 -0.09799775
*/
- val interceptR = 0.0
- val weightsR = Array(-0.06099165, 0.12857058, -0.04708770, -0.09799775)
+ val interceptR1 = 0.0
+ val weightsR1 = Vectors.dense(-0.06099165, 0.12857058, -0.04708770, -0.09799775)
+
+ assert(model1.intercept ~== interceptR1 absTol 1E-3)
+ assert(model1.weights ~= weightsR1 relTol 1E-2)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
+
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 0, lambda = 1.37,
+ intercept=FALSE, standardize=FALSE))
+ weights
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-2)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-2)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-3)
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) .
+ data.V2 -0.005679651
+ data.V3 0.048967094
+ data.V4 -0.093714016
+ data.V5 -0.053314311
+ */
+ val interceptR2 = 0.0
+ val weightsR2 = Vectors.dense(-0.005679651, 0.048967094, -0.093714016, -0.053314311)
+
+ assert(model2.intercept ~== interceptR2 absTol 1E-3)
+ assert(model2.weights ~= weightsR2 relTol 1E-2)
}
test("binary logistic regression with intercept with ElasticNet regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(true)
- .setElasticNetParam(0.38).setRegParam(0.21)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(0.38).setRegParam(0.21).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(0.38).setRegParam(0.21).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -429,20 +546,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.08849250
data.V5 -0.15458796
*/
- val interceptR = 0.57734851
- val weightsR = Array(-0.05310287, 0.0, -0.08849250, -0.15458796)
-
- assert(model.intercept ~== interceptR relTol 6E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 5E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
- assert(model.weights(2) ~== weightsR(2) relTol 5E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-3)
+ val interceptR1 = 0.57734851
+ val weightsR1 = Vectors.dense(-0.05310287, 0.0, -0.08849250, -0.15458796)
+
+ assert(model1.intercept ~== interceptR1 relTol 6E-3)
+ assert(model1.weights ~== weightsR1 absTol 5E-3)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
+
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 0.38, lambda = 0.21,
+ standardize=FALSE))
+ weights
+
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) 0.51555993
+ data.V2 .
+ data.V3 .
+ data.V4 -0.18807395
+ data.V5 -0.05350074
+ */
+ val interceptR2 = 0.51555993
+ val weightsR2 = Vectors.dense(0.0, 0.0, -0.18807395, -0.05350074)
+
+ assert(model2.intercept ~== interceptR2 relTol 6E-3)
+ assert(model2.weights ~= weightsR2 absTol 1E-3)
}
test("binary logistic regression without intercept with ElasticNet regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(false)
- .setElasticNetParam(0.38).setRegParam(0.21)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(0.38).setRegParam(0.21).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(false)
+ .setElasticNetParam(0.38).setRegParam(0.21).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -463,20 +606,46 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V4 -0.081203769
data.V5 -0.142534158
*/
- val interceptR = 0.0
- val weightsR = Array(-0.001005743, 0.072577857, -0.081203769, -0.142534158)
+ val interceptR1 = 0.0
+ val weightsR1 = Vectors.dense(-0.001005743, 0.072577857, -0.081203769, -0.142534158)
+
+ assert(model1.intercept ~== interceptR1 relTol 1E-3)
+ assert(model1.weights ~= weightsR1 absTol 1E-2)
+
+ /*
+ Using the following R code to load the data and train the model using glmnet package.
+
+ library("glmnet")
+ data <- read.csv("path", header=FALSE)
+ label = factor(data$V1)
+ features = as.matrix(data.frame(data$V2, data$V3, data$V4, data$V5))
+ weights = coef(glmnet(features,label, family="binomial", alpha = 0.38, lambda = 0.21,
+ intercept=FALSE, standardize=FALSE))
+ weights
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) absTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) absTol 1E-2)
- assert(model.weights(2) ~== weightsR(2) relTol 1E-3)
- assert(model.weights(3) ~== weightsR(3) relTol 1E-2)
+ 5 x 1 sparse Matrix of class "dgCMatrix"
+ s0
+ (Intercept) .
+ data.V2 .
+ data.V3 0.03345223
+ data.V4 -0.11304532
+ data.V5 .
+ */
+ val interceptR2 = 0.0
+ val weightsR2 = Vectors.dense(0.0, 0.03345223, -0.11304532, 0.0)
+
+ assert(model2.intercept ~== interceptR2 absTol 1E-3)
+ assert(model2.weights ~= weightsR2 absTol 1E-3)
}
test("binary logistic regression with intercept with strong L1 regularization") {
- val trainer = (new LogisticRegression).setFitIntercept(true)
- .setElasticNetParam(1.0).setRegParam(6.0)
- val model = trainer.fit(binaryDataset)
+ val trainer1 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(1.0).setRegParam(6.0).setStandardization(true)
+ val trainer2 = (new LogisticRegression).setFitIntercept(true)
+ .setElasticNetParam(1.0).setRegParam(6.0).setStandardization(false)
+
+ val model1 = trainer1.fit(binaryDataset)
+ val model2 = trainer2.fit(binaryDataset)
val histogram = binaryDataset.map { case Row(label: Double, features: Vector) => label }
.treeAggregate(new MultiClassSummarizer)(
@@ -500,13 +669,13 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
}}}
*/
val interceptTheory = math.log(histogram(1).toDouble / histogram(0).toDouble)
- val weightsTheory = Array(0.0, 0.0, 0.0, 0.0)
+ val weightsTheory = Vectors.dense(0.0, 0.0, 0.0, 0.0)
+
+ assert(model1.intercept ~== interceptTheory relTol 1E-5)
+ assert(model1.weights ~= weightsTheory absTol 1E-6)
- assert(model.intercept ~== interceptTheory relTol 1E-5)
- assert(model.weights(0) ~== weightsTheory(0) absTol 1E-6)
- assert(model.weights(1) ~== weightsTheory(1) absTol 1E-6)
- assert(model.weights(2) ~== weightsTheory(2) absTol 1E-6)
- assert(model.weights(3) ~== weightsTheory(3) absTol 1E-6)
+ assert(model2.intercept ~== interceptTheory relTol 1E-5)
+ assert(model2.weights ~= weightsTheory absTol 1E-6)
/*
Using the following R code to load the data and train the model using glmnet package.
@@ -527,12 +696,9 @@ class LogisticRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
data.V5 .
*/
val interceptR = -0.248065
- val weightsR = Array(0.0, 0.0, 0.0, 0.0)
+ val weightsR = Vectors.dense(0.0, 0.0, 0.0, 0.0)
- assert(model.intercept ~== interceptR relTol 1E-5)
- assert(model.weights(0) ~== weightsR(0) absTol 1E-6)
- assert(model.weights(1) ~== weightsR(1) absTol 1E-6)
- assert(model.weights(2) ~== weightsR(2) absTol 1E-6)
- assert(model.weights(3) ~== weightsR(3) absTol 1E-6)
+ assert(model1.intercept ~== interceptR relTol 1E-5)
+ assert(model1.weights ~= weightsR absTol 1E-6)
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizorSuite.scala
new file mode 100644
index 0000000000000..e90d9d4ef21ff
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizorSuite.scala
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.ml.feature
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.ml.param.ParamsSuite
+import org.apache.spark.mllib.linalg.{Vector, Vectors}
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+import org.apache.spark.mllib.util.TestingUtils._
+
+class CountVectorizerSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ test("params") {
+ ParamsSuite.checkParams(new CountVectorizerModel(Array("empty")))
+ }
+
+ test("CountVectorizerModel common cases") {
+ val df = sqlContext.createDataFrame(Seq(
+ (0, "a b c d".split(" ").toSeq,
+ Vectors.sparse(4, Seq((0, 1.0), (1, 1.0), (2, 1.0), (3, 1.0)))),
+ (1, "a b b c d a".split(" ").toSeq,
+ Vectors.sparse(4, Seq((0, 2.0), (1, 2.0), (2, 1.0), (3, 1.0)))),
+ (2, "a".split(" ").toSeq, Vectors.sparse(4, Seq((0, 1.0)))),
+ (3, "".split(" ").toSeq, Vectors.sparse(4, Seq())), // empty string
+ (4, "a notInDict d".split(" ").toSeq,
+ Vectors.sparse(4, Seq((0, 1.0), (3, 1.0)))) // with words not in vocabulary
+ )).toDF("id", "words", "expected")
+ val cv = new CountVectorizerModel(Array("a", "b", "c", "d"))
+ .setInputCol("words")
+ .setOutputCol("features")
+ val output = cv.transform(df).collect()
+ output.foreach { p =>
+ val features = p.getAs[Vector]("features")
+ val expected = p.getAs[Vector]("expected")
+ assert(features ~== expected absTol 1e-14)
+ }
+ }
+
+ test("CountVectorizerModel with minTermFreq") {
+ val df = sqlContext.createDataFrame(Seq(
+ (0, "a a a b b c c c d ".split(" ").toSeq, Vectors.sparse(4, Seq((0, 3.0), (2, 3.0)))),
+ (1, "c c c c c c".split(" ").toSeq, Vectors.sparse(4, Seq((2, 6.0)))),
+ (2, "a".split(" ").toSeq, Vectors.sparse(4, Seq())),
+ (3, "e e e e e".split(" ").toSeq, Vectors.sparse(4, Seq())))
+ ).toDF("id", "words", "expected")
+ val cv = new CountVectorizerModel(Array("a", "b", "c", "d"))
+ .setInputCol("words")
+ .setOutputCol("features")
+ .setMinTermFreq(3)
+ val output = cv.transform(df).collect()
+ output.foreach { p =>
+ val features = p.getAs[Vector]("features")
+ val expected = p.getAs[Vector]("expected")
+ assert(features ~== expected absTol 1e-14)
+ }
+ }
+}
+
+
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
index 8c85c96d5c6d8..03120c828ca96 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/VectorIndexerSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.ml.feature
import scala.beans.{BeanInfo, BeanProperty}
-import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.{Logging, SparkException, SparkFunSuite}
import org.apache.spark.ml.attribute._
import org.apache.spark.ml.param.ParamsSuite
import org.apache.spark.mllib.linalg.{SparseVector, Vector, Vectors}
@@ -27,7 +27,7 @@ import org.apache.spark.mllib.util.MLlibTestSparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.DataFrame
-class VectorIndexerSuite extends SparkFunSuite with MLlibTestSparkContext {
+class VectorIndexerSuite extends SparkFunSuite with MLlibTestSparkContext with Logging {
import VectorIndexerSuite.FeatureData
@@ -113,11 +113,11 @@ class VectorIndexerSuite extends SparkFunSuite with MLlibTestSparkContext {
model.transform(sparsePoints1) // should work
intercept[SparkException] {
model.transform(densePoints2).collect()
- println("Did not throw error when fit, transform were called on vectors of different lengths")
+ logInfo("Did not throw error when fit, transform were called on vectors of different lengths")
}
intercept[SparkException] {
vectorIndexer.fit(badPoints)
- println("Did not throw error when fitting vectors of different lengths in same RDD.")
+ logInfo("Did not throw error when fitting vectors of different lengths in same RDD.")
}
}
@@ -196,7 +196,7 @@ class VectorIndexerSuite extends SparkFunSuite with MLlibTestSparkContext {
}
} catch {
case e: org.scalatest.exceptions.TestFailedException =>
- println(errMsg)
+ logError(errMsg)
throw e
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
index 5f39d44f37352..cf120cf2a4b47 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.regression
import org.apache.spark.SparkFunSuite
-import org.apache.spark.mllib.linalg.DenseVector
+import org.apache.spark.mllib.linalg.{DenseVector, Vectors}
import org.apache.spark.mllib.util.{LinearDataGenerator, MLlibTestSparkContext}
import org.apache.spark.mllib.util.TestingUtils._
import org.apache.spark.sql.{DataFrame, Row}
@@ -75,11 +75,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 7.198257
*/
val interceptR = 6.298698
- val weightsR = Array(4.700706, 7.199082)
+ val weightsR = Vectors.dense(4.700706, 7.199082)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -104,11 +103,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V2. 6.995908
as.numeric.data.V3. 5.275131
*/
- val weightsR = Array(6.995908, 5.275131)
+ val weightsR = Vectors.dense(6.995908, 5.275131)
- assert(model.intercept ~== 0 relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== 0 absTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
/*
Then again with the data with no intercept:
> weightsWithoutIntercept
@@ -118,11 +116,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data3.V2. 4.70011
as.numeric.data3.V3. 7.19943
*/
- val weightsWithoutInterceptR = Array(4.70011, 7.19943)
+ val weightsWithoutInterceptR = Vectors.dense(4.70011, 7.19943)
- assert(modelWithoutIntercept.intercept ~== 0 relTol 1E-3)
- assert(modelWithoutIntercept.weights(0) ~== weightsWithoutInterceptR(0) relTol 1E-3)
- assert(modelWithoutIntercept.weights(1) ~== weightsWithoutInterceptR(1) relTol 1E-3)
+ assert(modelWithoutIntercept.intercept ~== 0 absTol 1E-3)
+ assert(modelWithoutIntercept.weights ~= weightsWithoutInterceptR relTol 1E-3)
}
test("linear regression with intercept with L1 regularization") {
@@ -139,11 +136,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 6.679841
*/
val interceptR = 6.24300
- val weightsR = Array(4.024821, 6.679841)
+ val weightsR = Vectors.dense(4.024821, 6.679841)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -169,11 +165,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.772913
*/
val interceptR = 0.0
- val weightsR = Array(6.299752, 4.772913)
+ val weightsR = Vectors.dense(6.299752, 4.772913)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-5)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -197,11 +192,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.926260
*/
val interceptR = 5.269376
- val weightsR = Array(3.736216, 5.712356)
+ val weightsR = Vectors.dense(3.736216, 5.712356)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -227,11 +221,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.214502
*/
val interceptR = 0.0
- val weightsR = Array(5.522875, 4.214502)
+ val weightsR = Vectors.dense(5.522875, 4.214502)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-3)
+ assert(model.weights ~== weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -255,11 +248,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 5.200403
*/
val interceptR = 5.696056
- val weightsR = Array(3.670489, 6.001122)
+ val weightsR = Vectors.dense(3.670489, 6.001122)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~== weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -285,11 +277,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.dataM.V3. 4.322251
*/
val interceptR = 0.0
- val weightsR = Array(5.673348, 4.322251)
+ val weightsR = Vectors.dense(5.673348, 4.322251)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -298,4 +289,63 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
assert(prediction1 ~== prediction2 relTol 1E-5)
}
}
+
+ test("linear regression model training summary") {
+ val trainer = new LinearRegression
+ val model = trainer.fit(dataset)
+
+ // Training results for the model should be available
+ assert(model.hasSummary)
+
+ // Residuals in [[LinearRegressionResults]] should equal those manually computed
+ val expectedResiduals = dataset.select("features", "label")
+ .map { case Row(features: DenseVector, label: Double) =>
+ val prediction =
+ features(0) * model.weights(0) + features(1) * model.weights(1) + model.intercept
+ prediction - label
+ }
+ .zip(model.summary.residuals.map(_.getDouble(0)))
+ .collect()
+ .foreach { case (manualResidual: Double, resultResidual: Double) =>
+ assert(manualResidual ~== resultResidual relTol 1E-5)
+ }
+
+ /*
+ Use the following R code to generate model training results.
+
+ predictions <- predict(fit, newx=features)
+ residuals <- predictions - label
+ > mean(residuals^2) # MSE
+ [1] 0.009720325
+ > mean(abs(residuals)) # MAD
+ [1] 0.07863206
+ > cor(predictions, label)^2# r^2
+ [,1]
+ s0 0.9998749
+ */
+ assert(model.summary.meanSquaredError ~== 0.00972035 relTol 1E-5)
+ assert(model.summary.meanAbsoluteError ~== 0.07863206 relTol 1E-5)
+ assert(model.summary.r2 ~== 0.9998749 relTol 1E-5)
+
+ // Objective function should be monotonically decreasing for linear regression
+ assert(
+ model.summary
+ .objectiveHistory
+ .sliding(2)
+ .forall(x => x(0) >= x(1)))
+ }
+
+ test("linear regression model testset evaluation summary") {
+ val trainer = new LinearRegression
+ val model = trainer.fit(dataset)
+
+ // Evaluating on training dataset should yield results summary equal to training summary
+ val testSummary = model.evaluate(dataset)
+ assert(model.summary.meanSquaredError ~== testSummary.meanSquaredError relTol 1E-5)
+ assert(model.summary.r2 ~== testSummary.r2 relTol 1E-5)
+ model.summary.residuals.select("residuals").collect()
+ .zip(testSummary.residuals.select("residuals").collect())
+ .forall { case (Row(r1: Double), Row(r2: Double)) => r1 ~== r2 relTol 1E-5 }
+ }
+
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
index f7fc8730606af..cffa1ab700f80 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/classification/NaiveBayesSuite.scala
@@ -19,13 +19,14 @@ package org.apache.spark.mllib.classification
import scala.util.Random
-import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, argmax => brzArgmax, sum => brzSum}
+import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, Vector => BV}
import breeze.stats.distributions.{Multinomial => BrzMultinomial}
import org.apache.spark.{SparkException, SparkFunSuite}
-import org.apache.spark.mllib.linalg.Vectors
+import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.util.{LocalClusterSparkContext, MLlibTestSparkContext}
+import org.apache.spark.mllib.util.TestingUtils._
import org.apache.spark.util.Utils
object NaiveBayesSuite {
@@ -154,6 +155,29 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
// Test prediction on Array.
validatePrediction(validationData.map(row => model.predict(row.features)), validationData)
+
+ // Test posteriors
+ validationData.map(_.features).foreach { features =>
+ val predicted = model.predictProbabilities(features).toArray
+ assert(predicted.sum ~== 1.0 relTol 1.0e-10)
+ val expected = expectedMultinomialProbabilities(model, features)
+ expected.zip(predicted).foreach { case (e, p) => assert(e ~== p relTol 1.0e-10) }
+ }
+ }
+
+ /**
+ * @param model Multinomial Naive Bayes model
+ * @param testData input to compute posterior probabilities for
+ * @return posterior class probabilities (in order of labels) for input
+ */
+ private def expectedMultinomialProbabilities(model: NaiveBayesModel, testData: Vector) = {
+ val piVector = new BDV(model.pi)
+ // model.theta is row-major; treat it as col-major representation of transpose, and transpose:
+ val thetaMatrix = new BDM(model.theta(0).length, model.theta.length, model.theta.flatten).t
+ val logClassProbs: BV[Double] = piVector + (thetaMatrix * testData.toBreeze)
+ val classProbs = logClassProbs.toArray.map(math.exp)
+ val classProbsSum = classProbs.sum
+ classProbs.map(_ / classProbsSum)
}
test("Naive Bayes Bernoulli") {
@@ -182,6 +206,33 @@ class NaiveBayesSuite extends SparkFunSuite with MLlibTestSparkContext {
// Test prediction on Array.
validatePrediction(validationData.map(row => model.predict(row.features)), validationData)
+
+ // Test posteriors
+ validationData.map(_.features).foreach { features =>
+ val predicted = model.predictProbabilities(features).toArray
+ assert(predicted.sum ~== 1.0 relTol 1.0e-10)
+ val expected = expectedBernoulliProbabilities(model, features)
+ expected.zip(predicted).foreach { case (e, p) => assert(e ~== p relTol 1.0e-10) }
+ }
+ }
+
+ /**
+ * @param model Bernoulli Naive Bayes model
+ * @param testData input to compute posterior probabilities for
+ * @return posterior class probabilities (in order of labels) for input
+ */
+ private def expectedBernoulliProbabilities(model: NaiveBayesModel, testData: Vector) = {
+ val piVector = new BDV(model.pi)
+ val thetaMatrix = new BDM(model.theta(0).length, model.theta.length, model.theta.flatten).t
+ val negThetaMatrix = new BDM(model.theta(0).length, model.theta.length,
+ model.theta.flatten.map(v => math.log(1.0 - math.exp(v)))).t
+ val testBreeze = testData.toBreeze
+ val negTestBreeze = new BDV(Array.fill(testBreeze.size)(1.0)) - testBreeze
+ val piTheta: BV[Double] = piVector + (thetaMatrix * testBreeze)
+ val logClassProbs: BV[Double] = piTheta + (negThetaMatrix * negTestBreeze)
+ val classProbs = logClassProbs.toArray.map(math.exp)
+ val classProbsSum = classProbs.sum
+ classProbs.map(_ / classProbsSum)
}
test("detect negative values") {
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
index 406affa25539d..03a8a2538b464 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/LDASuite.scala
@@ -99,9 +99,13 @@ class LDASuite extends SparkFunSuite with MLlibTestSparkContext {
// Check: per-doc topic distributions
val topicDistributions = model.topicDistributions.collect()
+
// Ensure all documents are covered.
- assert(topicDistributions.length === tinyCorpus.length)
- assert(tinyCorpus.map(_._1).toSet === topicDistributions.map(_._1).toSet)
+ // SPARK-5562. since the topicDistribution returns the distribution of the non empty docs
+ // over topics. Compare it against nonEmptyTinyCorpus instead of tinyCorpus
+ val nonEmptyTinyCorpus = getNonEmptyDoc(tinyCorpus)
+ assert(topicDistributions.length === nonEmptyTinyCorpus.length)
+ assert(nonEmptyTinyCorpus.map(_._1).toSet === topicDistributions.map(_._1).toSet)
// Ensure we have proper distributions
topicDistributions.foreach { case (docId, topicDistribution) =>
assert(topicDistribution.size === tinyK)
@@ -232,12 +236,17 @@ private[clustering] object LDASuite {
}
def tinyCorpus: Array[(Long, Vector)] = Array(
+ Vectors.dense(0, 0, 0, 0, 0), // empty doc
Vectors.dense(1, 3, 0, 2, 8),
Vectors.dense(0, 2, 1, 0, 4),
Vectors.dense(2, 3, 12, 3, 1),
+ Vectors.dense(0, 0, 0, 0, 0), // empty doc
Vectors.dense(0, 3, 1, 9, 8),
Vectors.dense(1, 1, 4, 2, 6)
).zipWithIndex.map { case (wordCounts, docId) => (docId.toLong, wordCounts) }
assert(tinyCorpus.forall(_._2.size == tinyVocabSize)) // sanity check for test data
+ def getNonEmptyDoc(corpus: Array[(Long, Vector)]): Array[(Long, Vector)] = corpus.filter {
+ case (_, wc: Vector) => Vectors.norm(wc, p = 1.0) != 0.0
+ }
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/fpm/AssociationRulesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/fpm/AssociationRulesSuite.scala
new file mode 100644
index 0000000000000..77a2773c36f56
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/mllib/fpm/AssociationRulesSuite.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.mllib.fpm
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.mllib.util.MLlibTestSparkContext
+
+class AssociationRulesSuite extends SparkFunSuite with MLlibTestSparkContext {
+
+ test("association rules using String type") {
+ val freqItemsets = sc.parallelize(Seq(
+ (Set("s"), 3L), (Set("z"), 5L), (Set("x"), 4L), (Set("t"), 3L), (Set("y"), 3L),
+ (Set("r"), 3L),
+ (Set("x", "z"), 3L), (Set("t", "y"), 3L), (Set("t", "x"), 3L), (Set("s", "x"), 3L),
+ (Set("y", "x"), 3L), (Set("y", "z"), 3L), (Set("t", "z"), 3L),
+ (Set("y", "x", "z"), 3L), (Set("t", "x", "z"), 3L), (Set("t", "y", "z"), 3L),
+ (Set("t", "y", "x"), 3L),
+ (Set("t", "y", "x", "z"), 3L)
+ ).map {
+ case (items, freq) => new FPGrowth.FreqItemset(items.toArray, freq)
+ })
+
+ val ar = new AssociationRules()
+
+ val results1 = ar
+ .setMinConfidence(0.9)
+ .run(freqItemsets)
+ .collect()
+
+ /* Verify results using the `R` code:
+ transactions = as(sapply(
+ list("r z h k p",
+ "z y x w v u t s",
+ "s x o n r",
+ "x z y m t s q e",
+ "z",
+ "x z y r q t p"),
+ FUN=function(x) strsplit(x," ",fixed=TRUE)),
+ "transactions")
+ ars = apriori(transactions,
+ parameter = list(support = 0.0, confidence = 0.5, target="rules", minlen=2))
+ arsDF = as(ars, "data.frame")
+ arsDF$support = arsDF$support * length(transactions)
+ names(arsDF)[names(arsDF) == "support"] = "freq"
+ > nrow(arsDF)
+ [1] 23
+ > sum(arsDF$confidence == 1)
+ [1] 23
+ */
+ assert(results1.size === 23)
+ assert(results1.count(rule => math.abs(rule.confidence - 1.0D) < 1e-6) == 23)
+
+ val results2 = ar
+ .setMinConfidence(0)
+ .run(freqItemsets)
+ .collect()
+
+ /* Verify results using the `R` code:
+ ars = apriori(transactions,
+ parameter = list(support = 0.5, confidence = 0.5, target="rules", minlen=2))
+ arsDF = as(ars, "data.frame")
+ arsDF$support = arsDF$support * length(transactions)
+ names(arsDF)[names(arsDF) == "support"] = "freq"
+ nrow(arsDF)
+ sum(arsDF$confidence == 1)
+ > nrow(arsDF)
+ [1] 30
+ > sum(arsDF$confidence == 1)
+ [1] 23
+ */
+ assert(results2.size === 30)
+ assert(results2.count(rule => math.abs(rule.confidence - 1.0D) < 1e-6) == 23)
+ }
+}
+
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/fpm/FPGrowthSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/fpm/FPGrowthSuite.scala
index 1a8a1e79f2810..4a9bfdb348d9f 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/fpm/FPGrowthSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/fpm/FPGrowthSuite.scala
@@ -22,7 +22,7 @@ import org.apache.spark.mllib.util.MLlibTestSparkContext
class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
- test("FP-Growth frequent itemsets using String type") {
+ test("FP-Growth using String type") {
val transactions = Seq(
"r z h k p",
"z y x w v u t s",
@@ -38,18 +38,59 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
val model6 = fpg
.setMinSupport(0.9)
.setNumPartitions(1)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ transactions = as(sapply(
+ list("r z h k p",
+ "z y x w v u t s",
+ "s x o n r",
+ "x z y m t s q e",
+ "z",
+ "x z y r q t p"),
+ FUN=function(x) strsplit(x," ",fixed=TRUE)),
+ "transactions")
+ > eclat(transactions, parameter = list(support = 0.9))
+ ...
+ eclat - zero frequent items
+ set of 0 itemsets
+ */
assert(model6.freqItemsets.count() === 0)
val model3 = fpg
.setMinSupport(0.5)
.setNumPartitions(2)
- .setOrdered(false)
.run(rdd)
val freqItemsets3 = model3.freqItemsets.collect().map { itemset =>
(itemset.items.toSet, itemset.freq)
}
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.5))
+ fpDF = as(sort(fp), "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > fpDF
+ items freq
+ 13 {z} 5
+ 14 {x} 4
+ 1 {s,x} 3
+ 2 {t,x,y,z} 3
+ 3 {t,y,z} 3
+ 4 {t,x,y} 3
+ 5 {x,y,z} 3
+ 6 {y,z} 3
+ 7 {x,y} 3
+ 8 {t,y} 3
+ 9 {t,x,z} 3
+ 10 {t,z} 3
+ 11 {t,x} 3
+ 12 {x,z} 3
+ 15 {t} 3
+ 16 {y} 3
+ 17 {s} 3
+ 18 {r} 3
+ */
val expected = Set(
(Set("s"), 3L), (Set("z"), 5L), (Set("x"), 4L), (Set("t"), 3L), (Set("y"), 3L),
(Set("r"), 3L),
@@ -63,19 +104,35 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
val model2 = fpg
.setMinSupport(0.3)
.setNumPartitions(4)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.3))
+ fpDF = as(fp, "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > nrow(fpDF)
+ [1] 54
+ */
assert(model2.freqItemsets.count() === 54)
val model1 = fpg
.setMinSupport(0.1)
.setNumPartitions(8)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.1))
+ fpDF = as(fp, "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > nrow(fpDF)
+ [1] 625
+ */
assert(model1.freqItemsets.count() === 625)
}
- test("FP-Growth frequent sequences using String type"){
+ test("FP-Growth String type association rule generation") {
val transactions = Seq(
"r z h k p",
"z y x w v u t s",
@@ -86,36 +143,38 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
.map(_.split(" "))
val rdd = sc.parallelize(transactions, 2).cache()
- val fpg = new FPGrowth()
-
- val model1 = fpg
+ /* Verify results using the `R` code:
+ transactions = as(sapply(
+ list("r z h k p",
+ "z y x w v u t s",
+ "s x o n r",
+ "x z y m t s q e",
+ "z",
+ "x z y r q t p"),
+ FUN=function(x) strsplit(x," ",fixed=TRUE)),
+ "transactions")
+ ars = apriori(transactions,
+ parameter = list(support = 0.0, confidence = 0.5, target="rules", minlen=2))
+ arsDF = as(ars, "data.frame")
+ arsDF$support = arsDF$support * length(transactions)
+ names(arsDF)[names(arsDF) == "support"] = "freq"
+ > nrow(arsDF)
+ [1] 23
+ > sum(arsDF$confidence == 1)
+ [1] 23
+ */
+ val rules = (new FPGrowth())
.setMinSupport(0.5)
.setNumPartitions(2)
- .setOrdered(true)
.run(rdd)
+ .generateAssociationRules(0.9)
+ .collect()
- /*
- Use the following R code to verify association rules using arulesSequences package.
-
- data = read_baskets("path", info = c("sequenceID","eventID","SIZE"))
- freqItemSeq = cspade(data, parameter = list(support = 0.5))
- resSeq = as(freqItemSeq, "data.frame")
- resSeq$support = resSeq$support * length(transactions)
- names(resSeq)[names(resSeq) == "support"] = "freq"
- resSeq
- */
- val expected = Set(
- (Seq("r"), 3L), (Seq("s"), 3L), (Seq("t"), 3L), (Seq("x"), 4L), (Seq("y"), 3L),
- (Seq("z"), 5L), (Seq("z", "y"), 3L), (Seq("x", "t"), 3L), (Seq("y", "t"), 3L),
- (Seq("z", "t"), 3L), (Seq("z", "y", "t"), 3L)
- )
- val freqItemseqs1 = model1.freqItemsets.collect().map { itemset =>
- (itemset.items.toSeq, itemset.freq)
- }.toSet
- assert(freqItemseqs1 == expected)
+ assert(rules.size === 23)
+ assert(rules.count(rule => math.abs(rule.confidence - 1.0D) < 1e-6) == 23)
}
- test("FP-Growth frequent itemsets using Int type") {
+ test("FP-Growth using Int type") {
val transactions = Seq(
"1 2 3",
"1 2 3 4",
@@ -132,20 +191,53 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
val model6 = fpg
.setMinSupport(0.9)
.setNumPartitions(1)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ transactions = as(sapply(
+ list("1 2 3",
+ "1 2 3 4",
+ "5 4 3 2 1",
+ "6 5 4 3 2 1",
+ "2 4",
+ "1 3",
+ "1 7"),
+ FUN=function(x) strsplit(x," ",fixed=TRUE)),
+ "transactions")
+ > eclat(transactions, parameter = list(support = 0.9))
+ ...
+ eclat - zero frequent items
+ set of 0 itemsets
+ */
assert(model6.freqItemsets.count() === 0)
val model3 = fpg
.setMinSupport(0.5)
.setNumPartitions(2)
- .setOrdered(false)
.run(rdd)
assert(model3.freqItemsets.first().items.getClass === Array(1).getClass,
"frequent itemsets should use primitive arrays")
val freqItemsets3 = model3.freqItemsets.collect().map { itemset =>
(itemset.items.toSet, itemset.freq)
}
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.5))
+ fpDF = as(sort(fp), "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > fpDF
+ items freq
+ 6 {1} 6
+ 3 {1,3} 5
+ 7 {2} 5
+ 8 {3} 5
+ 1 {2,4} 4
+ 2 {1,2,3} 4
+ 4 {2,3} 4
+ 5 {1,2} 4
+ 9 {4} 4
+ */
val expected = Set(
(Set(1), 6L), (Set(2), 5L), (Set(3), 5L), (Set(4), 4L),
(Set(1, 2), 4L), (Set(1, 3), 5L), (Set(2, 3), 4L),
@@ -155,15 +247,31 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext {
val model2 = fpg
.setMinSupport(0.3)
.setNumPartitions(4)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.3))
+ fpDF = as(fp, "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > nrow(fpDF)
+ [1] 15
+ */
assert(model2.freqItemsets.count() === 15)
val model1 = fpg
.setMinSupport(0.1)
.setNumPartitions(8)
- .setOrdered(false)
.run(rdd)
+
+ /* Verify results using the `R` code:
+ fp = eclat(transactions, parameter = list(support = 0.1))
+ fpDF = as(fp, "data.frame")
+ fpDF$support = fpDF$support * length(transactions)
+ names(fpDF)[names(fpDF) == "support"] = "freq"
+ > nrow(fpDF)
+ [1] 65
+ */
assert(model1.freqItemsets.count() === 65)
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
index c4ae0a16f7c04..178d95a7b94ec 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
@@ -21,10 +21,10 @@ import scala.util.Random
import breeze.linalg.{DenseMatrix => BDM, squaredDistance => breezeSquaredDistance}
-import org.apache.spark.{SparkException, SparkFunSuite}
+import org.apache.spark.{Logging, SparkException, SparkFunSuite}
import org.apache.spark.mllib.util.TestingUtils._
-class VectorsSuite extends SparkFunSuite {
+class VectorsSuite extends SparkFunSuite with Logging {
val arr = Array(0.1, 0.0, 0.3, 0.4)
val n = 4
@@ -142,7 +142,7 @@ class VectorsSuite extends SparkFunSuite {
malformatted.foreach { s =>
intercept[SparkException] {
Vectors.parse(s)
- println(s"Didn't detect malformatted string $s.")
+ logInfo(s"Didn't detect malformatted string $s.")
}
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/stat/CorrelationSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/stat/CorrelationSuite.scala
index c292ced75e870..c3eeda012571c 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/stat/CorrelationSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/stat/CorrelationSuite.scala
@@ -19,13 +19,13 @@ package org.apache.spark.mllib.stat
import breeze.linalg.{DenseMatrix => BDM, Matrix => BM}
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{Logging, SparkFunSuite}
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.stat.correlation.{Correlations, PearsonCorrelation,
SpearmanCorrelation}
import org.apache.spark.mllib.util.MLlibTestSparkContext
-class CorrelationSuite extends SparkFunSuite with MLlibTestSparkContext {
+class CorrelationSuite extends SparkFunSuite with MLlibTestSparkContext with Logging {
// test input data
val xData = Array(1.0, 0.0, -2.0)
@@ -146,7 +146,7 @@ class CorrelationSuite extends SparkFunSuite with MLlibTestSparkContext {
def matrixApproxEqual(A: BM[Double], B: BM[Double], threshold: Double = 1e-6): Boolean = {
for (i <- 0 until A.rows; j <- 0 until A.cols) {
if (!approxEqual(A(i, j), B(i, j), threshold)) {
- println("i, j = " + i + ", " + j + " actual: " + A(i, j) + " expected:" + B(i, j))
+ logInfo("i, j = " + i + ", " + j + " actual: " + A(i, j) + " expected:" + B(i, j))
return false
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/stat/HypothesisTestSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/stat/HypothesisTestSuite.scala
index b084a5fb4313f..142b90e764a7c 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/stat/HypothesisTestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/stat/HypothesisTestSuite.scala
@@ -19,6 +19,10 @@ package org.apache.spark.mllib.stat
import java.util.Random
+import org.apache.commons.math3.distribution.{ExponentialDistribution,
+ NormalDistribution, UniformRealDistribution}
+import org.apache.commons.math3.stat.inference.KolmogorovSmirnovTest
+
import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.mllib.linalg.{DenseVector, Matrices, Vectors}
import org.apache.spark.mllib.regression.LabeledPoint
@@ -153,4 +157,101 @@ class HypothesisTestSuite extends SparkFunSuite with MLlibTestSparkContext {
Statistics.chiSqTest(sc.parallelize(continuousFeature, 2))
}
}
+
+ test("1 sample Kolmogorov-Smirnov test: apache commons math3 implementation equivalence") {
+ // Create theoretical distributions
+ val stdNormalDist = new NormalDistribution(0, 1)
+ val expDist = new ExponentialDistribution(0.6)
+ val unifDist = new UniformRealDistribution()
+
+ // set seeds
+ val seed = 10L
+ stdNormalDist.reseedRandomGenerator(seed)
+ expDist.reseedRandomGenerator(seed)
+ unifDist.reseedRandomGenerator(seed)
+
+ // Sample data from the distributions and parallelize it
+ val n = 100000
+ val sampledNorm = sc.parallelize(stdNormalDist.sample(n), 10)
+ val sampledExp = sc.parallelize(expDist.sample(n), 10)
+ val sampledUnif = sc.parallelize(unifDist.sample(n), 10)
+
+ // Use a apache math commons local KS test to verify calculations
+ val ksTest = new KolmogorovSmirnovTest()
+ val pThreshold = 0.05
+
+ // Comparing a standard normal sample to a standard normal distribution
+ val result1 = Statistics.kolmogorovSmirnovTest(sampledNorm, "norm", 0, 1)
+ val referenceStat1 = ksTest.kolmogorovSmirnovStatistic(stdNormalDist, sampledNorm.collect())
+ val referencePVal1 = 1 - ksTest.cdf(referenceStat1, n)
+ // Verify vs apache math commons ks test
+ assert(result1.statistic ~== referenceStat1 relTol 1e-4)
+ assert(result1.pValue ~== referencePVal1 relTol 1e-4)
+ // Cannot reject null hypothesis
+ assert(result1.pValue > pThreshold)
+
+ // Comparing an exponential sample to a standard normal distribution
+ val result2 = Statistics.kolmogorovSmirnovTest(sampledExp, "norm", 0, 1)
+ val referenceStat2 = ksTest.kolmogorovSmirnovStatistic(stdNormalDist, sampledExp.collect())
+ val referencePVal2 = 1 - ksTest.cdf(referenceStat2, n)
+ // verify vs apache math commons ks test
+ assert(result2.statistic ~== referenceStat2 relTol 1e-4)
+ assert(result2.pValue ~== referencePVal2 relTol 1e-4)
+ // reject null hypothesis
+ assert(result2.pValue < pThreshold)
+
+ // Testing the use of a user provided CDF function
+ // Distribution is not serializable, so will have to create in the lambda
+ val expCDF = (x: Double) => new ExponentialDistribution(0.2).cumulativeProbability(x)
+
+ // Comparing an exponential sample with mean X to an exponential distribution with mean Y
+ // Where X != Y
+ val result3 = Statistics.kolmogorovSmirnovTest(sampledExp, expCDF)
+ val referenceStat3 = ksTest.kolmogorovSmirnovStatistic(new ExponentialDistribution(0.2),
+ sampledExp.collect())
+ val referencePVal3 = 1 - ksTest.cdf(referenceStat3, sampledNorm.count().toInt)
+ // verify vs apache math commons ks test
+ assert(result3.statistic ~== referenceStat3 relTol 1e-4)
+ assert(result3.pValue ~== referencePVal3 relTol 1e-4)
+ // reject null hypothesis
+ assert(result3.pValue < pThreshold)
+ }
+
+ test("1 sample Kolmogorov-Smirnov test: R implementation equivalence") {
+ /*
+ Comparing results with R's implementation of Kolmogorov-Smirnov for 1 sample
+ > sessionInfo()
+ R version 3.2.0 (2015-04-16)
+ Platform: x86_64-apple-darwin13.4.0 (64-bit)
+ > set.seed(20)
+ > v <- rnorm(20)
+ > v
+ [1] 1.16268529 -0.58592447 1.78546500 -1.33259371 -0.44656677 0.56960612
+ [7] -2.88971761 -0.86901834 -0.46170268 -0.55554091 -0.02013537 -0.15038222
+ [13] -0.62812676 1.32322085 -1.52135057 -0.43742787 0.97057758 0.02822264
+ [19] -0.08578219 0.38921440
+ > ks.test(v, pnorm, alternative = "two.sided")
+
+ One-sample Kolmogorov-Smirnov test
+
+ data: v
+ D = 0.18874, p-value = 0.4223
+ alternative hypothesis: two-sided
+ */
+
+ val rKSStat = 0.18874
+ val rKSPVal = 0.4223
+ val rData = sc.parallelize(
+ Array(
+ 1.1626852897838, -0.585924465893051, 1.78546500331661, -1.33259371048501,
+ -0.446566766553219, 0.569606122374976, -2.88971761441412, -0.869018343326555,
+ -0.461702683149641, -0.555540910137444, -0.0201353678515895, -0.150382224136063,
+ -0.628126755843964, 1.32322085193283, -1.52135057001199, -0.437427868856691,
+ 0.970577579543399, 0.0282226444247749, -0.0857821886527593, 0.389214404984942
+ )
+ )
+ val rCompResult = Statistics.kolmogorovSmirnovTest(rData, "norm", 0, 1)
+ assert(rCompResult.statistic ~== rKSStat relTol 1e-4)
+ assert(rCompResult.pValue ~== rKSPVal relTol 1e-4)
+ }
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
index 84dd3b342d4c0..2521b3342181a 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/tree/GradientBoostedTreesSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.mllib.tree
-import org.apache.spark.SparkFunSuite
+import org.apache.spark.{Logging, SparkFunSuite}
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.mllib.tree.configuration.Algo._
import org.apache.spark.mllib.tree.configuration.{BoostingStrategy, Strategy}
@@ -31,7 +31,7 @@ import org.apache.spark.util.Utils
/**
* Test suite for [[GradientBoostedTrees]].
*/
-class GradientBoostedTreesSuite extends SparkFunSuite with MLlibTestSparkContext {
+class GradientBoostedTreesSuite extends SparkFunSuite with MLlibTestSparkContext with Logging {
test("Regression with continuous features: SquaredError") {
GradientBoostedTreesSuite.testCombinations.foreach {
@@ -50,7 +50,7 @@ class GradientBoostedTreesSuite extends SparkFunSuite with MLlibTestSparkContext
EnsembleTestHelper.validateRegressor(gbt, GradientBoostedTreesSuite.data, 0.06)
} catch {
case e: java.lang.AssertionError =>
- println(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
+ logError(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
s" subsamplingRate=$subsamplingRate")
throw e
}
@@ -80,7 +80,7 @@ class GradientBoostedTreesSuite extends SparkFunSuite with MLlibTestSparkContext
EnsembleTestHelper.validateRegressor(gbt, GradientBoostedTreesSuite.data, 0.85, "mae")
} catch {
case e: java.lang.AssertionError =>
- println(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
+ logError(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
s" subsamplingRate=$subsamplingRate")
throw e
}
@@ -111,7 +111,7 @@ class GradientBoostedTreesSuite extends SparkFunSuite with MLlibTestSparkContext
EnsembleTestHelper.validateClassifier(gbt, GradientBoostedTreesSuite.data, 0.9)
} catch {
case e: java.lang.AssertionError =>
- println(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
+ logError(s"FAILED for numIterations=$numIterations, learningRate=$learningRate," +
s" subsamplingRate=$subsamplingRate")
throw e
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala
index fa4f74d71b7e7..16d7c3ab39b03 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala
@@ -33,7 +33,7 @@ class NumericParserSuite extends SparkFunSuite {
malformatted.foreach { s =>
intercept[SparkException] {
NumericParser.parse(s)
- println(s"Didn't detect malformatted string $s.")
+ throw new RuntimeException(s"Didn't detect malformatted string $s.")
}
}
}
diff --git a/pom.xml b/pom.xml
index ffa96128a3d61..370c95dd03632 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,7 +128,7 @@
${hadoop.version}
0.98.7-hadoop2
hbase
- 1.4.0
+ 1.6.0
3.4.5
2.4.0
org.spark-project.hive
@@ -161,6 +161,8 @@
2.4.4
1.1.1.7
1.1.2
+
+ false
${java.home}
@@ -177,6 +179,7 @@
compile
compile
compile
+ test
+
+ twttr-repo
+ Twttr Repository
+ http://maven.twttr.com
+
+ true
+
+
+ false
+
+
spark-1.4-staging
@@ -326,11 +341,6 @@
-
- ${jline.groupid}
- jline
- ${jline.version}
-
com.twitter
chill_${scala.binary.version}
@@ -1099,6 +1109,12 @@
${parquet.version}
${parquet.deps.scope}
+
+ org.apache.parquet
+ parquet-avro
+ ${parquet.version}
+ ${parquet.test.deps.scope}
+
org.apache.flume
flume-ng-core
@@ -1109,6 +1125,10 @@
io.netty
netty
+
+ org.apache.flume
+ flume-ng-auth
+
org.apache.thrift
libthrift
@@ -1304,6 +1324,7 @@
false
false
true
+ true