From 154164bc3aef49158451e169c3e7ccc13d267e09 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Mon, 30 Jun 2014 14:33:25 +0400
Subject: [PATCH 1/9] Multilabel evaluation metics and tests: macro precision
 and recall averaged by docs, micro and per-class precision and recall
 averaged by class

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 74 +++++++++++++++++
 .../evaluation/MultilabelMetricsSuite.scala   | 81 +++++++++++++++++++
 2 files changed, 155 insertions(+)
 create mode 100644 mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
 create mode 100644 mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
new file mode 100644
index 0000000000000..acb8c3d21d36f
--- /dev/null
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -0,0 +1,74 @@
+package org.apache.spark.mllib.evaluation
+
+import org.apache.spark.Logging
+import org.apache.spark.rdd.RDD
+import org.apache.spark.SparkContext._
+
+
+class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) extends Logging{
+
+  private lazy val numDocs = predictionAndLabels.count()
+
+  lazy val macroPrecisionDoc = (predictionAndLabels.map{ case(predictions, labels) =>
+    if (predictions.size >0)
+      predictions.intersect(labels).size.toDouble / predictions.size else 0}.fold(0.0)(_ + _)) / numDocs
+
+  lazy val macroRecallDoc = (predictionAndLabels.map{ case(predictions, labels) =>
+    predictions.intersect(labels).size.toDouble / labels.size}.fold(0.0)(_ + _)) / numDocs
+
+  lazy val microPrecisionDoc = {
+    val (sumTp, sumPredictions) = predictionAndLabels.map{ case(predictions, labels) =>
+      (predictions.intersect(labels).size, predictions.size)}.
+      fold((0, 0)){ case((tp1, predictions1), (tp2, predictions2)) =>
+      (tp1 + tp2, predictions1 + predictions2)}
+    sumTp.toDouble / sumPredictions
+  }
+
+  lazy val microRecallDoc = {
+    val (sumTp, sumLabels) = predictionAndLabels.map{ case(predictions, labels) =>
+      (predictions.intersect(labels).size, labels.size)}.
+      fold((0, 0)){ case((tp1, labels1), (tp2, labels2)) =>
+      (tp1 + tp2, labels1 + labels2)}
+    sumTp.toDouble / sumLabels
+  }
+
+  private lazy val tpPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
+    predictions.intersect(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+
+  private lazy val fpPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
+    predictions.diff(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+
+  private lazy val fnPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
+    labels.diff(predictions).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+
+  def precisionClass(label: Double) = if((tpPerClass(label) + fpPerClass.getOrElse(label, 0)) == 0) 0 else
+    tpPerClass(label).toDouble / (tpPerClass(label) + fpPerClass.getOrElse(label, 0))
+
+  def recallClass(label: Double) = if((tpPerClass(label) + fnPerClass.getOrElse(label, 0)) == 0) 0 else
+    tpPerClass(label).toDouble / (tpPerClass(label) + fnPerClass.getOrElse(label, 0))
+
+  def f1MeasureClass(label: Double) = {
+    val precision = precisionClass(label)
+    val recall = recallClass(label)
+    if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
+  }
+
+  private lazy val sumTp = tpPerClass.foldLeft(0L){ case(sumTp, (_, tp)) => sumTp + tp}
+
+  lazy val microPrecisionClass = {
+    val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
+    sumTp.toDouble / (sumTp + sumFp)
+  }
+
+  lazy val microRecallClass = {
+    val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
+    sumTp.toDouble / (sumTp + sumFn)
+  }
+
+  lazy val microF1MeasureClass = {
+    val precision = microPrecisionClass
+    val recall = microRecallClass
+    if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
+  }
+
+}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
new file mode 100644
index 0000000000000..62f6958639a74
--- /dev/null
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -0,0 +1,81 @@
+package org.apache.spark.mllib.evaluation
+
+import org.apache.spark.mllib.util.LocalSparkContext
+import org.apache.spark.rdd.RDD
+import org.scalatest.FunSuite
+
+
+class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
+  test("Multilabel evaluation metrics") {
+    /*
+    * Documents true labels (5x class0, 3x class1, 4x class2):
+    * doc 0 - predict 0, 1 - class 0, 2
+    * doc 1 - predict 0, 2 - class 0, 1
+    * doc 2 - predict none - class 0
+    * doc 3 - predict 2 - class 2
+    * doc 4 - predict 2, 0 - class 2, 0
+    * doc 5 - predict 0, 1, 2 - class 0, 1
+    * doc 6 - predict 1 - class 1, 2
+    *
+    * predicted classes
+    * class 0 - doc 0, 1, 4, 5 (total 4)
+    * class 1 - doc 0, 5, 6 (total 3)
+    * class 2 - doc 1, 3, 4, 5 (total 4)
+    *
+    * true classes
+    * class 0 - doc 0, 1, 2, 4, 5 (total 5)
+    * class 1 - doc 1, 5, 6 (total 3)
+    * class 2 - doc 0, 3, 4, 6 (total 4)
+    *
+    */
+    val scoreAndLabels:RDD[(Set[Double], Set[Double])] = sc.parallelize(
+      Seq((Set(0.0, 1.0), Set(0.0, 2.0)),
+        (Set(0.0, 2.0), Set(0.0, 1.0)),
+        (Set(), Set(0.0)),
+        (Set(2.0), Set(2.0)),
+        (Set(2.0, 0.0), Set(2.0, 0.0)),
+        (Set(0.0, 1.0, 2.0), Set(0.0, 1.0)),
+        (Set(1.0), Set(1.0, 2.0))), 2)
+    val metrics = new MultilabelMetrics(scoreAndLabels)
+    val delta = 0.00001
+    val precision0 = 4.0 / (4 + 0)
+    val precision1 = 2.0 / (2 + 1)
+    val precision2 = 2.0 / (2 + 2)
+    val recall0 = 4.0 / (4 + 1)
+    val recall1 = 2.0 / (2 + 1)
+    val recall2 = 2.0 / (2 + 2)
+    val f1measure0 = 2 * precision0 * recall0 / (precision0 + recall0)
+    val f1measure1 = 2 * precision1 * recall1 / (precision1 + recall1)
+    val f1measure2 = 2 * precision2 * recall2 / (precision2 + recall2)
+    val microPrecisionClass = (4.0 + 2.0 + 2.0) / (4 + 0 + 2 + 1 + 2 + 2)
+    val microRecallClass = (4.0 + 2.0 + 2.0) / (4 + 1 + 2 + 1 + 2 + 2)
+    val microF1MeasureClass = 2 * microPrecisionClass * microRecallClass / (microPrecisionClass + microRecallClass)
+
+    val macroPrecisionDoc = 1.0 / 7 * (1.0 / 2 + 1.0 / 2 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 1.0)
+    val macroRecallDoc = 1.0 / 7 * (1.0 / 2 + 1.0 / 2 + 0 / 1 + 1.0 / 1 + 2.0 / 2 + 2.0 / 2 + 1.0 / 2)
+
+    println("Ev" + metrics.macroPrecisionDoc)
+    println(macroPrecisionDoc)
+    println("Ev" + metrics.macroRecallDoc)
+    println(macroRecallDoc)
+    assert(math.abs(metrics.precisionClass(0.0) - precision0) < delta)
+    assert(math.abs(metrics.precisionClass(1.0) - precision1) < delta)
+    assert(math.abs(metrics.precisionClass(2.0) - precision2) < delta)
+    assert(math.abs(metrics.recallClass(0.0) - recall0) < delta)
+    assert(math.abs(metrics.recallClass(1.0) - recall1) < delta)
+    assert(math.abs(metrics.recallClass(2.0) - recall2) < delta)
+    assert(math.abs(metrics.f1MeasureClass(0.0) - f1measure0) < delta)
+    assert(math.abs(metrics.f1MeasureClass(1.0) - f1measure1) < delta)
+    assert(math.abs(metrics.f1MeasureClass(2.0) - f1measure2) < delta)
+
+    assert(math.abs(metrics.microPrecisionClass - microPrecisionClass) < delta)
+    assert(math.abs(metrics.microRecallClass - microRecallClass) < delta)
+    assert(math.abs(metrics.microF1MeasureClass - microF1MeasureClass) < delta)
+
+    assert(math.abs(metrics.macroPrecisionDoc - macroPrecisionDoc) < delta)
+    assert(math.abs(metrics.macroRecallDoc - macroRecallDoc) < delta)
+
+
+  }
+
+}

From ad62df02fe47042d54a7a3d2347262e88bba01f0 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Mon, 30 Jun 2014 16:38:15 +0400
Subject: [PATCH 2/9] Comments and scala style check

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 85 +++++++++++++++++--
 1 file changed, 77 insertions(+), 8 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index acb8c3d21d36f..32139e4fa6966 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -1,21 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.mllib.evaluation
 
 import org.apache.spark.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.SparkContext._
 
-
+/**
+ * Evaluator for multilabel classification.
+ * NB: type Double both for prediction and label is retained
+ * for compatibility with model.predict that returns Double
+ * and MLUtils.loadLibSVMFile that loads class labels as Double
+ *
+ * @param predictionAndLabels an RDD of pairs (predictions, labels) sets.
+ */
 class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) extends Logging{
 
   private lazy val numDocs = predictionAndLabels.count()
 
+  /**
+   * Returns Document-based Precision averaged by the number of documents
+   * @return macroPrecisionDoc.
+   */
   lazy val macroPrecisionDoc = (predictionAndLabels.map{ case(predictions, labels) =>
-    if (predictions.size >0)
-      predictions.intersect(labels).size.toDouble / predictions.size else 0}.fold(0.0)(_ + _)) / numDocs
-
+    if(predictions.size >0)
+      predictions.intersect(labels).size.toDouble / predictions.size else 0}.fold(0.0)(_ + _)) /
+    numDocs
+
+  /**
+   * Returns Document-based Recall averaged by the number of documents
+   * @return macroRecallDoc.
+   */
   lazy val macroRecallDoc = (predictionAndLabels.map{ case(predictions, labels) =>
     predictions.intersect(labels).size.toDouble / labels.size}.fold(0.0)(_ + _)) / numDocs
 
+  /**
+   * Returns micro-averaged document-based Precision
+   * @return microPrecisionDoc.
+   */
   lazy val microPrecisionDoc = {
     val (sumTp, sumPredictions) = predictionAndLabels.map{ case(predictions, labels) =>
       (predictions.intersect(labels).size, predictions.size)}.
@@ -24,6 +61,10 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
     sumTp.toDouble / sumPredictions
   }
 
+  /**
+   * Returns micro-averaged document-based Recall
+   * @return microRecallDoc.
+   */
   lazy val microRecallDoc = {
     val (sumTp, sumLabels) = predictionAndLabels.map{ case(predictions, labels) =>
       (predictions.intersect(labels).size, labels.size)}.
@@ -41,12 +82,28 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
   private lazy val fnPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
     labels.diff(predictions).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
 
-  def precisionClass(label: Double) = if((tpPerClass(label) + fpPerClass.getOrElse(label, 0)) == 0) 0 else
-    tpPerClass(label).toDouble / (tpPerClass(label) + fpPerClass.getOrElse(label, 0))
-
-  def recallClass(label: Double) = if((tpPerClass(label) + fnPerClass.getOrElse(label, 0)) == 0) 0 else
+  /**
+   * Returns Precision for a given label (category)
+   * @param label the label.
+   * @return Precision.
+   */
+  def precisionClass(label: Double) = if((tpPerClass(label) + fpPerClass.getOrElse(label, 0)) == 0)
+    0 else tpPerClass(label).toDouble / (tpPerClass(label) + fpPerClass.getOrElse(label, 0))
+
+  /**
+   * Returns Recall for a given label (category)
+   * @param label the label.
+   * @return Recall.
+   */
+  def recallClass(label: Double) = if((tpPerClass(label) + fnPerClass.getOrElse(label, 0)) == 0)
+    0 else
     tpPerClass(label).toDouble / (tpPerClass(label) + fnPerClass.getOrElse(label, 0))
 
+  /**
+   * Returns F1-measure for a given label (category)
+   * @param label the label.
+   * @return F1-measure.
+   */
   def f1MeasureClass(label: Double) = {
     val precision = precisionClass(label)
     val recall = recallClass(label)
@@ -55,16 +112,28 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
 
   private lazy val sumTp = tpPerClass.foldLeft(0L){ case(sumTp, (_, tp)) => sumTp + tp}
 
+  /**
+   * Returns micro-averaged label-based Precision
+   * @return microPrecisionClass.
+   */
   lazy val microPrecisionClass = {
     val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
     sumTp.toDouble / (sumTp + sumFp)
   }
 
+  /**
+   * Returns micro-averaged label-based Recall
+   * @return microRecallClass.
+   */
   lazy val microRecallClass = {
     val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
     sumTp.toDouble / (sumTp + sumFn)
   }
 
+  /**
+   * Returns micro-averaged label-based F1-measure
+   * @return microRecallClass.
+   */
   lazy val microF1MeasureClass = {
     val precision = microPrecisionClass
     val recall = microRecallClass

From 40593f5ac743feaa691e5ccdc0e6410d8dfa40d1 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Mon, 30 Jun 2014 19:28:56 +0400
Subject: [PATCH 3/9] Multi-label metrics: Hamming-loss, strict and normal
 accuracy, fix to macro measures, bunch of tests

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 73 +++++++++++++------
 .../evaluation/MultilabelMetricsSuite.scala   | 32 +++++---
 2 files changed, 74 insertions(+), 31 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index 32139e4fa6966..e45a611076a13 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -31,7 +31,33 @@ import org.apache.spark.SparkContext._
  */
 class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) extends Logging{
 
-  private lazy val numDocs = predictionAndLabels.count()
+  private lazy val numDocs = predictionAndLabels.count
+
+  private lazy val numLabels = predictionAndLabels.flatMap{case(_, labels) => labels}.distinct.count
+
+  /**
+   * Returns strict Accuracy
+   * (for equal sets of labels)
+   * @return strictAccuracy.
+   */
+  lazy val strictAccuracy = predictionAndLabels.filter{case(predictions, labels) =>
+    predictions == labels}.count.toDouble / numDocs
+
+  /**
+   * Returns Accuracy
+   * @return Accuracy.
+   */
+  lazy val accuracy = predictionAndLabels.map{ case(predictions, labels) =>
+    labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.
+    fold(0.0)(_ + _) / numDocs
+
+  /**
+   * Returns Hamming-loss
+   * @return hammingLoss.
+   */
+  lazy val hammingLoss = (predictionAndLabels.map{ case(predictions, labels) =>
+    labels.diff(predictions).size + predictions.diff(labels).size}.
+    fold(0)(_ + _)).toDouble / (numDocs * numLabels)
 
   /**
    * Returns Document-based Precision averaged by the number of documents
@@ -47,31 +73,36 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
    * @return macroRecallDoc.
    */
   lazy val macroRecallDoc = (predictionAndLabels.map{ case(predictions, labels) =>
-    predictions.intersect(labels).size.toDouble / labels.size}.fold(0.0)(_ + _)) / numDocs
+    labels.intersect(predictions).size.toDouble / labels.size}.fold(0.0)(_ + _)) / numDocs
+
+  /**
+   * Returns Document-based F1-measure averaged by the number of documents
+   * @return macroRecallDoc.
+   */
+  lazy val macroF1MeasureDoc = (predictionAndLabels.map{ case(predictions, labels) =>
+    2.0 * predictions.intersect(labels).size /
+      (predictions.size + labels.size)}.fold(0.0)(_ + _)) / numDocs
 
   /**
    * Returns micro-averaged document-based Precision
+   * (equals to label-based microPrecision)
    * @return microPrecisionDoc.
    */
-  lazy val microPrecisionDoc = {
-    val (sumTp, sumPredictions) = predictionAndLabels.map{ case(predictions, labels) =>
-      (predictions.intersect(labels).size, predictions.size)}.
-      fold((0, 0)){ case((tp1, predictions1), (tp2, predictions2)) =>
-      (tp1 + tp2, predictions1 + predictions2)}
-    sumTp.toDouble / sumPredictions
-  }
+  lazy val microPrecisionDoc = microPrecisionClass
 
   /**
    * Returns micro-averaged document-based Recall
+   * (equals to label-based microRecall)
    * @return microRecallDoc.
    */
-  lazy val microRecallDoc = {
-    val (sumTp, sumLabels) = predictionAndLabels.map{ case(predictions, labels) =>
-      (predictions.intersect(labels).size, labels.size)}.
-      fold((0, 0)){ case((tp1, labels1), (tp2, labels2)) =>
-      (tp1 + tp2, labels1 + labels2)}
-    sumTp.toDouble / sumLabels
-  }
+  lazy val microRecallDoc = microRecallClass
+
+  /**
+   * Returns micro-averaged document-based F1-measure
+   * (equals to label-based microF1measure)
+   * @return microF1MeasureDoc.
+   */
+  lazy val microF1MeasureDoc = microF1MeasureClass
 
   private lazy val tpPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
     predictions.intersect(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
@@ -110,7 +141,9 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
     if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
   }
 
-  private lazy val sumTp = tpPerClass.foldLeft(0L){ case(sumTp, (_, tp)) => sumTp + tp}
+  private lazy val sumTp = tpPerClass.foldLeft(0L){ case(sum, (_, tp)) => sum + tp}
+  private lazy val sumFpClass = fpPerClass.foldLeft(0L){ case(sum, (_, fp)) => sum + fp}
+  private lazy val sumFnClass = fnPerClass.foldLeft(0L){ case(sum, (_, fn)) => sum + fn}
 
   /**
    * Returns micro-averaged label-based Precision
@@ -134,10 +167,6 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
    * Returns micro-averaged label-based F1-measure
    * @return microRecallClass.
    */
-  lazy val microF1MeasureClass = {
-    val precision = microPrecisionClass
-    val recall = microRecallClass
-    if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
-  }
+  lazy val microF1MeasureClass = 2.0 * sumTp / (2 * sumTp + sumFnClass + sumFpClass)
 
 }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index 62f6958639a74..b36ddec489be9 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -47,17 +47,26 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     val f1measure0 = 2 * precision0 * recall0 / (precision0 + recall0)
     val f1measure1 = 2 * precision1 * recall1 / (precision1 + recall1)
     val f1measure2 = 2 * precision2 * recall2 / (precision2 + recall2)
-    val microPrecisionClass = (4.0 + 2.0 + 2.0) / (4 + 0 + 2 + 1 + 2 + 2)
-    val microRecallClass = (4.0 + 2.0 + 2.0) / (4 + 1 + 2 + 1 + 2 + 2)
-    val microF1MeasureClass = 2 * microPrecisionClass * microRecallClass / (microPrecisionClass + microRecallClass)
+    val sumTp = 4 + 2 + 2
+    assert(sumTp == (1 + 1 + 0 + 1 + 2 + 2 + 1))
+    val microPrecisionClass = sumTp.toDouble / (4 + 0 + 2 + 1 + 2 + 2)
+    val microRecallClass = sumTp.toDouble / (4 + 1 + 2 + 1 + 2 + 2)
+    val microF1MeasureClass = 2.0 * sumTp.toDouble /
+      (2 * sumTp.toDouble + (1 + 1 + 2) + (0 + 1 + 2))
 
-    val macroPrecisionDoc = 1.0 / 7 * (1.0 / 2 + 1.0 / 2 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 1.0)
-    val macroRecallDoc = 1.0 / 7 * (1.0 / 2 + 1.0 / 2 + 0 / 1 + 1.0 / 1 + 2.0 / 2 + 2.0 / 2 + 1.0 / 2)
+    val macroPrecisionDoc = 1.0 / 7 *
+      (1.0 / 2 + 1.0 / 2 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 1.0)
+    val macroRecallDoc = 1.0 / 7 *
+      (1.0 / 2 + 1.0 / 2 + 0 / 1 + 1.0 / 1 + 2.0 / 2 + 2.0 / 2 + 1.0 / 2)
+    val macroF1MeasureDoc = (1.0 / 7) *
+      2 * ( 1.0 / (2 + 2) + 1.0 / (2 + 2) + 0 + 1.0 / (1 + 1) +
+        2.0 / (2 + 2) + 2.0 / (3 + 2) + 1.0 / (1 + 2) )
+
+    val hammingLoss = (1.0 / (7 * 3)) * (2 + 2 + 1 + 0 + 0 + 1 + 1)
+
+    val strictAccuracy = 2.0 / 7
+    val accuracy = 1.0 / 7 * (1.0 / 3 + 1.0 /3 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 2)
 
-    println("Ev" + metrics.macroPrecisionDoc)
-    println(macroPrecisionDoc)
-    println("Ev" + metrics.macroRecallDoc)
-    println(macroRecallDoc)
     assert(math.abs(metrics.precisionClass(0.0) - precision0) < delta)
     assert(math.abs(metrics.precisionClass(1.0) - precision1) < delta)
     assert(math.abs(metrics.precisionClass(2.0) - precision2) < delta)
@@ -74,6 +83,11 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
 
     assert(math.abs(metrics.macroPrecisionDoc - macroPrecisionDoc) < delta)
     assert(math.abs(metrics.macroRecallDoc - macroRecallDoc) < delta)
+    assert(math.abs(metrics.macroF1MeasureDoc - macroF1MeasureDoc) < delta)
+
+    assert(math.abs(metrics.hammingLoss - hammingLoss) < delta)
+    assert(math.abs(metrics.strictAccuracy - strictAccuracy) < delta)
+    assert(math.abs(metrics.accuracy - accuracy) < delta)
 
 
   }

From ca467652011b517b2c5f63d287e221964e27ded2 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Mon, 30 Jun 2014 19:38:26 +0400
Subject: [PATCH 4/9] Cosmetic changes: Apache header and parameter explanation

---
 .../mllib/evaluation/MultilabelMetrics.scala    |  2 +-
 .../evaluation/MultilabelMetricsSuite.scala     | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index e45a611076a13..9b5b815d0495b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkContext._
  * for compatibility with model.predict that returns Double
  * and MLUtils.loadLibSVMFile that loads class labels as Double
  *
- * @param predictionAndLabels an RDD of pairs (predictions, labels) sets.
+ * @param predictionAndLabels an RDD of (predictions, labels) pairs, both are non-null sets.
  */
 class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) extends Logging{
 
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index b36ddec489be9..d67abc4f4df3d 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.spark.mllib.evaluation
 
 import org.apache.spark.mllib.util.LocalSparkContext

From 79e847656d8f062fad6a4c26a1f9a31dc59bed9d Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Wed, 2 Jul 2014 11:54:42 +0400
Subject: [PATCH 5/9] Replacing fold(_ + _) with sum as suggested by srowen

---
 .../spark/mllib/evaluation/MultilabelMetrics.scala  | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index 9b5b815d0495b..8e1afdd0ae17e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -48,8 +48,7 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
    * @return Accuracy.
    */
   lazy val accuracy = predictionAndLabels.map{ case(predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.
-    fold(0.0)(_ + _) / numDocs
+    labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.sum / numDocs
 
   /**
    * Returns Hamming-loss
@@ -57,7 +56,7 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
    */
   lazy val hammingLoss = (predictionAndLabels.map{ case(predictions, labels) =>
     labels.diff(predictions).size + predictions.diff(labels).size}.
-    fold(0)(_ + _)).toDouble / (numDocs * numLabels)
+    sum).toDouble / (numDocs * numLabels)
 
   /**
    * Returns Document-based Precision averaged by the number of documents
@@ -65,23 +64,21 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
    */
   lazy val macroPrecisionDoc = (predictionAndLabels.map{ case(predictions, labels) =>
     if(predictions.size >0)
-      predictions.intersect(labels).size.toDouble / predictions.size else 0}.fold(0.0)(_ + _)) /
-    numDocs
+      predictions.intersect(labels).size.toDouble / predictions.size else 0}.sum) / numDocs
 
   /**
    * Returns Document-based Recall averaged by the number of documents
    * @return macroRecallDoc.
    */
   lazy val macroRecallDoc = (predictionAndLabels.map{ case(predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.size}.fold(0.0)(_ + _)) / numDocs
+    labels.intersect(predictions).size.toDouble / labels.size}.sum) / numDocs
 
   /**
    * Returns Document-based F1-measure averaged by the number of documents
    * @return macroRecallDoc.
    */
   lazy val macroF1MeasureDoc = (predictionAndLabels.map{ case(predictions, labels) =>
-    2.0 * predictions.intersect(labels).size /
-      (predictions.size + labels.size)}.fold(0.0)(_ + _)) / numDocs
+    2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum) / numDocs
 
   /**
    * Returns micro-averaged document-based Precision

From 1843f739a6fc5cc341ef9f455859276f26f2b3b9 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Wed, 16 Jul 2014 12:41:11 +0400
Subject: [PATCH 6/9] Scala style fix

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 77 ++++++++-----------
 .../evaluation/MultilabelMetricsSuite.scala   | 16 +---
 2 files changed, 35 insertions(+), 58 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index 8e1afdd0ae17e..432cabf1c8a4d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -17,94 +17,83 @@
 
 package org.apache.spark.mllib.evaluation
 
-import org.apache.spark.Logging
 import org.apache.spark.rdd.RDD
 import org.apache.spark.SparkContext._
 
 /**
  * Evaluator for multilabel classification.
- * NB: type Double both for prediction and label is retained
- * for compatibility with model.predict that returns Double
- * and MLUtils.loadLibSVMFile that loads class labels as Double
- *
  * @param predictionAndLabels an RDD of (predictions, labels) pairs, both are non-null sets.
  */
-class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) extends Logging{
+class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
 
-  private lazy val numDocs = predictionAndLabels.count
+  private lazy val numDocs: Long = predictionAndLabels.count
 
-  private lazy val numLabels = predictionAndLabels.flatMap{case(_, labels) => labels}.distinct.count
+  private lazy val numLabels: Long = predictionAndLabels.flatMap { case (_, labels) =>
+    labels}.distinct.count
 
   /**
    * Returns strict Accuracy
    * (for equal sets of labels)
-   * @return strictAccuracy.
    */
-  lazy val strictAccuracy = predictionAndLabels.filter{case(predictions, labels) =>
+  lazy val strictAccuracy: Double = predictionAndLabels.filter { case (predictions, labels) =>
     predictions == labels}.count.toDouble / numDocs
 
   /**
    * Returns Accuracy
-   * @return Accuracy.
    */
-  lazy val accuracy = predictionAndLabels.map{ case(predictions, labels) =>
+  lazy val accuracy: Double = predictionAndLabels.map { case (predictions, labels) =>
     labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.sum / numDocs
 
   /**
    * Returns Hamming-loss
-   * @return hammingLoss.
    */
-  lazy val hammingLoss = (predictionAndLabels.map{ case(predictions, labels) =>
+  lazy val hammingLoss: Double = (predictionAndLabels.map { case (predictions, labels) =>
     labels.diff(predictions).size + predictions.diff(labels).size}.
     sum).toDouble / (numDocs * numLabels)
 
   /**
    * Returns Document-based Precision averaged by the number of documents
-   * @return macroPrecisionDoc.
    */
-  lazy val macroPrecisionDoc = (predictionAndLabels.map{ case(predictions, labels) =>
-    if(predictions.size >0)
-      predictions.intersect(labels).size.toDouble / predictions.size else 0}.sum) / numDocs
+  lazy val macroPrecisionDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
+    if (predictions.size > 0) {
+      predictions.intersect(labels).size.toDouble / predictions.size
+    } else 0
+  }.sum) / numDocs
 
   /**
    * Returns Document-based Recall averaged by the number of documents
-   * @return macroRecallDoc.
    */
-  lazy val macroRecallDoc = (predictionAndLabels.map{ case(predictions, labels) =>
+  lazy val macroRecallDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
     labels.intersect(predictions).size.toDouble / labels.size}.sum) / numDocs
 
   /**
    * Returns Document-based F1-measure averaged by the number of documents
-   * @return macroRecallDoc.
    */
-  lazy val macroF1MeasureDoc = (predictionAndLabels.map{ case(predictions, labels) =>
+  lazy val macroF1MeasureDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
     2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum) / numDocs
 
   /**
    * Returns micro-averaged document-based Precision
    * (equals to label-based microPrecision)
-   * @return microPrecisionDoc.
    */
-  lazy val microPrecisionDoc = microPrecisionClass
+  lazy val microPrecisionDoc: Double = microPrecisionClass
 
   /**
    * Returns micro-averaged document-based Recall
    * (equals to label-based microRecall)
-   * @return microRecallDoc.
    */
-  lazy val microRecallDoc = microRecallClass
+  lazy val microRecallDoc: Double = microRecallClass
 
   /**
    * Returns micro-averaged document-based F1-measure
    * (equals to label-based microF1measure)
-   * @return microF1MeasureDoc.
    */
-  lazy val microF1MeasureDoc = microF1MeasureClass
+  lazy val microF1MeasureDoc: Double = microF1MeasureClass
 
-  private lazy val tpPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
+  private lazy val tpPerClass = predictionAndLabels.flatMap { case (predictions, labels) =>
     predictions.intersect(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
 
-  private lazy val fpPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
+  private lazy val fpPerClass = predictionAndLabels.flatMap { case(predictions, labels) =>
     predictions.diff(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
 
   private lazy val fnPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
@@ -113,24 +102,26 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
   /**
    * Returns Precision for a given label (category)
    * @param label the label.
-   * @return Precision.
    */
-  def precisionClass(label: Double) = if((tpPerClass(label) + fpPerClass.getOrElse(label, 0)) == 0)
-    0 else tpPerClass(label).toDouble / (tpPerClass(label) + fpPerClass.getOrElse(label, 0))
+  def precisionClass(label: Double) = {
+    val tp = tpPerClass(label)
+    val fp = fpPerClass.getOrElse(label, 0)
+    if (tp + fp == 0) 0 else tp.toDouble / (tp + fp)
+  }
 
   /**
    * Returns Recall for a given label (category)
    * @param label the label.
-   * @return Recall.
    */
-  def recallClass(label: Double) = if((tpPerClass(label) + fnPerClass.getOrElse(label, 0)) == 0)
-    0 else
-    tpPerClass(label).toDouble / (tpPerClass(label) + fnPerClass.getOrElse(label, 0))
+  def recallClass(label: Double) = {
+    val tp = tpPerClass(label)
+    val fn = fnPerClass.getOrElse(label, 0)
+    if (tp + fn == 0) 0 else tp.toDouble / (tp + fn)
+  }
 
   /**
    * Returns F1-measure for a given label (category)
    * @param label the label.
-   * @return F1-measure.
    */
   def f1MeasureClass(label: Double) = {
     val precision = precisionClass(label)
@@ -138,13 +129,12 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
     if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
   }
 
-  private lazy val sumTp = tpPerClass.foldLeft(0L){ case(sum, (_, tp)) => sum + tp}
-  private lazy val sumFpClass = fpPerClass.foldLeft(0L){ case(sum, (_, fp)) => sum + fp}
-  private lazy val sumFnClass = fnPerClass.foldLeft(0L){ case(sum, (_, fn)) => sum + fn}
+  private lazy val sumTp = tpPerClass.foldLeft(0L){ case (sum, (_, tp)) => sum + tp}
+  private lazy val sumFpClass = fpPerClass.foldLeft(0L){ case (sum, (_, fp)) => sum + fp}
+  private lazy val sumFnClass = fnPerClass.foldLeft(0L){ case (sum, (_, fn)) => sum + fn}
 
   /**
    * Returns micro-averaged label-based Precision
-   * @return microPrecisionClass.
    */
   lazy val microPrecisionClass = {
     val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
@@ -153,7 +143,6 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
 
   /**
    * Returns micro-averaged label-based Recall
-   * @return microRecallClass.
    */
   lazy val microRecallClass = {
     val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
@@ -162,8 +151,6 @@ class MultilabelMetrics(predictionAndLabels:RDD[(Set[Double], Set[Double])]) ext
 
   /**
    * Returns micro-averaged label-based F1-measure
-   * @return microRecallClass.
    */
   lazy val microF1MeasureClass = 2.0 * sumTp / (2 * sumTp + sumFnClass + sumFpClass)
-
 }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index d67abc4f4df3d..4d33aa3e5ed53 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -17,10 +17,10 @@
 
 package org.apache.spark.mllib.evaluation
 
-import org.apache.spark.mllib.util.LocalSparkContext
-import org.apache.spark.rdd.RDD
 import org.scalatest.FunSuite
 
+import org.apache.spark.mllib.util.LocalSparkContext
+import org.apache.spark.rdd.RDD
 
 class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
   test("Multilabel evaluation metrics") {
@@ -45,7 +45,7 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     * class 2 - doc 0, 3, 4, 6 (total 4)
     *
     */
-    val scoreAndLabels:RDD[(Set[Double], Set[Double])] = sc.parallelize(
+    val scoreAndLabels: RDD[(Set[Double], Set[Double])] = sc.parallelize(
       Seq((Set(0.0, 1.0), Set(0.0, 2.0)),
         (Set(0.0, 2.0), Set(0.0, 1.0)),
         (Set(), Set(0.0)),
@@ -70,7 +70,6 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     val microRecallClass = sumTp.toDouble / (4 + 1 + 2 + 1 + 2 + 2)
     val microF1MeasureClass = 2.0 * sumTp.toDouble /
       (2 * sumTp.toDouble + (1 + 1 + 2) + (0 + 1 + 2))
-
     val macroPrecisionDoc = 1.0 / 7 *
       (1.0 / 2 + 1.0 / 2 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 1.0)
     val macroRecallDoc = 1.0 / 7 *
@@ -78,12 +77,9 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     val macroF1MeasureDoc = (1.0 / 7) *
       2 * ( 1.0 / (2 + 2) + 1.0 / (2 + 2) + 0 + 1.0 / (1 + 1) +
         2.0 / (2 + 2) + 2.0 / (3 + 2) + 1.0 / (1 + 2) )
-
     val hammingLoss = (1.0 / (7 * 3)) * (2 + 2 + 1 + 0 + 0 + 1 + 1)
-
     val strictAccuracy = 2.0 / 7
     val accuracy = 1.0 / 7 * (1.0 / 3 + 1.0 /3 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 2)
-
     assert(math.abs(metrics.precisionClass(0.0) - precision0) < delta)
     assert(math.abs(metrics.precisionClass(1.0) - precision1) < delta)
     assert(math.abs(metrics.precisionClass(2.0) - precision2) < delta)
@@ -93,20 +89,14 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     assert(math.abs(metrics.f1MeasureClass(0.0) - f1measure0) < delta)
     assert(math.abs(metrics.f1MeasureClass(1.0) - f1measure1) < delta)
     assert(math.abs(metrics.f1MeasureClass(2.0) - f1measure2) < delta)
-
     assert(math.abs(metrics.microPrecisionClass - microPrecisionClass) < delta)
     assert(math.abs(metrics.microRecallClass - microRecallClass) < delta)
     assert(math.abs(metrics.microF1MeasureClass - microF1MeasureClass) < delta)
-
     assert(math.abs(metrics.macroPrecisionDoc - macroPrecisionDoc) < delta)
     assert(math.abs(metrics.macroRecallDoc - macroRecallDoc) < delta)
     assert(math.abs(metrics.macroF1MeasureDoc - macroF1MeasureDoc) < delta)
-
     assert(math.abs(metrics.hammingLoss - hammingLoss) < delta)
     assert(math.abs(metrics.strictAccuracy - strictAccuracy) < delta)
     assert(math.abs(metrics.accuracy - accuracy) < delta)
-
-
   }
-
 }

From cf4222bc49671014add8dbda9872ff41d922edfc Mon Sep 17 00:00:00 2001
From: avulanov <nashb@yandex.ru>
Date: Wed, 10 Sep 2014 15:39:02 +0400
Subject: [PATCH 7/9] Addressing reviewers comments: renaming. Added label
 method that returns the list of labels

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 73 ++++++++-----------
 .../evaluation/MultilabelMetricsSuite.scala   | 33 +++++----
 2 files changed, 49 insertions(+), 57 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index 432cabf1c8a4d..d7a23ce65d8c1 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -32,14 +32,14 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
     labels}.distinct.count
 
   /**
-   * Returns strict Accuracy
+   * Returns subset accuracy
    * (for equal sets of labels)
    */
-  lazy val strictAccuracy: Double = predictionAndLabels.filter { case (predictions, labels) =>
+  lazy val subsetAccuracy: Double = predictionAndLabels.filter { case (predictions, labels) =>
     predictions == labels}.count.toDouble / numDocs
 
   /**
-   * Returns Accuracy
+   * Returns accuracy
    */
   lazy val accuracy: Double = predictionAndLabels.map { case (predictions, labels) =>
     labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.sum / numDocs
@@ -52,43 +52,26 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
     sum).toDouble / (numDocs * numLabels)
 
   /**
-   * Returns Document-based Precision averaged by the number of documents
+   * Returns document-based precision averaged by the number of documents
    */
-  lazy val macroPrecisionDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val precision: Double = (predictionAndLabels.map { case (predictions, labels) =>
     if (predictions.size > 0) {
       predictions.intersect(labels).size.toDouble / predictions.size
     } else 0
   }.sum) / numDocs
 
   /**
-   * Returns Document-based Recall averaged by the number of documents
+   * Returns document-based recall averaged by the number of documents
    */
-  lazy val macroRecallDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val recall: Double = (predictionAndLabels.map { case (predictions, labels) =>
     labels.intersect(predictions).size.toDouble / labels.size}.sum) / numDocs
 
   /**
-   * Returns Document-based F1-measure averaged by the number of documents
+   * Returns document-based f1-measure averaged by the number of documents
    */
-  lazy val macroF1MeasureDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val f1Measure: Double = (predictionAndLabels.map { case (predictions, labels) =>
     2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum) / numDocs
 
-  /**
-   * Returns micro-averaged document-based Precision
-   * (equals to label-based microPrecision)
-   */
-  lazy val microPrecisionDoc: Double = microPrecisionClass
-
-  /**
-   * Returns micro-averaged document-based Recall
-   * (equals to label-based microRecall)
-   */
-  lazy val microRecallDoc: Double = microRecallClass
-
-  /**
-   * Returns micro-averaged document-based F1-measure
-   * (equals to label-based microF1measure)
-   */
-  lazy val microF1MeasureDoc: Double = microF1MeasureClass
 
   private lazy val tpPerClass = predictionAndLabels.flatMap { case (predictions, labels) =>
     predictions.intersect(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
@@ -100,33 +83,33 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
     labels.diff(predictions).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
 
   /**
-   * Returns Precision for a given label (category)
+   * Returns precision for a given label (category)
    * @param label the label.
    */
-  def precisionClass(label: Double) = {
+  def precision(label: Double) = {
     val tp = tpPerClass(label)
     val fp = fpPerClass.getOrElse(label, 0)
     if (tp + fp == 0) 0 else tp.toDouble / (tp + fp)
   }
 
   /**
-   * Returns Recall for a given label (category)
+   * Returns recall for a given label (category)
    * @param label the label.
    */
-  def recallClass(label: Double) = {
+  def recall(label: Double) = {
     val tp = tpPerClass(label)
     val fn = fnPerClass.getOrElse(label, 0)
     if (tp + fn == 0) 0 else tp.toDouble / (tp + fn)
   }
 
   /**
-   * Returns F1-measure for a given label (category)
+   * Returns f1-measure for a given label (category)
    * @param label the label.
    */
-  def f1MeasureClass(label: Double) = {
-    val precision = precisionClass(label)
-    val recall = recallClass(label)
-    if((precision + recall) == 0) 0 else 2 * precision * recall / (precision + recall)
+  def f1Measure(label: Double) = {
+    val p = precision(label)
+    val r = recall(label)
+    if((p + r) == 0) 0 else 2 * p * r / (p + r)
   }
 
   private lazy val sumTp = tpPerClass.foldLeft(0L){ case (sum, (_, tp)) => sum + tp}
@@ -134,23 +117,31 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
   private lazy val sumFnClass = fnPerClass.foldLeft(0L){ case (sum, (_, fn)) => sum + fn}
 
   /**
-   * Returns micro-averaged label-based Precision
+   * Returns micro-averaged label-based precision
+   * (equals to micro-averaged document-based precision)
    */
-  lazy val microPrecisionClass = {
+  lazy val microPrecision = {
     val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
     sumTp.toDouble / (sumTp + sumFp)
   }
 
   /**
-   * Returns micro-averaged label-based Recall
+   * Returns micro-averaged label-based recall
+   * (equals to micro-averaged document-based recall)
    */
-  lazy val microRecallClass = {
+  lazy val microRecall = {
     val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
     sumTp.toDouble / (sumTp + sumFn)
   }
 
   /**
-   * Returns micro-averaged label-based F1-measure
+   * Returns micro-averaged label-based f1-measure
+   * (equals to micro-averaged document-based f1-measure)
+   */
+  lazy val microF1Measure = 2.0 * sumTp / (2 * sumTp + sumFnClass + sumFpClass)
+
+  /**
+   * Returns the sequence of labels in ascending order
    */
-  lazy val microF1MeasureClass = 2.0 * sumTp / (2 * sumTp + sumFnClass + sumFpClass)
+  lazy val labels: Array[Double] = tpPerClass.keys.toArray.sorted
 }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index 4d33aa3e5ed53..01bcbf0fc55a4 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -80,23 +80,24 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     val hammingLoss = (1.0 / (7 * 3)) * (2 + 2 + 1 + 0 + 0 + 1 + 1)
     val strictAccuracy = 2.0 / 7
     val accuracy = 1.0 / 7 * (1.0 / 3 + 1.0 /3 + 0 + 1.0 / 1 + 2.0 / 2 + 2.0 / 3 + 1.0 / 2)
-    assert(math.abs(metrics.precisionClass(0.0) - precision0) < delta)
-    assert(math.abs(metrics.precisionClass(1.0) - precision1) < delta)
-    assert(math.abs(metrics.precisionClass(2.0) - precision2) < delta)
-    assert(math.abs(metrics.recallClass(0.0) - recall0) < delta)
-    assert(math.abs(metrics.recallClass(1.0) - recall1) < delta)
-    assert(math.abs(metrics.recallClass(2.0) - recall2) < delta)
-    assert(math.abs(metrics.f1MeasureClass(0.0) - f1measure0) < delta)
-    assert(math.abs(metrics.f1MeasureClass(1.0) - f1measure1) < delta)
-    assert(math.abs(metrics.f1MeasureClass(2.0) - f1measure2) < delta)
-    assert(math.abs(metrics.microPrecisionClass - microPrecisionClass) < delta)
-    assert(math.abs(metrics.microRecallClass - microRecallClass) < delta)
-    assert(math.abs(metrics.microF1MeasureClass - microF1MeasureClass) < delta)
-    assert(math.abs(metrics.macroPrecisionDoc - macroPrecisionDoc) < delta)
-    assert(math.abs(metrics.macroRecallDoc - macroRecallDoc) < delta)
-    assert(math.abs(metrics.macroF1MeasureDoc - macroF1MeasureDoc) < delta)
+    assert(math.abs(metrics.precision(0.0) - precision0) < delta)
+    assert(math.abs(metrics.precision(1.0) - precision1) < delta)
+    assert(math.abs(metrics.precision(2.0) - precision2) < delta)
+    assert(math.abs(metrics.recall(0.0) - recall0) < delta)
+    assert(math.abs(metrics.recall(1.0) - recall1) < delta)
+    assert(math.abs(metrics.recall(2.0) - recall2) < delta)
+    assert(math.abs(metrics.f1Measure(0.0) - f1measure0) < delta)
+    assert(math.abs(metrics.f1Measure(1.0) - f1measure1) < delta)
+    assert(math.abs(metrics.f1Measure(2.0) - f1measure2) < delta)
+    assert(math.abs(metrics.microPrecision - microPrecisionClass) < delta)
+    assert(math.abs(metrics.microRecall - microRecallClass) < delta)
+    assert(math.abs(metrics.microF1Measure - microF1MeasureClass) < delta)
+    assert(math.abs(metrics.precision - macroPrecisionDoc) < delta)
+    assert(math.abs(metrics.recall - macroRecallDoc) < delta)
+    assert(math.abs(metrics.f1Measure - macroF1MeasureDoc) < delta)
     assert(math.abs(metrics.hammingLoss - hammingLoss) < delta)
-    assert(math.abs(metrics.strictAccuracy - strictAccuracy) < delta)
+    assert(math.abs(metrics.subsetAccuracy - strictAccuracy) < delta)
     assert(math.abs(metrics.accuracy - accuracy) < delta)
+    assert(metrics.labels.sameElements(Array(0.0, 1.0, 2.0)))
   }
 }

From 517a594c3779cfe68d90d5208b2f72e29f7fcedb Mon Sep 17 00:00:00 2001
From: avulanov <nashb@yandex.ru>
Date: Mon, 15 Sep 2014 21:05:47 +0400
Subject: [PATCH 8/9] Addressing reviewers comments: Scala style

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 59 +++++++++++--------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index d7a23ce65d8c1..dab919208eca8 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -26,7 +26,7 @@ import org.apache.spark.SparkContext._
  */
 class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
 
-  private lazy val numDocs: Long = predictionAndLabels.count
+  private lazy val numDocs: Long = predictionAndLabels.count()
 
   private lazy val numLabels: Long = predictionAndLabels.flatMap { case (_, labels) =>
     labels}.distinct.count
@@ -36,51 +36,60 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    * (for equal sets of labels)
    */
   lazy val subsetAccuracy: Double = predictionAndLabels.filter { case (predictions, labels) =>
-    predictions == labels}.count.toDouble / numDocs
+    predictions == labels
+  }.count().toDouble / numDocs
 
   /**
    * Returns accuracy
    */
   lazy val accuracy: Double = predictionAndLabels.map { case (predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.sum / numDocs
+    labels.intersect(predictions).size.toDouble / labels.union(predictions).size
+  }.sum / numDocs
 
   /**
    * Returns Hamming-loss
    */
-  lazy val hammingLoss: Double = (predictionAndLabels.map { case (predictions, labels) =>
-    labels.diff(predictions).size + predictions.diff(labels).size}.
-    sum).toDouble / (numDocs * numLabels)
+  lazy val hammingLoss: Double = predictionAndLabels.map { case (predictions, labels) =>
+    labels.size + predictions.size - 2 * labels.intersect(predictions).size
+  }.sum / (numDocs * numLabels)
 
   /**
    * Returns document-based precision averaged by the number of documents
    */
-  lazy val precision: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val precision: Double = predictionAndLabels.map { case (predictions, labels) =>
     if (predictions.size > 0) {
       predictions.intersect(labels).size.toDouble / predictions.size
-    } else 0
-  }.sum) / numDocs
+    } else {
+      0
+    }
+  }.sum / numDocs
 
   /**
    * Returns document-based recall averaged by the number of documents
    */
-  lazy val recall: Double = (predictionAndLabels.map { case (predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.size}.sum) / numDocs
+  lazy val recall: Double = predictionAndLabels.map { case (predictions, labels) =>
+    labels.intersect(predictions).size.toDouble / labels.size
+  }.sum / numDocs
 
   /**
    * Returns document-based f1-measure averaged by the number of documents
    */
-  lazy val f1Measure: Double = (predictionAndLabels.map { case (predictions, labels) =>
-    2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum) / numDocs
+  lazy val f1Measure: Double = predictionAndLabels.map { case (predictions, labels) =>
+    2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)
+  }.sum / numDocs
 
 
   private lazy val tpPerClass = predictionAndLabels.flatMap { case (predictions, labels) =>
-    predictions.intersect(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+    predictions.intersect(labels)
+  }.countByValue()
 
-  private lazy val fpPerClass = predictionAndLabels.flatMap { case(predictions, labels) =>
-    predictions.diff(labels).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+  private lazy val fpPerClass = predictionAndLabels.flatMap { case (predictions, labels) =>
+    predictions.diff(labels)
+  }.countByValue()
 
-  private lazy val fnPerClass = predictionAndLabels.flatMap{ case(predictions, labels) =>
-    labels.diff(predictions).map(category => (category, 1))}.reduceByKey(_ + _).collectAsMap()
+  private lazy val fnPerClass = predictionAndLabels.flatMap { case(predictions, labels) =>
+    labels.diff(predictions)
+  }.countByValue()
 
   /**
    * Returns precision for a given label (category)
@@ -88,7 +97,7 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    */
   def precision(label: Double) = {
     val tp = tpPerClass(label)
-    val fp = fpPerClass.getOrElse(label, 0)
+    val fp = fpPerClass.getOrElse(label, 0L)
     if (tp + fp == 0) 0 else tp.toDouble / (tp + fp)
   }
 
@@ -98,7 +107,7 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    */
   def recall(label: Double) = {
     val tp = tpPerClass(label)
-    val fn = fnPerClass.getOrElse(label, 0)
+    val fn = fnPerClass.getOrElse(label, 0L)
     if (tp + fn == 0) 0 else tp.toDouble / (tp + fn)
   }
 
@@ -112,16 +121,16 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
     if((p + r) == 0) 0 else 2 * p * r / (p + r)
   }
 
-  private lazy val sumTp = tpPerClass.foldLeft(0L){ case (sum, (_, tp)) => sum + tp}
-  private lazy val sumFpClass = fpPerClass.foldLeft(0L){ case (sum, (_, fp)) => sum + fp}
-  private lazy val sumFnClass = fnPerClass.foldLeft(0L){ case (sum, (_, fn)) => sum + fn}
+  private lazy val sumTp = tpPerClass.foldLeft(0L) { case (sum, (_, tp)) => sum + tp }
+  private lazy val sumFpClass = fpPerClass.foldLeft(0L) { case (sum, (_, fp)) => sum + fp }
+  private lazy val sumFnClass = fnPerClass.foldLeft(0L) { case (sum, (_, fn)) => sum + fn }
 
   /**
    * Returns micro-averaged label-based precision
    * (equals to micro-averaged document-based precision)
    */
   lazy val microPrecision = {
-    val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
+    val sumFp = fpPerClass.foldLeft(0L){ case(cum, (_, fp)) => cum + fp}
     sumTp.toDouble / (sumTp + sumFp)
   }
 
@@ -130,7 +139,7 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    * (equals to micro-averaged document-based recall)
    */
   lazy val microRecall = {
-    val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
+    val sumFn = fnPerClass.foldLeft(0.0){ case(cum, (_, fn)) => cum + fn}
     sumTp.toDouble / (sumTp + sumFn)
   }
 

From 43a613edf9d25237cf18498b64821eedb4b0d404 Mon Sep 17 00:00:00 2001
From: Alexander Ulanov <nashb@yandex.ru>
Date: Fri, 31 Oct 2014 11:43:54 -0700
Subject: [PATCH 9/9] Addressing reviewers comments: change Set to Array

---
 .../mllib/evaluation/MultilabelMetrics.scala  | 34 ++++++++++---------
 .../evaluation/MultilabelMetricsSuite.scala   | 16 ++++-----
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
index 432cabf1c8a4d..b31719c11ea31 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/MultilabelMetrics.scala
@@ -22,55 +22,57 @@ import org.apache.spark.SparkContext._
 
 /**
  * Evaluator for multilabel classification.
- * @param predictionAndLabels an RDD of (predictions, labels) pairs, both are non-null sets.
+ * @param predictionAndLabels an RDD of (predictions, labels) pairs,
+ * both are non-null Arrays, each with unique elements.
  */
-class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
+class MultilabelMetrics(predictionAndLabels: RDD[(Array[Double], Array[Double])]) {
 
-  private lazy val numDocs: Long = predictionAndLabels.count
+  private lazy val numDocs: Long = predictionAndLabels.count()
 
   private lazy val numLabels: Long = predictionAndLabels.flatMap { case (_, labels) =>
-    labels}.distinct.count
+    labels}.distinct().count()
 
   /**
    * Returns strict Accuracy
    * (for equal sets of labels)
    */
   lazy val strictAccuracy: Double = predictionAndLabels.filter { case (predictions, labels) =>
-    predictions == labels}.count.toDouble / numDocs
+    predictions.deep == labels.deep }.count().toDouble / numDocs
 
   /**
    * Returns Accuracy
    */
   lazy val accuracy: Double = predictionAndLabels.map { case (predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.union(predictions).size}.sum / numDocs
+    labels.intersect(predictions).size.toDouble /
+      (labels.size + predictions.size - labels.intersect(predictions).size)}.sum / numDocs
 
   /**
    * Returns Hamming-loss
    */
-  lazy val hammingLoss: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val hammingLoss: Double = predictionAndLabels.map { case (predictions, labels) =>
     labels.diff(predictions).size + predictions.diff(labels).size}.
-    sum).toDouble / (numDocs * numLabels)
+    sum / (numDocs * numLabels)
 
   /**
    * Returns Document-based Precision averaged by the number of documents
    */
-  lazy val macroPrecisionDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
+  lazy val macroPrecisionDoc: Double = predictionAndLabels.map { case (predictions, labels) =>
     if (predictions.size > 0) {
       predictions.intersect(labels).size.toDouble / predictions.size
     } else 0
-  }.sum) / numDocs
+  }.sum / numDocs
 
   /**
    * Returns Document-based Recall averaged by the number of documents
    */
-  lazy val macroRecallDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
-    labels.intersect(predictions).size.toDouble / labels.size}.sum) / numDocs
+  lazy val macroRecallDoc: Double = predictionAndLabels.map { case (predictions, labels) =>
+    labels.intersect(predictions).size.toDouble / labels.size}.sum / numDocs
 
   /**
    * Returns Document-based F1-measure averaged by the number of documents
    */
-  lazy val macroF1MeasureDoc: Double = (predictionAndLabels.map { case (predictions, labels) =>
-    2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum) / numDocs
+  lazy val macroF1MeasureDoc: Double = predictionAndLabels.map { case (predictions, labels) =>
+    2.0 * predictions.intersect(labels).size / (predictions.size + labels.size)}.sum / numDocs
 
   /**
    * Returns micro-averaged document-based Precision
@@ -137,7 +139,7 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    * Returns micro-averaged label-based Precision
    */
   lazy val microPrecisionClass = {
-    val sumFp = fpPerClass.foldLeft(0L){ case(sumFp, (_, fp)) => sumFp + fp}
+    val sumFp = fpPerClass.foldLeft(0L){ case(cum, (_, fp)) => cum + fp}
     sumTp.toDouble / (sumTp + sumFp)
   }
 
@@ -145,7 +147,7 @@ class MultilabelMetrics(predictionAndLabels: RDD[(Set[Double], Set[Double])]) {
    * Returns micro-averaged label-based Recall
    */
   lazy val microRecallClass = {
-    val sumFn = fnPerClass.foldLeft(0.0){ case(sumFn, (_, fn)) => sumFn + fn}
+    val sumFn = fnPerClass.foldLeft(0.0){ case(cum, (_, fn)) => cum + fn}
     sumTp.toDouble / (sumTp + sumFn)
   }
 
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
index 4d33aa3e5ed53..5ace9d9a59d6e 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/evaluation/MultilabelMetricsSuite.scala
@@ -45,14 +45,14 @@ class MultilabelMetricsSuite extends FunSuite with LocalSparkContext {
     * class 2 - doc 0, 3, 4, 6 (total 4)
     *
     */
-    val scoreAndLabels: RDD[(Set[Double], Set[Double])] = sc.parallelize(
-      Seq((Set(0.0, 1.0), Set(0.0, 2.0)),
-        (Set(0.0, 2.0), Set(0.0, 1.0)),
-        (Set(), Set(0.0)),
-        (Set(2.0), Set(2.0)),
-        (Set(2.0, 0.0), Set(2.0, 0.0)),
-        (Set(0.0, 1.0, 2.0), Set(0.0, 1.0)),
-        (Set(1.0), Set(1.0, 2.0))), 2)
+    val scoreAndLabels: RDD[(Array[Double], Array[Double])] = sc.parallelize(
+      Seq((Array(0.0, 1.0), Array(0.0, 2.0)),
+        (Array(0.0, 2.0), Array(0.0, 1.0)),
+        (Array(), Array(0.0)),
+        (Array(2.0), Array(2.0)),
+        (Array(2.0, 0.0), Array(2.0, 0.0)),
+        (Array(0.0, 1.0, 2.0), Array(0.0, 1.0)),
+        (Array(1.0), Array(1.0, 2.0))), 2)
     val metrics = new MultilabelMetrics(scoreAndLabels)
     val delta = 0.00001
     val precision0 = 4.0 / (4 + 0)