From d4022d49514cc1f8ffc5bfe243186ec3748df475 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Tue, 16 May 2017 10:08:23 +0800 Subject: [PATCH] [SPARK-20707][ML] ML deprecated APIs should be removed in major release. ## What changes were proposed in this pull request? Before 2.2, MLlib keep to remove APIs deprecated in last feature/minor release. But from Spark 2.2, we decide to remove deprecated APIs in a major release, so we need to change corresponding annotations to tell users those will be removed in 3.0. Meanwhile, this fixed bugs in ML documents. The original ML docs can't show deprecated annotations in ```MLWriter``` and ```MLReader``` related class, we correct it in this PR. Before: ![image](https://cloud.githubusercontent.com/assets/1962026/25939889/f8c55f20-3666-11e7-9fa2-0605bfb3ed06.png) After: ![image](https://cloud.githubusercontent.com/assets/1962026/25939870/e9b0d5be-3666-11e7-9765-5e04885e4b32.png) ## How was this patch tested? Existing tests. Author: Yanbo Liang Closes #17946 from yanboliang/spark-20707. --- .../org/apache/spark/ml/tree/treeParams.scala | 60 +++++++++---------- .../org/apache/spark/ml/util/ReadWrite.scala | 4 +- python/docs/pyspark.ml.rst | 8 +++ python/pyspark/ml/util.py | 16 +++-- 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala index cd1950bd76c05..3fc3ac58b7795 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala @@ -110,77 +110,77 @@ private[ml] trait DecisionTreeParams extends PredictorParams maxMemoryInMB -> 256, cacheNodeIds -> false, checkpointInterval -> 10) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMaxDepth(value: Int): this.type = set(maxDepth, value) /** @group getParam */ final def getMaxDepth: Int = $(maxDepth) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMaxBins(value: Int): this.type = set(maxBins, value) /** @group getParam */ final def getMaxBins: Int = $(maxBins) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMinInstancesPerNode(value: Int): this.type = set(minInstancesPerNode, value) /** @group getParam */ final def getMinInstancesPerNode: Int = $(minInstancesPerNode) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMinInfoGain(value: Double): this.type = set(minInfoGain, value) /** @group getParam */ final def getMinInfoGain: Double = $(minInfoGain) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setSeed(value: Long): this.type = set(seed, value) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group expertSetParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMaxMemoryInMB(value: Int): this.type = set(maxMemoryInMB, value) /** @group expertGetParam */ final def getMaxMemoryInMB: Int = $(maxMemoryInMB) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group expertSetParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setCacheNodeIds(value: Boolean): this.type = set(cacheNodeIds, value) /** @group expertGetParam */ final def getCacheNodeIds: Boolean = $(cacheNodeIds) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setCheckpointInterval(value: Int): this.type = set(checkpointInterval, value) /** (private[ml]) Create a Strategy instance to use with the old API. */ @@ -226,10 +226,10 @@ private[ml] trait TreeClassifierParams extends Params { setDefault(impurity -> "gini") /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setImpurity(value: String): this.type = set(impurity, value) /** @group getParam */ @@ -277,10 +277,10 @@ private[ml] trait TreeRegressorParams extends Params { setDefault(impurity -> "variance") /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setImpurity(value: String): this.type = set(impurity, value) /** @group getParam */ @@ -339,10 +339,10 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams { setDefault(subsamplingRate -> 1.0) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setSubsamplingRate(value: Double): this.type = set(subsamplingRate, value) /** @group getParam */ @@ -383,10 +383,10 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams { setDefault(numTrees -> 20) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setNumTrees(value: Int): this.type = set(numTrees, value) /** @group getParam */ @@ -431,10 +431,10 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams { setDefault(featureSubsetStrategy -> "auto") /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setFeatureSubsetStrategy(value: String): this.type = set(featureSubsetStrategy, value) /** @group getParam */ @@ -472,10 +472,10 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter { // validationTol -> 1e-5 /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setMaxIter(value: Int): this.type = set(maxIter, value) /** @@ -492,10 +492,10 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter { final def getStepSize: Double = $(stepSize) /** - * @deprecated This method is deprecated and will be removed in 2.2.0. + * @deprecated This method is deprecated and will be removed in 3.0.0. * @group setParam */ - @deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0") + @deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0") def setStepSize(value: Double): this.type = set(stepSize, value) setDefault(maxIter -> 20, stepSize -> 0.1) diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala index a8b80031faf86..b54e258cff2f8 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala @@ -44,9 +44,11 @@ private[util] sealed trait BaseReadWrite { /** * Sets the Spark SQLContext to use for saving/loading. + * + * @deprecated Use session instead. This method will be removed in 3.0.0. */ @Since("1.6.0") - @deprecated("Use session instead, This method will be removed in 2.2.0.", "2.0.0") + @deprecated("Use session instead. This method will be removed in 3.0.0.", "2.0.0") def context(sqlContext: SQLContext): this.type = { optionSparkSession = Option(sqlContext.sparkSession) this diff --git a/python/docs/pyspark.ml.rst b/python/docs/pyspark.ml.rst index 930646de9cd86..01627ba92b633 100644 --- a/python/docs/pyspark.ml.rst +++ b/python/docs/pyspark.ml.rst @@ -96,3 +96,11 @@ pyspark.ml.fpm module :members: :undoc-members: :inherited-members: + +pyspark.ml.util module +---------------------------- + +.. automodule:: pyspark.ml.util + :members: + :undoc-members: + :inherited-members: diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py index 02016f172aebc..7863edda7e7ab 100644 --- a/python/pyspark/ml/util.py +++ b/python/pyspark/ml/util.py @@ -79,7 +79,8 @@ def overwrite(self): def context(self, sqlContext): """ Sets the SQL context to use for saving. - .. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead. + + .. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead. """ raise NotImplementedError("MLWriter is not yet implemented for type: %s" % type(self)) @@ -113,9 +114,10 @@ def overwrite(self): def context(self, sqlContext): """ Sets the SQL context to use for saving. - .. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead. + + .. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead. """ - warnings.warn("Deprecated in 2.1 and will be removed in 2.2, use session instead.") + warnings.warn("Deprecated in 2.1 and will be removed in 3.0, use session instead.") self._jwrite.context(sqlContext._ssql_ctx) return self @@ -168,7 +170,8 @@ def load(self, path): def context(self, sqlContext): """ Sets the SQL context to use for loading. - .. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead. + + .. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead. """ raise NotImplementedError("MLReader is not yet implemented for type: %s" % type(self)) @@ -200,9 +203,10 @@ def load(self, path): def context(self, sqlContext): """ Sets the SQL context to use for loading. - .. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead. + + .. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead. """ - warnings.warn("Deprecated in 2.1 and will be removed in 2.2, use session instead.") + warnings.warn("Deprecated in 2.1 and will be removed in 3.0, use session instead.") self._jread.context(sqlContext._ssql_ctx) return self