Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-20707] [ML] ML deprecated APIs should be removed in major release. #17946

Closed
wants to merge 1 commit into from
Closed

[SPARK-20707] [ML] ML deprecated APIs should be removed in major release. #17946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions mllib/src/main/scala/org/apache/spark/ml/tree/treeParams.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,77 +110,77 @@ private[ml] trait DecisionTreeParams extends PredictorParams
maxMemoryInMB -> 256, cacheNodeIds -> false, checkpointInterval -> 10)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMaxDepth(value: Int): this.type = set(maxDepth, value)

/** @group getParam */
final def getMaxDepth: Int = $(maxDepth)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMaxBins(value: Int): this.type = set(maxBins, value)

/** @group getParam */
final def getMaxBins: Int = $(maxBins)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMinInstancesPerNode(value: Int): this.type = set(minInstancesPerNode, value)

/** @group getParam */
final def getMinInstancesPerNode: Int = $(minInstancesPerNode)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMinInfoGain(value: Double): this.type = set(minInfoGain, value)

/** @group getParam */
final def getMinInfoGain: Double = $(minInfoGain)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setSeed(value: Long): this.type = set(seed, value)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group expertSetParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMaxMemoryInMB(value: Int): this.type = set(maxMemoryInMB, value)

/** @group expertGetParam */
final def getMaxMemoryInMB: Int = $(maxMemoryInMB)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group expertSetParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setCacheNodeIds(value: Boolean): this.type = set(cacheNodeIds, value)

/** @group expertGetParam */
final def getCacheNodeIds: Boolean = $(cacheNodeIds)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setCheckpointInterval(value: Int): this.type = set(checkpointInterval, value)

/** (private[ml]) Create a Strategy instance to use with the old API. */
Expand Down Expand Up @@ -226,10 +226,10 @@ private[ml] trait TreeClassifierParams extends Params {
setDefault(impurity -> "gini")

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setImpurity(value: String): this.type = set(impurity, value)

/** @group getParam */
Expand Down Expand Up @@ -277,10 +277,10 @@ private[ml] trait TreeRegressorParams extends Params {
setDefault(impurity -> "variance")

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setImpurity(value: String): this.type = set(impurity, value)

/** @group getParam */
Expand Down Expand Up @@ -339,10 +339,10 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams {
setDefault(subsamplingRate -> 1.0)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setSubsamplingRate(value: Double): this.type = set(subsamplingRate, value)

/** @group getParam */
Expand Down Expand Up @@ -383,10 +383,10 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
setDefault(numTrees -> 20)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setNumTrees(value: Int): this.type = set(numTrees, value)

/** @group getParam */
Expand Down Expand Up @@ -431,10 +431,10 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
setDefault(featureSubsetStrategy -> "auto")

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setFeatureSubsetStrategy(value: String): this.type = set(featureSubsetStrategy, value)

/** @group getParam */
Expand Down Expand Up @@ -472,10 +472,10 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter {
// validationTol -> 1e-5

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setMaxIter(value: Int): this.type = set(maxIter, value)

/**
Expand All @@ -492,10 +492,10 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter {
final def getStepSize: Double = $(stepSize)

/**
* @deprecated This method is deprecated and will be removed in 2.2.0.
* @deprecated This method is deprecated and will be removed in 3.0.0.
* @group setParam
*/
@deprecated("This method is deprecated and will be removed in 2.2.0.", "2.1.0")
@deprecated("This method is deprecated and will be removed in 3.0.0.", "2.1.0")
def setStepSize(value: Double): this.type = set(stepSize, value)

setDefault(maxIter -> 20, stepSize -> 0.1)
Expand Down
4 changes: 3 additions & 1 deletion mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,11 @@ private[util] sealed trait BaseReadWrite {

/**
* Sets the Spark SQLContext to use for saving/loading.
*
* @deprecated Use session instead. This method will be removed in 3.0.0.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add this can make Scala MLWriter and MLReader produce correct deprecated annotation, just like the screenshot in PR description.

*/
@Since("1.6.0")
@deprecated("Use session instead, This method will be removed in 2.2.0.", "2.0.0")
@deprecated("Use session instead. This method will be removed in 3.0.0.", "2.0.0")
def context(sqlContext: SQLContext): this.type = {
optionSparkSession = Option(sqlContext.sparkSession)
this
Expand Down
8 changes: 8 additions & 0 deletions python/docs/pyspark.ml.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,11 @@ pyspark.ml.fpm module
:members:
:undoc-members:
:inherited-members:

pyspark.ml.util module
----------------------------

.. automodule:: pyspark.ml.util
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original PySpark miss out docs of pyspark.ml.util module which includes MLWriter and MLReader.

:members:
:undoc-members:
:inherited-members:
16 changes: 10 additions & 6 deletions python/pyspark/ml/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ def overwrite(self):
def context(self, sqlContext):
"""
Sets the SQL context to use for saving.
.. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead.

.. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead.
"""
raise NotImplementedError("MLWriter is not yet implemented for type: %s" % type(self))

Expand Down Expand Up @@ -113,9 +114,10 @@ def overwrite(self):
def context(self, sqlContext):
"""
Sets the SQL context to use for saving.
.. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead.

.. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead.
"""
warnings.warn("Deprecated in 2.1 and will be removed in 2.2, use session instead.")
warnings.warn("Deprecated in 2.1 and will be removed in 3.0, use session instead.")
self._jwrite.context(sqlContext._ssql_ctx)
return self

Expand Down Expand Up @@ -168,7 +170,8 @@ def load(self, path):
def context(self, sqlContext):
"""
Sets the SQL context to use for loading.
.. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead.

.. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead.
"""
raise NotImplementedError("MLReader is not yet implemented for type: %s" % type(self))

Expand Down Expand Up @@ -200,9 +203,10 @@ def load(self, path):
def context(self, sqlContext):
"""
Sets the SQL context to use for loading.
.. note:: Deprecated in 2.1 and will be removed in 2.2, use session instead.

.. note:: Deprecated in 2.1 and will be removed in 3.0, use session instead.
"""
warnings.warn("Deprecated in 2.1 and will be removed in 2.2, use session instead.")
warnings.warn("Deprecated in 2.1 and will be removed in 3.0, use session instead.")
self._jread.context(sqlContext._ssql_ctx)
return self

Expand Down