Skip to content

[SPARK-25838][ML] Remove formatVersion from Saveable #22830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mgaido91 wants to merge 4 commits into from
Closed

[SPARK-25838][ML] Remove formatVersion from Saveable #22830

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7f17c6e
[SPARK-25838][ML] Remove formatVersion from Saveable
mgaido91 Oct 25, 2018
492f6ed
fix mima
mgaido91 Oct 26, 2018
e9d9e0e
Merge branch 'master' of github.com:apache/spark into SPARK-25838
mgaido91 Oct 26, 2018
db63b5b
Merge branch 'master' into SPARK-25838
mgaido91 Mar 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/classification/LogisticRegression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,6 @@ class LogisticRegressionModel @Since("1.3.0") (
numFeatures, numClasses, weights, intercept, threshold)
}

override protected def formatVersion: String = "1.0"

Copy link
Member

@dongjoon-hyun dongjoon-hyun Oct 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems visible to the users. Although this occurs in Spark 3.0.0, I guess we had better do deprecation processes first for this kind of removals.

cc @srowen , @mengxr , @rxin .

Copy link
Contributor Author

@mgaido91 mgaido91 Oct 29, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you mean by visible to users? It is protected, it is visible only if a user extends the class...

override def toString: String = {
s"${super.toString}, numClasses = ${numClasses}, threshold = ${threshold.getOrElse("None")}"
}
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/classification/NaiveBayes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,6 @@ class NaiveBayesModel private[spark] (
val data = NaiveBayesModel.SaveLoadV2_0.Data(labels, pi, theta, modelType)
NaiveBayesModel.SaveLoadV2_0.save(sc, path, data)
}

override protected def formatVersion: String = "2.0"
}

@Since("1.3.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/classification/SVM.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,6 @@ class SVMModel @Since("1.1.0") (
numFeatures = weights.size, numClasses = 2, weights, intercept, threshold)
}

override protected def formatVersion: String = "1.0"

override def toString: String = {
s"${super.toString}, numClasses = 2, threshold = ${threshold.getOrElse("None")}"
}
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,6 @@ class BisectingKMeansModel private[clustering] (
override def save(sc: SparkContext, path: String): Unit = {
BisectingKMeansModel.SaveLoadV3_0.save(sc, this, path)
}

override protected def formatVersion: String = "3.0"
}

@Since("2.0.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/clustering/GaussianMixtureModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ class GaussianMixtureModel @Since("1.3.0") (

require(weights.length == gaussians.length, "Length of weight and Gaussian arrays must match")

override protected def formatVersion = "1.0"

@Since("1.4.0")
override def save(sc: SparkContext, path: String): Unit = {
GaussianMixtureModel.SaveLoadV1_0.save(sc, path, weights, gaussians)
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,6 @@ class KMeansModel (@Since("1.0.0") val clusterCenters: Array[Vector],
override def save(sc: SparkContext, path: String): Unit = {
KMeansModel.SaveLoadV2_0.save(sc, this, path)
}

override protected def formatVersion: String = "2.0"
}

@Since("1.4.0")
Expand Down
4 changes: 0 additions & 4 deletions mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,6 @@ class LocalLDAModel private[spark] (
}.toArray
}

override protected def formatVersion = "1.0"

/**
* Random seed for cluster initialization.
*/
Expand Down Expand Up @@ -835,8 +833,6 @@ class DistributedLDAModel private[clustering] (
// TODO:
// override def topicDistributions(documents: RDD[(Long, Vector)]): RDD[(Long, Vector)] = ???

override protected def formatVersion = "1.0"

@Since("1.5.0")
override def save(sc: SparkContext, path: String): Unit = {
// Note: This intentionally does not save checkpointFiles.
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/clustering/PowerIterationClustering.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ class PowerIterationClusteringModel @Since("1.3.0") (
override def save(sc: SparkContext, path: String): Unit = {
PowerIterationClusteringModel.SaveLoadV1_0.save(sc, this, path)
}

override protected def formatVersion: String = "1.0"
}

@Since("1.4.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,6 @@ class ChiSqSelectorModel @Since("1.3.0") (
override def save(sc: SparkContext, path: String): Unit = {
ChiSqSelectorModel.SaveLoadV1_0.save(sc, this, path)
}

override protected def formatVersion: String = "1.0"
}

object ChiSqSelectorModel extends Loader[ChiSqSelectorModel] {
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -511,8 +511,6 @@ class Word2VecModel private[spark] (
this(Word2VecModel.buildWordIndex(model), Word2VecModel.buildWordVectors(model))
}

override protected def formatVersion = "1.0"

@Since("1.4.0")
def save(sc: SparkContext, path: String): Unit = {
Word2VecModel.SaveLoadV1_0.save(sc, path, getVectors)
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/fpm/FPGrowth.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,6 @@ class FPGrowthModel[Item: ClassTag] @Since("2.4.0") (
override def save(sc: SparkContext, path: String): Unit = {
FPGrowthModel.SaveLoadV1_0.save(this, path)
}

override protected val formatVersion: String = "1.0"
}

@Since("2.0.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,8 +628,6 @@ class PrefixSpanModel[Item] @Since("1.5.0") (
override def save(sc: SparkContext, path: String): Unit = {
PrefixSpanModel.SaveLoadV1_0.save(this, path)
}

override protected val formatVersion: String = "1.0"
}

@Since("2.0.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,6 @@ class MatrixFactorizationModel @Since("0.8.0") (
.map(t => Rating(t._1, product, t._2))
}

protected override val formatVersion: String = "1.0"

/**
* Save this model to the given path.
*
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/regression/IsotonicRegression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,8 +160,6 @@ class IsotonicRegressionModel @Since("1.3.0") (
override def save(sc: SparkContext, path: String): Unit = {
IsotonicRegressionModel.SaveLoadV1_0.save(sc, path, boundaries, predictions, isotonic)
}

override protected def formatVersion: String = "1.0"
}

@Since("1.4.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/regression/Lasso.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ class LassoModel @Since("1.1.0") (
override def save(sc: SparkContext, path: String): Unit = {
GLMRegressionModel.SaveLoadV1_0.save(sc, path, this.getClass.getName, weights, intercept)
}

override protected def formatVersion: String = "1.0"
}

@Since("1.3.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/regression/LinearRegression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,6 @@ class LinearRegressionModel @Since("1.1.0") (
override def save(sc: SparkContext, path: String): Unit = {
GLMRegressionModel.SaveLoadV1_0.save(sc, path, this.getClass.getName, weights, intercept)
}

override protected def formatVersion: String = "1.0"
}

@Since("1.3.0")
Expand Down
2 changes: 0 additions & 2 deletions mllib/src/main/scala/org/apache/spark/mllib/regression/RidgeRegression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,6 @@ class RidgeRegressionModel @Since("1.1.0") (
override def save(sc: SparkContext, path: String): Unit = {
GLMRegressionModel.SaveLoadV1_0.save(sc, path, this.getClass.getName, weights, intercept)
}

override protected def formatVersion: String = "1.0"
}

@Since("1.3.0")
Expand Down
4 changes: 0 additions & 4 deletions mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,15 +126,11 @@ class DecisionTreeModel @Since("1.0.0") (
override def save(sc: SparkContext, path: String): Unit = {
DecisionTreeModel.SaveLoadV1_0.save(sc, path, this)
}

override protected def formatVersion: String = DecisionTreeModel.formatVersion
}

@Since("1.3.0")
object DecisionTreeModel extends Loader[DecisionTreeModel] with Logging {

private[spark] def formatVersion: String = "1.0"

private[tree] object SaveLoadV1_0 {

def thisFormatVersion: String = "1.0"
Expand Down
8 changes: 0 additions & 8 deletions mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,11 @@ class RandomForestModel @Since("1.2.0") (
TreeEnsembleModel.SaveLoadV1_0.save(sc, path, this,
RandomForestModel.SaveLoadV1_0.thisClassName)
}

override protected def formatVersion: String = RandomForestModel.formatVersion
}

@Since("1.3.0")
object RandomForestModel extends Loader[RandomForestModel] {

private[mllib] def formatVersion: String = TreeEnsembleModel.SaveLoadV1_0.thisFormatVersion

/**
*
* @param sc Spark context used for loading model files.
Expand Down Expand Up @@ -170,8 +166,6 @@ class GradientBoostedTreesModel @Since("1.2.0") (
broadcastTrees.destroy()
evaluation.toArray
}

override protected def formatVersion: String = GradientBoostedTreesModel.formatVersion
}

/**
Expand Down Expand Up @@ -235,8 +229,6 @@ object GradientBoostedTreesModel extends Loader[GradientBoostedTreesModel] {
newPredError
}

private[mllib] def formatVersion: String = TreeEnsembleModel.SaveLoadV1_0.thisFormatVersion

/**
* @param sc Spark context used for loading model files.
* @param path Path specifying the directory to which the model was saved.
Expand Down
3 changes: 0 additions & 3 deletions mllib/src/main/scala/org/apache/spark/mllib/util/modelSaveLoad.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,6 @@ trait Saveable {
@Since("1.3.0")
def save(sc: SparkContext, path: String): Unit

/** Current version of model save/load format. */
protected def formatVersion: String

}

/**
Expand Down
21 changes: 21 additions & 0 deletions project/MimaExcludes.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,27 @@ object MimaExcludes {

// Exclude rules for 3.0.x
lazy val v30excludes = v24excludes ++ Seq(
// [SPARK-25838] Remove formatVersion from Saveable
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.DistributedLDAModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.LocalLDAModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.BisectingKMeansModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.KMeansModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.PowerIterationClusteringModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.clustering.GaussianMixtureModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.recommendation.MatrixFactorizationModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.feature.ChiSqSelectorModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.feature.Word2VecModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.classification.SVMModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.classification.LogisticRegressionModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.classification.NaiveBayesModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.util.Saveable.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.fpm.FPGrowthModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.fpm.PrefixSpanModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.regression.IsotonicRegressionModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.regression.RidgeRegressionModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.regression.LassoModel.formatVersion"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.mllib.regression.LinearRegressionModel.formatVersion"),

// [SPARK-26254][CORE] Extract Hive + Kafka dependencies from Core.
ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.deploy.security.HiveDelegationTokenProvider"),

Expand Down