add llk in summary

Author: zhengruifeng

mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala

@@ -129,25 +129,6 @@ class GaussianMixtureModel private[ml] (
     Vectors.dense(probs)
   }
 
-  /**
-   * Return the total log-likelihood for this model on the given data.
-   */
-  @Since("2.2.0")
-  def computeLogLikelihood(dataset: Dataset[_]): Double = {
-    SchemaUtils.checkColumnType(dataset.schema, $(featuresCol), new VectorUDT)
-    val spark = dataset.sparkSession
-    import spark.implicits._
-
-    val bcWeightAndDists = spark.sparkContext.broadcast(weights.zip(gaussians))
-    dataset.select(col($(featuresCol))).map {
-      case Row(feature: Vector) =>
-        val likelihood = bcWeightAndDists.value.map {
-          case (weight, dist) => EPSILON + weight * dist.pdf(feature)
-        }.sum
-        math.log(likelihood)
-    }.reduce(_ + _)
-  }
-
   /**
    * Retrieve Gaussian distributions as a DataFrame.
    * Each row represents a Gaussian Distribution.
@@ -435,7 +416,7 @@ class GaussianMixture @Since("2.0.0") (
 
     val model = copyValues(new GaussianMixtureModel(uid, weights, gaussianDists)).setParent(this)
     val summary = new GaussianMixtureSummary(model.transform(dataset),
-      $(predictionCol), $(probabilityCol), $(featuresCol), $(k))
+      $(predictionCol), $(probabilityCol), $(featuresCol), $(k), logLikelihood)
     model.setSummary(Some(summary))
     instr.logSuccess(model)
     model
@@ -693,6 +674,7 @@ private class ExpectationAggregator(
  *                        in `predictions`.
  * @param featuresCol  Name for column of features in `predictions`.
  * @param k  Number of clusters.
+ * @param logLikelihood  Total log-likelihood for this model on the given data.
  */
 @Since("2.0.0")
 @Experimental
@@ -701,7 +683,9 @@ class GaussianMixtureSummary private[clustering] (
     predictionCol: String,
     @Since("2.0.0") val probabilityCol: String,
     featuresCol: String,
-    k: Int) extends ClusteringSummary(predictions, predictionCol, featuresCol, k) {
+    k: Int,
+    @Since("2.2.0") val logLikelihood: Double)
+  extends ClusteringSummary(predictions, predictionCol, featuresCol, k) {
 
   /**
    * Probability of each cluster.

mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala

@@ -224,7 +224,7 @@ class GaussianMixtureSuite extends SparkFunSuite with MLlibTestSparkContext
     val actual = new GaussianMixture().setK(2).setSeed(seed).fit(rDataset)
     modelEquals(expected, actual)
 
-    val llk = expected.computeLogLikelihood(rDataset)
+    val llk = actual.summary.logLikelihood
     assert(llk ~== -46.89499 absTol 1E-5)
   }
 

python/pyspark/ml/clustering.py

@@ -281,6 +281,14 @@ def probability(self):
         """
         return self._call_java("probability")
 
+    @property
+    @since("2.2.0")
+    def logLikelihood(self):
+        """
+        Total log-likelihood for this model on the given data.
+        """
+        return self._call_java("logLikelihood")
+
 
 class KMeansSummary(ClusteringSummary):
     """