Patch examples

Author: zero323

examples/src/main/python/ml/estimator_transformer_param_example.py

@@ -19,6 +19,8 @@
 Estimator Transformer Param Example.
 """
 # $example on$
+from typing import Any, Dict
+from pyspark.ml.param import Param
 from pyspark.ml.linalg import Vectors
 from pyspark.ml.classification import LogisticRegression
 # $example off$
@@ -54,7 +56,7 @@
     print(model1.extractParamMap())
 
     # We may alternatively specify parameters using a Python dictionary as a paramMap
-    paramMap = {lr.maxIter: 20}
+    paramMap: Dict[Param, Any] = {lr.maxIter: 20}
     paramMap[lr.maxIter] = 30  # Specify 1 Param, overwriting the original maxIter.
     paramMap.update({lr.regParam: 0.1, lr.threshold: 0.55})  # Specify multiple Params.
 

examples/src/main/python/ml/fm_classifier_example.py

@@ -20,7 +20,7 @@
 """
 # $example on$
 from pyspark.ml import Pipeline
-from pyspark.ml.classification import FMClassifier
+from pyspark.ml.classification import FMClassifier, FMClassificationModel
 from pyspark.ml.feature import MinMaxScaler, StringIndexer
 from pyspark.ml.evaluation import MulticlassClassificationEvaluator
 # $example off$
@@ -66,7 +66,7 @@
     accuracy = evaluator.evaluate(predictions)
     print("Test set accuracy = %g" % accuracy)
 
-    fmModel = model.stages[2]
+    fmModel: FMClassificationModel = model.stages[2]  # type: ignore[assignment]
     print("Factors: " + str(fmModel.factors))
     print("Linear: " + str(fmModel.linear))
     print("Intercept: " + str(fmModel.intercept))

examples/src/main/python/ml/fm_regressor_example.py

@@ -20,7 +20,7 @@
 """
 # $example on$
 from pyspark.ml import Pipeline
-from pyspark.ml.regression import FMRegressor
+from pyspark.ml.regression import FMRegressor, FMRegressionModel
 from pyspark.ml.feature import MinMaxScaler
 from pyspark.ml.evaluation import RegressionEvaluator
 # $example off$
@@ -63,7 +63,7 @@
     rmse = evaluator.evaluate(predictions)
     print("Root Mean Squared Error (RMSE) on test data = %g" % rmse)
 
-    fmModel = model.stages[1]
+    fmModel: FMRegressionModel = model.stages[1]  # type: ignore[assignment]
     print("Factors: " + str(fmModel.factors))
     print("Linear: " + str(fmModel.linear))
     print("Intercept: " + str(fmModel.intercept))

examples/src/main/python/ml/logistic_regression_summary_example.py

@@ -21,7 +21,7 @@
   bin/spark-submit examples/src/main/python/ml/logistic_regression_summary_example.py
 """
 # $example on$
-from pyspark.ml.classification import LogisticRegression
+from pyspark.ml.classification import LogisticRegression, BinaryLogisticRegressionTrainingSummary
 # $example off$
 from pyspark.sql import SparkSession
 
@@ -42,7 +42,9 @@
     # $example on$
     # Extract the summary from the returned LogisticRegressionModel instance trained
     # in the earlier example
-    trainingSummary = lrModel.summary
+    trainingSummary: BinaryLogisticRegressionTrainingSummary = (
+        lrModel.summary
+    )  # type: ignore[assignment]
 
     # Obtain the objective per iteration
     objectiveHistory = trainingSummary.objectiveHistory

examples/src/main/python/ml/pipeline_example.py

@@ -62,8 +62,12 @@
     prediction = model.transform(test)
     selected = prediction.select("id", "text", "probability", "prediction")
     for row in selected.collect():
-        rid, text, prob, prediction = row
-        print("(%d, %s) --> prob=%s, prediction=%f" % (rid, text, str(prob), prediction))
+        rid, text, prob, prediction = row  # type: ignore[no-redef]
+        print(
+            "(%d, %s) --> prob=%s, prediction=%f" % (
+                rid, text, str(prob), prediction   # type: ignore
+            )
+        )
     # $example off$
 
     spark.stop()

examples/src/main/python/sql/arrow.py

@@ -32,8 +32,8 @@
 
 
 def dataframe_with_arrow_example(spark):
-    import numpy as np
-    import pandas as pd
+    import numpy as np  # type: ignore[import]
+    import pandas as pd  # type: ignore[import]
 
     # Enable Arrow-based columnar data transfers
     spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")