Skip to content

Commit

Permalink
convert options in include statment to set statments (#1633)
Browse files Browse the repository at this point in the history
* convert options in include statment to set statments

* add test and fix exception throws when no suffix in  include statement

* ignore result check in pr_1633.mlsql

revise the code example

add checking multi group params

revise the remind msg when setting multi-group params in the Discretizer et

rename the checking functions
  • Loading branch information
allwefantasy authored and ckeys committed Dec 30, 2021
1 parent ab07ba8 commit f66fe6b
Showing 1 changed file with 17 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,17 @@ class SQLDiscretizer(override val uid: String) extends SQLAlg with Functions wit
return metas
}

def checkWithoutGroupParams(params: Map[String, String]): Unit = {
val group = params.map(p => {
val key = p._1
key.contains(DiscretizerParamsConstrant.PARAMS_PREFIX + ".")
}).filter(_ == true)
require(group.size == 0, "The multi-group params are not available for the Discretizer Et!")
}

def trainWithoutGroup(df: DataFrame, params: Map[String, String], _method: String, metaPath: String): DataFrame = {
val dfWithId = df.withColumn("id", monotonically_increasing_id)
checkWithoutGroupParams(params)
var transformedDF = dfWithId
// we need save metadatas with index, because we need index
val metas: Array[(Int, DiscretizerTrainData)] = {
Expand Down Expand Up @@ -220,10 +229,14 @@ class SQLDiscretizer(override val uid: String) extends SQLAlg with Functions wit
|
|train data1 as Discretizer.`/tmp/model`
|where method="bucketizer"
|and `fitParam.0.inputCol`="a"
|and `fitParam.0.splitArray`="-inf,0.0,1.0,inf"
|and `fitParam.1.inputCol`="b"
|and `fitParam.1.splitArray`="-inf,0.0,1.0,inf";
|and `inputCol`="a"
|and `splits`="-inf,0.0,1.0,inf";
|
|train data1 as Discretizer.`/tmp/discretizer`
|where method="quantile"
|and `inputCol`="a"
|and `outputCol`="a_v"
|and `numBuckets`="3";
|;
""".stripMargin)

Expand Down

0 comments on commit f66fe6b

Please sign in to comment.