Skip to content

[SPARK-21284][SQL] rename SessionCatalog.registerFunction parameter name #18510

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cloud-fan wants to merge 1 commit into from
Closed

[SPARK-21284][SQL] rename SessionCatalog.registerFunction parameter name #18510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1104,10 +1104,10 @@ class SessionCatalog(
*/
def registerFunction(
Copy link
Contributor Author

@cloud-fan cloud-fan Jul 3, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ouyangxiaochen , you can call this method in https://github.com/apache/spark/pull/17681/files#diff-d2a203f08c862bd762e6740c16e972f7R84

Copy link

@ouyangxiaochen ouyangxiaochen Jul 3, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, I will do this change after your PR is merged into master.

funcDefinition: CatalogFunction,
ignoreIfExists: Boolean,
overrideIfExists: Boolean,
functionBuilder: Option[FunctionBuilder] = None): Unit = {
val func = funcDefinition.identifier
if (functionRegistry.functionExists(func) && !ignoreIfExists) {
if (functionRegistry.functionExists(func) && !overrideIfExists) {
throw new AnalysisException(s"Function $func already exists")
}
val info = new ExpressionInfo(funcDefinition.className, func.database.orNull, func.funcName)
Expand Down Expand Up @@ -1219,7 +1219,7 @@ class SessionCatalog(
// catalog. So, it is possible that qualifiedName is not exactly the same as
// catalogFunction.identifier.unquotedString (difference is on case-sensitivity).
// At here, we preserve the input from the user.
registerFunction(catalogFunction.copy(identifier = qualifiedName), ignoreIfExists = false)
registerFunction(catalogFunction.copy(identifier = qualifiedName), overrideIfExists = false)
// Now, we need to create the Expression.
functionRegistry.lookupFunction(qualifiedName, children)
}
Expand Down
20 changes: 11 additions & 9 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1175,9 +1175,9 @@ abstract class SessionCatalogSuite extends AnalysisTest {
val tempFunc1 = (e: Seq[Expression]) => e.head
val tempFunc2 = (e: Seq[Expression]) => e.last
catalog.registerFunction(
newFunc("temp1", None), ignoreIfExists = false, functionBuilder = Some(tempFunc1))
newFunc("temp1", None), overrideIfExists = false, functionBuilder = Some(tempFunc1))
catalog.registerFunction(
newFunc("temp2", None), ignoreIfExists = false, functionBuilder = Some(tempFunc2))
newFunc("temp2", None), overrideIfExists = false, functionBuilder = Some(tempFunc2))
val arguments = Seq(Literal(1), Literal(2), Literal(3))
assert(catalog.lookupFunction(FunctionIdentifier("temp1"), arguments) === Literal(1))
assert(catalog.lookupFunction(FunctionIdentifier("temp2"), arguments) === Literal(3))
Expand All @@ -1189,12 +1189,12 @@ abstract class SessionCatalogSuite extends AnalysisTest {
// Temporary function already exists
val e = intercept[AnalysisException] {
catalog.registerFunction(
newFunc("temp1", None), ignoreIfExists = false, functionBuilder = Some(tempFunc3))
newFunc("temp1", None), overrideIfExists = false, functionBuilder = Some(tempFunc3))
}.getMessage
assert(e.contains("Function temp1 already exists"))
// Temporary function is overridden
catalog.registerFunction(
newFunc("temp1", None), ignoreIfExists = true, functionBuilder = Some(tempFunc3))
newFunc("temp1", None), overrideIfExists = true, functionBuilder = Some(tempFunc3))
assert(
catalog.lookupFunction(
FunctionIdentifier("temp1"), arguments) === Literal(arguments.length))
Expand All @@ -1208,7 +1208,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {

val tempFunc1 = (e: Seq[Expression]) => e.head
catalog.registerFunction(
newFunc("temp1", None), ignoreIfExists = false, functionBuilder = Some(tempFunc1))
newFunc("temp1", None), overrideIfExists = false, functionBuilder = Some(tempFunc1))

// Returns true when the function is temporary
assert(catalog.isTemporaryFunction(FunctionIdentifier("temp1")))
Expand Down Expand Up @@ -1259,7 +1259,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
withBasicCatalog { catalog =>
val tempFunc = (e: Seq[Expression]) => e.head
catalog.registerFunction(
newFunc("func1", None), ignoreIfExists = false, functionBuilder = Some(tempFunc))
newFunc("func1", None), overrideIfExists = false, functionBuilder = Some(tempFunc))
val arguments = Seq(Literal(1), Literal(2), Literal(3))
assert(catalog.lookupFunction(FunctionIdentifier("func1"), arguments) === Literal(1))
catalog.dropTempFunction("func1", ignoreIfNotExists = false)
Expand Down Expand Up @@ -1300,7 +1300,7 @@ abstract class SessionCatalogSuite extends AnalysisTest {
withBasicCatalog { catalog =>
val tempFunc1 = (e: Seq[Expression]) => e.head
catalog.registerFunction(
newFunc("func1", None), ignoreIfExists = false, functionBuilder = Some(tempFunc1))
newFunc("func1", None), overrideIfExists = false, functionBuilder = Some(tempFunc1))
assert(catalog.lookupFunction(
FunctionIdentifier("func1"), Seq(Literal(1), Literal(2), Literal(3))) == Literal(1))
catalog.dropTempFunction("func1", ignoreIfNotExists = false)
Expand All @@ -1318,8 +1318,10 @@ abstract class SessionCatalogSuite extends AnalysisTest {
val tempFunc2 = (e: Seq[Expression]) => e.last
catalog.createFunction(newFunc("func2", Some("db2")), ignoreIfExists = false)
catalog.createFunction(newFunc("not_me", Some("db2")), ignoreIfExists = false)
catalog.registerFunction(funcMeta1, ignoreIfExists = false, functionBuilder = Some(tempFunc1))
catalog.registerFunction(funcMeta2, ignoreIfExists = false, functionBuilder = Some(tempFunc2))
catalog.registerFunction(
funcMeta1, overrideIfExists = false, functionBuilder = Some(tempFunc1))
catalog.registerFunction(
funcMeta2, overrideIfExists = false, functionBuilder = Some(tempFunc2))
assert(catalog.listFunctions("db1", "*").map(_._1).toSet ==
Set(FunctionIdentifier("func1"),
FunctionIdentifier("yes_me")))
Expand Down
3 changes: 1 addition & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,8 @@ case class CreateFunctionCommand(
s"is not allowed: '${databaseName.get}'")
}
// We first load resources and then put the builder in the function registry.
// Please note that it is allowed to overwrite an existing temp function.
catalog.loadFunctionResources(resources)
catalog.registerFunction(func, ignoreIfExists = false)
catalog.registerFunction(func, overrideIfExists = false)
} else {
// For a permanent, we will store the metadata into underlying external catalog.
// This function will be loaded into the FunctionRegistry when a query uses it.
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class CatalogSuite
val tempFunc = (e: Seq[Expression]) => e.head
val funcMeta = CatalogFunction(FunctionIdentifier(name, None), "className", Nil)
sessionCatalog.registerFunction(
funcMeta, ignoreIfExists = false, functionBuilder = Some(tempFunc))
funcMeta, overrideIfExists = false, functionBuilder = Some(tempFunc))
}

private def dropFunction(name: String, db: Option[String] = None): Unit = {
Expand Down
2 changes: 1 addition & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveSessionCatalog.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ private[sql] class HiveSessionCatalog(
FunctionIdentifier(functionName.toLowerCase(Locale.ROOT), database)
val func = CatalogFunction(functionIdentifier, className, Nil)
// Put this Hive built-in function to our function registry.
registerFunction(func, ignoreIfExists = false)
registerFunction(func, overrideIfExists = false)
// Now, we need to create the Expression.
functionRegistry.lookupFunction(functionIdentifier, children)
}
Expand Down
2 changes: 1 addition & 1 deletion ...est/scala/org/apache/spark/sql/execution/benchmark/ObjectHashAggregateExecBenchmark.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ class ObjectHashAggregateExecBenchmark extends BenchmarkBase with TestHiveSingle
val sessionCatalog = sparkSession.sessionState.catalog.asInstanceOf[HiveSessionCatalog]
val functionIdentifier = FunctionIdentifier(functionName, database = None)
val func = CatalogFunction(functionIdentifier, clazz.getName, resources = Nil)
sessionCatalog.registerFunction(func, ignoreIfExists = false)
sessionCatalog.registerFunction(func, overrideIfExists = false)
}

private def percentile_approx(
Expand Down