Skip to content

Commit

Permalink
parse classpath
Browse files Browse the repository at this point in the history
  • Loading branch information
@yikf
yikf committed Dec 28, 2023
1 parent 6b97c42 commit 5474ebf
Show file tree
Hide file tree
Showing 8 changed files with 99 additions and 37 deletions.
1 change: 0 additions & 1 deletion docs/configuration/settings.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,6 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.hive.deploy.mode | LOCAL | Configures the hive engine deploy mode, The value can be 'local', 'yarn'. In local mode, the engine operates on the same node as the KyuubiServer. In YARN mode, the engine runs within the Application Master (AM) container of YARN. | string | 1.9.0 |
| kyuubi.engine.hive.event.loggers | JSON | A comma-separated list of engine history loggers, where engine/session/operation etc events go.<ul> <li>JSON: the events will be written to the location of kyuubi.engine.event.json.log.path</li> <li>JDBC: to be done</li> <li>CUSTOM: to be done.</li></ul> | seq | 1.7.0 |
| kyuubi.engine.hive.extra.classpath | (none) | The extra classpath for the Hive query engine, for configuring location of the hadoop client jars and etc. | string | 1.6.0 |
| kyuubi.engine.hive.extra.libDir | (none) | The extra lib directory for the Hive query engine, for configuring location of the hadoop client jars and etc. | string | 1.9.0 |
| kyuubi.engine.hive.java.options | (none) | The extra Java options for the Hive query engine | string | 1.6.0 |
| kyuubi.engine.hive.memory | 1g | The heap memory for the Hive query engine | string | 1.6.0 |
| kyuubi.engine.initialize.sql | SHOW DATABASES | SemiColon-separated list of SQL statements to be initialized in the newly created engine before queries. i.e. use `SHOW DATABASES` to eagerly active HiveClient. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver. | seq | 1.2.0 |
Expand Down
32 changes: 32 additions & 0 deletions ...ql-engine/src/main/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,12 @@
*/
package org.apache.kyuubi.engine.hive.deploy

import java.io.File

import scala.collection.mutable.ListBuffer

import org.apache.kyuubi.Utils
import org.apache.kyuubi.config.KyuubiConf.ENGINE_HIVE_EXTRA_CLASSPATH
import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter
import org.apache.kyuubi.engine.hive.HiveSQLEngine

Expand All @@ -30,4 +35,31 @@ object HiveYarnModeSubmitter extends EngineYarnModeSubmitter {
override var engineType: String = "hive"

override def engineMainClass(): String = HiveSQLEngine.getClass.getName

/**
* Jar list for the Hive engine.
*/
override def engineExtraJars(): Seq[File] = {
val hadoopCp = sys.env.get("HIVE_HADOOP_CLASSPATH")
val extraCp = kyuubiConf.get(ENGINE_HIVE_EXTRA_CLASSPATH)
val jars = new ListBuffer[File]
hadoopCp.foreach(cp => parseClasspath(cp, jars))
extraCp.foreach(cp => parseClasspath(cp, jars))
jars
}

private[hive] def parseClasspath(classpath: String, jars: ListBuffer[File]): Unit = {
classpath.split(":").filter(_.nonEmpty).foreach { cp =>
if (cp.endsWith("/*")) {
val dir = cp.substring(0, cp.length - 2)
new File(dir) match {
case f if f.isDirectory =>
f.listFiles().filter(_.getName.endsWith(".jar")).foreach(jars += _)
case _ =>
}
} else {
jars += new File(cp)
}
}
}
}
39 changes: 39 additions & 0 deletions ...gine/src/test/scala/org/apache/kyuubi/engine/hive/deploy/HiveYarnModeSubmitterSuite.scala
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kyuubi.engine.hive.deploy

import java.io.File

import scala.collection.mutable.ListBuffer

import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils}

class HiveYarnModeSubmitterSuite extends KyuubiFunSuite {
val hiveEngineHome: String = Utils.getCodeSourceLocation(getClass).split("/target")(0)

test("hadoop class path") {
val jars = new ListBuffer[File]
val classpath =
s"$hiveEngineHome/target/scala-$SCALA_COMPILE_VERSION/jars/*:" +
s"$hiveEngineHome/target/kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar"
HiveYarnModeSubmitter.parseClasspath(classpath, jars)
assert(jars.size > 1)
assert(jars.exists(
_.getName == s"kyuubi-hive-sql-engine-$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar"))
}

}
8 changes: 0 additions & 8 deletions kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2695,14 +2695,6 @@ object KyuubiConf {
.stringConf
.createOptional

val ENGINE_HIVE_EXTRA_LIB_DIR: OptionalConfigEntry[String] =
buildConf("kyuubi.engine.hive.extra.libDir")
.doc("The extra lib directory for the Hive query engine, for configuring location" +
" of the hadoop client jars and etc.")
.version("1.9.0")
.stringConf
.createOptional

val ENGINE_HIVE_DEPLOY_MODE: ConfigEntry[String] =
buildConf("kyuubi.engine.hive.deploy.mode")
.doc("Configures the hive engine deploy mode, The value can be 'local', 'yarn'. " +
Expand Down
11 changes: 8 additions & 3 deletions ...-common/src/main/scala/org/apache/kyuubi/engine/deploy/yarn/EngineYarnModeSubmitter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,14 @@ abstract class EngineYarnModeSubmitter extends Logging {
var yarnConf: Configuration = _
var hadoopConf: Configuration = _

var engineType: String

def engineMainClass(): String

var engineType: String
/**
* The extra jars that will be added to the classpath of the engine.
*/
def engineExtraJars(): Seq[File] = Seq.empty

protected def submitApplication(): Unit = {
yarnConf = KyuubiHadoopUtils.newYarnConfiguration(kyuubiConf)
Expand Down Expand Up @@ -195,10 +200,10 @@ abstract class EngineYarnModeSubmitter extends Logging {
val jars = kyuubiConf.getOption(KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY)
val putedEntry = new ListBuffer[String]
jars.get.split(KYUUBI_ENGINE_DEPLOY_YARN_MODE_ARCHIVE_SEPARATOR).foreach { path =>
val jars = Utils.listFilesRecursively(new File(path))
val jars = Utils.listFilesRecursively(new File(path)) ++ engineExtraJars()
jars.foreach { f =>
if (!putedEntry.contains(f.getName) && f.isFile &&
f.getName.toLowerCase(Locale.ROOT).endsWith(".jar") && f.canRead) {
f.getName.toLowerCase(Locale.ROOT).endsWith(".jar") && f.canRead) {
jarsStream.putNextEntry(new ZipEntry(f.getName))
Files.copy(f.toPath, jarsStream)
jarsStream.closeEntry()
Expand Down
11 changes: 4 additions & 7 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveProcessBuilder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import com.google.common.annotations.VisibleForTesting

import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_DEPLOY_YARN_MODE_APP_NAME, ENGINE_HIVE_DEPLOY_MODE, ENGINE_HIVE_EXTRA_CLASSPATH, ENGINE_HIVE_EXTRA_LIB_DIR, ENGINE_HIVE_JAVA_OPTIONS, ENGINE_HIVE_MEMORY}
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_DEPLOY_YARN_MODE_APP_NAME, ENGINE_HIVE_DEPLOY_MODE, ENGINE_HIVE_EXTRA_CLASSPATH, ENGINE_HIVE_JAVA_OPTIONS, ENGINE_HIVE_MEMORY}
import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_SESSION_USER_KEY}
import org.apache.kyuubi.engine.{KyuubiApplicationManager, ProcBuilder}
import org.apache.kyuubi.engine.deploy.DeployMode
Expand Down Expand Up @@ -85,10 +85,7 @@ class HiveProcessBuilder(
hadoopCp.foreach(classpathEntries.add)
val extraCp = conf.get(ENGINE_HIVE_EXTRA_CLASSPATH)
extraCp.foreach(classpathEntries.add)
val extraLibDir = conf.get(ENGINE_HIVE_EXTRA_LIB_DIR)
extraLibDir.foreach(libs => classpathEntries.add(s"$libs${File.separator}*"))
classpathEntries.add(s"$extraLibDir${File.separator}*")
if (hadoopCp.isEmpty && extraCp.isEmpty && extraLibDir.isEmpty) {
if (hadoopCp.isEmpty && extraCp.isEmpty) {
warn(s"The conf of ${HIVE_HADOOP_CLASSPATH_KEY} and ${ENGINE_HIVE_EXTRA_CLASSPATH.key}" +
s" is empty.")
debug("Detected development environment")
Expand All @@ -98,8 +95,8 @@ class HiveProcessBuilder(
.resolve("jars")
if (!Files.exists(devHadoopJars)) {
throw new KyuubiException(s"The path $devHadoopJars does not exists. " +
s"Please set ${HIVE_HADOOP_CLASSPATH_KEY}, ${ENGINE_HIVE_EXTRA_CLASSPATH.key} " +
s"or ${ENGINE_HIVE_EXTRA_LIB_DIR} for configuring location of hadoop client jars, etc")
s"Please set ${HIVE_HADOOP_CLASSPATH_KEY} or ${ENGINE_HIVE_EXTRA_CLASSPATH.key} for " +
s"configuring location of hadoop client jars, etc")
}
classpathEntries.add(s"$devHadoopJars${File.separator}*")
}
Expand Down
22 changes: 15 additions & 7 deletions kyuubi-server/src/main/scala/org/apache/kyuubi/engine/hive/HiveYarnModeProcessBuilder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@ import scala.collection.mutable.ArrayBuffer

import org.apache.kyuubi.{KyuubiException, Logging, SCALA_COMPILE_VERSION}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_HIVE_EXTRA_LIB_DIR, ENGINE_HIVE_MEMORY}
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_HIVE_EXTRA_CLASSPATH, ENGINE_HIVE_MEMORY}
import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_SESSION_USER_KEY}
import org.apache.kyuubi.engine.{ApplicationManagerInfo, KyuubiApplicationManager}
import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter._
import org.apache.kyuubi.engine.hive.HiveProcessBuilder.HIVE_HADOOP_CLASSPATH_KEY
import org.apache.kyuubi.operation.log.OperationLog
import org.apache.kyuubi.util.command.CommandLineUtils.{confKeyValue, confKeyValues}

Expand Down Expand Up @@ -100,18 +101,25 @@ class HiveYarnModeProcessBuilder(

jarEntries.add(s"$hiveHome${File.separator}lib${appendClasspathSuffix(isClasspath)}")

val extraLibDir = conf.get(ENGINE_HIVE_EXTRA_LIB_DIR)
extraLibDir.foreach(libs => jarEntries.add(s"$libs${appendClasspathSuffix(isClasspath)}"))

if (extraLibDir.isEmpty) {
val hadoopCp = env.get(HIVE_HADOOP_CLASSPATH_KEY)
val extraCp = conf.get(ENGINE_HIVE_EXTRA_CLASSPATH)
// the classpath of the ApplicationMaster is resolved when submit hive engine to YARN.
if (isClasspath) {
extraCp.foreach(jarEntries.add)
hadoopCp.foreach(jarEntries.add)
}
if (hadoopCp.isEmpty && extraCp.isEmpty) {
warn(s"The conf of ${HIVE_HADOOP_CLASSPATH_KEY} and ${ENGINE_HIVE_EXTRA_CLASSPATH.key}" +
s" is empty.")
debug("Detected development environment")
mainResource.foreach { path =>
val devHadoopJars = Paths.get(path).getParent
.resolve(s"scala-$SCALA_COMPILE_VERSION")
.resolve("jars")
if (!Files.exists(devHadoopJars)) {
throw new KyuubiException(s"The path $devHadoopJars does not exists. " +
s"Please set $ENGINE_HIVE_EXTRA_LIB_DIR for configuring location " +
s"of hadoop client jars, etc")
s"Please set ${HIVE_HADOOP_CLASSPATH_KEY} or ${ENGINE_HIVE_EXTRA_CLASSPATH.key} for " +
s"configuring location of hadoop client jars, etc")
}
jarEntries.add(s"$devHadoopJars${appendClasspathSuffix(isClasspath)}")
}
Expand Down
12 changes: 1 addition & 11 deletions ...server/src/test/scala/org/apache/kyuubi/engine/hive/HiveYarnModeProcessBuilderSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ package org.apache.kyuubi.engine.hive

import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.ENGINE_HIVE_EXTRA_LIB_DIR
import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.{KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY}
import org.apache.kyuubi.engine.deploy.yarn.EngineYarnModeSubmitter.{KYUUBI_ENGINE_DEPLOY_YARN_MODE_HADOOP_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY, KYUUBI_ENGINE_DEPLOY_YARN_MODE_YARN_CONF_KEY}

class HiveYarnModeProcessBuilderSuite extends KyuubiFunSuite {

Expand All @@ -37,15 +36,6 @@ class HiveYarnModeProcessBuilderSuite extends KyuubiFunSuite {
s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_HIVE_CONF_KEY=/etc/hive/conf"))
}

test("hive extra lib dir") {
val conf = KyuubiConf()
.set(ENGINE_HIVE_EXTRA_LIB_DIR, "/hadoop/jars")
val builder = new HiveYarnModeProcessBuilder("kyuubi", conf, "")
val command = builder.toString
assert(command.contains(
s"--conf $KYUUBI_ENGINE_DEPLOY_YARN_MODE_JARS_KEY") && command.contains("/hadoop/jars"))
}

test("hadoop conf dir") {
val conf = KyuubiConf().set("kyuubi.on", "off")
val builder = new HiveYarnModeProcessBuilder("kyuubi", conf, "") {
Expand Down

0 comments on commit 5474ebf

Please sign in to comment.