diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala index 6036af855af..335253925e6 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.config object KyuubiReservedKeys { final val KYUUBI_CLIENT_IP_KEY = "kyuubi.client.ipAddress" + final val KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version" final val KYUUBI_SERVER_IP_KEY = "kyuubi.server.ipAddress" final val KYUUBI_SESSION_USER_KEY = "kyuubi.session.user" final val KYUUBI_SESSION_SIGN_PUBLICKEY = "kyuubi.session.sign.publickey" diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml index 7b45c27bbf4..36ea7acc274 100644 --- a/kyuubi-hive-jdbc/pom.xml +++ b/kyuubi-hive-jdbc/pom.xml @@ -171,6 +171,14 @@ + + + + true + src/main/resources + + + org.apache.maven.plugins diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java index 9931dcec2d0..0932ea56585 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java @@ -738,6 +738,7 @@ private void openSession() throws SQLException { } catch (UnknownHostException e) { LOG.debug("Error getting Kyuubi session local client ip address", e); } + openConf.put(Utils.KYUUBI_CLIENT_VERSION_KEY, Utils.getVersion()); openReq.setConfiguration(openConf); // Store the user name in the open request in case no non-sasl authentication diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java index 59cc67f9dc7..1daa322ecd2 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java @@ -486,4 +486,21 @@ public static String getCanonicalHostName(String hostName) { public static boolean isKyuubiOperationHint(String hint) { return KYUUBI_OPERATION_HINT_PATTERN.matcher(hint).matches(); } + + public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version"; + private static String KYUUBI_CLIENT_VERSION; + + public static synchronized String getVersion() { + if (KYUUBI_CLIENT_VERSION == null) { + try { + Properties prop = new Properties(); + prop.load(Utils.class.getClassLoader().getResourceAsStream("version.properties")); + KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown"); + } catch (Exception e) { + LOG.error("Error getting kyuubi client version", e); + KYUUBI_CLIENT_VERSION = "unknown"; + } + } + return KYUUBI_CLIENT_VERSION; + } } diff --git a/kyuubi-hive-jdbc/src/main/resources/version.properties b/kyuubi-hive-jdbc/src/main/resources/version.properties new file mode 100644 index 00000000000..82ae50cfbf6 --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/resources/version.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kyuubi.client.version = ${project.version} diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java index e0594dde0b2..b01957b3e43 100644 --- a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java +++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.Map; import java.util.Properties; +import java.util.regex.Pattern; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -138,4 +139,10 @@ public void testExtractURLComponents() throws JdbcUriParseException { assertEquals(expectedDb, jdbcConnectionParams1.getDbName()); assertEquals(expectedHiveConf, jdbcConnectionParams1.getHiveConfs()); } + + @Test + public void testGetVersion() { + Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*"); + assert pattern.matcher(Utils.getVersion()).matches(); + } } diff --git a/kyuubi-rest-client/pom.xml b/kyuubi-rest-client/pom.xml index 0af949125e5..6ba88e8dc8b 100644 --- a/kyuubi-rest-client/pom.xml +++ b/kyuubi-rest-client/pom.xml @@ -115,6 +115,13 @@ + + + true + src/main/resources + + + net.alchim31.maven diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java index a09ab562b87..f5099568b21 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.kyuubi.client.api.v1.dto.*; import org.apache.kyuubi.client.util.JsonUtils; +import org.apache.kyuubi.client.util.VersionUtils; public class BatchRestApi { @@ -36,11 +37,13 @@ public BatchRestApi(KyuubiRestClient client) { } public Batch createBatch(BatchRequest request) { + setClientVersion(request); String requestBody = JsonUtils.toJson(request); return this.getClient().post(API_BASE_PATH, requestBody, Batch.class, client.getAuthHeader()); } public Batch createBatch(BatchRequest request, File resourceFile) { + setClientVersion(request); Map multiPartMap = new HashMap<>(); multiPartMap.put("batchRequest", new MultiPart(MultiPart.MultiPartType.JSON, request)); multiPartMap.put("resourceFile", new MultiPart(MultiPart.MultiPartType.FILE, resourceFile)); @@ -96,4 +99,13 @@ public CloseBatchResponse deleteBatch(String batchId, String hs2ProxyUser) { private IRestClient getClient() { return this.client.getHttpClient(); } + + private void setClientVersion(BatchRequest request) { + if (request != null) { + Map newConf = new HashMap<>(); + newConf.putAll(request.getConf()); + newConf.put(VersionUtils.KYUUBI_CLIENT_VERSION_KEY, VersionUtils.getVersion()); + request.setConf(newConf); + } + } } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java new file mode 100644 index 00000000000..bcabca5b9f8 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.util; + +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VersionUtils { + static final Logger LOG = LoggerFactory.getLogger(VersionUtils.class); + + public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version"; + private static String KYUUBI_CLIENT_VERSION; + + public static synchronized String getVersion() { + if (KYUUBI_CLIENT_VERSION == null) { + try { + Properties prop = new Properties(); + prop.load(VersionUtils.class.getClassLoader().getResourceAsStream("version.properties")); + KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown"); + } catch (Exception e) { + LOG.error("Error getting kyuubi client version", e); + KYUUBI_CLIENT_VERSION = "unknown"; + } + } + return KYUUBI_CLIENT_VERSION; + } +} diff --git a/kyuubi-rest-client/src/main/resources/version.properties b/kyuubi-rest-client/src/main/resources/version.properties new file mode 100644 index 00000000000..82ae50cfbf6 --- /dev/null +++ b/kyuubi-rest-client/src/main/resources/version.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kyuubi.client.version = ${project.version} diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java new file mode 100644 index 00000000000..d4675f34039 --- /dev/null +++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.util; + +import java.util.regex.Pattern; +import org.junit.Test; + +public class VersionUtilsTest { + + @Test + public void testGetClientVersion() { + Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*"); + assert pattern.matcher(VersionUtils.getVersion()).matches(); + } +} diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala index 341f0063978..669475b6cba 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerConnectionSuite.scala @@ -26,8 +26,8 @@ import scala.collection.JavaConverters._ import org.apache.hive.service.rpc.thrift._ import org.scalatest.time.SpanSugar.convertIntToGrainOfTime -import org.apache.kyuubi.WithKyuubiServer -import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.{KYUUBI_VERSION, WithKyuubiServer} +import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} import org.apache.kyuubi.config.KyuubiConf.SESSION_CONF_ADVISOR import org.apache.kyuubi.engine.ApplicationState import org.apache.kyuubi.jdbc.KyuubiHiveDriver @@ -272,6 +272,14 @@ class KyuubiOperationPerConnectionSuite extends WithKyuubiServer with HiveJDBCTe assert(MetricsSystem.counterValue(connFailedMetric).getOrElse(0L) > connFailedCount) } } + + test("support to transfer client version when opening jdbc connection") { + withJdbcStatement() { stmt => + val rs = stmt.executeQuery(s"set spark.${KyuubiReservedKeys.KYUUBI_CLIENT_VERSION_KEY}") + assert(rs.next()) + assert(rs.getString(2) === KYUUBI_VERSION) + } + } } class TestSessionConfAdvisor extends SessionConfAdvisor { diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala index 6110ccb1e22..cb7905286f9 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/rest/client/BatchRestApiSuite.scala @@ -22,10 +22,11 @@ import java.util.Base64 import org.scalatest.time.SpanSugar.convertIntToGrainOfTime -import org.apache.kyuubi.{BatchTestHelper, RestClientTestHelper} +import org.apache.kyuubi.{BatchTestHelper, KYUUBI_VERSION, RestClientTestHelper} import org.apache.kyuubi.client.{BatchRestApi, KyuubiRestClient} import org.apache.kyuubi.client.api.v1.dto.Batch import org.apache.kyuubi.client.exception.KyuubiRestException +import org.apache.kyuubi.config.KyuubiReservedKeys import org.apache.kyuubi.metrics.{MetricsConstants, MetricsSystem} import org.apache.kyuubi.session.{KyuubiSession, SessionHandle} @@ -215,4 +216,22 @@ class BatchRestApiSuite extends RestClientTestHelper with BatchTestHelper { batchRestApi.listBatches(null, null, null, 0, 0, 0, 1) batchRestApi.listBatches(null, null, null, 0, 0, 0, 1) } + + test("support to transfer client version when creating batch") { + val spnegoKyuubiRestClient: KyuubiRestClient = + KyuubiRestClient.builder(baseUri.toString) + .authHeaderMethod(KyuubiRestClient.AuthHeaderMethod.SPNEGO) + .spnegoHost("localhost") + .build() + val batchRestApi: BatchRestApi = new BatchRestApi(spnegoKyuubiRestClient) + // create batch + val requestObj = + newSparkBatchRequest(Map("spark.master" -> "local")) + + val batch = batchRestApi.createBatch(requestObj) + val batchSession = + server.backendService.sessionManager.getSession(SessionHandle.fromUUID(batch.getId)) + assert( + batchSession.conf.get(KyuubiReservedKeys.KYUUBI_CLIENT_VERSION_KEY) == Some(KYUUBI_VERSION)) + } }