From 8d108ada585a33da04f15093480be53c0ce8b95e Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 22 Oct 2021 09:02:29 +0800 Subject: [PATCH 1/6] Backport HIVE-21498 --- pom.xml | 2 +- .../org/apache/hive/service/auth/TSetIpAddressProcessor.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 5bae4d280038..097c072601f4 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ 2.10.12 3.5.2 3.0.0 - 0.12.0 + 0.13.0 4.8 1.1 3.141.59 diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java index 1205d21be6be..b727b4e27de8 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/TSetIpAddressProcessor.java @@ -45,11 +45,12 @@ public TSetIpAddressProcessor(Iface iface) { } @Override - public boolean process(final TProtocol in, final TProtocol out) throws TException { + public void process(final TProtocol in, final TProtocol out) throws TException { setIpAddress(in); setUserName(in); try { - return super.process(in, out); + super.process(in, out); + return; } finally { THREAD_LOCAL_USER_NAME.remove(); THREAD_LOCAL_IP_ADDRESS.remove(); From d0e7777c98028582a0cb309d7c96345b2322f755 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 22 Oct 2021 12:58:08 +0800 Subject: [PATCH 2/6] Remove javax.annotation-api --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 097c072601f4..baa0093b8d61 100644 --- a/pom.xml +++ b/pom.xml @@ -2602,6 +2602,10 @@ org.slf4j slf4j-api + + javax.annotation + javax.annotation-api + From 017f937c282c9791956258b9d7671a5f6b29c3bc Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 28 Jan 2022 10:27:57 +0800 Subject: [PATCH 3/6] 0.16.0 --- dev/deps/spark-deps-hadoop-2-hive-2.3 | 2 +- dev/deps/spark-deps-hadoop-3-hive-2.3 | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3 index 5efdca980992..3664f2c74523 100644 --- a/dev/deps/spark-deps-hadoop-2-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-2-hive-2.3 @@ -186,7 +186,7 @@ kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar lapack/2.2.1//lapack-2.2.1.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar +libthrift/0.16.0//libthrift-0.16.0.jar log4j-1.2-api/2.17.1//log4j-1.2-api-2.17.1.jar log4j-api/2.17.1//log4j-api-2.17.1.jar log4j-core/2.17.1//log4j-core-2.17.1.jar diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index a79a71b846dd..99bc78b5b1d4 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -172,7 +172,7 @@ kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar lapack/2.2.1//lapack-2.2.1.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar -libthrift/0.12.0//libthrift-0.12.0.jar +libthrift/0.16.0//libthrift-0.16.0.jar log4j-1.2-api/2.17.1//log4j-1.2-api-2.17.1.jar log4j-api/2.17.1//log4j-api-2.17.1.jar log4j-core/2.17.1//log4j-core-2.17.1.jar diff --git a/pom.xml b/pom.xml index baa0093b8d61..c6a24ab1a5ff 100644 --- a/pom.xml +++ b/pom.xml @@ -187,7 +187,7 @@ 2.10.12 3.5.2 3.0.0 - 0.13.0 + 0.16.0 4.8 1.1 3.141.59 From ccc25d0f84adaf0f9ea33d81a2721ac26a4ba23f Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 18 Feb 2022 14:54:41 +0800 Subject: [PATCH 4/6] Port HIVE-25098 --- .../apache/hive/service/auth/KerberosSaslHelper.java | 5 +++-- .../org/apache/hive/service/auth/PlainSaslHelper.java | 3 ++- .../service/cli/thrift/ThriftBinaryCLIService.java | 6 ------ .../hive/service/cli/thrift/ThriftCLIService.java | 10 ++++++++++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java index 175412ed98c6..ef91f94eeec2 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/KerberosSaslHelper.java @@ -30,6 +30,7 @@ import org.apache.thrift.TProcessorFactory; import org.apache.thrift.transport.TSaslClientTransport; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; public final class KerberosSaslHelper { @@ -68,8 +69,8 @@ public static TTransport createSubjectAssumedTransport(String principal, new TSaslClientTransport("GSSAPI", null, names[0], names[1], saslProps, null, underlyingTransport); return new TSubjectAssumingTransport(saslTransport); - } catch (SaslException se) { - throw new IOException("Could not instantiate SASL transport", se); + } catch (SaslException | TTransportException se) { + throw new IOException("Could not instantiate transport", se); } } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java index c06f6ec34653..5ac29950f4f8 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/auth/PlainSaslHelper.java @@ -38,6 +38,7 @@ import org.apache.thrift.transport.TSaslClientTransport; import org.apache.thrift.transport.TSaslServerTransport; import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; import org.apache.thrift.transport.TTransportFactory; public final class PlainSaslHelper { @@ -64,7 +65,7 @@ public static TTransportFactory getPlainTransportFactory(String authTypeStr) } public static TTransport getPlainTransport(String username, String password, - TTransport underlyingTransport) throws SaslException { + TTransport underlyingTransport) throws SaslException, TTransportException { return new TSaslClientTransport("PLAIN", null, null, null, new HashMap(), new PlainCallbackHandler(username, password), underlyingTransport); } diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java index a980b5118be2..025c85eb6580 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java @@ -90,16 +90,10 @@ protected void initializeServer() { // Server args int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE); - int requestTimeout = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_TIMEOUT, TimeUnit.SECONDS); - int beBackoffSlotLength = (int) hiveConf.getTimeVar( - HiveConf.ConfVars.HIVE_SERVER2_THRIFT_LOGIN_BEBACKOFF_SLOT_LENGTH, TimeUnit.MILLISECONDS); TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket) .processorFactory(processorFactory).transportFactory(transportFactory) .protocolFactory(new TBinaryProtocol.Factory()) .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize, maxMessageSize)) - .requestTimeout(requestTimeout).requestTimeoutUnit(TimeUnit.SECONDS) - .beBackoffSlotLength(beBackoffSlotLength).beBackoffSlotLengthUnit(TimeUnit.MILLISECONDS) .executorService(executorService); // TCP Server diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 4a223c8666a1..ddbe89b0b721 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -83,6 +83,16 @@ public void setSessionHandle(SessionHandle sessionHandle) { public SessionHandle getSessionHandle() { return sessionHandle; } + + @Override + public T unwrap(Class aClass) { + return null; + } + + @Override + public boolean isWrapperFor(Class aClass) { + return false; + } } public ThriftCLIService(CLIService service, String serviceName) { From 476958f08c0303d46085b2ec778fbd7e4449e9d9 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 18 Feb 2022 15:19:51 +0800 Subject: [PATCH 5/6] Init --- .../thrift/transport/TFramedTransport.java | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java diff --git a/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java b/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java new file mode 100644 index 000000000000..6f3ff212de47 --- /dev/null +++ b/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.thrift.transport; + + +/** + * This is from libthrift-0.12.0 {@link org.apache.thrift.transport.TFramedTransport}. + * To fix class of org.apache.thrift.transport.TFramedTransport not found after upgrading libthrift. + * + * TFramedTransport is a buffered TTransport that ensures a fully read message + * every time by preceding messages with a 4-byte frame size. + */ +public class TFramedTransport extends TTransport { + + protected static final int DEFAULT_MAX_LENGTH = 16384000; + + private int maxLength_; + + /** + * Underlying transport + */ + private TTransport transport_ = null; + + /** + * Buffer for output + */ + private final TByteArrayOutputStream writeBuffer_ = + new TByteArrayOutputStream(1024); + + /** + * Buffer for input + */ + private final TMemoryInputTransport readBuffer_ = + new TMemoryInputTransport(new byte[0]); + + public static class Factory extends TTransportFactory { + private int maxLength_; + + public Factory() { + maxLength_ = TFramedTransport.DEFAULT_MAX_LENGTH; + } + + public Factory(int maxLength) { + maxLength_ = maxLength; + } + + @Override + public TTransport getTransport(TTransport base) { + return new TFramedTransport(base, maxLength_); + } + } + + /** + * Constructor wraps around another transport + */ + public TFramedTransport(TTransport transport, int maxLength) { + transport_ = transport; + maxLength_ = maxLength; + } + + public TFramedTransport(TTransport transport) { + transport_ = transport; + maxLength_ = TFramedTransport.DEFAULT_MAX_LENGTH; + } + + public void open() throws TTransportException { + transport_.open(); + } + + public boolean isOpen() { + return transport_.isOpen(); + } + + public void close() { + transport_.close(); + } + + public int read(byte[] buf, int off, int len) throws TTransportException { + int got = readBuffer_.read(buf, off, len); + if (got > 0) { + return got; + } + + // Read another frame of data + readFrame(); + + return readBuffer_.read(buf, off, len); + } + + @Override + public byte[] getBuffer() { + return readBuffer_.getBuffer(); + } + + @Override + public int getBufferPosition() { + return readBuffer_.getBufferPosition(); + } + + @Override + public int getBytesRemainingInBuffer() { + return readBuffer_.getBytesRemainingInBuffer(); + } + + @Override + public void consumeBuffer(int len) { + readBuffer_.consumeBuffer(len); + } + + public void clear() { + readBuffer_.clear(); + } + + private final byte[] i32buf = new byte[4]; + + private void readFrame() throws TTransportException { + transport_.readAll(i32buf, 0, 4); + int size = decodeFrameSize(i32buf); + + if (size < 0) { + close(); + throw new TTransportException(TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + } + + if (size > maxLength_) { + close(); + throw new TTransportException(TTransportException.CORRUPTED_DATA, + "Frame size (" + size + ") larger than max length (" + maxLength_ + ")!"); + } + + byte[] buff = new byte[size]; + transport_.readAll(buff, 0, size); + readBuffer_.reset(buff); + } + + public void write(byte[] buf, int off, int len) throws TTransportException { + writeBuffer_.write(buf, off, len); + } + + @Override + public void flush() throws TTransportException { + byte[] buf = writeBuffer_.get(); + int len = writeBuffer_.len(); + writeBuffer_.reset(); + + encodeFrameSize(len, i32buf); + transport_.write(i32buf, 0, 4); + transport_.write(buf, 0, len); + transport_.flush(); + } + + public static final void encodeFrameSize(final int frameSize, final byte[] buf) { + buf[0] = (byte)(0xff & (frameSize >> 24)); + buf[1] = (byte)(0xff & (frameSize >> 16)); + buf[2] = (byte)(0xff & (frameSize >> 8)); + buf[3] = (byte)(0xff & (frameSize)); + } + + public static final int decodeFrameSize(final byte[] buf) { + return + ((buf[0] & 0xff) << 24) | + ((buf[1] & 0xff) << 16) | + ((buf[2] & 0xff) << 8) | + ((buf[3] & 0xff)); + } +} From 442f9a38228b2dceb11e71993765745852cd7c29 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 18 Feb 2022 15:26:07 +0800 Subject: [PATCH 6/6] Workaround: 14:53:54.715 WARN org.apache.hadoop.util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Exception in thread "HiveServer2-Handler-Pool: Thread-164" java.lang.NoClassDefFoundError: org/apache/thrift/transport/TFramedTransport | => hat java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.hadoop.hive.metastore.MetaStoreUtils.getClass(MetaStoreUtils.java:1708) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:131) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3607) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3659) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3639) at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3901) at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:248) at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:231) at org.apache.hadoop.hive.ql.metadata.Hive.(Hive.java:395) at org.apache.hadoop.hive.ql.metadata.Hive.create(Hive.java:339) at org.apache.hadoop.hive.ql.metadata.Hive.getInternal(Hive.java:319) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:288) --- .../thrift/transport/TFramedTransport.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java b/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java index 6f3ff212de47..4b32108c7d20 100644 --- a/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java +++ b/sql/hive/src/main/java/org/apache/thrift/transport/TFramedTransport.java @@ -18,8 +18,11 @@ package org.apache.thrift.transport; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TConfiguration; + /** - * This is from libthrift-0.12.0 {@link org.apache.thrift.transport.TFramedTransport}. + * This is based on libthrift-0.12.0 {@link org.apache.thrift.transport.TFramedTransport}. * To fix class of org.apache.thrift.transport.TFramedTransport not found after upgrading libthrift. * * TFramedTransport is a buffered TTransport that ensures a fully read message @@ -60,7 +63,7 @@ public Factory(int maxLength) { } @Override - public TTransport getTransport(TTransport base) { + public TTransport getTransport(TTransport base) throws TTransportException { return new TFramedTransport(base, maxLength_); } } @@ -68,12 +71,12 @@ public TTransport getTransport(TTransport base) { /** * Constructor wraps around another transport */ - public TFramedTransport(TTransport transport, int maxLength) { + public TFramedTransport(TTransport transport, int maxLength) throws TTransportException { transport_ = transport; maxLength_ = maxLength; } - public TFramedTransport(TTransport transport) { + public TFramedTransport(TTransport transport) throws TTransportException { transport_ = transport; maxLength_ = TFramedTransport.DEFAULT_MAX_LENGTH; } @@ -122,6 +125,21 @@ public void consumeBuffer(int len) { readBuffer_.consumeBuffer(len); } + @Override + public TConfiguration getConfiguration() { + return null; + } + + @Override + public void updateKnownMessageSize(long l) throws TTransportException { + + } + + @Override + public void checkReadBytesAvailable(long l) throws TTransportException { + + } + public void clear() { readBuffer_.clear(); } @@ -134,7 +152,8 @@ private void readFrame() throws TTransportException { if (size < 0) { close(); - throw new TTransportException(TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + throw new TTransportException(TTransportException.CORRUPTED_DATA, + "Read a negative frame size (" + size + ")!"); } if (size > maxLength_) {