Merge branch 'trunk' into renameErrorMapping

Author: manika137

LICENSE-binary

@@ -341,20 +341,20 @@ org.apache.solr:solr-solrj:8.11.2
 org.apache.yetus:audience-annotations:0.5.0
 org.apache.zookeeper:zookeeper:3.8.4
 org.codehaus.jettison:jettison:1.5.4
-org.eclipse.jetty:jetty-annotations:9.4.53.v20231009
-org.eclipse.jetty:jetty-http:9.4.53.v20231009
-org.eclipse.jetty:jetty-io:9.4.53.v20231009
-org.eclipse.jetty:jetty-jndi:9.4.53.v20231009
-org.eclipse.jetty:jetty-plus:9.4.53.v20231009
-org.eclipse.jetty:jetty-security:9.4.53.v20231009
-org.eclipse.jetty:jetty-server:9.4.53.v20231009
-org.eclipse.jetty:jetty-servlet:9.4.53.v20231009
-org.eclipse.jetty:jetty-util:9.4.53.v20231009
-org.eclipse.jetty:jetty-util-ajax:9.4.53.v20231009
-org.eclipse.jetty:jetty-webapp:9.4.53.v20231009
-org.eclipse.jetty:jetty-xml:9.4.53.v20231009
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.53.v20231009
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.53.v20231009
+org.eclipse.jetty:jetty-annotations:9.4.57.v20241219
+org.eclipse.jetty:jetty-http:9.4.57.v20241219
+org.eclipse.jetty:jetty-io:9.4.57.v20241219
+org.eclipse.jetty:jetty-jndi:9.4.57.v20241219
+org.eclipse.jetty:jetty-plus:9.4.57.v20241219
+org.eclipse.jetty:jetty-security:9.4.57.v20241219
+org.eclipse.jetty:jetty-server:9.4.57.v20241219
+org.eclipse.jetty:jetty-servlet:9.4.57.v20241219
+org.eclipse.jetty:jetty-util:9.4.57.v20241219
+org.eclipse.jetty:jetty-util-ajax:9.4.57.v20241219
+org.eclipse.jetty:jetty-webapp:9.4.57.v20241219
+org.eclipse.jetty:jetty-xml:9.4.57.v20241219
+org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.57.v20241219
+org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.57.v20241219
 org.ehcache:ehcache:3.8.2
 org.ini4j:ini4j:0.5.4
 org.lz4:lz4-java:1.7.1
@@ -363,16 +363,6 @@ org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:2.1.4.Final
 software.amazon.awssdk:bundle:2.25.53
-org.glassfish.jersey.core:jersey-common:2.46
-org.glassfish.jersey.core:jersey-server:2.46
-org.glassfish.jersey.inject:jersey-hk2:2.46
-org.glassfish.jersey.core:jersey-client:2.46
-org.glassfish.jersey.test-framework:jersey-test-framework-core:2.46
-org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2:2.46
-org.glassfish.jersey.containers:jersey-container-servlet:2.46
-org.glassfish.jersey.containers:jersey-container-servlet-core:2.46
-org.glassfish.jersey.media:jersey-media-json-jettison:2.46
-org.glassfish.jersey.media:jersey-media-jaxb:2.46
 net.jodah:failsafe:2.4.4
 
 --------------------------------------------------------------------------------
@@ -405,7 +395,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
 leveldb v1.13
 
 com.google.protobuf:protobuf-java:2.5.0
-com.google.protobuf:protobuf-java:3.25.3
+com.google.protobuf:protobuf-java:3.25.5
 com.google.re2j:re2j:1.1
 com.jcraft:jsch:0.1.55
 com.thoughtworks.paranamer:paranamer:2.3
@@ -535,6 +525,24 @@ jakarta.ws.rs-api:jakarta.ws.rs-api:2.1.6
 jakarta.servlet.jsp:jakarta.servlet.jsp-api:2.3.6
 jakarta.servlet:jakarta.servlet-api:4.0.4
 
+Eclipse Public License (EPL) 2.0 with some parts being
+GNU General Public License (GPL), Version 2, With Classpath Exception,
+Apache License, 2.0 or Public Domain
+--------------------------
+
+See also https://github.com/eclipse-ee4j/jersey/blob/master/NOTICE.md
+
+org.glassfish.jersey.core:jersey-common:2.46
+org.glassfish.jersey.core:jersey-server:2.46
+org.glassfish.jersey.inject:jersey-hk2:2.46
+org.glassfish.jersey.core:jersey-client:2.46
+org.glassfish.jersey.test-framework:jersey-test-framework-core:2.46
+org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2:2.46
+org.glassfish.jersey.containers:jersey-container-servlet:2.46
+org.glassfish.jersey.containers:jersey-container-servlet-core:2.46
+org.glassfish.jersey.media:jersey-media-json-jettison:2.46
+org.glassfish.jersey.media:jersey-media-jaxb:2.46
+
 
 HSQL License
 ------------

hadoop-common-project/hadoop-common/pom.xml

@@ -381,6 +381,26 @@
       <artifactId>lz4-java</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java

@@ -215,12 +215,12 @@ protected void processPath(PathData item) throws IOException {
   }
 
   protected class TextRecordInputStream extends InputStream {
-    SequenceFile.Reader r;
-    Object key;
-    Object val;
+    private final SequenceFile.Reader r;
+    private Object key;
+    private Object val;
 
-    DataInputBuffer inbuf;
-    DataOutputBuffer outbuf;
+    private final DataInputBuffer inbuf;
+    private final DataOutputBuffer outbuf;
 
     public TextRecordInputStream(FileStatus f) throws IOException {
       final Path fpath = f.getPath();
@@ -237,30 +237,67 @@ public TextRecordInputStream(FileStatus f) throws IOException {
     public int read() throws IOException {
       int ret;
       if (null == inbuf || -1 == (ret = inbuf.read())) {
-        key = r.next(key);
-        if (key == null) {
-          return -1;
+        if (!readNextFromSequenceFile()) {
+          ret = -1;
         } else {
-          val = r.getCurrentValue(val);
+          ret = inbuf.read();
         }
-        byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8);
-        outbuf.write(tmp, 0, tmp.length);
-        outbuf.write('\t');
-        tmp = val.toString().getBytes(StandardCharsets.UTF_8);
-        outbuf.write(tmp, 0, tmp.length);
-        outbuf.write('\n');
-        inbuf.reset(outbuf.getData(), outbuf.getLength());
-        outbuf.reset();
-        ret = inbuf.read();
       }
       return ret;
     }
 
+    @Override
+    public int read(byte[] dest, int destPos, int destLen) throws IOException {
+      validateInputStreamReadArguments(dest, destPos, destLen);
+
+      if (destLen == 0) {
+        return 0;
+      }
+
+      int bytesRead = 0;
+      while (destLen > 0) {
+        // Attempt to copy buffered data.
+        int copyLen = inbuf.read(dest, destPos, destLen);
+        if (-1 == copyLen) {
+          // There was no buffered data.
+          if (!readNextFromSequenceFile()) {
+            // There is also no data remaining in the file.
+            break;
+          }
+          // Reattempt copy now that we have buffered data.
+          copyLen = inbuf.read(dest, destPos, destLen);
+        }
+        bytesRead += copyLen;
+        destPos += copyLen;
+        destLen -= copyLen;
+      }
+
+      return bytesRead > 0 ? bytesRead : -1;
+    }
+
     @Override
     public void close() throws IOException {
       r.close();
       super.close();
     }
+
+    private boolean readNextFromSequenceFile() throws IOException {
+      key = r.next(key);
+      if (key == null) {
+        return false;
+      } else {
+        val = r.getCurrentValue(val);
+      }
+      byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8);
+      outbuf.write(tmp, 0, tmp.length);
+      outbuf.write('\t');
+      tmp = val.toString().getBytes(StandardCharsets.UTF_8);
+      outbuf.write(tmp, 0, tmp.length);
+      outbuf.write('\n');
+      inbuf.reset(outbuf.getData(), outbuf.getLength());
+      outbuf.reset();
+      return true;
+    }
   }
 
   /**
@@ -270,10 +307,11 @@ public void close() throws IOException {
   protected static class AvroFileInputStream extends InputStream {
     private int pos;
     private byte[] buffer;
-    private ByteArrayOutputStream output;
-    private FileReader<?> fileReader;
-    private DatumWriter<Object> writer;
-    private JsonEncoder encoder;
+    private final ByteArrayOutputStream output;
+    private final FileReader<?> fileReader;
+    private final DatumWriter<Object> writer;
+    private final JsonEncoder encoder;
+    private final byte[] finalSeparator;
 
     public AvroFileInputStream(FileStatus status) throws IOException {
       pos = 0;
@@ -286,31 +324,96 @@ public AvroFileInputStream(FileStatus status) throws IOException {
       writer = new GenericDatumWriter<Object>(schema);
       output = new ByteArrayOutputStream();
       encoder = EncoderFactory.get().jsonEncoder(schema, output);
+      finalSeparator = System.getProperty("line.separator").getBytes(StandardCharsets.UTF_8);
     }
 
     /**
      * Read a single byte from the stream.
      */
     @Override
     public int read() throws IOException {
+      if (buffer == null) {
+        return -1;
+      }
+
       if (pos < buffer.length) {
         return buffer[pos++];
       }
+
       if (!fileReader.hasNext()) {
+        // Unset buffer to signal EOF on future calls.
+        buffer = null;
         return -1;
       }
+
       writer.write(fileReader.next(), encoder);
       encoder.flush();
+
       if (!fileReader.hasNext()) {
-        // Write a new line after the last Avro record.
-        output.write(System.getProperty("line.separator")
-                         .getBytes(StandardCharsets.UTF_8));
-        output.flush();
+        if (buffer.length > 0) {
+          // Write a new line after the last Avro record.
+          output.write(finalSeparator);
+          output.flush();
+        }
       }
+
+      swapBuffer();
+      return read();
+    }
+
+    @Override
+    public int read(byte[] dest, int destPos, int destLen) throws IOException {
+      validateInputStreamReadArguments(dest, destPos, destLen);
+
+      if (destLen == 0) {
+        return 0;
+      }
+
+      if (buffer == null) {
+        return -1;
+      }
+
+      int bytesRead = 0;
+      while (destLen > 0 && buffer != null) {
+        if (pos < buffer.length) {
+          // We have buffered data available, either from the Avro file or the final separator.
+          int copyLen = Math.min(buffer.length - pos, destLen);
+          System.arraycopy(buffer, pos, dest, destPos, copyLen);
+          pos += copyLen;
+          bytesRead += copyLen;
+          destPos += copyLen;
+          destLen -= copyLen;
+        } else if (buffer == finalSeparator) {
+          // There is no buffered data, and the last buffer processed was the final separator.
+          // Unset buffer to signal EOF on future calls.
+          buffer = null;
+        } else if (!fileReader.hasNext()) {
+          if (buffer.length > 0) {
+            // There is no data remaining in the file. Get ready to write the final separator on
+            // the next iteration.
+            buffer = finalSeparator;
+            pos = 0;
+          } else {
+            // We never read data into the buffer. This must be an empty file.
+            // Immediate EOF, no separator needed.
+            buffer = null;
+            return -1;
+          }
+        } else {
+          // Read the next data from the file into the buffer.
+          writer.write(fileReader.next(), encoder);
+          encoder.flush();
+          swapBuffer();
+        }
+      }
+
+      return bytesRead;
+    }
+
+    private void swapBuffer() {
       pos = 0;
       buffer = output.toByteArray();
       output.reset();
-      return read();
     }
 
     /**
@@ -323,4 +426,14 @@ public void close() throws IOException {
       super.close();
     }
   }
+
+  private static void validateInputStreamReadArguments(byte[] dest, int destPos, int destLen)
+      throws IOException {
+    if (dest == null) {
+      throw new NullPointerException("null destination buffer");
+    } else if (destPos < 0 || destLen < 0 || destLen > dest.length - destPos) {
+      throw new IndexOutOfBoundsException(String.format(
+          "invalid destination buffer range: destPos = %d, destLen = %d", destPos, destLen));
+    }
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UGIExceptionMessages.java

@@ -27,6 +27,7 @@ final class UGIExceptionMessages {
   public static final String FOR_USER = " for user: ";
   public static final String FOR_PRINCIPAL = " for principal: ";
   public static final String FROM_KEYTAB = " from keytab ";
+  public static final String INVALID_UID = "Invalid UID, could not determine effective user";
   public static final String LOGIN_FAILURE = "Login failure";
   public static final String LOGOUT_FAILURE = "Logout failure";
   public static final String MUST_FIRST_LOGIN =

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -2075,6 +2075,12 @@ private static UserGroupInformation doSubjectLogin(
       }
       return ugi;
     } catch (LoginException le) {
+      String msg = le.getMessage();
+      if (msg != null && msg.contains("invalid null input")) {
+        // This error from the JDK indicates that the OS couldn't map the UID of this process to an
+        // actual user. Throw this as an IOException, because it's not related to Kerberos.
+        throw new IOException(INVALID_UID, le);
+      }
       KerberosAuthException kae =
         new KerberosAuthException(FAILURE_TO_LOGIN, le);
       if (params != null) {