From 4b0c37da04755f251a34bf6a8acf80d57af3af1b Mon Sep 17 00:00:00 2001 From: xerial-bot Date: Tue, 19 Mar 2024 20:17:10 +0000 Subject: [PATCH 1/3] Update hadoop-common to 3.4.0 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 133227b1..ad23758a 100644 --- a/build.sbt +++ b/build.sbt @@ -78,7 +78,7 @@ libraryDependencies ++= Seq( "org.wvlet.airframe" %% "airframe-log" % "24.2.3" % "test", "org.osgi" % "org.osgi.core" % "6.0.0" % "provided", "com.github.sbt" % "junit-interface" % "0.13.3" % "test", - "org.apache.hadoop" % "hadoop-common" % "2.10.2" % "test" exclude ("org.xerial.snappy", "snappy-java") + "org.apache.hadoop" % "hadoop-common" % "3.4.0" % "test" exclude ("org.xerial.snappy", "snappy-java") ) enablePlugins(SbtOsgi) From adf9dcacd8e2ec90b659f0910cf00eef588b0292 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Thu, 21 Mar 2024 09:59:43 -0700 Subject: [PATCH 2/3] Fix tests --- .../snappy/SnappyHadoopCompatibleOutputStreamTest.java | 5 ++--- src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java index 72422bc3..e9c66bf9 100644 --- a/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyHadoopCompatibleOutputStreamTest.java @@ -2,7 +2,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.SystemUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.compress.SnappyCodec; import org.junit.AfterClass; @@ -25,13 +24,13 @@ public static void loadHadoopNativeLibrary() throws Exception { final String libResourceFolder; Map libraryNames = new LinkedHashMap<>(); - if (SystemUtils.IS_OS_LINUX) { + if (OSInfo.getOSName() == "Linux") { libResourceFolder = "/lib/Linux"; libraryNames.put("libhadoop.so", "libhadoop.so"); // certain Linux systems need these shared library be copied before the JVM started, see build.sbt libraryNames.put("libsnappy.so", "libsnappy.so"); libraryNames.put("libsnappy.so.1", "libsnappy.so"); - } else if (SystemUtils.IS_OS_MAC_OSX) { + } else if (OSInfo.getOSName() == "Mac") { libResourceFolder = "/lib/MacOSX"; libraryNames.put("libhadoop.dylib", "libhadoop.dylib"); libraryNames.put("libsnappy.dylib", "libsnappy.dylib"); diff --git a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java index d7831cbe..a79eb198 100755 --- a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java @@ -180,7 +180,7 @@ public void batchingOfWritesShouldNotAffectCompressedDataSize() if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) assertEquals(90992, expectedCompressedData.length); else - assertEquals(91080, expectedCompressedData.length); + assertEquals(91051, expectedCompressedData.length); // The chunk size should not affect the size of the compressed output: int[] chunkSizes = new int[] {1, 100, 1023, 1024, 10000}; for (int chunkSize : chunkSizes) { From ff7e1b5ee90e706dbd95c3eab7e9af671d0e3238 Mon Sep 17 00:00:00 2001 From: "Taro L. Saito" Date: Thu, 21 Mar 2024 10:05:18 -0700 Subject: [PATCH 3/3] Fix for arm --- src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java index a79eb198..36ef552e 100755 --- a/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java +++ b/src/test/java/org/xerial/snappy/SnappyOutputStreamTest.java @@ -179,8 +179,11 @@ public void batchingOfWritesShouldNotAffectCompressedDataSize() // compression quality: if (ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN) assertEquals(90992, expectedCompressedData.length); - else + else if(OSInfo.getArchName() == "aarch64") + // Arm has a better compression ratio assertEquals(91051, expectedCompressedData.length); + else + assertEquals(91080, expectedCompressedData.length); // The chunk size should not affect the size of the compressed output: int[] chunkSizes = new int[] {1, 100, 1023, 1024, 10000}; for (int chunkSize : chunkSizes) {