diff --git a/LICENSE-binary b/LICENSE-binary
index a090e5205cc1..e8ad55c93545 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -546,6 +546,7 @@ jakarta.annotation:jakarta-annotation-api https://projects.eclipse.org/projects/
jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet
jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api
org.glassfish.hk2.external:jakarta.inject
+com.github.jnr:jnr-posix
Public Domain
diff --git a/core/pom.xml b/core/pom.xml
index 7d4bab556596..80578417b05e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -61,6 +61,11 @@
com.twitter
chill-java
+
+ com.github.jnr
+ jnr-posix
+ test
+
org.apache.xbean
xbean-asm9-shaded
diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
index 58fe40f9adeb..3e4002614ca4 100644
--- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala
@@ -24,6 +24,7 @@ import java.util.HashMap
import com.fasterxml.jackson.core.`type`.TypeReference
import com.fasterxml.jackson.databind.ObjectMapper
+import jnr.posix.{POSIX, POSIXFactory}
import org.apache.commons.io.FileUtils
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
@@ -141,28 +142,46 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B
assert(attemptId.equals("1"))
}
+ // Use jnr to get and override the current process umask.
+ // Expects the input mask to be an octal number
+ private def getAndSetUmask(posix: POSIX, mask: String): String = {
+ val prev = posix.umask(BigInt(mask, 8).toInt)
+ "0" + "%o".format(prev)
+ }
+
test("SPARK-37618: Sub dirs are group writable when removing from shuffle service enabled") {
val conf = testConf.clone
conf.set("spark.local.dir", rootDirs)
conf.set("spark.shuffle.service.enabled", "true")
conf.set("spark.shuffle.service.removeShuffle", "false")
- val diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true, isDriver = false)
- val blockId = new TestBlockId("test")
- val newFile = diskBlockManager.getFile(blockId)
- val parentDir = newFile.getParentFile()
- assert(parentDir.exists && parentDir.isDirectory)
- val permission = Files.getPosixFilePermissions(parentDir.toPath)
- assert(!permission.contains(PosixFilePermission.GROUP_WRITE))
-
- assert(parentDir.delete())
-
- conf.set("spark.shuffle.service.removeShuffle", "true")
- val diskBlockManager2 = new DiskBlockManager(conf, deleteFilesOnStop = true, isDriver = false)
- val newFile2 = diskBlockManager2.getFile(blockId)
- val parentDir2 = newFile2.getParentFile()
- assert(parentDir2.exists && parentDir2.isDirectory)
- val permission2 = Files.getPosixFilePermissions(parentDir2.toPath)
- assert(permission2.contains(PosixFilePermission.GROUP_WRITE))
+ val posix = POSIXFactory.getPOSIX
+
+ assume(posix.isNative, "Skipping test for SPARK-37618, native posix support not found")
+
+ val oldUmask = getAndSetUmask(posix, "077")
+ try {
+ val diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true,
+ isDriver = false)
+ val blockId = new TestBlockId("test")
+ val newFile = diskBlockManager.getFile(blockId)
+ val parentDir = newFile.getParentFile()
+ assert(parentDir.exists && parentDir.isDirectory)
+ val permission = Files.getPosixFilePermissions(parentDir.toPath)
+ assert(!permission.contains(PosixFilePermission.GROUP_WRITE))
+
+ assert(parentDir.delete())
+
+ conf.set("spark.shuffle.service.removeShuffle", "true")
+ val diskBlockManager2 = new DiskBlockManager(conf, deleteFilesOnStop = true,
+ isDriver = false)
+ val newFile2 = diskBlockManager2.getFile(blockId)
+ val parentDir2 = newFile2.getParentFile()
+ assert(parentDir2.exists && parentDir2.isDirectory)
+ val permission2 = Files.getPosixFilePermissions(parentDir2.toPath)
+ assert(permission2.contains(PosixFilePermission.GROUP_WRITE))
+ } finally {
+ getAndSetUmask(posix, oldUmask)
+ }
}
def writeToFile(file: File, numBytes: Int): Unit = {
diff --git a/pom.xml b/pom.xml
index 6081f700b628..a95a84e7d66e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -457,6 +457,12 @@
chill-java
${chill.version}
+
+ com.github.jnr
+ jnr-posix
+ 3.1.15
+ test
+