diff --git a/LICENSE-binary b/LICENSE-binary index a090e5205cc1..e8ad55c93545 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -546,6 +546,7 @@ jakarta.annotation:jakarta-annotation-api https://projects.eclipse.org/projects/ jakarta.servlet:jakarta.servlet-api https://projects.eclipse.org/projects/ee4j.servlet jakarta.ws.rs:jakarta.ws.rs-api https://github.com/eclipse-ee4j/jaxrs-api org.glassfish.hk2.external:jakarta.inject +com.github.jnr:jnr-posix Public Domain diff --git a/core/pom.xml b/core/pom.xml index 7d4bab556596..80578417b05e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -61,6 +61,11 @@ com.twitter chill-java + + com.github.jnr + jnr-posix + test + org.apache.xbean xbean-asm9-shaded diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala index 58fe40f9adeb..3e4002614ca4 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala @@ -24,6 +24,7 @@ import java.util.HashMap import com.fasterxml.jackson.core.`type`.TypeReference import com.fasterxml.jackson.databind.ObjectMapper +import jnr.posix.{POSIX, POSIXFactory} import org.apache.commons.io.FileUtils import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} @@ -141,28 +142,46 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B assert(attemptId.equals("1")) } + // Use jnr to get and override the current process umask. + // Expects the input mask to be an octal number + private def getAndSetUmask(posix: POSIX, mask: String): String = { + val prev = posix.umask(BigInt(mask, 8).toInt) + "0" + "%o".format(prev) + } + test("SPARK-37618: Sub dirs are group writable when removing from shuffle service enabled") { val conf = testConf.clone conf.set("spark.local.dir", rootDirs) conf.set("spark.shuffle.service.enabled", "true") conf.set("spark.shuffle.service.removeShuffle", "false") - val diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true, isDriver = false) - val blockId = new TestBlockId("test") - val newFile = diskBlockManager.getFile(blockId) - val parentDir = newFile.getParentFile() - assert(parentDir.exists && parentDir.isDirectory) - val permission = Files.getPosixFilePermissions(parentDir.toPath) - assert(!permission.contains(PosixFilePermission.GROUP_WRITE)) - - assert(parentDir.delete()) - - conf.set("spark.shuffle.service.removeShuffle", "true") - val diskBlockManager2 = new DiskBlockManager(conf, deleteFilesOnStop = true, isDriver = false) - val newFile2 = diskBlockManager2.getFile(blockId) - val parentDir2 = newFile2.getParentFile() - assert(parentDir2.exists && parentDir2.isDirectory) - val permission2 = Files.getPosixFilePermissions(parentDir2.toPath) - assert(permission2.contains(PosixFilePermission.GROUP_WRITE)) + val posix = POSIXFactory.getPOSIX + + assume(posix.isNative, "Skipping test for SPARK-37618, native posix support not found") + + val oldUmask = getAndSetUmask(posix, "077") + try { + val diskBlockManager = new DiskBlockManager(conf, deleteFilesOnStop = true, + isDriver = false) + val blockId = new TestBlockId("test") + val newFile = diskBlockManager.getFile(blockId) + val parentDir = newFile.getParentFile() + assert(parentDir.exists && parentDir.isDirectory) + val permission = Files.getPosixFilePermissions(parentDir.toPath) + assert(!permission.contains(PosixFilePermission.GROUP_WRITE)) + + assert(parentDir.delete()) + + conf.set("spark.shuffle.service.removeShuffle", "true") + val diskBlockManager2 = new DiskBlockManager(conf, deleteFilesOnStop = true, + isDriver = false) + val newFile2 = diskBlockManager2.getFile(blockId) + val parentDir2 = newFile2.getParentFile() + assert(parentDir2.exists && parentDir2.isDirectory) + val permission2 = Files.getPosixFilePermissions(parentDir2.toPath) + assert(permission2.contains(PosixFilePermission.GROUP_WRITE)) + } finally { + getAndSetUmask(posix, oldUmask) + } } def writeToFile(file: File, numBytes: Int): Unit = { diff --git a/pom.xml b/pom.xml index 6081f700b628..a95a84e7d66e 100644 --- a/pom.xml +++ b/pom.xml @@ -457,6 +457,12 @@ chill-java ${chill.version} + + com.github.jnr + jnr-posix + 3.1.15 + test +