From 36a8b1fc5f4cac0c17933775f44af379e01f817c Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Sun, 8 Jan 2023 02:25:42 +0800 Subject: [PATCH 1/3] [CORE] Use java api to create files and grant permissions --- .../apache/spark/storage/DiskBlockManager.scala | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index e29f3fc1b805..1780ee70b57c 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -19,7 +19,7 @@ package org.apache.spark.storage import java.io.{File, IOException} import java.nio.file.Files -import java.nio.file.attribute.PosixFilePermission +import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions} import java.util.UUID import scala.collection.mutable.HashMap @@ -301,9 +301,6 @@ private[spark] class DiskBlockManager( * Create a directory that is writable by the group. * Grant the permission 770 "rwxrwx---" to the directory so the shuffle server can * create subdirs/files within the merge folder. - * TODO: Find out why can't we create a dir using java api with permission 770 - * Files.createDirectories(mergeDir.toPath, PosixFilePermissions.asFileAttribute( - * PosixFilePermissions.fromString("rwxrwx---"))) */ def createDirWithPermission770(dirToCreate: File): Unit = { var attempts = 0 @@ -315,16 +312,15 @@ private[spark] class DiskBlockManager( throw SparkCoreErrors.failToCreateDirectoryError(dirToCreate.getAbsolutePath, maxAttempts) } try { - val builder = new ProcessBuilder().command( - "mkdir", "-p", "-m770", dirToCreate.getAbsolutePath) - val proc = builder.start() - val exitCode = proc.waitFor() + dirToCreate.mkdirs() + Files.setPosixFilePermissions(dirToCreate.toPath, + PosixFilePermissions.fromString("rwxrwx---")) if (dirToCreate.exists()) { created = dirToCreate } logDebug( s"Created directory at ${dirToCreate.getAbsolutePath} with permission " + - s"770 and exitCode $exitCode") + s"770") } catch { case e: SecurityException => logWarning(s"Failed to create directory ${dirToCreate.getAbsolutePath} " + From f2e02e86da2baacebbf9446bce8048c49f5ffcb5 Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Sun, 8 Jan 2023 18:01:30 +0800 Subject: [PATCH 2/3] fix log --- .../main/scala/org/apache/spark/storage/DiskBlockManager.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 1780ee70b57c..ec0101185457 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -319,8 +319,7 @@ private[spark] class DiskBlockManager( created = dirToCreate } logDebug( - s"Created directory at ${dirToCreate.getAbsolutePath} with permission " + - s"770") + s"Created directory at ${dirToCreate.getAbsolutePath} with permission 770") } catch { case e: SecurityException => logWarning(s"Failed to create directory ${dirToCreate.getAbsolutePath} " + From 696b7f5d635992d89498ccec40e4f76438bec8c6 Mon Sep 17 00:00:00 2001 From: smallzhongfeng Date: Sun, 8 Jan 2023 21:55:45 +0800 Subject: [PATCH 3/3] just trigger GA --- .../scala/org/apache/spark/storage/DiskBlockManager.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index ec0101185457..a7ed9226c574 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -313,13 +313,12 @@ private[spark] class DiskBlockManager( } try { dirToCreate.mkdirs() - Files.setPosixFilePermissions(dirToCreate.toPath, - PosixFilePermissions.fromString("rwxrwx---")) + Files.setPosixFilePermissions( + dirToCreate.toPath, PosixFilePermissions.fromString("rwxrwx---")) if (dirToCreate.exists()) { created = dirToCreate } - logDebug( - s"Created directory at ${dirToCreate.getAbsolutePath} with permission 770") + logDebug(s"Created directory at ${dirToCreate.getAbsolutePath} with permission 770") } catch { case e: SecurityException => logWarning(s"Failed to create directory ${dirToCreate.getAbsolutePath} " +