diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 56808c75ff62f..165e2629e4326 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -74,6 +74,7 @@ public class FSDownload implements Callable { private static final Logger LOG = LoggerFactory.getLogger(FSDownload.class); + private String containerId; private FileContext files; private final UserGroupInformation userUgi; private Configuration conf; @@ -94,12 +95,27 @@ public class FSDownload implements Callable { public FSDownload(FileContext files, UserGroupInformation ugi, Configuration conf, Path destDirPath, LocalResource resource) { - this(files, ugi, conf, destDirPath, resource, null); + this(files, ugi, conf, destDirPath, resource, null, ""); } + public FSDownload(String containerId, FileContext files, UserGroupInformation ugi, + Configuration conf, + Path destDirPath, LocalResource resource) { + this(files, ugi, conf, destDirPath, resource, null, containerId); + } + + public FSDownload(FileContext files, + UserGroupInformation ugi, Configuration conf, + Path destDirPath, LocalResource resource, + LoadingCache> statCache) { + this(files, ugi, conf, destDirPath, resource, statCache, ""); + } + public FSDownload(FileContext files, UserGroupInformation ugi, Configuration conf, Path destDirPath, LocalResource resource, - LoadingCache> statCache) { + LoadingCache> statCache, + String containerId) { + this.containerId = containerId; this.conf = conf; this.destDirPath = destDirPath; this.files = files; @@ -408,8 +424,8 @@ public Path call() throws Exception { throw new IOException("Invalid resource", e); } - LOG.debug("Starting to download {} {} {}", sCopy, - resource.getType(), resource.getPattern()); + LOG.info("Starting to download {} {} {} for containerId: {}", sCopy, + resource.getType(), resource.getPattern(), containerId); final Path destinationTmp = new Path(destDirPath + "_tmp"); createDir(destinationTmp, cachePerms); @@ -430,8 +446,8 @@ public Void run() throws Exception { changePermissions(dFinal.getFileSystem(conf), dFinal); files.rename(destinationTmp, destDirPath, Rename.OVERWRITE); - LOG.debug("File has been downloaded to {} from {}", - new Path(destDirPath, sCopy.getName()), sCopy); + LOG.info("File has been downloaded to {} from {} for containerId: {}", + new Path(destDirPath, sCopy.getName()), sCopy, containerId); } catch (Exception e) { try { files.delete(destDirPath, true); @@ -478,7 +494,7 @@ private void changePermissions(FileSystem fs, final Path path) perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS; } - LOG.debug("Changing permissions for path {} to perm {}", path, perm); + LOG.info("Changing permissions for path {} to perm {}", path, perm); final FsPermission fPerm = perm; if (null == userUgi) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 705ef88dee357..1b1cc8e877b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -171,6 +171,7 @@ public void startLocalizer(LocalizerStartContext ctx) String user = ctx.getUser(); String appId = ctx.getAppId(); String locId = ctx.getLocId(); + String containerId = ctx.getContainerId(); LocalDirsHandlerService dirsHandler = ctx.getDirsHandler(); List localDirs = dirsHandler.getLocalDirs(); @@ -199,7 +200,7 @@ public void startLocalizer(LocalizerStartContext ctx) ContainerLocalizer localizer = createContainerLocalizer(user, appId, locId, tokenFn, localDirs, - localizerFc); + localizerFc, containerId); // TODO: DO it over RPC for maintaining similarity? localizer.runLocalization(nmAddr); } @@ -224,11 +225,11 @@ public void startLocalizer(LocalizerStartContext ctx) @VisibleForTesting protected ContainerLocalizer createContainerLocalizer(String user, String appId, String locId, String tokenFileName, List localDirs, - FileContext localizerFc) throws IOException { + FileContext localizerFc, String containerId) throws IOException { ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId, tokenFileName, getPaths(localDirs), - RecordFactoryProvider.getRecordFactory(getConf())); + RecordFactoryProvider.getRecordFactory(getConf()), containerId); return localizer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 604a810ec9468..77060d3fd568a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -105,6 +105,7 @@ public class ContainerLocalizer { private final String user; private final String appId; + private final String containerId; private final List localDirs; private final String localizerId; private final FileContext lfs; @@ -120,7 +121,7 @@ public class ContainerLocalizer { public ContainerLocalizer(FileContext lfs, String user, String appId, String localizerId, String tokenFileName, List localDirs, - RecordFactory recordFactory) throws IOException { + RecordFactory recordFactory, String containerId) throws IOException { if (null == user) { throw new IOException("Cannot initialize for null user"); } @@ -130,6 +131,7 @@ public ContainerLocalizer(FileContext lfs, String user, String appId, this.lfs = lfs; this.user = user; this.appId = appId; + this.containerId = containerId; this.localDirs = localDirs; this.localizerId = localizerId; this.recordFactory = recordFactory; @@ -142,6 +144,12 @@ public ContainerLocalizer(FileContext lfs, String user, String appId, "token file name cannot be null"); } + public ContainerLocalizer(FileContext lfs, String user, String appId, + String localizerId, List localDirs, + RecordFactory recordFactory) throws IOException { + this(lfs, user, appId, localizerId, localDirs, recordFactory, ""); + } + @VisibleForTesting @Private Configuration initConfiguration() { @@ -232,6 +240,12 @@ class FSDownloadWrapper extends FSDownload { super(files, ugi, conf, destDirPath, resource); } + FSDownloadWrapper(FileContext files, UserGroupInformation ugi, + Configuration conf, Path destDirPath, LocalResource resource, + String containerId) { + super(containerId, files, ugi, conf, destDirPath, resource); + } + @Override public Path call() throws Exception { Thread currentThread = Thread.currentThread(); @@ -258,7 +272,7 @@ Callable download(Path destDirPath, LocalResource rsrc, } diskValidator .checkStatus(new File(destDirPath.getParent().toUri().getRawPath())); - return new FSDownloadWrapper(lfs, ugi, conf, destDirPath, rsrc); + return new FSDownloadWrapper(lfs, ugi, conf, destDirPath, rsrc, containerId); } private void createParentDirs(Path destDirPath) throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 9c7a8e811f9a5..f983ff452f5c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -1269,6 +1269,7 @@ public void run() { .setAppId(context.getContainerId() .getApplicationAttemptId().getApplicationId().toString()) .setLocId(localizerId) + .setContainerId(context.getContainerId().toString()) .setDirsHandler(dirsHandler) .build()); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/LocalizerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/LocalizerStartContext.java index 276297624e111..11ff05c713257 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/LocalizerStartContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/LocalizerStartContext.java @@ -39,6 +39,7 @@ public final class LocalizerStartContext { private final String user; private final String appId; private final String locId; + public final String containerId; private final LocalDirsHandlerService dirsHandler; public static final class Builder { @@ -47,6 +48,7 @@ public static final class Builder { private String user; private String appId; private String locId; + private String containerId; private LocalDirsHandlerService dirsHandler; public Builder() { @@ -72,6 +74,11 @@ public Builder setAppId(String appId) { return this; } + public Builder setContainerId(String containerId) { + this.containerId = containerId; + return this; + } + public Builder setLocId(String locId) { this.locId = locId; return this; @@ -93,6 +100,7 @@ private LocalizerStartContext(Builder builder) { this.user = builder.user; this.appId = builder.appId; this.locId = builder.locId; + this.containerId = builder.containerId; this.dirsHandler = builder.dirsHandler; } @@ -116,6 +124,10 @@ public String getLocId() { return this.locId; } + public String getContainerId() { + return this.containerId; + } + public LocalDirsHandlerService getDirsHandler() { return this.dirsHandler; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index 473292f35f749..971f6207d0140 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -549,14 +549,17 @@ public Object answer(InvocationOnMock invocationOnMock) @Override public ContainerLocalizer createContainerLocalizer(String user, String appId, String locId, String tokenFileName, - List localDirs, FileContext localizerFc) - throws IOException { + List localDirs, FileContext localizerFc, + String containerId) throws IOException { // Spy on the localizer and make it return valid heart-beat // responses even though there is no real NodeManager. ContainerLocalizer localizer = super.createContainerLocalizer(user, appId, locId, - tokenFileName, localDirs, localizerFc); + tokenFileName, localDirs, localizerFc, appId); + // in the above line passing appId in place of container Id as + // container id is just for logging purposes and has not other + // use ContainerLocalizer spyLocalizer = spy(localizer); LocalizationProtocol nmProxy = mock(LocalizationProtocol.class); try {