HDFS-16791. Add getEnclosingRoot() API to filesystem interface and implementations (#6198)

The enclosing root path is a common ancestor that should be used for temp and staging dirs
as well as within encryption zones and other restricted directories.

Contributed by Tom McCormick

Author: mccormickt12

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java

@@ -1638,6 +1638,24 @@ public MultipartUploaderBuilder createMultipartUploader(Path basePath)
     return null;
   }
 
+  /**
+   * Return path of the enclosing root for a given path
+   * The enclosing root path is a common ancestor that should be used for temp and staging dirs
+   * as well as within encryption zones and other restricted directories.
+   *
+   * Call makeQualified on the param path to ensure its part of the correct filesystem
+   *
+   * @param path file path to find the enclosing root path for
+   * @return a path to the enclosing root
+   * @throws IOException early checks like failure to resolve path cause IO failures
+   */
+  @InterfaceAudience.Public
+  @InterfaceStability.Unstable
+  public Path getEnclosingRoot(Path path) throws IOException {
+    makeQualified(path);
+    return makeQualified(new Path("/"));
+  }
+
   /**
    * Helper method that throws an {@link UnsupportedOperationException} for the
    * current {@link FileSystem} method being called.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -4944,6 +4944,24 @@ public CompletableFuture<FSDataInputStream> build() throws IOException {
 
   }
 
+  /**
+   * Return path of the enclosing root for a given path.
+   * The enclosing root path is a common ancestor that should be used for temp and staging dirs
+   * as well as within encryption zones and other restricted directories.
+   *
+   * Call makeQualified on the param path to ensure its part of the correct filesystem.
+   *
+   * @param path file path to find the enclosing root path for
+   * @return a path to the enclosing root
+   * @throws IOException early checks like failure to resolve path cause IO failures
+   */
+  @InterfaceAudience.Public
+  @InterfaceStability.Unstable
+  public Path getEnclosingRoot(Path path) throws IOException {
+    this.makeQualified(path);
+    return this.makeQualified(new Path("/"));
+  }
+
   /**
    * Create a multipart uploader.
    * @param basePath file path under which all files are uploaded

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -732,6 +732,11 @@ protected CompletableFuture<FSDataInputStream> openFileWithOptions(
     return fs.openFileWithOptions(pathHandle, parameters);
   }
 
+  @Override
+  public Path getEnclosingRoot(Path path) throws IOException {
+    return fs.getEnclosingRoot(path);
+  }
+
   @Override
   public boolean hasPathCapability(final Path path, final String capability)
       throws IOException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java

@@ -459,4 +459,9 @@ public MultipartUploaderBuilder createMultipartUploader(final Path basePath)
       throws IOException {
     return myFs.createMultipartUploader(basePath);
   }
+
+  @Override
+  public Path getEnclosingRoot(Path path) throws IOException {
+    return myFs.getEnclosingRoot(path);
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java

@@ -1370,6 +1370,24 @@ public boolean hasPathCapability(Path path, String capability)
     }
   }
 
+  @Override
+  public Path getEnclosingRoot(Path path) throws IOException {
+    InodeTree.ResolveResult<FileSystem> res;
+    try {
+      res = fsState.resolve(getUriPath(path), true);
+    } catch (FileNotFoundException ex) {
+      NotInMountpointException mountPointEx =
+          new NotInMountpointException(path,
+              String.format("getEnclosingRoot - %s", ex.getMessage()));
+      mountPointEx.initCause(ex);
+      throw mountPointEx;
+    }
+    Path mountPath = new Path(res.resolvedPath);
+    Path enclosingPath = res.targetFileSystem.getEnclosingRoot(new Path(getUriPath(path)));
+    return fixRelativePart(this.makeQualified(enclosingPath.depth() > mountPath.depth()
+        ?  enclosingPath : mountPath));
+  }
+
   /**
    * An instance of this class represents an internal dir of the viewFs
    * that is internal dir of the mount table.
@@ -1919,6 +1937,25 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
       }
       return allPolicies;
     }
+
+    @Override
+    public Path getEnclosingRoot(Path path) throws IOException {
+      InodeTree.ResolveResult<FileSystem> res;
+      try {
+        res = fsState.resolve((path.toString()), true);
+      } catch (FileNotFoundException ex) {
+        NotInMountpointException mountPointEx =
+            new NotInMountpointException(path,
+            String.format("getEnclosingRoot - %s", ex.getMessage()));
+        mountPointEx.initCause(ex);
+        throw mountPointEx;
+      }
+      Path fullPath = new Path(res.resolvedPath);
+      Path enclosingPath = res.targetFileSystem.getEnclosingRoot(path);
+      return enclosingPath.depth() > fullPath.depth()
+          ?  enclosingPath
+          : fullPath;
+    }
   }
 
   enum RenameStrategy {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java

@@ -1477,5 +1477,22 @@ public void setStoragePolicy(Path path, String policyName)
         throws IOException {
       throw readOnlyMountTable("setStoragePolicy", path);
     }
+
+    @Override
+    public Path getEnclosingRoot(Path path) throws IOException {
+      InodeTree.ResolveResult<AbstractFileSystem> res;
+      try {
+        res = fsState.resolve((path.toString()), true);
+      } catch (FileNotFoundException ex) {
+        NotInMountpointException mountPointEx =
+            new NotInMountpointException(path,
+                String.format("getEnclosingRoot - %s", ex.getMessage()));
+        mountPointEx.initCause(ex);
+        throw mountPointEx;
+      }
+      Path fullPath = new Path(res.resolvedPath);
+      Path enclosingPath = res.targetFileSystem.getEnclosingRoot(path);
+      return enclosingPath.depth() > fullPath.depth() ?  enclosingPath : fullPath;
+    }
   }
 }

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -601,7 +601,40 @@ on the filesystem.
 
 1. The outcome of this operation MUST be identical to the value of
    `getFileStatus(P).getBlockSize()`.
-1. By inference, it MUST be > 0 for any file of length > 0.
+2. By inference, it MUST be > 0 for any file of length > 0.
+
+###  `Path getEnclosingRoot(Path p)`
+
+This method is used to find a root directory for a path given. This is useful for creating
+staging and temp directories in the same enclosing root directory. There are constraints around how
+renames are allowed to atomically occur (ex. across hdfs volumes or across encryption zones).
+
+For any two paths p1 and p2 that do not have the same enclosing root, `rename(p1, p2)` is expected to fail or will not
+be atomic.
+
+For object stores, even with the same enclosing root, there is no guarantee file or directory rename is atomic
+
+The following statement is always true:
+`getEnclosingRoot(p) == getEnclosingRoot(getEnclosingRoot(p))`
+
+
+```python
+path in ancestors(FS, p) or path == p:
+isDir(FS, p)
+```
+
+#### Preconditions
+
+The path does not have to exist, but the path does need to be valid and reconcilable by the filesystem
+* if a linkfallback is used all paths are reconcilable
+* if a linkfallback is not used there must be a mount point covering the path
+
+
+#### Postconditions
+
+* The path returned will not be null, if there is no deeper enclosing root, the root path ("/") will be returned.
+* The path returned is a directory
+
 
 ## <a name="state_changing_operations"></a> State Changing Operations
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestGetEnclosingRoot.java

@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.HadoopTestBase;
+import org.junit.Test;
+
+public class TestGetEnclosingRoot extends HadoopTestBase {
+  @Test
+  public void testEnclosingRootEquivalence() throws IOException {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+    Path foobar = path("/foo/bar");
+
+    assertEquals(root, fs.getEnclosingRoot(root));
+    assertEquals(root, fs.getEnclosingRoot(foobar));
+    assertEquals(root, fs.getEnclosingRoot(fs.getEnclosingRoot(foobar)));
+    assertEquals(fs.getEnclosingRoot(root), fs.getEnclosingRoot(foobar));
+
+    assertEquals(root, fs.getEnclosingRoot(path(foobar.toString())));
+    assertEquals(root, fs.getEnclosingRoot(fs.getEnclosingRoot(path(foobar.toString()))));
+    assertEquals(fs.getEnclosingRoot(root), fs.getEnclosingRoot(path(foobar.toString())));
+  }
+
+  @Test
+  public void testEnclosingRootPathExists() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+    Path foobar = path("/foo/bar");
+    fs.mkdirs(foobar);
+
+    assertEquals(root, fs.getEnclosingRoot(foobar));
+    assertEquals(root, fs.getEnclosingRoot(path(foobar.toString())));
+  }
+
+  @Test
+  public void testEnclosingRootPathDNE() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path foobar = path("/foo/bar");
+    Path root = path("/");
+
+    assertEquals(root, fs.getEnclosingRoot(foobar));
+    assertEquals(root, fs.getEnclosingRoot(path(foobar.toString())));
+  }
+
+  @Test
+  public void testEnclosingRootWrapped() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+
+    assertEquals(root, fs.getEnclosingRoot(new Path("/foo/bar")));
+
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("foo");
+    Path p = ugi.doAs((PrivilegedExceptionAction<Path>) () -> {
+      FileSystem wFs = getFileSystem();
+      return wFs.getEnclosingRoot(new Path("/foo/bar"));
+    });
+    assertEquals(root, p);
+  }
+
+  private FileSystem getFileSystem() throws IOException {
+    return FileSystem.get(new Configuration());
+  }
+
+  /**
+   * Create a path under the test path provided by
+   * the FS contract.
+   * @param filepath path string in
+   * @return a path qualified by the test filesystem
+   * @throws IOException IO problems
+   */
+  private Path path(String filepath) throws IOException {
+    return getFileSystem().makeQualified(
+        new Path(filepath));
+  }}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java

@@ -255,6 +255,8 @@ MultipartUploaderBuilder createMultipartUploader(Path basePath)
 
     FSDataOutputStream append(Path f, int bufferSize,
         Progressable progress, boolean appendToNewBlock) throws IOException;
+
+    Path getEnclosingRoot(Path path) throws IOException;
   }
 
   @Test

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetEnclosingRoot.java

@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.contract;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public abstract class AbstractContractGetEnclosingRoot extends AbstractFSContractTestBase {
+  private static final Logger LOG = LoggerFactory.getLogger(AbstractContractGetEnclosingRoot.class);
+
+  @Test
+  public void testEnclosingRootEquivalence() throws IOException {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+    Path foobar = path("/foo/bar");
+
+    assertEquals("Ensure getEnclosingRoot on the root directory returns the root directory",
+        root, fs.getEnclosingRoot(foobar));
+    assertEquals("Ensure getEnclosingRoot called on itself returns the root directory",
+        root, fs.getEnclosingRoot(fs.getEnclosingRoot(foobar)));
+    assertEquals(
+        "Ensure getEnclosingRoot for different paths in the same enclosing root "
+            + "returns the same path",
+        fs.getEnclosingRoot(root), fs.getEnclosingRoot(foobar));
+    assertEquals("Ensure getEnclosingRoot on a path returns the root directory",
+        root, fs.getEnclosingRoot(methodPath()));
+    assertEquals("Ensure getEnclosingRoot called on itself on a path returns the root directory",
+        root, fs.getEnclosingRoot(fs.getEnclosingRoot(methodPath())));
+    assertEquals(
+        "Ensure getEnclosingRoot for different paths in the same enclosing root "
+            + "returns the same path",
+        fs.getEnclosingRoot(root),
+        fs.getEnclosingRoot(methodPath()));
+  }
+
+
+  @Test
+  public void testEnclosingRootPathExists() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+    Path foobar = methodPath();
+    fs.mkdirs(foobar);
+
+    assertEquals(
+        "Ensure getEnclosingRoot returns the root directory when the root directory exists",
+        root, fs.getEnclosingRoot(foobar));
+    assertEquals("Ensure getEnclosingRoot returns the root directory when the directory exists",
+        root, fs.getEnclosingRoot(foobar));
+  }
+
+  @Test
+  public void testEnclosingRootPathDNE() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path foobar = path("/foo/bar");
+    Path root = path("/");
+
+    // .
+    assertEquals(
+        "Ensure getEnclosingRoot returns the root directory even when the path does not exist",
+        root, fs.getEnclosingRoot(foobar));
+    assertEquals(
+        "Ensure getEnclosingRoot returns the root directory even when the path does not exist",
+        root, fs.getEnclosingRoot(methodPath()));
+  }
+
+  @Test
+  public void testEnclosingRootWrapped() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path root = path("/");
+
+    assertEquals("Ensure getEnclosingRoot returns the root directory when the directory exists",
+        root, fs.getEnclosingRoot(new Path("/foo/bar")));
+
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("foo");
+    Path p = ugi.doAs((PrivilegedExceptionAction<Path>) () -> {
+      FileSystem wFs = getContract().getTestFileSystem();
+      return wFs.getEnclosingRoot(new Path("/foo/bar"));
+    });
+    assertEquals("Ensure getEnclosingRoot works correctly within a wrapped FileSystem", root, p);
+  }
+}