HADOOP-18373. IOStatisticsContext tuning (#4705)

The name of the option to enable/disable thread level statistics is
"fs.iostatistics.thread.level.enabled";

There is also an enabled() probe in IOStatisticsContext which can
be used to see if the thread level statistics is active.

Contributed by Viraj Jasani

Author: virajjasani

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java

@@ -480,13 +480,13 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
    * Thread-level IOStats Support.
    * {@value}
    */
-  public static final String THREAD_LEVEL_IOSTATISTICS_ENABLED =
-      "fs.thread.level.iostatistics.enabled";
+  public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
+      "fs.iostatistics.thread.level.enabled";
 
   /**
    * Default value for Thread-level IOStats Support is true.
    */
-  public static final boolean THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT =
+  public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
       true;
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java

@@ -80,4 +80,14 @@ static void setThreadIOStatisticsContext(
     IOStatisticsContextIntegration.setThreadIOStatisticsContext(
         statisticsContext);
   }
+
+  /**
+   * Static probe to check if the thread-level IO statistics enabled.
+   *
+   * @return if the thread-level IO statistics enabled.
+   */
+  static boolean enabled() {
+    return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled();
+  }
+
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java

@@ -29,8 +29,8 @@
 import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
 import org.apache.hadoop.fs.statistics.IOStatisticsContext;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT;
 
 /**
  * A Utility class for IOStatisticsContext, which helps in creating and
@@ -76,8 +76,17 @@ public final class IOStatisticsContextIntegration {
     // Work out if the current context has thread level IOStatistics enabled.
     final Configuration configuration = new Configuration();
     isThreadIOStatsEnabled =
-        configuration.getBoolean(THREAD_LEVEL_IOSTATISTICS_ENABLED,
-            THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT);
+        configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED,
+            IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT);
+  }
+
+  /**
+   * Static probe to check if the thread-level IO statistics enabled.
+   *
+   * @return if the thread-level IO statistics enabled.
+   */
+  public static boolean isIOStatisticsThreadLevelEnabled() {
+    return isThreadIOStatsEnabled;
   }
 
   /**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java

@@ -153,6 +153,8 @@ public void testS3AInputStreamIOStatisticsContext()
    * @return thread context
    */
   private static IOStatisticsContext getAndResetThreadStatisticsContext() {
+    assertTrue("thread-level IOStatistics should be enabled by default",
+        IOStatisticsContext.enabled());
     IOStatisticsContext context =
         IOStatisticsContext.getCurrentIOStatisticsContext();
     context.reset();

hadoop-tools/hadoop-aws/src/test/resources/core-site.xml

@@ -178,6 +178,12 @@
     <value>true</value>
   </property>
 
+  <!-- Enable IOStatisticsContext support for Thread level. -->
+  <property>
+    <name>fs.iostatistics.thread.level.enabled</name>
+    <value>true</value>
+  </property>
+
   <!--
   To run these tests.