diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 2db880976c3a..71426cd1f13d 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -568,7 +568,7 @@ class SparkContext(config: SparkConf) extends Logging { // The metrics system for Driver need to be set spark.app.id to app ID. // So it should start after we get app ID from the task scheduler and set spark.app.id. - _env.metricsSystem.start() + _env.metricsSystem.start(_conf.get(METRICS_STATIC_SOURCES_ENABLED)) // Attach the driver metrics servlet handler to the web ui after the metrics system is started. _env.metricsSystem.getServletHandlers.foreach(handler => ui.foreach(_.attachHandler(handler))) diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala index 78ac00909ea1..9232938464e0 100644 --- a/core/src/main/scala/org/apache/spark/SparkEnv.scala +++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala @@ -383,7 +383,7 @@ object SparkEnv extends Logging { conf.set(EXECUTOR_ID, executorId) val ms = MetricsSystem.createMetricsSystem(MetricsSystemInstances.EXECUTOR, conf, securityManager) - ms.start() + ms.start(conf.get(METRICS_STATIC_SOURCES_ENABLED)) ms } diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 444a1544777a..c7c3c8f2cdcd 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -615,6 +615,12 @@ package object config { .stringConf .createOptional + private[spark] val METRICS_STATIC_SOURCES_ENABLED = + ConfigBuilder("spark.metrics.static.sources.enabled") + .doc("Whether to register static sources with the metrics system.") + .booleanConf + .createWithDefault(true) + private[spark] val PYSPARK_DRIVER_PYTHON = ConfigBuilder("spark.pyspark.driver.python") .stringConf .createOptional diff --git a/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala b/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala new file mode 100644 index 000000000000..76c568056aee --- /dev/null +++ b/core/src/test/scala/org/apache/spark/metrics/source/SourceConfigSuite.scala @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.metrics.source + +import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkFunSuite} +import org.apache.spark.internal.config.METRICS_STATIC_SOURCES_ENABLED + +class SourceConfigSuite extends SparkFunSuite with LocalSparkContext { + + test("Test configuration for adding static sources registration") { + val conf = new SparkConf() + conf.set(METRICS_STATIC_SOURCES_ENABLED, true) + val sc = new SparkContext("local", "test", conf) + try { + val metricsSystem = sc.env.metricsSystem + + // Static sources should be registered + assert (metricsSystem.getSourcesByName("CodeGenerator").nonEmpty) + assert (metricsSystem.getSourcesByName("HiveExternalCatalog").nonEmpty) + } finally { + sc.stop() + } + } + + test("Test configuration for skipping static sources registration") { + val conf = new SparkConf() + conf.set(METRICS_STATIC_SOURCES_ENABLED, false) + val sc = new SparkContext("local", "test", conf) + try { + val metricsSystem = sc.env.metricsSystem + + // Static sources should not be registered + assert (metricsSystem.getSourcesByName("CodeGenerator").isEmpty) + assert (metricsSystem.getSourcesByName("HiveExternalCatalog").isEmpty) + } finally { + sc.stop() + } + } + +} diff --git a/docs/monitoring.md b/docs/monitoring.md index 8cb237df0ba7..f284d3be9b58 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -923,6 +923,8 @@ This is the component with the largest amount of instrumented metrics - memory.remainingOnHeapMem_MB - namespace=HiveExternalCatalog + - **note:**: these metrics are conditional to a configuration parameter: + `spark.metrics.static.sources.enabled` (default is true) - fileCacheHits.count - filesDiscovered.count - hiveClientCalls.count @@ -930,6 +932,8 @@ This is the component with the largest amount of instrumented metrics - partitionsFetched.count - namespace=CodeGenerator + - **note:**: these metrics are conditional to a configuration parameter: + `spark.metrics.static.sources.enabled` (default is true) - compilationTime (histogram) - generatedClassSize (histogram) - generatedMethodSize (histogram) @@ -1047,6 +1051,8 @@ when running in local mode. - shuffle-server.usedHeapMemory - namespace=HiveExternalCatalog + - **note:**: these metrics are conditional to a configuration parameter: + `spark.metrics.static.sources.enabled` (default is true) - fileCacheHits.count - filesDiscovered.count - hiveClientCalls.count @@ -1054,6 +1060,8 @@ when running in local mode. - partitionsFetched.count - namespace=CodeGenerator + - **note:**: these metrics are conditional to a configuration parameter: + `spark.metrics.static.sources.enabled` (default is true) - compilationTime (histogram) - generatedClassSize (histogram) - generatedMethodSize (histogram)