diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 1dad20fdb5130..a896311043e53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -6665,14 +6665,15 @@ object SQLConf { .createWithDefault(false) val UI_EXPLAIN_MODE = buildConf("spark.sql.ui.explainMode") - .doc("Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', " + - "'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.") + .doc("Configures the query explain mode used in the Spark SQL UI. The value can be 'none', " + + "'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.") .version("3.1.0") .stringConf .transform(_.toUpperCase(Locale.ROOT)) - .checkValue(mode => Set("SIMPLE", "EXTENDED", "CODEGEN", "COST", "FORMATTED").contains(mode), - "Invalid value for 'spark.sql.ui.explainMode'. Valid values are 'simple', 'extended', " + - "'codegen', 'cost' and 'formatted'.") + .checkValue(mode => + Set("NONE", "SIMPLE", "EXTENDED", "CODEGEN", "COST", "FORMATTED").contains(mode), + "Invalid value for 'spark.sql.ui.explainMode'. Valid values are 'none', 'simple', " + + "'extended', 'codegen', 'cost' and 'formatted'.") .createWithDefault("formatted") val SOURCES_BINARY_FILE_MAX_LENGTH = buildConf("spark.sql.sources.binaryFile.maxLength") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala index f25e908a9cdb7..e296e00ba48d7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala @@ -120,6 +120,27 @@ object SQLExecution extends Logging { } } + private[sql] val NONE_EXPLAIN_MODE = "none" + + private[sql] val NO_PLAN_DESCRIPTION = + s"No plan description because ${SQLConf.UI_EXPLAIN_MODE.key}=$NONE_EXPLAIN_MODE" + + /** + * Returns the plan description carried by the SQL UI events, or a placeholder when the UI + * explain mode is `none`, in which case the explain string, which is expensive to build for + * large plans, is not generated at all. + * + * `none` is intentionally not an [[ExplainMode]]: it only makes sense for the UI events, and + * keeping it here avoids making `df.explain("none")` a valid public API. + */ + private[sql] def planDescription(qe: QueryExecution, uiExplainMode: String): String = { + if (uiExplainMode.equalsIgnoreCase(NONE_EXPLAIN_MODE)) { + NO_PLAN_DESCRIPTION + } else { + qe.explainString(ExplainMode.fromString(uiExplainMode)) + } + } + /** * Wrap an action that will execute "queryExecution" to track all Spark jobs in the body so that * we can connect them with an execution. @@ -209,9 +230,8 @@ object SQLExecution extends Logging { sc.listenerBus.post(startEvent) throw e case Right(f) => - val planDescriptionMode = - ExplainMode.fromString(sparkSession.sessionState.conf.uiExplainMode) - val planDesc = queryExecution.explainString(planDescriptionMode) + val planDesc = planDescription( + queryExecution, sparkSession.sessionState.conf.uiExplainMode) val planInfo = try { SparkPlanInfo.fromSparkPlan(queryExecution.executedPlan) } catch { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala index a12b202b5b009..0ef9b02d32d0b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala @@ -1145,10 +1145,9 @@ case class AdaptiveSparkPlanExec( context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates( executionId, newMetrics)) } else { - val planDescriptionMode = ExplainMode.fromString(conf.uiExplainMode) context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate( executionId, - context.qe.explainString(planDescriptionMode), + SQLExecution.planDescription(context.qe, conf.uiExplainMode), SparkPlanInfo.fromSparkPlan(context.qe.executedPlan))) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala index 8e98d7785f6c0..3ac16235333c1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala @@ -2541,7 +2541,8 @@ class AdaptiveQueryExecSuite "== Optimized Logical Plan ==", "== Physical Plan ==")), ("codegen", Seq("WholeStageCodegen subtrees")), ("cost", Seq("== Optimized Logical Plan ==", "Statistics(sizeInBytes")), - ("formatted", Seq("== Physical Plan ==", "Output", "Arguments"))).foreach { + ("formatted", Seq("== Physical Plan ==", "Output", "Arguments")), + ("none", Seq(SQLExecution.NO_PLAN_DESCRIPTION))).foreach { case (mode, expected) => checkPlanDescription(mode, expected) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala index 9230eb48dcca0..7d9c79ab92363 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala @@ -369,7 +369,8 @@ abstract class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTes "== Optimized Logical Plan ==", "== Physical Plan ==")), ("codegen", Seq("WholeStageCodegen subtrees")), ("cost", Seq("== Optimized Logical Plan ==", "Statistics(sizeInBytes")), - ("formatted", Seq("== Physical Plan ==", "Output", "Arguments"))).foreach { + ("formatted", Seq("== Physical Plan ==", "Output", "Arguments")), + ("none", Seq(SQLExecution.NO_PLAN_DESCRIPTION))).foreach { case (mode, expected) => checkPlanDescription(mode, expected) }