Skip to content

[SPARK-59023][SQL] Support none mode in spark.sql.ui.explainMode to skip plan description in SQL events #58313

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pan3793 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[SPARK-59023][SQL] Support none mode in spark.sql.ui.explainMode to skip plan description in SQL events #58313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6665,14 +6665,15 @@ object SQLConf {
.createWithDefault(false)

val UI_EXPLAIN_MODE = buildConf("spark.sql.ui.explainMode")
.doc("Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', " +
"'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.")
.doc("Configures the query explain mode used in the Spark SQL UI. The value can be 'none', " +
"'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.")
.version("3.1.0")
.stringConf
.transform(_.toUpperCase(Locale.ROOT))
.checkValue(mode => Set("SIMPLE", "EXTENDED", "CODEGEN", "COST", "FORMATTED").contains(mode),
"Invalid value for 'spark.sql.ui.explainMode'. Valid values are 'simple', 'extended', " +
"'codegen', 'cost' and 'formatted'.")
.checkValue(mode =>
Set("NONE", "SIMPLE", "EXTENDED", "CODEGEN", "COST", "FORMATTED").contains(mode),
"Invalid value for 'spark.sql.ui.explainMode'. Valid values are 'none', 'simple', " +
"'extended', 'codegen', 'cost' and 'formatted'.")
.createWithDefault("formatted")

val SOURCES_BINARY_FILE_MAX_LENGTH = buildConf("spark.sql.sources.binaryFile.maxLength")
Expand Down
9 changes: 6 additions & 3 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,12 @@ object SQLExecution extends Logging {
sc.listenerBus.post(startEvent)
throw e
case Right(f) =>
val planDescriptionMode =
ExplainMode.fromString(sparkSession.sessionState.conf.uiExplainMode)
val planDesc = queryExecution.explainString(planDescriptionMode)
val planDescriptionMode = {
val mode = sparkSession.sessionState.conf.uiExplainMode
if (mode == "NONE") None else Some(ExplainMode.fromString(mode))
}
val planDesc = planDescriptionMode.map(queryExecution.explainString).getOrElse(
"No plan description because spark.sql.ui.explainMode=none")
val planInfo = try {
SparkPlanInfo.fromSparkPlan(queryExecution.executedPlan)
} catch {
Expand Down
8 changes: 6 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/adaptive/AdaptiveSparkPlanExec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1145,10 +1145,14 @@ case class AdaptiveSparkPlanExec(
context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveSQLMetricUpdates(
executionId, newMetrics))
} else {
val planDescriptionMode = ExplainMode.fromString(conf.uiExplainMode)
val planDescription = if (conf.uiExplainMode == "NONE") {
"No plan description because spark.sql.ui.explainMode=none"
} else {
context.qe.explainString(ExplainMode.fromString(conf.uiExplainMode))
}
context.session.sparkContext.listenerBus.post(SparkListenerSQLAdaptiveExecutionUpdate(
executionId,
context.qe.explainString(planDescriptionMode),
planDescription,
SparkPlanInfo.fromSparkPlan(context.qe.executedPlan)))
}
}
Expand Down
3 changes: 2 additions & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2541,7 +2541,8 @@ class AdaptiveQueryExecSuite
"== Optimized Logical Plan ==", "== Physical Plan ==")),
("codegen", Seq("WholeStageCodegen subtrees")),
("cost", Seq("== Optimized Logical Plan ==", "Statistics(sizeInBytes")),
("formatted", Seq("== Physical Plan ==", "Output", "Arguments"))).foreach {
("formatted", Seq("== Physical Plan ==", "Output", "Arguments")),
("none", Seq("No plan description because spark.sql.ui.explainMode=none"))).foreach {
case (mode, expected) =>
checkPlanDescription(mode, expected)
}
Expand Down
3 changes: 2 additions & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SQLAppStatusListenerSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,8 @@ abstract class SQLAppStatusListenerSuite extends SharedSparkSession with JsonTes
"== Optimized Logical Plan ==", "== Physical Plan ==")),
("codegen", Seq("WholeStageCodegen subtrees")),
("cost", Seq("== Optimized Logical Plan ==", "Statistics(sizeInBytes")),
("formatted", Seq("== Physical Plan ==", "Output", "Arguments"))).foreach {
("formatted", Seq("== Physical Plan ==", "Output", "Arguments")),
("none", Seq("No plan description because spark.sql.ui.explainMode=none"))).foreach {
case (mode, expected) =>
checkPlanDescription(mode, expected)
}
Expand Down