From 2b41748aeac1f124170abfb70e88cbc01639a77f Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 26 Aug 2026 23:22:15 +0800 Subject: [PATCH 1/6] [SPARK-59024][SQL] Support sequential cached name for anonymous cached tables Add spark.sql.useSequentialCacheName. When it is true and the cached table has no name, CachedRDDBuilder uses a sequential number like 'CachedRDD 1' as the cached name instead of the abbreviated plan tree string. Rendering the plan tree string can be expensive for large plans. Assisted-by: Qwen3.8 Max --- .../apache/spark/sql/internal/SQLConf.scala | 9 +++++++++ .../execution/columnar/InMemoryRelation.scala | 16 ++++++++++++++-- .../columnar/InMemoryRelationSuite.scala | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 9398d14eb94e8..ff34415a3faa1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2647,6 +2647,15 @@ object SQLConf { .enumConf(classOf[Level]) .createWithDefault(Level.TRACE) + val USE_SEQUENTIAL_CACHE_NAME = buildConf("spark.sql.useSequentialCacheName") + .internal() + .doc("When true and the cached table has no name, use a sequential number like " + + "'CachedRDD 1' as the cached name instead of the abbreviated plan tree string. " + + "Rendering the plan tree string can be expensive for large plans.") + .version("4.4.0") + .booleanConf + .createWithDefault(false) + val DROP_TABLE_VIEW_ENABLED = buildConf("spark.sql.dropTableOnView.enabled") .doc("When true, DROP TABLE command will work on VIEW as well.") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala index 584eaa9ecbc82..2ce8646f0911a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.execution.columnar +import java.util.concurrent.atomic.AtomicLong + import com.esotericsoftware.kryo.{DefaultSerializer, Kryo, Serializer => KryoSerializer} import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput} @@ -255,6 +257,11 @@ class DefaultCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer { } } +private[sql] object CachedRDDBuilder { + private val _nextCachedRDDId = new AtomicLong(0) + def nextCachedRDDId(): Long = _nextCachedRDDId.getAndIncrement +} + private[sql] case class CachedRDDBuilder( serializer: CachedBatchSerializer, @@ -289,8 +296,13 @@ case class CachedRDDBuilder( // late updates from making a rebuilt cache appear complete. private var partitionStats = newPartitionStats() - val cachedName = tableName.map(n => s"In-memory table $n") - .getOrElse(Utils.abbreviate(cachedPlan.toString, 1024)) + val cachedName: String = tableName.map(n => s"In-memory table $n").getOrElse { + if (cachedPlan.session.conf.get(SQLConf.USE_SEQUENTIAL_CACHE_NAME)) { + s"CachedRDD ${CachedRDDBuilder.nextCachedRDDId()}" + } else { + Utils.abbreviate(cachedPlan.toString, 1024) + } + } val supportsColumnarInput: Boolean = { cachedPlan.supportsColumnar && diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala index 6a9c8964e4dbf..f6dd079637f71 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala @@ -22,6 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.AttributeSet import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.expr +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSessionBase import org.apache.spark.storage.StorageLevel @@ -51,6 +52,23 @@ class InMemoryRelationSuite extends SparkFunSuite assert(r1.sameResult(r2)) } + test("sequential cached name for anonymous cached tables") { + val d = spark.range(1) + withSQLConf(SQLConf.USE_SEQUENTIAL_CACHE_NAME.key -> "true") { + val r1 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) + val r2 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) + assert(r1.cacheBuilder.cachedName.matches("CachedRDD \\d+")) + assert(r2.cacheBuilder.cachedName.matches("CachedRDD \\d+")) + assert(r1.cacheBuilder.cachedName != r2.cacheBuilder.cachedName) + // Named tables keep the usual name. + val r3 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, Some("t1")) + assert(r3.cacheBuilder.cachedName == "In-memory table t1") + } + // The default keeps the abbreviated plan tree string. + val r4 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) + assert(!r4.cacheBuilder.cachedName.startsWith("CachedRDD ")) + } + test("SPARK-47177: Cached SQL plan do not display final AQE plan in explain string") { def findIMRInnerChild(p: SparkPlan): SparkPlan = { val tableCache = find(p) { From 3f9e96edfccf2cf0a7d07e82ed3e30d58484bed4 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Wed, 26 Aug 2026 23:31:24 +0800 Subject: [PATCH 2/6] [SPARK-59024][SQL] Make spark.sql.useSequentialCacheName public Remove .internal() from spark.sql.useSequentialCacheName, and pin the disabled case in the test with an explicit conf value instead of relying on the default. --- .../scala/org/apache/spark/sql/internal/SQLConf.scala | 1 - .../sql/execution/columnar/InMemoryRelationSuite.scala | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index ff34415a3faa1..293e38a9e5536 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2648,7 +2648,6 @@ object SQLConf { .createWithDefault(Level.TRACE) val USE_SEQUENTIAL_CACHE_NAME = buildConf("spark.sql.useSequentialCacheName") - .internal() .doc("When true and the cached table has no name, use a sequential number like " + "'CachedRDD 1' as the cached name instead of the abbreviated plan tree string. " + "Rendering the plan tree string can be expensive for large plans.") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala index f6dd079637f71..2720025851492 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala @@ -64,9 +64,11 @@ class InMemoryRelationSuite extends SparkFunSuite val r3 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, Some("t1")) assert(r3.cacheBuilder.cachedName == "In-memory table t1") } - // The default keeps the abbreviated plan tree string. - val r4 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) - assert(!r4.cacheBuilder.cachedName.startsWith("CachedRDD ")) + // When disabled, the cached name keeps the abbreviated plan tree string. + withSQLConf(SQLConf.USE_SEQUENTIAL_CACHE_NAME.key -> "false") { + val r4 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) + assert(!r4.cacheBuilder.cachedName.startsWith("CachedRDD ")) + } } test("SPARK-47177: Cached SQL plan do not display final AQE plan in explain string") { From 4dc1753cb21c0dfacaa9de851e7c062eff95fcad Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 27 Aug 2026 01:35:59 +0800 Subject: [PATCH 3/6] Retrigger CI From f0d76431a69c090f44a396870a482d4d9161e7f7 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Thu, 27 Aug 2026 22:44:30 +0800 Subject: [PATCH 4/6] [SPARK-59024][SQL] Address review comments - Make cachedName a lazy val so the plan tree string is only rendered when the name is needed - Rename the config to spark.sql.dataframeCache.sequentialName.enabled, keep it internal, and declare ConfigBindingPolicy.NOT_APPLICABLE - Use cachedPlan.conf instead of cachedPlan.session.conf - Follow the SparkPlan.newPlanId naming for the id generator and document the id-consumption side effect - Prefix the test with SPARK-59024 and assert the disabled-case name exactly --- .../org/apache/spark/sql/internal/SQLConf.scala | 17 ++++++++++------- .../execution/columnar/InMemoryRelation.scala | 12 +++++++----- .../columnar/InMemoryRelationSuite.scala | 10 ++++++---- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 293e38a9e5536..912eb27b12df0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2647,13 +2647,16 @@ object SQLConf { .enumConf(classOf[Level]) .createWithDefault(Level.TRACE) - val USE_SEQUENTIAL_CACHE_NAME = buildConf("spark.sql.useSequentialCacheName") - .doc("When true and the cached table has no name, use a sequential number like " + - "'CachedRDD 1' as the cached name instead of the abbreviated plan tree string. " + - "Rendering the plan tree string can be expensive for large plans.") - .version("4.4.0") - .booleanConf - .createWithDefault(false) + val DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED = + buildConf("spark.sql.dataframeCache.sequentialName.enabled") + .internal() + .doc("When true and the cached table has no name, use a sequential number like " + + "'CachedRDD 0' as the cached name instead of the abbreviated plan tree string. " + + "Rendering the plan tree string can be expensive for large plans.") + .version("4.4.0") + .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE) + .booleanConf + .createWithDefault(false) val DROP_TABLE_VIEW_ENABLED = buildConf("spark.sql.dropTableOnView.enabled") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala index 2ce8646f0911a..eac6058fb18fc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala @@ -258,8 +258,8 @@ class DefaultCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer { } private[sql] object CachedRDDBuilder { - private val _nextCachedRDDId = new AtomicLong(0) - def nextCachedRDDId(): Long = _nextCachedRDDId.getAndIncrement + private val nextCachedRDDId = new AtomicLong(0) + def newCachedRDDId(): Long = nextCachedRDDId.getAndIncrement() } private[sql] @@ -296,9 +296,11 @@ case class CachedRDDBuilder( // late updates from making a rebuilt cache appear complete. private var partitionStats = newPartitionStats() - val cachedName: String = tableName.map(n => s"In-memory table $n").getOrElse { - if (cachedPlan.session.conf.get(SQLConf.USE_SEQUENTIAL_CACHE_NAME)) { - s"CachedRDD ${CachedRDDBuilder.nextCachedRDDId()}" + // The sequential id is consumed when this lazy val is first forced; a copy of the builder + // would draw a new id. + lazy val cachedName: String = tableName.map(n => s"In-memory table $n").getOrElse { + if (cachedPlan.conf.getConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED)) { + s"CachedRDD ${CachedRDDBuilder.newCachedRDDId()}" } else { Utils.abbreviate(cachedPlan.toString, 1024) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala index 2720025851492..5ff6d184b9ec4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.functions.expr import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSessionBase import org.apache.spark.storage.StorageLevel +import org.apache.spark.util.Utils class InMemoryRelationSuite extends SparkFunSuite with SharedSparkSessionBase with AdaptiveSparkPlanHelper { @@ -52,9 +53,9 @@ class InMemoryRelationSuite extends SparkFunSuite assert(r1.sameResult(r2)) } - test("sequential cached name for anonymous cached tables") { + test("SPARK-59024: sequential cached name for anonymous cached tables") { val d = spark.range(1) - withSQLConf(SQLConf.USE_SEQUENTIAL_CACHE_NAME.key -> "true") { + withSQLConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED.key -> "true") { val r1 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) val r2 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) assert(r1.cacheBuilder.cachedName.matches("CachedRDD \\d+")) @@ -65,9 +66,10 @@ class InMemoryRelationSuite extends SparkFunSuite assert(r3.cacheBuilder.cachedName == "In-memory table t1") } // When disabled, the cached name keeps the abbreviated plan tree string. - withSQLConf(SQLConf.USE_SEQUENTIAL_CACHE_NAME.key -> "false") { + withSQLConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED.key -> "false") { val r4 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) - assert(!r4.cacheBuilder.cachedName.startsWith("CachedRDD ")) + assert(r4.cacheBuilder.cachedName == + Utils.abbreviate(r4.cacheBuilder.cachedPlan.toString, 1024)) } } From 25ab5c6e11da6a1d6541b3033ea27fd0969d3903 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 28 Aug 2026 16:06:10 +0800 Subject: [PATCH 5/6] [SPARK-59024][SQL] Use the physical plan id for the anonymous cached name - Rename the config to spark.sql.dataframeCache.planIdName.enabled and use 'CachedRDD (plan_id=)', dropping the dedicated id generator - Note in the config doc that the name is resolved at first materialization - Rework the tests: same-plan caches share the name, and the tree string is not rendered at construction --- .../apache/spark/sql/internal/SQLConf.scala | 11 ++--- .../execution/columnar/InMemoryRelation.scala | 15 ++----- .../columnar/InMemoryRelationSuite.scala | 45 +++++++++++++++---- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 912eb27b12df0..2e866508c8675 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2647,12 +2647,13 @@ object SQLConf { .enumConf(classOf[Level]) .createWithDefault(Level.TRACE) - val DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED = - buildConf("spark.sql.dataframeCache.sequentialName.enabled") + val DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED = + buildConf("spark.sql.dataframeCache.planIdName.enabled") .internal() - .doc("When true and the cached table has no name, use a sequential number like " + - "'CachedRDD 0' as the cached name instead of the abbreviated plan tree string. " + - "Rendering the plan tree string can be expensive for large plans.") + .doc("When true and the cached table has no name, use the physical plan id, e.g. " + + "'CachedRDD (plan_id=42)', as the cached name instead of the abbreviated plan tree " + + "string. Rendering the plan tree string can be expensive for large plans. The name " + + "is resolved when the cache is first materialized.") .version("4.4.0") .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE) .booleanConf diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala index eac6058fb18fc..68a5e74032d8c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.execution.columnar -import java.util.concurrent.atomic.AtomicLong - import com.esotericsoftware.kryo.{DefaultSerializer, Kryo, Serializer => KryoSerializer} import com.esotericsoftware.kryo.io.{Input => KryoInput, Output => KryoOutput} @@ -257,11 +255,6 @@ class DefaultCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer { } } -private[sql] object CachedRDDBuilder { - private val nextCachedRDDId = new AtomicLong(0) - def newCachedRDDId(): Long = nextCachedRDDId.getAndIncrement() -} - private[sql] case class CachedRDDBuilder( serializer: CachedBatchSerializer, @@ -296,11 +289,11 @@ case class CachedRDDBuilder( // late updates from making a rebuilt cache appear complete. private var partitionStats = newPartitionStats() - // The sequential id is consumed when this lazy val is first forced; a copy of the builder - // would draw a new id. + // Resolved on first access, which happens at cache materialization; for adaptive plans the + // name therefore reflects the final plan. lazy val cachedName: String = tableName.map(n => s"In-memory table $n").getOrElse { - if (cachedPlan.conf.getConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED)) { - s"CachedRDD ${CachedRDDBuilder.newCachedRDDId()}" + if (cachedPlan.conf.getConf(SQLConf.DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED)) { + s"CachedRDD (plan_id=${cachedPlan.id})" } else { Utils.abbreviate(cachedPlan.toString, 1024) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala index 5ff6d184b9ec4..955b1e5be3257 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala @@ -17,9 +17,13 @@ package org.apache.spark.sql.execution.columnar +import java.util.concurrent.atomic.AtomicInteger + import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.expressions.AttributeSet -import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet} +import org.apache.spark.sql.execution.{LeafExecNode, SparkPlan} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.expr import org.apache.spark.sql.internal.SQLConf @@ -53,26 +57,35 @@ class InMemoryRelationSuite extends SparkFunSuite assert(r1.sameResult(r2)) } - test("SPARK-59024: sequential cached name for anonymous cached tables") { + test("SPARK-59024: plan id cached name for anonymous cached tables") { val d = spark.range(1) - withSQLConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED.key -> "true") { + withSQLConf(SQLConf.DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED.key -> "true") { val r1 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) - val r2 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) - assert(r1.cacheBuilder.cachedName.matches("CachedRDD \\d+")) - assert(r2.cacheBuilder.cachedName.matches("CachedRDD \\d+")) + // Caches of the same plan share the plan id. + val r1Again = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) + val r2 = InMemoryRelation(StorageLevel.MEMORY_ONLY, spark.range(2).queryExecution, None) + assert(r1.cacheBuilder.cachedName.matches("CachedRDD \\(plan_id=\\d+\\)")) + assert(r1Again.cacheBuilder.cachedName == r1.cacheBuilder.cachedName) assert(r1.cacheBuilder.cachedName != r2.cacheBuilder.cachedName) // Named tables keep the usual name. val r3 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, Some("t1")) assert(r3.cacheBuilder.cachedName == "In-memory table t1") } // When disabled, the cached name keeps the abbreviated plan tree string. - withSQLConf(SQLConf.DATAFRAME_CACHE_SEQUENTIAL_NAME_ENABLED.key -> "false") { + withSQLConf(SQLConf.DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED.key -> "false") { val r4 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) assert(r4.cacheBuilder.cachedName == Utils.abbreviate(r4.cacheBuilder.cachedPlan.toString, 1024)) } } + test("SPARK-59024: anonymous cached name is not rendered before materialization") { + val plan = ToStringCountingPlan() + InMemoryRelation(new DefaultCachedBatchSerializer, StorageLevel.MEMORY_ONLY, plan, None, + spark.range(1).queryExecution.optimizedPlan) + assert(plan.toStringCount == 0) + } + test("SPARK-47177: Cached SQL plan do not display final AQE plan in explain string") { def findIMRInnerChild(p: SparkPlan): SparkPlan = { val tableCache = find(p) { @@ -96,3 +109,19 @@ class InMemoryRelationSuite extends SparkFunSuite .contains("AdaptiveSparkPlan isFinalPlan=true")) } } + +case class ToStringCountingPlan() extends LeafExecNode { + private val _toStringCount = new AtomicInteger(0) + + def toStringCount: Int = _toStringCount.get() + + override def output: Seq[Attribute] = Seq.empty + + override protected def doExecute(): RDD[InternalRow] = + throw new UnsupportedOperationException + + override def toString: String = { + _toStringCount.incrementAndGet() + "ToStringCountingPlan" + } +} From a37aec19bb352f524676823b713fd91e39c02c72 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Sat, 29 Aug 2026 04:14:09 +0800 Subject: [PATCH 6/6] [SPARK-59024][SQL] Address second-round review comments - Reword the cachedName comment: named caches force the name at scan construction - Reword the test comment: the same physical plan instance shares the plan id - Also assert the fallback path renders the tree string exactly once --- .../spark/sql/execution/columnar/InMemoryRelation.scala | 4 ++-- .../sql/execution/columnar/InMemoryRelationSuite.scala | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala index 68a5e74032d8c..3de3e41e110bc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala @@ -289,8 +289,8 @@ case class CachedRDDBuilder( // late updates from making a rebuilt cache appear complete. private var partitionStats = newPartitionStats() - // Resolved on first access, which happens at cache materialization; for adaptive plans the - // name therefore reflects the final plan. + // Resolved on first access (cache materialization for anonymous caches). For adaptive plans, + // the name reflects the final plan. lazy val cachedName: String = tableName.map(n => s"In-memory table $n").getOrElse { if (cachedPlan.conf.getConf(SQLConf.DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED)) { s"CachedRDD (plan_id=${cachedPlan.id})" diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala index 955b1e5be3257..706e81d8588cd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryRelationSuite.scala @@ -61,7 +61,7 @@ class InMemoryRelationSuite extends SparkFunSuite val d = spark.range(1) withSQLConf(SQLConf.DATAFRAME_CACHE_PLAN_ID_NAME_ENABLED.key -> "true") { val r1 = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) - // Caches of the same plan share the plan id. + // Caches of the same physical plan instance share the plan id. val r1Again = InMemoryRelation(StorageLevel.MEMORY_ONLY, d.queryExecution, None) val r2 = InMemoryRelation(StorageLevel.MEMORY_ONLY, spark.range(2).queryExecution, None) assert(r1.cacheBuilder.cachedName.matches("CachedRDD \\(plan_id=\\d+\\)")) @@ -81,9 +81,12 @@ class InMemoryRelationSuite extends SparkFunSuite test("SPARK-59024: anonymous cached name is not rendered before materialization") { val plan = ToStringCountingPlan() - InMemoryRelation(new DefaultCachedBatchSerializer, StorageLevel.MEMORY_ONLY, plan, None, - spark.range(1).queryExecution.optimizedPlan) + val relation = InMemoryRelation(new DefaultCachedBatchSerializer, StorageLevel.MEMORY_ONLY, + plan, None, spark.range(1).queryExecution.optimizedPlan) assert(plan.toStringCount == 0) + // Forcing the name renders the tree string exactly once. + relation.cacheBuilder.cachedName + assert(plan.toStringCount == 1) } test("SPARK-47177: Cached SQL plan do not display final AQE plan in explain string") {