[#8771] feat(spark-connector): Add Spark 4.0 support, drop 3.3 and 3.4

.github/workflows/backend-integration-test-action.yml

@@ -1,3 +1,3 @@
 name: Backend Integration Test Action
 
 # run backend integration test
@@ -72,9 +72,8 @@
             -x :flink-connector:flink-1.18:test -x :flink-connector:flink-runtime-1.18:test \
             -x :flink-connector:flink-1.19:test -x :flink-connector:flink-runtime-1.19:test \
             -x :flink-connector:flink-1.20:test -x :flink-connector:flink-runtime-1.20:test \
-            -x :spark-connector:spark-common:test \
-            -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test \
-            -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test \
+            -x :spark-connector:spark-3.5:test -x :spark-connector:spark-4.0:test \
+            -x :spark-connector:spark-runtime-3.5:test -x :spark-connector:spark-runtime-4.0:test \
             -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test \
             -x :iceberg:iceberg-rest-trino-it:test \
             -x :plugins:idp-basic:test \

.github/workflows/build.yml

@@ -133,8 +133,6 @@ jobs:
 
       - name: Build with Scala2.13
         run: |
-          ./gradlew :spark-connector:spark-3.3:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false -PskipWeb=true
-          ./gradlew :spark-connector:spark-3.4:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false -PskipWeb=true
           ./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false -PskipWeb=true
 
       - name: Upload unit tests report
@@ -152,7 +150,12 @@ jobs:
     strategy:
       matrix:
         java-version: [ 17 ]
-    timeout-minutes: 90
+    # This job compiles and unit-tests every module. It has been finishing in 75-84 minutes against
+    # a 90-minute cap, and :core:test alone has varied from 27 to 45 minutes across runs on
+    # identical code, so the margin was already thinner than the runner's own variance. Adding a
+    # second Spark line spends some of what is left. 120 minutes matches the backend and cron
+    # integration test jobs.
+    timeout-minutes: 120
     needs: changes
     if: needs.changes.outputs.source_changes == 'true'
     # Steps represent a sequence of tasks that will be executed as part of the job

.github/workflows/cron-integration-test.yml

@@ -1,3 +1,3 @@
 name: Cron Integration Test
 
 # Controls when the workflow will run
@@ -104,8 +104,7 @@
             catalogs/**/*.log
             catalogs/**/*.tar
             distribution/**/*.log
-            spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
-            spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
             spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
+            spark-connector/v4.0/spark/build/spark-4.0-integration-test.log
             flink-connector/build/flink-connector-integration-test.log
             flink-connector/build/*.tar

.github/workflows/spark-integration-test-action.yml

@@ -25,7 +25,9 @@ jobs:
   start-runner:
     name: JDK${{ inputs.java-version }}-${{ inputs.test-mode }}-Scala${{ inputs.scala-version }}
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # Two Spark lines, each running the whole suite, so the budget is roughly double the
+    # single-version job's.
+    timeout-minutes: 90
     env:
       PLATFORM: ${{ inputs.architecture }}
     steps:
@@ -56,11 +58,12 @@ jobs:
       - name: Spark Integration Test
         id: integrationTest
         run: |
-          if [ "${{ inputs.scala-version }}" == "2.12" ];then 
-            ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false -PskipWeb=true :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
-          fi
-          ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false -PskipWeb=true :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
           ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PscalaVersion=${{ inputs.scala-version }} -PskipDockerTests=false -PskipWeb=true :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
+          # Spark 4 is Scala 2.13 only, so its module ignores -PscalaVersion. Guard on 2.12 so that
+          # adding a 2.13 entry to the matrix later does not run the Spark 4 suite twice.
+          if [ "${{ inputs.scala-version }}" == "2.12" ]; then
+            ./gradlew -PskipTests -PtestMode=${{ inputs.test-mode }} -PskipDockerTests=false -PskipWeb=true :spark-connector:spark-4.0:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
+          fi
 
       - name: Upload integrate tests reports
         uses: actions/upload-artifact@v7
@@ -69,8 +72,7 @@ jobs:
           name: spark-connector-integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}
           path: |
             build/reports
-            spark-connector/v3.3/spark/build/spark-3.3-integration-test.log
-            spark-connector/v3.4/spark/build/spark-3.4-integration-test.log
             spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
+            spark-connector/v4.0/spark/build/spark-4.0-integration-test.log
             distribution/package/logs/*.out
             distribution/package/logs/*.log

.github/workflows/trino-integration-test.yml

@@ -87,9 +87,8 @@ jobs:
             -x :flink-connector:flink-runtime-1.19:build \
             -x :flink-connector:flink-1.20:build \
             -x :flink-connector:flink-runtime-1.20:build \
-            -x :spark-connector:spark-common:build \
-            -x :spark-connector:spark-3.3:build -x :spark-connector:spark-3.4:build -x :spark-connector:spark-3.5:build \
-            -x :spark-connector:spark-runtime-3.3:build -x :spark-connector:spark-runtime-3.4:build -x :spark-connector:spark-runtime-3.5:build \
+            -x :spark-connector:spark-3.5:build -x :spark-connector:spark-4.0:build \
+            -x :spark-connector:spark-runtime-3.5:build -x :spark-connector:spark-runtime-4.0:build \
             -x :mcp-server:build -x :lineage:build \
             -x :maintenance:optimizer:build -x :maintenance:jobs:build \
             -x :lance:lance-common:build -x :lance:lance-rest-server:build

.github/workflows/trino-multi-version-test.yml

@@ -33,9 +33,8 @@ jobs:
             -x :flink-connector:flink-runtime-1.19:build \
             -x :flink-connector:flink-1.20:build \
             -x :flink-connector:flink-runtime-1.20:build \
-            -x :spark-connector:spark-common:build \
-            -x :spark-connector:spark-3.3:build -x :spark-connector:spark-3.4:build -x :spark-connector:spark-3.5:build \
-            -x :spark-connector:spark-runtime-3.3:build -x :spark-connector:spark-runtime-3.4:build -x :spark-connector:spark-runtime-3.5:build \
+            -x :spark-connector:spark-3.5:build -x :spark-connector:spark-4.0:build \
+            -x :spark-connector:spark-runtime-3.5:build -x :spark-connector:spark-runtime-4.0:build \
             -x :mcp-server:build -x :lineage:build \
             -x :maintenance:optimizer:build -x :maintenance:jobs:build \
             -x :lance:lance-common:build -x :lance:lance-rest-server:build

build.gradle.kts

@@ -377,8 +377,18 @@ subprojects {
     ":flink-connector"
   )
 
+  // Spark 4 requires JDK 17, so the Spark 4 connector modules opt out of the JDK 8 target even
+  // though the rest of :spark-connector (3.x) stays on Java 8.
+  val jdk17OnlyProjectPaths = setOf(
+    ":spark-connector:spark-4.0",
+    ":spark-connector:spark-runtime-4.0"
+  )
+
   fun compatibleWithJDK8(project: Project): Boolean {
     val path = project.path.lowercase()
+    if (jdk17OnlyProjectPaths.any { path == it.lowercase() }) {
+      return false
+    }
     return jdk8CompatibleProjectPathPrefixes.any { path.startsWith(it) }
   }
   extensions.extraProperties.set("excludePackagesForSparkConnector", ::excludePackagesForSparkConnector)

docs/how-to-build.md

@@ -74,14 +74,12 @@ Gravitino builds from source on Linux and macOS natively, and on Windows through
   If you want to build a module on its own, like the Spark connector, you can use Gradle to build a module with a specific name, like so:
 
    ```shell
-   ./gradlew spark-connector:spark-runtime-3.4:build -PscalaVersion=2.12
+   ./gradlew spark-connector:spark-runtime-3.5:build -PscalaVersion=2.12
    ```
 
-  This creates `gravitino-spark-connector-runtime-{sparkVersion}_{scalaVersion}-{version}.jar` under the `spark-connector/v3.4/spark-runtime/build/libs` directory. You could replace `3.4` with  `3.3` or `3.5` to specify different Spark versions and replace `2.12` with `2.13` for different Scala versions. The default Scala version is `2.12` if `-PscalaVersion` is not specified.
+  This creates `gravitino-spark-connector-runtime-{sparkVersion}_{scalaVersion}-{version}.jar` under the `spark-connector/v3.5/spark-runtime/build/libs` directory. Replace `2.12` with `2.13` to build against a different Scala version. The default Scala version is `2.12` if `-PscalaVersion` is not specified.
 
-  :::info
-  Gravitino Spark connector doesn't support Scala 2.13 for Spark 3.3.
-  :::
+  Replace `spark-runtime-3.5` with `spark-runtime-4.0` for Spark 4. Spark 4 is Scala 2.13 only, so that module ignores `-PscalaVersion` and always builds against 2.13.
 
   :::note
   The first time you build the project, downloading the dependencies may take a while.

docs/spark-connector/spark-catalog-glue.md

@@ -34,10 +34,15 @@ Table routing is based on the `table-format` property in Glue table parameters.
 - Network access to the AWS Glue API and Amazon S3
 - AWS IAM credentials with necessary Glue and S3 permissions.
   See [AWS IAM permissions](../aws-glue-catalog.md#aws-iam-permissions) for the required policy
-- Apache Spark 3.3, 3.4, or 3.5
+- Apache Spark 3.5
 - Patched Hive and AWS Glue client JARs (see [Setup](#setup); pre-installed on Amazon EMR)
 - `iceberg-spark-runtime` and `iceberg-aws-bundle` JARs on the Spark classpath for Iceberg table support (not required on Amazon EMR)
 
+:::note
+The connector resolves a Glue catalog class on Spark 4 as well, but the patched Hive JARs this
+catalog needs are only published for Spark 3, so Glue is verified on Spark 3.5 only.
+:::
+
 ## Setup
 
 Spark's bundled Hive 2.3.9 does not include the `HiveMetaStoreClientFactory` interface
@@ -75,7 +80,7 @@ that include the patched Hive 2.3.10 and the AWS Glue Data Catalog client for Sp
 Download all JARs from the `spark3/glue-3.4.0` directory of
 [spark-hive-glue-libs](https://github.com/datastrato/spark-hive-glue-libs).
 The directory name refers to the Glue client version (3.4.0), not the Spark version;
-these JARs are compatible with Spark 3.3, 3.4, and 3.5.
+these JARs are compatible with Spark 3.5.
 
 ```bash
 mkdir -p /opt/glue-hive-jars

docs/spark-connector/spark-catalog-iceberg.md

@@ -18,9 +18,8 @@ Spark clients use a different Iceberg version than the Gravitino server (1.11.0)
 
 | Spark version | Scala          | Iceberg version | Iceberg client runtime artifact                         | Gravitino connector runtime artifact                                              |
 |---------------|----------------|-----------------|---------------------------------------------------------|-----------------------------------------------------------------------------------|
-| 3.3           | 2.12 or 2.13   | 1.8.1           | `iceberg-spark-runtime-3.3_${scala-version}-1.8.1.jar`  | `gravitino-spark-connector-runtime-3.3_${scala-version}-${gravitino-version}.jar` |
-| 3.4           | 2.12 or 2.13   | 1.11.0          | `iceberg-spark-runtime-3.4_${scala-version}-1.11.0.jar` | `gravitino-spark-connector-runtime-3.4_${scala-version}-${gravitino-version}.jar` |
 | 3.5           | 2.12 or 2.13   | 1.11.0          | `iceberg-spark-runtime-3.5_${scala-version}-1.11.0.jar` | `gravitino-spark-connector-runtime-3.5_${scala-version}-${gravitino-version}.jar` |
+| 4.0           | 2.13           | 1.11.0          | `iceberg-spark-runtime-4.0_2.13-1.11.0.jar`             | `gravitino-spark-connector-runtime-4.0_2.13-${gravitino-version}.jar`             |
 
 Replace `${scala-version}` with `2.12` or `2.13`, and `${gravitino-version}` with your Gravitino release version.
 

docs/spark-connector/spark-catalog-paimon.md

@@ -14,6 +14,12 @@ The Apache Gravitino Spark connector offers the capability to read and write Pai
 1. Set `spark.sql.gravitino.enablePaimonSupport` to `true` in Spark configuration.
 2. Download Paimon Spark runtime jar to Spark classpath.
 
+:::info
+The Paimon catalog is available on Spark 3.5 only. Paimon first published `paimon-spark-4.0` in
+Paimon 1.3.0, above the version Gravitino currently depends on, so the Spark 4 connector builds
+without the Paimon classes.
+:::
+
 ## Capabilities
 
 ### DDL and DML Operations

docs/spark-connector/spark-connector.md

@@ -17,13 +17,13 @@ The Apache Gravitino Spark connector leverages the Spark DataSourceV2 interface
 
 ## Requirement
 
-* Spark 3.3 or 3.4 or 3.5
-* Scala 2.12 or 2.13
-* JDK 8, 11 or 17
+* Spark 3.5 or 4.0
+* Scala 2.12 or 2.13 on Spark 3.5; Spark 4 is Scala 2.13 only
+* JDK 8, 11 or 17 on Spark 3.5; Spark 4 requires JDK 17
 
 ## Usage
 
-1. [Build](../how-to-build.md) or download the package ([gravitino-spark-connector-runtime-3.3](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-spark-connector-runtime-3.3), [gravitino-spark-connector-runtime-3.4](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-spark-connector-runtime-3.4), [gravitino-spark-connector-runtime-3.5](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-spark-connector-runtime-3.5)), and place it to the classpath of Spark.
+1. [Build](../how-to-build.md) or download the package matching your Spark minor version ([gravitino-spark-connector-runtime-3.5](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-spark-connector-runtime-3.5), [gravitino-spark-connector-runtime-4.0](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-spark-connector-runtime-4.0)), and place it to the classpath of Spark.
 2. Configure the Spark session to use the Gravitino spark connector.
 
 | Property                                 | Type   | Default Value | Description                                                                                     | Required |
@@ -98,7 +98,7 @@ Gravitino spark connector support the following datatype mapping between Spark a
 | `CharType`                        | `char`                        |
 | `VarcharType`                     | `varchar`                     |
 | `TimestampType`                   | `timestamp with time zone`    |
-| `TimestampNTZType` *(Spark 3.4+)* | `timestamp without time zone` |
+| `TimestampNTZType`                | `timestamp without time zone` |
 | `DateType`                        | `date`                        |
 | `BinaryType`                      | `binary`                      |
 | `ArrayType`                       | `array`                       |

docs/spark-connector/spark-integration-test.md

@@ -14,15 +14,17 @@ There are two types of integration tests in spark connector, normal integration
 Normal integration test are mainly used to test the correctness of the metadata, it's enabled in the GitHub CI. You could run tests with specific Spark version like:
 
 ```
-./gradlew :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.hive.SparkHiveCatalogIT33.testCreateHiveFormatPartitionTable"
+./gradlew :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.hive.SparkHiveCatalogIT35.testCreateHiveFormatPartitionTable"
 ```
 
+Every version module carries its own subclass of each shared IT, named after the Spark minor version, so the Spark 4.0 equivalent is `:spark-connector:spark-4.0:test --tests "...SparkHiveCatalogIT40.testCreateHiveFormatPartitionTable"`.
+
 ## Golden File Integration Test
 
 Golden file integration test are mainly to test the correctness of the SQL result with massive data, it's disabled in the GitHub CI, you could run tests with following command:
 
 ```
-./gradlew :spark-connector:spark-3.3:test --tests  "org.apache.gravitino.spark.connector.integration.test.sql.SparkSQLRegressionTest33" -PenableSparkSQLITs
+./gradlew :spark-connector:spark-3.5:test --tests  "org.apache.gravitino.spark.connector.integration.test.sql.SparkSQLRegressionTest35" -PenableSparkSQLITs
 ```
 
 Please change the Spark version number if you want to test other Spark versions.

gradle/libs.versions.toml

@@ -67,14 +67,17 @@ commons-configuration1 = "1.6"
 commons-dbcp2 = "2.11.0"
 caffeine = "2.9.3"
 iceberg = '1.11.0'
-# Last Iceberg release compatible with Spark 3.3; Spark 3.3 deploy ITs cover basic CRUD only.
+# Last Iceberg release compatible with Spark 3.3.
 iceberg4spark33 = "1.8.1"
-iceberg4spark34 = "1.11.0"
 iceberg4spark35 = "1.11.0"
+iceberg4spark40 = "1.11.0"
 paimon = '1.2.0'
+# spark33 and spark34 are pinned by modules other than the Spark connector for their own tests, so
+# they stay after the connector dropped 3.3 and 3.4; they say nothing about connector support.
 spark33 = "3.3.4"
 spark34 = "3.4.3"
 spark35 = "3.5.3"
+spark40 = "4.0.3"
 kyuubi4spark = "1.11.0"
 kyuubi4authz = "1.10.2"
 trino = '435'

settings.gradle.kts

@@ -86,7 +86,6 @@ if (!skipTrinoConnector) {
 } else {
   println("Skipping trino-connector modules since skipTrinoConnector is set to true")
 }
-include("spark-connector:spark-common")
 if (scalaVersion == "2.12") {
   // flink only support scala 2.12
   include("flink-connector:flink-common")
@@ -108,14 +107,14 @@ if (scalaVersion == "2.12") {
   project(":flink-connector:flink-runtime-1.20").projectDir =
     file("flink-connector/v1.20/flink-runtime")
 }
-include("spark-connector:spark-3.3", "spark-connector:spark-runtime-3.3")
-project(":spark-connector:spark-3.3").projectDir = file("spark-connector/v3.3/spark")
-project(":spark-connector:spark-runtime-3.3").projectDir = file("spark-connector/v3.3/spark-runtime")
-include("spark-connector:spark-3.4", "spark-connector:spark-runtime-3.4", "spark-connector:spark-3.5", "spark-connector:spark-runtime-3.5")
-project(":spark-connector:spark-3.4").projectDir = file("spark-connector/v3.4/spark")
-project(":spark-connector:spark-runtime-3.4").projectDir = file("spark-connector/v3.4/spark-runtime")
+include("spark-connector:spark-3.5", "spark-connector:spark-runtime-3.5")
 project(":spark-connector:spark-3.5").projectDir = file("spark-connector/v3.5/spark")
 project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector/v3.5/spark-runtime")
+// Spark 4.0 support (#8771). Spark 4 is Scala 2.13 only and needs JDK 17, so these modules pin
+// 2.13 rather than reading -PscalaVersion.
+include("spark-connector:spark-4.0", "spark-connector:spark-runtime-4.0")
+project(":spark-connector:spark-4.0").projectDir = file("spark-connector/v4.0/spark")
+project(":spark-connector:spark-runtime-4.0").projectDir = file("spark-connector/v4.0/spark-runtime")
 include("web:web", "web:integration-test")
 include("web-v2:web", "web-v2:integration-test")
 include("docs")