[WIP][POC][GRAPHFRAMES] Add GraphFrames as an in-tree Spark module

.github/workflows/build_and_test.yml

@@ -164,7 +164,7 @@ jobs:
             docs=false
             java25=false
           fi
-          build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,credential-aws,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"`
+          build=`./dev/is-changed.py -m "core,unsafe,kvstore,avro,utils,utils-java,network-common,network-shuffle,repl,launcher,examples,sketch,variant,api,catalyst,hive-thriftserver,mllib-local,mllib,graphx,graphframes,streaming,sql-kafka-0-10,streaming-kafka-0-10,streaming-kinesis-asl,credential-aws,kubernetes,hadoop-cloud,spark-ganglia-lgpl,profiler,protobuf,yarn,connect,sql,hive,pipelines"`
           build_core_utils=`./dev/is-changed.py -m "core,unsafe,kvstore,utils,utils-java,network-common,network-shuffle,sketch,variant,launcher"`
           precondition="
             {
@@ -288,7 +288,7 @@ jobs:
           - >-
             api, catalyst, hive-thriftserver
           - >-
-            mllib-local, mllib, graphx, profiler, pipelines, repl, examples
+            mllib-local, mllib, graphx, graphframes, profiler, pipelines, repl, examples
           - >-
             streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl,
             credential-aws, kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf, connect, avro
@@ -625,6 +625,8 @@ jobs:
             pyspark-streaming, pyspark-structured-streaming, pyspark-structured-streaming-connect
           - >-
             pyspark-connect
+          - >-
+            pyspark-graphframes,pyspark-graphframes-connect
           - >-
             pyspark-connect-old-client
           - >-
@@ -642,6 +644,7 @@ jobs:
           - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark != 'true' && 'pyspark-mllib, pyspark-ml, pyspark-ml-connect' }}
           - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark != 'true' && 'pyspark-streaming, pyspark-structured-streaming, pyspark-structured-streaming-connect' }}
           - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark != 'true' && 'pyspark-connect' }}
+          - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark != 'true' && 'pyspark-graphframes,pyspark-graphframes-connect' }}
           - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark-connect-old-client != 'true' &&  'pyspark-connect-old-client'}}
           # pyspark-install is very slow so we only run it when it's changed or explicity requested
           - modules: ${{ fromJson(needs.precondition.outputs.required).pyspark-install != 'true' && 'pyspark-install' }}

.github/workflows/build_python_connect.yml

@@ -94,7 +94,7 @@ jobs:
           mv python/pyspark pyspark.back
 
           # Several tests related to catalog requires to run them sequencially, e.g., writing a table in a listener.
-          ./python/run-tests --parallelism=1 --python-executables=python3 --modules pyspark-connect,pyspark-ml-connect
+          ./python/run-tests --parallelism=1 --python-executables=python3 --modules pyspark-connect,pyspark-ml-connect,pyspark-graphframes-connect
           # None of tests are dependent on each other in Pandas API on Spark so run them in parallel
           ./python/run-tests --parallelism=1 --python-executables=python3 --modules pyspark-pandas-connect,pyspark-pandas-slow-connect
 

NOTICE

@@ -38,4 +38,13 @@ LongAdder), which was released with the following comments:
 
     Written by Doug Lea with assistance from members of JCP JSR-166
     Expert Group and released to the public domain, as explained at
-    http://creativecommons.org/publicdomain/zero/1.0/
+    http://creativecommons.org/publicdomain/zero/1.0/
+
+
+LDBC datasets
+-------------
+
+The LDBC datasets used by GraphFrames integration tests are licensed under the Apache
+Software License, Version 2.0. They are used for testing and evaluation purposes only.
+Per the LDBC fair-use policy, results must not be described using the words "LDBC benchmark"
+or any equivalent phrase.

assembly/pom.xml

@@ -64,6 +64,11 @@
       <artifactId>spark-graphx_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-graphframes_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.binary.version}</artifactId>

dev/sparktestsupport/modules.py

@@ -540,6 +540,17 @@ def __hash__(self):
     ],
 )
 
+graphframes = Module(
+    name="graphframes",
+    dependencies=[graphx, mllib],
+    source_file_regexes=[
+        "graphframes/",
+    ],
+    sbt_test_goals=[
+        "graphframes/test",
+    ],
+)
+
 pipelines = Module(
     name="pipelines",
     dependencies=[sql],
@@ -551,7 +562,7 @@ def __hash__(self):
 
 connect = Module(
     name="connect",
-    dependencies=[hive, avro, protobuf, mllib],
+    dependencies=[hive, avro, protobuf, graphframes, mllib],
     source_file_regexes=[
         "sql/connect",
     ],
@@ -576,7 +587,9 @@ def __hash__(self):
 pyspark_core = Module(
     name="pyspark-core",
     dependencies=[core],
-    source_file_regexes=["python/(?!pyspark/(ml|mllib|sql|streaming|pandas|resource|testing))"],
+    source_file_regexes=[
+        "python/(?!pyspark/(graphframes|ml|mllib|sql|streaming|pandas|resource|testing))"
+    ],
     python_test_goals=[
         # doctests
         "pyspark.conf",
@@ -1355,6 +1368,35 @@ def __hash__(self):
     ],
 )
 
+pyspark_graphframes = Module(
+    name="pyspark-graphframes",
+    dependencies=[pyspark_sql, graphframes],
+    source_file_regexes=[
+        "python/pyspark/graphframes",
+    ],
+    python_test_goals=[
+        "pyspark.graphframes.tests.test_graphframe",
+        "pyspark.graphframes.tests.test_graphframes",
+        "pyspark.graphframes.tests.pg.test_property_graphframe",
+    ],
+)
+
+pyspark_graphframes_connect = Module(
+    name="pyspark-graphframes-connect",
+    dependencies=[pyspark_connect, pyspark_graphframes],
+    source_file_regexes=[
+        "python/pyspark/graphframes/connect",
+        "python/pyspark/graphframes/tests/connect",
+    ],
+    python_test_goals=[
+        "pyspark.graphframes.tests.connect.test_client_imports",
+        "pyspark.graphframes.tests.connect.test_parity_graphframe",
+        "pyspark.graphframes.tests.connect.test_all_algorithms",
+        "pyspark.graphframes.tests.connect.test_upstream_graphframes",
+        "pyspark.graphframes.tests.connect.test_property_graphframe",
+    ],
+)
+
 pyspark_structured_streaming_connect = Module(
     name="pyspark-structured-streaming-connect",
     dependencies=[pyspark_connect, pyspark_structured_streaming],

dev/sparktestsupport/utils.py

@@ -105,20 +105,26 @@ def determine_modules_to_test(changed_modules, deduplicated=True):
     >>> [x.name for x in determine_modules_to_test([modules.launcher])]
     ['root']
     >>> [x.name for x in determine_modules_to_test([modules.graphx])]
-    ['graphx', 'examples']
+    ... # doctest: +NORMALIZE_WHITESPACE
+    ['graphx', 'examples', 'graphframes', 'connect', 'pyspark-graphframes', 'pyspark-connect',
+     'pyspark-graphframes-connect', 'pyspark-ml-connect', 'pyspark-pandas-connect',
+     'pyspark-pandas-slow-connect',
+     'pyspark-pipelines', 'pyspark-structured-streaming-connect']
     >>> sorted([x.name for x in determine_modules_to_test([modules.sql])])
     ... # doctest: +NORMALIZE_WHITESPACE
-    ['avro', 'connect', 'docker-integration-tests', 'examples', 'hive', 'hive-thriftserver',
-     'mllib', 'pipelines', 'protobuf', 'pyspark-connect', 'pyspark-ml', 'pyspark-ml-connect',
+    ['avro', 'connect', 'docker-integration-tests', 'examples', 'graphframes', 'hive',
+     'hive-thriftserver', 'mllib', 'pipelines', 'protobuf', 'pyspark-connect',
+     'pyspark-graphframes', 'pyspark-graphframes-connect', 'pyspark-ml', 'pyspark-ml-connect',
      'pyspark-mllib', 'pyspark-pandas', 'pyspark-pandas-connect', 'pyspark-pandas-slow',
      'pyspark-pandas-slow-connect', 'pyspark-pipelines', 'pyspark-sql',
      'pyspark-structured-streaming', 'pyspark-structured-streaming-connect',
      'pyspark-testing', 'repl', 'sparkr', 'sql', 'sql-kafka-0-10']
     >>> sorted([x.name for x in determine_modules_to_test(
     ...     [modules.sparkr, modules.sql], deduplicated=False)])
     ... # doctest: +NORMALIZE_WHITESPACE
-    ['avro', 'connect', 'docker-integration-tests', 'examples', 'hive', 'hive-thriftserver',
-     'mllib', 'pipelines', 'protobuf', 'pyspark-connect', 'pyspark-ml', 'pyspark-ml-connect',
+    ['avro', 'connect', 'docker-integration-tests', 'examples', 'graphframes', 'hive',
+     'hive-thriftserver', 'mllib', 'pipelines', 'protobuf', 'pyspark-connect',
+     'pyspark-graphframes', 'pyspark-graphframes-connect', 'pyspark-ml', 'pyspark-ml-connect',
      'pyspark-mllib', 'pyspark-pandas', 'pyspark-pandas-connect', 'pyspark-pandas-slow',
      'pyspark-pandas-slow-connect', 'pyspark-pipelines', 'pyspark-sql',
      'pyspark-structured-streaming', 'pyspark-structured-streaming-connect',
@@ -127,9 +133,10 @@ def determine_modules_to_test(changed_modules, deduplicated=True):
     ...     [modules.sql, modules.core], deduplicated=False)])
     ... # doctest: +NORMALIZE_WHITESPACE
     ['avro', 'catalyst', 'connect', 'core', 'credential-aws', 'docker-integration-tests',
-     'examples', 'graphx',
+     'examples', 'graphframes', 'graphx',
      'hive', 'hive-thriftserver', 'mllib', 'mllib-local', 'pipelines', 'protobuf',
-     'pyspark-connect', 'pyspark-core', 'pyspark-errors', 'pyspark-ml', 'pyspark-ml-connect',
+     'pyspark-connect', 'pyspark-core', 'pyspark-errors', 'pyspark-graphframes',
+     'pyspark-graphframes-connect', 'pyspark-ml', 'pyspark-ml-connect',
      'pyspark-mllib', 'pyspark-pandas', 'pyspark-pandas-connect', 'pyspark-pandas-slow',
      'pyspark-pandas-slow-connect', 'pyspark-pipelines', 'pyspark-resource', 'pyspark-sql',
      'pyspark-streaming', 'pyspark-structured-streaming', 'pyspark-structured-streaming-connect',

graphframes/pom.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+         http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.spark</groupId>
+    <artifactId>spark-parent_2.13</artifactId>
+    <version>5.0.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>spark-graphframes_2.13</artifactId>
+  <properties>
+    <sbt.project.name>graphframes</sbt.project.name>
+  </properties>
+  <packaging>jar</packaging>
+  <name>Spark Project GraphFrames</name>
+  <url>https://spark.apache.org/</url>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-graphx_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-mllib_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-core_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-sql_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-tags_${scala.binary.version}</artifactId>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+  </build>
+</project>