graphframes · rjurney · Feb 20, 2025 · Feb 16, 2025 · Feb 16, 2025 · Feb 16, 2025
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -31,12 +31,20 @@ jobs:
     - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install python dependencies
+    - name: Install and configure Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 2.1.1
+        virtualenvs-create: true
+        virtualenvs-in-project: false
+        installer-parallel: true
+    - name: Build Python package and its dependencies
+      working-directory: ./python
       run: |
-        python -m pip install --upgrade pip wheel
-        pip install -r ./python/requirements.txt
-        pip install pyspark==${{ matrix.spark-version }}
+        poetry build
+        poetry install
     - name: Test
+      working-directory: ./python
       run: |
-        export SPARK_HOME=$(python -c "import os; from importlib.util import find_spec; print(os.path.join(os.path.dirname(find_spec('pyspark').origin)))")
-        ./python/run-tests.sh
+        export SPARK_HOME=$(poetry run python -c "import os; from importlib.util import find_spec; spec = find_spec('pyspark'); print(os.path.join(os.path.dirname(spec.origin)))")
+        ./run-tests.sh
diff --git a/.gitignore b/.gitignore
@@ -27,3 +27,12 @@ project/plugins/project/
 
 # Mac
 *.DS_Store
+
+# Python specific
+python/build
+python/dist
+build/lib
+python/graphframes.egg-info
+python/graphframes/tutorials/data
+python/docs/_build
+python/docs/_site
diff --git a/VERSION b/VERSION
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
@@ -2,3 +2,8 @@
 # https://github.com/pypa/sampleproject/blob/master/MANIFEST.in
 # For more details about the MANIFEST file, you may read the docs at
 # https://docs.python.org/2/distutils/sourcedist.html#the-manifest-in-template
+recursive-include python/graphframes *.py
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+include README.md
+include LICENSE
diff --git a/python/README.md b/python/README.md
@@ -0,0 +1,17 @@
+# GraphFrames `graphframes-py` Python Package
+
+The is the officila [graphframes-py PyPI package](https://pypi.org/project/graphframes-py/), which is a Python wrapper for the Scala GraphFrames library. This package is maintained by the GraphFrames project and is available on PyPI.
+
+For instructions on GraphFrames, check the project [../README.md](../README.md). See [Installation and Quick-Start](#installation-and-quick-start) for the best way to install and use GraphFrames.
+
+## Running `graphframes-py`
+
+You should use GraphFrames via the `--packages` argument to `pyspark` or `spark-submit`, but this package is helpful in development environments.
+
+```bash
+# Interactive Python
+$ pyspark --packages graphframes:graphframes:0.8.4-spark3.5-s_2.12
+
+# Submit a script in Scala/Java/Python
+$ spark-submit --packages graphframes:graphframes:0.8.4-spark3.5-s_2.12 script.py
+```
diff --git a/python/VERSION b/python/VERSION
@@ -0,0 +1 @@
+0.8.4
diff --git a/python/poetry.lock b/python/poetry.lock
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -0,0 +1,48 @@
+[tool.poetry]
+name = "graphframes-py"
+version = "0.8.4"
+description = "GraphFrames: Graph Processing Framework for Apache Spark"
+authors = ["GraphFrames Contributors <[email protected]>"]
+license = "Apache 2.0"
+readme = "README.md"
+packages = [{include = "graphframes"}]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12"
+]
+
+[tool.poetry.urls]
+"Project Homepage" = "https://graphframes.github.io/graphframes"
+"PyPi Homepage" = "https://pypi.org/project/graphframes-py"
+"Code Repository" = "https://github.com/graphframes/graphframes"
+"Bug Tracker" = "https://github.com/graphframes/graphframes/issues"
+
+[tool.poetry.dependencies]
+python = ">=3.9 <3.13"
+nose = "1.3.7"
+pyspark = "^3.4"
+numpy = ">= 1.7"
+
+[tool.poetry.group.dev.dependencies]
+black = "^25.1.0"
+flake8 = "^7.1.1"
+isort = "^6.0.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+line-length = 100
+target-version = ["py39"]
+include = ["graphframes"]
+
+[tool.isort]
+profile = "black"
+src_paths = ["graphframes"]
diff --git a/python/requirements.txt b/python/requirements.txt
diff --git a/python/run-tests.sh b/python/run-tests.sh
@@ -38,7 +38,7 @@ echo $pyver
 
 LIBS=""
 for lib in "$SPARK_HOME/python/lib"/*zip ; do
-  LIBS=$LIBS:$lib
+    LIBS=$LIBS:$lib
 done
 
 # The current directory of the script.
@@ -51,7 +51,7 @@ assembly_path="$DIR/../target/scala-$scala_version_major_minor"
 echo `ls $assembly_path/graphframes-assembly*.jar`
 JAR_PATH=""
 for assembly in $assembly_path/graphframes-assembly*.jar ; do
-  JAR_PATH=$assembly
+    JAR_PATH=$assembly
 done
 
 export PYSPARK_SUBMIT_ARGS="--driver-memory 2g --executor-memory 2g --jars $JAR_PATH pyspark-shell "
@@ -64,14 +64,14 @@ export PYTHONPATH=$PYTHONPATH:graphframes
 # Run test suites
 
 if [[ "$python_major" == "2" ]]; then
-
-  # Horrible hack for spark 1.x: we manually remove some log lines to stay below the 4MB log limit on Travis.
-  $PYSPARK_DRIVER_PYTHON `which nosetests` -v --all-modules -w $DIR 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
-
+    
+    # Horrible hack for spark 1.x: we manually remove some log lines to stay below the 4MB log limit on Travis.
+    poetry run python `which nosetests` -v --all-modules -w $DIR 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
+    
 else
-
-  $PYSPARK_DRIVER_PYTHON -m "nose" -v --all-modules -w $DIR 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
-
+    
+    poetry run python -m "nose" -v --all-modules -w $DIR 2>&1 | grep -vE "INFO (ParquetOutputFormat|SparkContext|ContextCleaner|ShuffleBlockFetcherIterator|MapOutputTrackerMaster|TaskSetManager|Executor|MemoryStore|CacheManager|BlockManager|DAGScheduler|PythonRDD|TaskSchedulerImpl|ZippedPartitionsRDD2)";
+    
 fi
 
 # Exit immediately if the tests fail.
@@ -83,4 +83,4 @@ test ${PIPESTATUS[0]} -eq 0 || exit 1;
 
 cd "$DIR"
 
-$PYSPARK_PYTHON -u ./graphframes/graphframe.py "$@"
+poetry run python -u ./graphframes/graphframe.py "$@"
diff --git a/python/setup.cfg b/python/setup.cfg
diff --git a/python/setup.py b/python/setup.py