graphframes · SemyonSinchenko · Apr 15, 2025 · Apr 6, 2025 · Apr 6, 2025 · Apr 6, 2025
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -0,0 +1,95 @@
+# This file is inspired by:
+# https://github.com/actions/starter-workflows/blob/main/pages/jekyll.yml
+# MIT License
+#
+# Copyright (c) 2020 GitHub
+
+name: Deploy Docs
+
+on:
+  push:
+    branches: [$default-branch]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.1'
+          bundler-cache: true
+          cache-version: 0
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: "11"
+
+      - name: Setup Jekyll
+        run: |
+          gem install jekyll jekyll-redirect-from bundler
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 2.1.1
+          virtualenvs-create: true
+          virtualenvs-in-project: false
+          installer-parallel: true
+
+      - name: Configure poetry shell and deps
+        working-directory: ./python
+        run: |
+          poetry self add poetry-plugin-shell
+          poetry install --with=dev,docs
+
+      - name: Build Python API docs
+        working-directory: ./python
+        run: |
+          poetry run bash -c "cd docs && make clean && make html"
+
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
+
+      - name: Build with Jekyll
+        working-directory: ./docs
+        run: jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          SKIP_SCALADOC: 0
+          SKIP_PYTHONDOC: 0
+          PRODUCTION: 1
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./docs/_site
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
diff --git a/docs/Dockerfile b/docs/Dockerfile
diff --git a/docs/README.md b/docs/README.md
@@ -1,56 +1,25 @@
 Welcome to the GraphFrames Spark Package documentation!
 
-This readme will walk you through navigating and building the GraphFrames documentation, which is included here with the source code.
+# Build and deploy documentation
 
-Read on to learn more about viewing documentation in plain text (i.e., markdown) or building the documentation yourself. Why build it yourself? So that you have the docs that correspond to whichever version of GraphFrames you currently have checked out of revision control.
+Documentation of GraphFrames package is build automatically during the CI.
 
-## Generating the Documentation HTML
+# Build the documentation locally
 
-We include the GraphFrames documentation as part of the source (as opposed to using a hosted wiki, such as the github wiki, as the definitive documentation) to enable the documentation to evolve along with the source code and be captured by revision control (currently git). This way the code automatically
-includes the version of the documentation that is relevant regardless of which version or release you have checked out or downloaded.
+## Build Python API docs
 
-In this directory you will find textfiles formatted using Markdown, with an ".md" suffix. You can read those text files directly if you want. Start with index.md.
+From the `./python` directory run the following:
 
-The markdown code can be compiled to HTML using the [Jekyll tool](http://jekyllrb.com).
-`Jekyll` and a few dependencies must be installed for this to work. We recommend installing via the Ruby Gem dependency manager. Since the exact HTML output varies between versions of Jekyll and its dependencies, we list specific versions here in some cases:
+```sh
+poetry run bash -c "cd docs && make clean && make html"
+```
 
-    $ gem install jekyll
-    $ gem install jekyll-redirect-from
+## Build core documentation
 
-On macOS, with the default Ruby, please install Jekyll with Bundler as [instructed on offical website](https://jekyllrb.com/docs/quickstart/). Otherwise the build script might fail to resolve dependencies.
+From the root of the project:
 
-    $ gem install jekyll bundler
-    $ gem install jekyll-redirect-from
-
-Execute `jekyll build` from the `docs/` directory to compile the site. Compiling the site with Jekyll will create a directory called `_site` containing index.html as well as the rest of the compiled files.
-
-You can modify the default Jekyll build as follows:
-
-    # Skip generating API docs (which takes a while)
-    $ SKIP_API=1 jekyll build
-    # Serve content locally on port 4000
-    $ jekyll serve --watch
-    # Build the site with extra features used on the live page
-    $ PRODUCTION=1 jekyll build
-
-Note that `SPARK_HOME` must be set to your local Spark installation in order to generate the docs.
-
-To manually point to a specific `Spark` installation,
-    $ SPARK_HOME=<your-path-to-spark-home> PRODUCTION=1 jekyll build
-
-## Sphinx
-
-We use Sphinx to generate Python API docs, so you will need to install it by running (once we upgrade to Python 3.10 it will get added to the dev requirements):
-
-    pip install sphinx
-
-## API Docs (Scaladoc, Sphinx)
-
-You can build just the scaladoc by running `build/sbt unidoc` from the GRAPHFRAMES_PROJECT_ROOT directory.
-
-Similarly, you can build just the Python docs by running `make html` from the GRAPHFRAMES_PROJECT_ROOT/python/docs directory. Documentation is only generated for classes that are listed as public in `__init__.py`.
-
-When you run `jekyll` in the `docs` directory, it will also copy over the scaladoc for the various subprojects into the `docs` directory (and then also into the `_site` directory). We use a jekyll plugin to run `build/sbt unidoc` before building the site so if you haven't run it (recently) it may take some time as it generates all of the scaladoc.  The jekyll plugin also generates the
-Python docs [Sphinx](http://sphinx-doc.org/).
-
-NOTE: To skip the step of building and copying over the Scala, Python API docs, run `SKIP_API=1 jekyll build`. To skip building Scala API docs, run `SKIP_SCALADOC=1 jekyll build`; to skip building Python API docs, run `SKIP_PYTHONDOC=1 jekyll build`.
+```sh
+gem install jekyll jekyll-redicrect-from bundler
+cd docs
+jukyll build
+```
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
@@ -59,22 +59,8 @@
   end
 
   if not (ENV['SKIP_PYTHONDOC'] == '1')
-    # Build Sphinx docs for Python
-
-    # Get and set release version
-    version = File.foreach('_config.yml').grep(/^GRAPHFRAMES_VERSION: (.+)$/){$1}.first
-    version ||= 'Unknown'
-
-    puts "Moving to python/docs directory and building sphinx."
-    cd("../python/docs")
-    if not (ENV['SPARK_HOME'])
-      raise("Python API docs cannot be generated if SPARK_HOME is not set.")
-    end
-    system({"PACKAGE_VERSION"=>version}, "make clean") || raise("Python doc clean failed")
-    system({"PACKAGE_VERSION"=>version}, "make html") || raise("Python doc generation failed")
-
     puts "Moving back into home dir."
-    cd("../../")
+    cd("..")
 
     puts "Making directory api/python"
     mkdir_p "docs/api/python"