From 2c0b53a7cf6cf828b41bc9757f97a74e45cfe69c Mon Sep 17 00:00:00 2001 From: Jonathan Indig Date: Tue, 7 Jan 2025 18:31:41 -0500 Subject: [PATCH 1/9] Add get_active_kernel() fn --- .../python/PythonInterpreter.scala | 20 +++++-- .../python/PythonInterpreterSpec.scala | 58 +++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala index d6ac84cbb..b3fde3b6c 100644 --- a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala +++ b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala @@ -16,6 +16,7 @@ import polynote.kernel.task.TaskManager import polynote.kernel.util.DepsParser.flattenDeps import polynote.kernel.{BaseEnv, CompileErrors, Completion, CompletionType, GlobalEnv, InterpreterEnv, KernelReport, ParameterHint, ParameterHints, ResultValue, ScalaCompiler, Signatures} import polynote.messages.{CellID, DefinitionLocation, Notebook, NotebookConfig, ShortString, TinyList, TinyString} +import polynote.runtime.KernelRuntime import polynote.runtime.python.{PythonFunction, PythonObject, TypedPythonObject} import zio.ZIO.effect import zio.blocking.{Blocking, effectBlocking} @@ -666,6 +667,13 @@ class PythonInterpreter private[python] ( | import pandas as pd | return pd.MultiIndex.from_arrays([[x for x in list(l1)], [x for x in list(l2)]]) | + | def get_active_kernel(): + | ''' + | Returns the instance of the kernel runtime provided to the cell currently being executed. + | The value in `globals` gets set by the interpreter when it runs the cell. + | ''' + | return globals()["kernel"] + | |except Exception as e: | import sys, traceback | print("An error occurred while initializing the Python interpreter.", e, file=sys.stderr) @@ -703,13 +711,13 @@ class PythonInterpreter private[python] ( | def show(self): | fmt = PolynoteBackend.output_format | if fmt == 'svg': - | kernel.display.content("image/svg", self.svg()) + | get_active_kernel().display.content("image/svg", self.svg()) | elif fmt == 'png': - | kernel.display.content("image/png", self.png()) + | get_active_kernel().display.content("image/png", self.png()) | else: | print(f"PolynoteBackend: Unknown output format. Accepted values for PolynoteBackend.output_format are 'png' or 'svg' but got '{fmt}'. Defaulting to 'png'.",file=sys.stderr) | sys.stderr.flush() - | kernel.display.content("image/png", self.png()) + | get_active_kernel().display.content("image/png", self.png()) | | | @_Backend.export @@ -747,11 +755,11 @@ class PythonInterpreter private[python] ( jep => val setItem = globals.getAttr("__setitem__", classOf[PyCallable]) setItem.call("kernel", runtime) - // TODO: Also update the global `kernel` so matplotlib integration (and other libraries?) can access the - // correct kernel. We may want to have a better way to do this though, like a first class getKernel() - // function for libraries (this only solves the problem for python) jep.set("kernel", runtime) + // make visible to cell globals + setItem.call("get_active_kernel", jep.getValue("get_active_kernel")) + // expose `PolynoteBackend` so users can set `PolynoteBackend.output_format` // `PolynoteBackend` is not defined if `matplotlib` is not present. Try(jep.getValue("PolynoteBackend", classOf[PyObject])).foreach { diff --git a/polynote-kernel/src/test/scala/polynote/kernel/interpreter/python/PythonInterpreterSpec.scala b/polynote-kernel/src/test/scala/polynote/kernel/interpreter/python/PythonInterpreterSpec.scala index 0e6b0fd87..88373972d 100644 --- a/polynote-kernel/src/test/scala/polynote/kernel/interpreter/python/PythonInterpreterSpec.scala +++ b/polynote-kernel/src/test/scala/polynote/kernel/interpreter/python/PythonInterpreterSpec.scala @@ -479,7 +479,65 @@ class PythonInterpreterSpec extends FreeSpec with Matchers with InterpreterSpec } } + "should properly handle kernel display APIs" in { + val code = + """ + |kernel.display.html("

HTML

") + |kernel.display.content("image/png", "base64encodedpng") + |mykernel = kernel + |""".stripMargin + assertOutput(code) { case (vars, output) => + vars("mykernel").toString should include("polynote.runtime.KernelRuntime") + output should contain theSameElementsInOrderAs(Seq( + Output("text/html", Vector("

HTML

")), + Output("image/png", Vector("base64encodedpng")), + )) + } + } + + "should be able to retrieve the current active kernel dynamically" in { + val cell1 = interp( + """ + |cell1_kernel = kernel + |cell1_active_kernel = get_active_kernel() + |def get_cell1_kernel(): + | return kernel + |def get_cell1_active_kernel(): + | return get_active_kernel() + |def print_fancy(s): + | get_active_kernel().display.html(f"{s}")""".stripMargin).run(cellState).runIO() + val cell2 = interp( + """ + |cell2_kernel = kernel + |cell2_active_kernel = get_active_kernel() + |kernel_from_cell1 = get_cell1_kernel() + |active_kernel_from_cell1 = get_cell1_active_kernel() + |print_fancy("hi there") + |""".stripMargin).run(cell1._1).runIO() + + val cell1Kernel :: cell1ActiveKernel :: getCell1Kernel :: getCell1ActiveKernel :: printFancy :: Nil = cell1._2.state.values + cell1Kernel.value.toString should include("polynote.runtime.KernelRuntime") + cell1ActiveKernel.value.toString should include("polynote.runtime.KernelRuntime") + cell1Kernel.value shouldEqual cell1ActiveKernel.value + getCell1Kernel.value shouldBe a[PythonFunction] + getCell1ActiveKernel.value shouldBe a[PythonFunction] + printFancy.value shouldBe a[PythonFunction] + // note - empty even though print_fancy is defined here! + cell1._2.env.publishResult.toList.runIO() shouldBe empty + + val cell2Kernel :: cell2ActiveKernel :: kernelFromCell1 :: activeKernelFromCell1 :: Nil = cell2._2.state.values + cell2Kernel.value.toString should include("polynote.runtime.KernelRuntime") + cell2ActiveKernel.value.toString should include("polynote.runtime.KernelRuntime") + cell2Kernel.value shouldEqual cell2ActiveKernel.value + kernelFromCell1.value.toString should include("polynote.runtime.KernelRuntime") + activeKernelFromCell1.value.toString should include("polynote.runtime.KernelRuntime") + cell2ActiveKernel.value shouldEqual activeKernelFromCell1.value + // note - print_fancy correctly found the active kernel + cell2._2.env.publishResult.toList.runIO() should contain theSameElementsInOrderAs(Seq( + Output("text/html", Vector("hi there")), + )) + } } "PythonObject" - { From 8a30e515fd3ffe2bc6e2524d8fe84a587c80e495 Mon Sep 17 00:00:00 2001 From: Jonathan Indig Date: Tue, 7 Jan 2025 20:23:31 -0500 Subject: [PATCH 2/9] forgot to add the figure canvas --- .../interpreter/python/PythonInterpreter.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala index b3fde3b6c..b8b64c9ad 100644 --- a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala +++ b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala @@ -693,6 +693,9 @@ class PythonInterpreter private[python] ( | import io, base64 | | class PolynoteFigureManager(FigureManagerBase): + | def _display(self, mime, body): + | get_active_kernel().display.content(mime, body) + | | def png(self): | # Save figure as PNG for display | buf = io.BytesIO() @@ -711,20 +714,23 @@ class PythonInterpreter private[python] ( | def show(self): | fmt = PolynoteBackend.output_format | if fmt == 'svg': - | get_active_kernel().display.content("image/svg", self.svg()) + | self._display("image/svg", self.svg()) | elif fmt == 'png': - | get_active_kernel().display.content("image/png", self.png()) + | self._display("image/png", self.png()) | else: | print(f"PolynoteBackend: Unknown output format. Accepted values for PolynoteBackend.output_format are 'png' or 'svg' but got '{fmt}'. Defaulting to 'png'.",file=sys.stderr) | sys.stderr.flush() - | get_active_kernel().display.content("image/png", self.png()) + | self._display("image/png", self.png()) | + | class PolynoteFigureCanvas(FigureCanvasBase): + | manager_class = PolynoteFigureManager | | @_Backend.export | class PolynoteBackend(_BackendAgg): | __module__ = "polynote" | | FigureManager = PolynoteFigureManager + | FigureCanvas = PolynoteFigureCanvas | | output_format = 'png' # options are ['png', 'svg'] | From fbaa125239de913253cedc3895e230cb2446beee Mon Sep 17 00:00:00 2001 From: Jonathan Indig Date: Wed, 8 Jan 2025 10:51:23 -0500 Subject: [PATCH 3/9] oops --- .../polynote/kernel/interpreter/python/PythonInterpreter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala index b8b64c9ad..3bd60fa3f 100644 --- a/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala +++ b/polynote-kernel/src/main/scala/polynote/kernel/interpreter/python/PythonInterpreter.scala @@ -688,7 +688,7 @@ class PythonInterpreter private[python] ( | import matplotlib | | from matplotlib._pylab_helpers import Gcf - | from matplotlib.backend_bases import (_Backend, FigureManagerBase) + | from matplotlib.backend_bases import (_Backend, FigureCanvasBase, FigureManagerBase) | from matplotlib.backends.backend_agg import _BackendAgg | import io, base64 | From be1998e348d103b4d363927538e7a4a8026ccbc2 Mon Sep 17 00:00:00 2001 From: Jonathan Indig Date: Tue, 28 Jan 2025 16:18:07 -0500 Subject: [PATCH 4/9] add python 310 --- .github/workflows/ci-backend-2.12.yml | 2 +- .github/workflows/ci-backend-2.13.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-backend-2.12.yml b/.github/workflows/ci-backend-2.12.yml index 6f00bc6c7..33c535ce8 100644 --- a/.github/workflows/ci-backend-2.12.yml +++ b/.github/workflows/ci-backend-2.12.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.10' architecture: 'x64' - uses: actions/setup-java@v1 with: diff --git a/.github/workflows/ci-backend-2.13.yml b/.github/workflows/ci-backend-2.13.yml index fcdfad6ac..66cb3e7e7 100644 --- a/.github/workflows/ci-backend-2.13.yml +++ b/.github/workflows/ci-backend-2.13.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.10' architecture: 'x64' - uses: actions/setup-java@v1 with: From f7227de8d4e98097b9735933d952c7cbaed3abc0 Mon Sep 17 00:00:00 2001 From: jeremyrsmith Date: Wed, 29 Jan 2025 10:03:41 -0800 Subject: [PATCH 5/9] Use wget to download spark dist --- build.sbt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 0c86fb858..3bbaac43c 100644 --- a/build.sbt +++ b/build.sbt @@ -291,9 +291,10 @@ val sparkSettings = Seq( } else { val pkgFile = baseDir / filename if (!pkgFile.exists()) { - pkgFile.createNewFile() - println(s"Downloading $distUrl to $pkgFile...") - (distUrl #> pkgFile).!! + println(s"Downloading $distUrl to $pkgFile") + val cmd = Seq("wget", "-O", pkgFile.toString, "-nv", distUrl.toString) + println(cmd.mkString(" ")) + cmd.! } println(s"Verifying checksum for $pkgFile for $distVersion...") From fda97c5468c8a3377082c0171fd9ab532cbf3bfc Mon Sep 17 00:00:00 2001 From: jeremyrsmith Date: Wed, 29 Jan 2025 10:15:48 -0800 Subject: [PATCH 6/9] Limit concurrency of tests Trying to eliminate crashing in CI --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 3bbaac43c..1219f8436 100644 --- a/build.sbt +++ b/build.sbt @@ -83,6 +83,7 @@ val commonSettings = Seq( ), Test / fork := true, Test / javaOptions += s"-Djava.library.path=$nativeLibraryPath", + Global / concurrentRestrictions += Tags.limit(Tags.Test, 1), libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.0.8" % "test", "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" From 599537f83168d72ff3ff7ff1557a278cb76dc531 Mon Sep 17 00:00:00 2001 From: jeremyrsmith Date: Wed, 29 Jan 2025 10:49:16 -0800 Subject: [PATCH 7/9] Disable parallel execution altogether --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1219f8436..697e9e8f8 100644 --- a/build.sbt +++ b/build.sbt @@ -83,7 +83,7 @@ val commonSettings = Seq( ), Test / fork := true, Test / javaOptions += s"-Djava.library.path=$nativeLibraryPath", - Global / concurrentRestrictions += Tags.limit(Tags.Test, 1), + Test / parallelExecution := false, libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.0.8" % "test", "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" From 04805d97bd85784429bf9d78b05f6e44af8f9deb Mon Sep 17 00:00:00 2001 From: jeremyrsmith Date: Wed, 29 Jan 2025 10:59:09 -0800 Subject: [PATCH 8/9] Add more memory to test fork --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 697e9e8f8..643d16d07 100644 --- a/build.sbt +++ b/build.sbt @@ -82,7 +82,7 @@ val commonSettings = Seq( if (sys.props.get("java.version").exists(_.startsWith("1.8"))) Nil else Seq("-release", "8") ), Test / fork := true, - Test / javaOptions += s"-Djava.library.path=$nativeLibraryPath", + Test / javaOptions ++= Seq(s"-Djava.library.path=$nativeLibraryPath", "-Xmx8G") Test / parallelExecution := false, libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.0.8" % "test", From e10ba25f565006e6db57927a945d43b6b59773a0 Mon Sep 17 00:00:00 2001 From: jeremyrsmith Date: Wed, 29 Jan 2025 11:02:35 -0800 Subject: [PATCH 9/9] missed a comma --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 643d16d07..d1d04a7e2 100644 --- a/build.sbt +++ b/build.sbt @@ -82,7 +82,7 @@ val commonSettings = Seq( if (sys.props.get("java.version").exists(_.startsWith("1.8"))) Nil else Seq("-release", "8") ), Test / fork := true, - Test / javaOptions ++= Seq(s"-Djava.library.path=$nativeLibraryPath", "-Xmx8G") + Test / javaOptions ++= Seq(s"-Djava.library.path=$nativeLibraryPath", "-Xmx8G"), Test / parallelExecution := false, libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "3.0.8" % "test",