From ca551c23dfc0a70e79e8c863c7ea63def0ac24b5 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Wed, 6 Aug 2025 22:41:51 -0700 Subject: [PATCH 1/4] check engine context stopped --- .../spark/session/SparkSQLSessionManager.scala | 4 ++++ .../apache/kyuubi/session/SessionManager.scala | 15 +++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala index a4048b78b94..2cb7ca9a08f 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala @@ -227,4 +227,8 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession) opHandle: OperationHandle): Path = { new Path(getSessionResultSavePath(sessionHandle), opHandle.identifier.toString) } + + override private[kyuubi] def isEngineContextStopped = { + spark.sparkContext.isStopped + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 7751b7298e4..432a1d85633 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -345,10 +345,15 @@ abstract class SessionManager(name: String) extends CompositeService(name) { if (idleTimeout > 0) { val checkTask = new Runnable { override def run(): Unit = { - if (!shutdown && System.currentTimeMillis() - latestLogoutTime > idleTimeout && - getActiveUserSessionCount <= 0) { - info(s"Idled for more than $idleTimeout ms, terminating") - stop() + if (!shutdown) { + if (System.currentTimeMillis() - latestLogoutTime > idleTimeout && + getActiveUserSessionCount <= 0) { + info(s"Idled for more than $idleTimeout ms, terminating") + stop() + } else if (isEngineContextStopped) { + error(s"Engine context is stopped, terminating") + stop() + } } } } @@ -360,4 +365,6 @@ abstract class SessionManager(name: String) extends CompositeService(name) { TimeUnit.MILLISECONDS) } } + + private[kyuubi] def isEngineContextStopped: Boolean = false } From cf9e40ef6a455399fb3e96115ca36fc3e045f766 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Thu, 7 Aug 2025 01:08:09 -0700 Subject: [PATCH 2/4] no hard code --- .../main/scala/org/apache/kyuubi/session/SessionManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 432a1d85633..507b295a80a 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -351,7 +351,7 @@ abstract class SessionManager(name: String) extends CompositeService(name) { info(s"Idled for more than $idleTimeout ms, terminating") stop() } else if (isEngineContextStopped) { - error(s"Engine context is stopped, terminating") + error(s"Engine's SparkContext is stopped, terminating") stop() } } @@ -366,5 +366,5 @@ abstract class SessionManager(name: String) extends CompositeService(name) { } } - private[kyuubi] def isEngineContextStopped: Boolean = false + private[kyuubi] def isEngineContextStopped: Boolean = !isServer && shutdown } From cd542decb713a010624ad9a1a273a0ed3727c709 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Thu, 7 Aug 2025 01:19:59 -0700 Subject: [PATCH 3/4] Revert "no hard code" This reverts commit cf9e40ef6a455399fb3e96115ca36fc3e045f766. --- .../main/scala/org/apache/kyuubi/session/SessionManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 507b295a80a..432a1d85633 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -351,7 +351,7 @@ abstract class SessionManager(name: String) extends CompositeService(name) { info(s"Idled for more than $idleTimeout ms, terminating") stop() } else if (isEngineContextStopped) { - error(s"Engine's SparkContext is stopped, terminating") + error(s"Engine context is stopped, terminating") stop() } } @@ -366,5 +366,5 @@ abstract class SessionManager(name: String) extends CompositeService(name) { } } - private[kyuubi] def isEngineContextStopped: Boolean = !isServer && shutdown + private[kyuubi] def isEngineContextStopped: Boolean = false } From 835cb3dec3f66ed4d65fc0e9c24adc64e7cd06d5 Mon Sep 17 00:00:00 2001 From: "Wang, Fei" Date: Thu, 7 Aug 2025 01:24:07 -0700 Subject: [PATCH 4/4] SparkContext --- .../main/scala/org/apache/kyuubi/session/SessionManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 432a1d85633..217cfc707c9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -351,7 +351,7 @@ abstract class SessionManager(name: String) extends CompositeService(name) { info(s"Idled for more than $idleTimeout ms, terminating") stop() } else if (isEngineContextStopped) { - error(s"Engine context is stopped, terminating") + error(s"Engine's SparkContext is stopped, terminating") stop() } }