Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Improve handling of ERROR Watch events #352

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
doriordan merged 1 commit into from
May 11, 2023
Merged

Improve handling of ERROR Watch events #352

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions client/src/main/scala/skuber/api/client/KubernetesClient.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,10 +232,11 @@ trait KubernetesClient {
* applicable type (e.g. PodList, DeploymentList) and then supplies that to this method to receive any future updates. If no resource version is specified,
* a single ADDED event will be produced for an already existing object followed by events for any future changes.
* @param bufSize optional buffer size for received object updates, normally the default is more than enough
* @param errorHandler an optional function that takes a single string parameter - it will be invoked with the error details whenever ERROR events are received
* @tparam O the type of the resource
* @return A future containing an Akka streams Source of WatchEvents that will be emitted
*/
def watchContinuously[O <: ObjectResource](name: String, sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
def watchContinuously[O <: ObjectResource](name: String, sinceResourceVersion: Option[String] = None, bufSize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _]

/**
Expand All @@ -248,10 +249,11 @@ trait KubernetesClient {
* applicable type (e.g. PodList, DeploymentList) and then supplies that to this method to receive any future updates. If no resource version is specified,
* a single ADDED event will be produced for an already existing object followed by events for any future changes.
* @param bufSize optional buffer size for received object updates, normally the default is more than enough
* @param errorHandler an optional function that takes a single string parameter - it will be invoked with the error details whenever ERROR events are received
* @tparam O the type pf the resource
* @return A future containing an Akka streams Source of WatchEvents that will be emitted
*/
def watchAllContinuously[O <: ObjectResource](sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
def watchAllContinuously[O <: ObjectResource](sinceResourceVersion: Option[String] = None, bufSize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _]

/**
Expand All @@ -261,10 +263,11 @@ trait KubernetesClient {
* for the meaning of the options. Note that the `watch` flag in the options will be ignored / overridden by the client, which
* ensures a watch is always requested on the server.
* @param bufsize optional buffer size for received object updates, normally the default is more than enough
* @param errorHandler an optional function that takes a single string parameter - it will be invoked with the error details whenever ERROR events are received
* @tparam O the resource type to watch
* @return A future containing an Akka streams Source of WatchEvents that will be emitted
*/
def watchWithOptions[O <: ObjectResource](options: ListOptions, bufsize: Int = 10000)(
def watchWithOptions[O <: ObjectResource](options: ListOptions, bufsize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _]

/**
Expand Down
16 changes: 8 additions & 8 deletions client/src/main/scala/skuber/api/client/impl/KubernetesClientImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,14 +470,14 @@ class KubernetesClientImpl private[client] (
override def watch[O <: ObjectResource](name: String, sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Future[Source[WatchEvent[O], _]] =
{
Watch.events(this, name, sinceResourceVersion, bufSize)
Watch.events(this, name, sinceResourceVersion, bufSize, None)
}

// watch events on all objects of specified kind in current namespace
override def watchAll[O <: ObjectResource](sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Future[Source[WatchEvent[O], _]] =
{
Watch.eventsOnKind[O](this, sinceResourceVersion, bufSize)
Watch.eventsOnKind[O](this, sinceResourceVersion, bufSize, None)
}

override def watchContinuously[O <: ObjectResource](obj: O)(
Expand All @@ -486,24 +486,24 @@ class KubernetesClientImpl private[client] (
watchContinuously(obj.name)
}

override def watchContinuously[O <: ObjectResource](name: String, sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
override def watchContinuously[O <: ObjectResource](name: String, sinceResourceVersion: Option[String] = None, bufSize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _] =
{
val options=ListOptions(resourceVersion = sinceResourceVersion, timeoutSeconds = Some(watchContinuouslyRequestTimeout.toSeconds) )
WatchSource(this, buildLongPollingPool(), Some(name), options, bufSize)
WatchSource(this, buildLongPollingPool(), Some(name), options, bufSize, errorHandler)
}

override def watchAllContinuously[O <: ObjectResource](sinceResourceVersion: Option[String] = None, bufSize: Int = 10000)(
override def watchAllContinuously[O <: ObjectResource](sinceResourceVersion: Option[String] = None, bufSize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _] =
{
val options=ListOptions(resourceVersion = sinceResourceVersion, timeoutSeconds = Some(watchContinuouslyRequestTimeout.toSeconds))
WatchSource(this, buildLongPollingPool(), None, options, bufSize)
WatchSource(this, buildLongPollingPool(), None, options, bufSize, errorHandler)
}

override def watchWithOptions[O <: skuber.ObjectResource](options: ListOptions, bufsize: Int = 10000)(
override def watchWithOptions[O <: skuber.ObjectResource](options: ListOptions, bufsize: Int = 10000, errorHandler: Option[String => _] = None)(
implicit fmt: Format[O], rd: ResourceDefinition[O], lc: LoggingContext): Source[WatchEvent[O], _] =
{
WatchSource(this, buildLongPollingPool(), None, options, bufsize)
WatchSource(this, buildLongPollingPool(), None, options, bufsize, errorHandler)
}

private def buildLongPollingPool[O <: ObjectResource]() = {
Expand Down
29 changes: 24 additions & 5 deletions client/src/main/scala/skuber/api/watch/BytesToWatchEventSource.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,42 @@
package skuber.api.watch

import akka.stream.scaladsl.{JsonFraming, Source}
import akka.NotUsed
import akka.stream.scaladsl.{JsonFraming, Sink, Source}
import akka.util.ByteString
import play.api.libs.json.{Format, JsError, JsSuccess, Json}
import play.api.libs.json.{Format, JsError, JsObject, JsString, JsSuccess, JsValue, Json}
import skuber.ObjectResource
import skuber.api.client.{K8SException, Status, WatchEvent}
import skuber.api.client.impl.KubernetesClientImpl
import skuber.api.client.{K8SException, LoggingContext, Status, WatchEvent}

import scala.concurrent.ExecutionContext

/**
* Convert a source of bytes to a source of watch events of type O
*/
private[api] object BytesToWatchEventSource {
def apply[O <: ObjectResource](bytesSource: Source[ByteString, _], bufSize: Int)(implicit ec: ExecutionContext, format: Format[O]): Source[WatchEvent[O], _] = {
def apply[O <: ObjectResource](client: KubernetesClientImpl, bytesSource: Source[ByteString, _], bufSize: Int, errorHandlerOpt: Option[String => _] = None)(implicit ec: ExecutionContext, format: Format[O], lc: LoggingContext): Source[WatchEvent[O], _] = {
import skuber.json.format.apiobj.watchEventFormat
bytesSource.via(
JsonFraming.objectScanner(bufSize)
).map { singleEventBytes =>
Json.parse(singleEventBytes.utf8String).validate(watchEventFormat[O]) match {
Json.parse(singleEventBytes.utf8String).as[JsObject]
}.filter {
case JsObject(underlying) if underlying.get("type").contains(JsString("ERROR")) =>
// handle error events, removing them from downstream emitted elements
val handled = for {
errorHandler <- errorHandlerOpt
errorObj <- underlying.get("object").flatMap(_.asOpt[JsObject])
} yield {
errorHandler(errorObj.toString)
}
if (!handled.isDefined) {
// no error handler, just log instead
client.logWarn(s"Watcher received ERROR event: $underlying")
}
false
case _ => true
}.map { eventJson =>
eventJson.validate(watchEventFormat[O]) match {
case JsSuccess(value, _) => value
case JsError(e) => throw new K8SException(Status(message = Some("Error parsing watched object"), details = Some(e.toString)))
}
Expand Down
14 changes: 7 additions & 7 deletions client/src/main/scala/skuber/api/watch/Watch.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package skuber.api.watch

import akka.http.scaladsl.model.{HttpMethods, _}
import akka.stream.scaladsl.Source
import play.api.libs.json.Format
import play.api.libs.json.{Format, JsObject}
import skuber.api.client._
import skuber.api.client.impl.KubernetesClientImpl
import skuber.{ListOptions, ObjectResource, ResourceDefinition}
Expand All @@ -28,7 +28,7 @@ object Watch {
* @tparam O the object resource type
* @return
*/
def events[O <: ObjectResource](context: KubernetesClientImpl, name: String, sinceResourceVersion: Option[String] = None, bufSize: Int)(
def events[O <: ObjectResource](context: KubernetesClientImpl, name: String, sinceResourceVersion: Option[String] = None, bufSize: Int, errorHandler: Option[String => _])(
implicit format: Format[O], rd: ResourceDefinition[O], lc: LoggingContext) : Future[Source[WatchEvent[O], _]] =
{
context.logInfo(context.logConfig.logRequestBasic, s"creating watch on resource $name of kind ${rd.spec.names.kind}")
Expand All @@ -41,7 +41,7 @@ object Watch {
)
val request = context.buildRequest(HttpMethods.GET, rd, None, query = Some(Uri.Query(watchOptions.asMap)))
val responseFut = context.invokeWatch(request)
toFutureWatchEventSource(context, responseFut, bufSize)
toFutureWatchEventSource(context, responseFut, bufSize, errorHandler)
}

/**
Expand All @@ -54,15 +54,15 @@ object Watch {
* @tparam O the object resource type
* @return a Future which will eventually return a Source of events
*/
def eventsOnKind[O <: ObjectResource](context: KubernetesClientImpl, sinceResourceVersion: Option[String] = None, bufSize: Int)(
def eventsOnKind[O <: ObjectResource](context: KubernetesClientImpl, sinceResourceVersion: Option[String] = None, bufSize: Int, errorHandler: Option[String => _])(
implicit format: Format[O], rd: ResourceDefinition[O], lc: LoggingContext) : Future[Source[WatchEvent[O], _]] =
{
context.logInfo(context.logConfig.logRequestBasic, s"creating skuber watch on kind ${rd.spec.names.kind}")

val watchOptions=ListOptions(resourceVersion = sinceResourceVersion, watch = Some(true))
val request = context.buildRequest(HttpMethods.GET, rd, None, query = Some(Uri.Query(watchOptions.asMap)))
val responseFut = context.invokeWatch(request)
toFutureWatchEventSource(context, responseFut, bufSize)
toFutureWatchEventSource(context, responseFut, bufSize, errorHandler)
}

/**
Expand All @@ -74,13 +74,13 @@ object Watch {
* @tparam O the Kubernetes kind of each events object
* @return a Future which will eventually return a Source of events
*/
private def toFutureWatchEventSource[O <: ObjectResource](context: KubernetesClientImpl, eventStreamResponseFut: Future[HttpResponse], bufSize: Int)(
private def toFutureWatchEventSource[O <: ObjectResource](context: KubernetesClientImpl, eventStreamResponseFut: Future[HttpResponse], bufSize: Int, errorHandler: Option[String => _])(
implicit format: Format[O],lc: LoggingContext): Future[Source[WatchEvent[O], _]] =
{
implicit val ec: ExecutionContext = context.actorSystem.dispatcher

eventStreamResponseFut.map { eventStreamResponse =>
BytesToWatchEventSource(eventStreamResponse.entity.dataBytes, bufSize)
BytesToWatchEventSource(context, eventStreamResponse.entity.dataBytes, bufSize, errorHandler)
}
}
}
Expand Down
6 changes: 4 additions & 2 deletions client/src/main/scala/skuber/api/watch/WatchSource.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ private[api] object WatchSource {
pool: Pool[Start[O]],
name: Option[String],
options: ListOptions,
bufSize: Int)(implicit sys: ActorSystem,
bufSize: Int,
errorHandler: Option[String => _])
(implicit sys: ActorSystem,
format: Format[O],
rd: ResourceDefinition[O],
lc: LoggingContext): Source[WatchEvent[O], NotUsed] = {
Expand Down Expand Up @@ -69,7 +71,7 @@ private[api] object WatchSource {
case (Success(HttpResponse(StatusCodes.OK, _, entity, _)), se) =>
client.logInfo(client.logConfig.logResponseBasic, s"received response with HTTP status 200")
singleStart(se).concat(
BytesToWatchEventSource[O](entity.dataBytes, bufSize).map { event =>
BytesToWatchEventSource[O](client, entity.dataBytes, bufSize, errorHandler).map { event =>
Result[O](event._object.resourceVersion, event)
}
).concat(singleEnd)
Expand Down
Loading