Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Support a custom executor in StreamEffect #1483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Support a custom executor in StreamEffect #1483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion streams-tests/jvm/src/test/scala/zio/stream/StreamSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1083,9 +1083,11 @@ class StreamSpec(implicit ee: org.specs2.concurrent.ExecutionEnv) extends TestRu

private def fromInputStream = unsafeRun {
import java.io.ByteArrayInputStream

val chunkSize = ZStreamChunk.DefaultChunkSize
val data = Array.tabulate[Byte](chunkSize * 5 / 2)(_.toByte)
val is = new ByteArrayInputStream(data)
val is = UIO(new ByteArrayInputStream(data))

ZStream.fromInputStream(is, chunkSize).run(Sink.collectAll[Chunk[Byte]]) map { chunks =>
chunks.flatMap(_.toArray[Byte]).toArray must_=== data
}
Expand Down
2 changes: 2 additions & 0 deletions streams/js/src/main/scala/zio/platform.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
package zio.stream

trait StreamEffectPlatformSpecific
trait StreamPlatformSpecific
trait ZStreamPlatformSpecific
trait ZSinkPlatformSpecific
85 changes: 83 additions & 2 deletions streams/jvm/src/main/scala/zio/stream/platform.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,91 @@
package zio.stream

import java.io.{ IOException, OutputStream }
import java.io.{ IOException, InputStream, OutputStream }

import zio._
import zio.{ blocking => _, _ }
import zio.blocking._

import scala.util.control.NonFatal

trait StreamEffectPlatformSpecific {
import StreamEffect.{ end, fail }

final def fromInputStream[R <: Blocking](
is: ZManaged[R, Throwable, InputStream],
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): StreamEffectChunk[R, Throwable, Byte] =
StreamEffectChunk {
StreamEffect(
is.flatMap { is =>
Managed.effectTotal { () =>
{
val buf = Array.ofDim[Byte](chunkSize)
val bytesRead = try {
is.read(buf)
} catch {
case NonFatal(e) => fail(e)
}

if (bytesRead < 0) end
else if (0 < bytesRead && bytesRead < buf.length) Chunk.fromArray(buf).take(bytesRead)
else if (bytesRead == buf.length) Chunk.fromArray(buf)
else fail(new RuntimeException("Misbehaved InputStream: read more bytes than buffer length"))
}
}
},
blockingExecutor
)
}

final def fromInputStream[R <: Blocking](
is: ZIO[R, Throwable, InputStream],
chunkSize: Int
): StreamEffectChunk[R, Throwable, Byte] =
fromInputStream(is.toManaged(is => blocking(Task(is.close())).orDie), chunkSize)
}

trait ZStreamPlatformSpecific {

/**
* Creates a stream from a [[java.io.InputStream]]
*/
final def fromInputStream[R <: Blocking](
is: ZManaged[R, Throwable, InputStream],
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): ZStreamChunk[R, Throwable, Byte] =
StreamEffect.fromInputStream(is, chunkSize)

/**
* Creates a stream from a [[java.io.InputStream]]
*/
final def fromInputStream[R <: Blocking](
is: ZIO[R, Throwable, InputStream],
chunkSize: Int
): ZStreamChunk[R, Throwable, Byte] =
StreamEffect.fromInputStream(is, chunkSize)
}

trait StreamPlatformSpecific {

/**
* See [[ZStream.fromInputStream]]
*/
final def fromInputStream(
is: Managed[Throwable, InputStream],
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): ZStreamChunk[Blocking, Throwable, Byte] =
StreamEffect.fromInputStream(is, chunkSize)

/**
* See [[ZStream.fromInputStream]]
*/
final def fromInputStream(
is: IO[Throwable, InputStream],
chunkSize: Int
): ZStreamChunk[Blocking, Throwable, Byte] =
StreamEffect.fromInputStream(is, chunkSize)
}

trait ZSinkPlatformSpecific {

/**
Expand Down
13 changes: 1 addition & 12 deletions streams/shared/src/main/scala/zio/stream/Stream.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,11 @@

package zio.stream

import java.io.{ IOException, InputStream }

import zio._
import zio.clock.Clock
import zio.Cause

object Stream {
object Stream extends StreamPlatformSpecific {
import ZStream.Pull

/**
Expand Down Expand Up @@ -127,15 +125,6 @@ object Stream {
): Stream[E, A] =
ZStream.flattenPar(n, outputBuffer)(fa)

/**
* See [[ZStream.fromInputStream]]
*/
final def fromInputStream(
is: InputStream,
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): StreamEffectChunk[Any, IOException, Byte] =
ZStream.fromInputStream(is, chunkSize)

/**
* See [[ZStream.fromChunk]]
*/
Expand Down
102 changes: 48 additions & 54 deletions streams/shared/src/main/scala/zio/stream/StreamEffect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,34 @@

package zio.stream

import java.io.{ IOException, InputStream }

import zio._

private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E, () => A])
extends ZStream[R, E, A](
processEffect.map { thunk =>
UIO.effectTotal {
try UIO.succeed(thunk())
catch {
case StreamEffect.Failure(e) => IO.fail(Some(e.asInstanceOf[E]))
case StreamEffect.End => IO.fail(None)
}
}.flatten
import zio.internal.Executor

private[stream] class StreamEffect[-R, +E, +A](
val processEffect: ZManaged[R, E, () => A],
val executor: ZIO[R, Nothing, Executor]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I can tell, this structure doesn't compose with well-defined semantics. Consider, e.g. x ++ y — what is the executor for such a composite?

It seems like we need to say three things:

  • Invocation of "open" occurs on the blocking thread pool, which we can do in the reserve of the managed
  • Invocation of "close" occurs on the blocking thread pool, which we can do in the reserve of managed (on finalization)
  • Invocation of the "pull" occurs on the blocking thread pool, which we can also do (below, using effectBlocking instead of effectTotal, etc.).

So unless I'm missing something, we can do this w/o new machinery. Right? Or wrong? 😄

Copy link
Member Author

@iravid iravid Aug 26, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Say x: StreamEffect uses a Blocking executor, y: StreamEffect uses a regular executor, and z: Stream that is asynchronous.

(x ++ y ++ z).process does the following:

  • opens x and starts pulling it. The Pull on x evaluates the () => A thunk while locked to the Blocking executor (StreamEffect.scala:34), so that pull runs on the blocking executor;
  • opens y and starts pulling it. Similarly, it evaluates the thunk while locked to the default executor, so that pull runs there.
  • opens z and starts pulling it. That Pull does whatever it wants because it's effectful.

From how I'm reading things, the semantics here are identical to the semantics of lock, as long as we always propagate the executor when composing StreamEffect values. Wdyt?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jdegoes

Sorry, I realized I didn't respond to your questions about whether it's possible to do this without new machinery.

The problem is as follows: how does a constructor (like fromInputStream), which only produces a thunk Managed[E, () => A], indicate that when a stream which is based on it is run, it should be run on the Blocking threadpool? We can't use effectBlocking because we want to produce synchronous code that runs in the thunk.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jdegoes Any thoughts on the above? :-)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was busy with the huge ZSink rewrite PR, where are we with this?

@iravid @jdegoes

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We hashed it over deadlift sets. I believe @jdegoes is onboard. I'll pick this back up next week :-)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome. Please keep me in the review loop.

) extends ZStream[R, E, A](
processEffect.flatMap { thunk =>
executor.toManaged_.map { exec =>
UIO.effectTotal {
try UIO.succeed(thunk())
catch {
case StreamEffect.Failure(e) => IO.fail(Some(e.asInstanceOf[E]))
case StreamEffect.End => IO.fail(None)
}
}.flatten.lock(exec)
}
}
) { self =>

def copy[R1, E1, A1](
processEffect0: ZManaged[R1, E1, () => A1] = processEffect,
executor0: ZIO[R1, Nothing, Executor] = executor
): StreamEffect[R1, E1, A1] =
new StreamEffect[R1, E1, A1](processEffect0, executor0)

override def collect[B](pf: PartialFunction[A, B]): StreamEffect[R, E, B] =
StreamEffect[R, E, B] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal { () =>
{
Expand All @@ -51,7 +60,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def collectWhile[B](pred: PartialFunction[A, B]): StreamEffect[R, E, B] =
StreamEffect[R, E, B] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var done = false
Expand All @@ -65,7 +74,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def dropWhile(pred: A => Boolean): StreamEffect[R, E, A] =
StreamEffect[R, E, A] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var drop = true
Expand All @@ -86,7 +95,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def filter(pred: A => Boolean): StreamEffect[R, E, A] =
StreamEffect[R, E, A] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
@annotation.tailrec
Expand Down Expand Up @@ -120,11 +129,16 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
if (error == null) Right(state) else Left(error)
}

Managed.effectTotal(Managed.fromEither(fold())).flatten
executor.flatMap { exec =>
ZIO.effectTotal(ZIO.fromEither(fold())).flatten.lock(exec)
}.toManaged_
}

final override def foldWhile[A1 >: A, S](s: S)(cont: S => Boolean)(f: (S, A1) => S): ZIO[R, E, S] =
foldWhileManaged(s)(cont)(f).use(UIO.succeed)

override def map[B](f0: A => B): StreamEffect[R, E, B] =
StreamEffect[R, E, B] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal { () =>
f0(thunk())
Expand All @@ -133,7 +147,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def mapAccum[S1, B](s1: S1)(f1: (S1, A) => (S1, B)): StreamEffect[R, E, B] =
StreamEffect[R, E, B] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var state = s1
Expand All @@ -148,7 +162,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def mapConcat[B](f: A => Chunk[B]): StreamEffect[R, E, B] =
StreamEffect[R, E, B] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var chunk: Chunk[B] = Chunk.empty
Expand All @@ -170,15 +184,18 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
override def run[R1 <: R, E1 >: E, A0, A1 >: A, B](sink: ZSink[R1, E1, A0, A1, B]): ZIO[R1, E1, B] =
sink match {
case sink: SinkPure[E1, A0, A1, B] =>
foldWhileManaged[A1, sink.State](sink.initialPure)(sink.cont)(sink.stepPure).use[R1, E1, B] { state =>
ZIO.fromEither(sink.extractPure(state).map(_._1))
executor flatMap { exec =>
(for {
state <- foldWhile(sink.initialPure)(sink.cont)(sink.stepPure)
result <- ZIO.effectTotal(ZIO.fromEither(sink.extractPure(state))).flatten
} yield result._1).lock(exec)
}

case sink: ZSink[R1, E1, A0, A1, B] => super.run(sink)
}

override def take(n: Int): StreamEffect[R, E, A] =
StreamEffect[R, E, A] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var counter = 0
Expand All @@ -195,7 +212,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
}

override def takeWhile(pred: A => Boolean): StreamEffect[R, E, A] =
StreamEffect[R, E, A] {
copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal { () =>
{
Expand All @@ -212,8 +229,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
): ZStream[R1, E1, B] =
sink match {
case sink: SinkPure[E1, A1, A1, B] =>
StreamEffect[R1, E1, B] {

copy {
self.processEffect.flatMap { thunk =>
Managed.effectTotal {
var done = false
Expand Down Expand Up @@ -280,8 +296,7 @@ private[stream] class StreamEffect[-R, +E, +A](val processEffect: ZManaged[R, E,
} yield javaStream
}

private[stream] object StreamEffect extends Serializable {

private[stream] object StreamEffect extends StreamEffectPlatformSpecific with Serializable {
case class Failure[E](e: E) extends Throwable(e.toString, null, true, false) {
override def fillInStackTrace() = this
}
Expand All @@ -302,7 +317,10 @@ private[stream] object StreamEffect extends Serializable {
}

final def apply[R, E, A](pull: ZManaged[R, E, () => A]): StreamEffect[R, E, A] =
new StreamEffect[R, E, A](pull)
new StreamEffect[R, E, A](pull, ZIO.runtime[R].map(_.Platform.executor))

final def apply[R, E, A](pull: ZManaged[R, E, () => A], executor: ZIO[R, Nothing, Executor]): StreamEffect[R, E, A] =
new StreamEffect[R, E, A](pull, executor)

final def fromChunk[A](c: Chunk[A]): StreamEffect[Any, Nothing, A] =
StreamEffect[Any, Nothing, A] {
Expand Down Expand Up @@ -339,30 +357,6 @@ private[stream] object StreamEffect extends Serializable {
}
}

final def fromInputStream(
is: InputStream,
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): StreamEffectChunk[Any, IOException, Byte] =
StreamEffectChunk[Any, IOException, Byte] {
StreamEffect[Any, IOException, Chunk[Byte]] {
Managed.effectTotal {
def pull(): Chunk[Byte] = {
val buf = Array.ofDim[Byte](chunkSize)
try {
val bytesRead = is.read(buf)
if (bytesRead < 0) end
else if (0 < bytesRead && bytesRead < buf.length) Chunk.fromArray(buf).take(bytesRead)
else Chunk.fromArray(buf)
} catch {
case e: IOException => fail(e)
}
}

() => pull()
}
}
}

/**
* Creates a stream by effectfully peeling off the "layers" of a value of type `S`
*/
Expand Down
15 changes: 2 additions & 13 deletions streams/shared/src/main/scala/zio/stream/ZStream.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

package zio.stream

import java.io.{ IOException, InputStream }

import com.github.ghik.silencer.silent
import zio._
import zio.clock.Clock
Expand Down Expand Up @@ -1237,7 +1235,7 @@ class ZStream[-R, +E, +A](val process: ZManaged[R, E, Pull[R, E, A]]) extends Se
* Stream(1).forever.foldWhile(0)(_ <= 4)(_ + _) // UIO[Int] == 5
* }}}
*/
final def foldWhile[A1 >: A, S](s: S)(cont: S => Boolean)(f: (S, A1) => S): ZIO[R, E, S] =
def foldWhile[A1 >: A, S](s: S)(cont: S => Boolean)(f: (S, A1) => S): ZIO[R, E, S] =
foldWhileManagedM[R, E, A1, S](s)(cont)((s, a) => ZIO.succeed(f(s, a))).use(ZIO.succeed)

/**
Expand Down Expand Up @@ -2398,7 +2396,7 @@ class ZStream[-R, +E, +A](val process: ZManaged[R, E, Pull[R, E, A]]) extends Se
self zipRight that
}

object ZStream {
object ZStream extends ZStreamPlatformSpecific {

/**
* Describes an effectful pull from a stream. The optionality of the error channel denotes
Expand Down Expand Up @@ -2688,15 +2686,6 @@ object ZStream {
): ZStream[R, E, A] =
flattenPar(Int.MaxValue, outputBuffer)(fa)

/**
* Creates a stream from a [[java.io.InputStream]]
*/
final def fromInputStream(
is: InputStream,
chunkSize: Int = ZStreamChunk.DefaultChunkSize
): StreamEffectChunk[Any, IOException, Byte] =
StreamEffect.fromInputStream(is, chunkSize)

/**
* Creates a stream from a [[zio.Chunk]] of values
*/
Expand Down