diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 6f3dbb49e1..092cc5e7d6 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -21,3 +21,6 @@ e5525d3f0da44052fdcfbe844993260bdc044270 # Scala Steward: Reformat with scalafmt 3.8.2 a0a37ece16ee55056270b4d9ba5c1505ead8af17 + +# Scala Steward: Reformat with scalafmt 3.8.6 +52e52b013db077ecb5b5a8f5b6e6113f912556d8 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1dc77e73a0..7977b8fd60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,17 @@ jobs: matrix: os: [ubuntu-latest] scala: [2.12, 2.13, 3] - java: [temurin@17] + java: [temurin@17, temurin@21] project: [rootJS, rootJVM, rootNative] + exclude: + - scala: 2.12 + java: temurin@21 + - scala: 3 + java: temurin@21 + - project: rootJS + java: temurin@21 + - project: rootNative + java: temurin@21 runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -55,6 +64,19 @@ jobs: if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' run: sbt +update + - name: Setup Java (temurin@21) + id: setup-java-temurin-21 + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@21' && steps.setup-java-temurin-21.outputs.cache-hit == 'false' + run: sbt +update + - name: Install brew formulae (ubuntu) if: (matrix.project == 'rootNative') && startsWith(matrix.os, 'ubuntu') run: /home/linuxbrew/.linuxbrew/bin/brew install openssl s2n @@ -137,6 +159,19 @@ jobs: if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' run: sbt +update + - name: Setup Java (temurin@21) + id: setup-java-temurin-21 + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@21' && steps.setup-java-temurin-21.outputs.cache-hit == 'false' + run: sbt +update + - name: Download target directories (2.12, rootJS) uses: actions/download-artifact@v4 with: @@ -281,6 +316,19 @@ jobs: if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' run: sbt +update + - name: Setup Java (temurin@21) + id: setup-java-temurin-21 + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@21' && steps.setup-java-temurin-21.outputs.cache-hit == 'false' + run: sbt +update + - name: Submit Dependencies uses: scalacenter/sbt-dependency-submission@v2 with: @@ -317,6 +365,19 @@ jobs: if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' run: sbt +update + - name: Setup Java (temurin@21) + id: setup-java-temurin-21 + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@21' && steps.setup-java-temurin-21.outputs.cache-hit == 'false' + run: sbt +update + - if: matrix.project == 'ioNative' run: brew install s2n @@ -351,6 +412,19 @@ jobs: if: matrix.java == 'temurin@17' && steps.setup-java-temurin-17.outputs.cache-hit == 'false' run: sbt +update + - name: Setup Java (temurin@21) + id: setup-java-temurin-21 + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: sbt update + if: matrix.java == 'temurin@21' && steps.setup-java-temurin-21.outputs.cache-hit == 'false' + run: sbt +update + - name: Generate site run: sbt microsite/tlSite diff --git a/.scalafmt.conf b/.scalafmt.conf index a9703df6a1..be98dbc4ce 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.8.2" +version = "3.9.4" style = default diff --git a/build.sbt b/build.sbt index 9f907b26f7..c3abc2909e 100644 --- a/build.sbt +++ b/build.sbt @@ -8,14 +8,14 @@ ThisBuild / organization := "co.fs2" ThisBuild / organizationName := "Functional Streams for Scala" ThisBuild / startYear := Some(2013) -val Scala213 = "2.13.15" +val Scala213 = "2.13.16" ThisBuild / scalaVersion := Scala213 -ThisBuild / crossScalaVersions := Seq("2.12.20", Scala213, "3.3.4") +ThisBuild / crossScalaVersions := Seq("2.12.20", Scala213, "3.3.5") ThisBuild / tlVersionIntroduced := Map("3" -> "3.0.3") ThisBuild / githubWorkflowOSes := Seq("ubuntu-latest") -ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("17")) +ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("17"), JavaSpec.temurin("21")) ThisBuild / githubWorkflowBuildPreamble ++= nativeBrewInstallWorkflowSteps.value ThisBuild / nativeBrewInstallCond := Some("matrix.project == 'rootNative'") @@ -299,7 +299,7 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.scodec" %%% "scodec-bits" % "1.1.38", "org.typelevel" %%% "cats-core" % "2.11.0", - "org.typelevel" %%% "cats-effect" % "3.6.0-RC1", + "org.typelevel" %%% "cats-effect" % "3.6.0-RC2", "org.typelevel" %%% "cats-effect-laws" % "3.6.0-RC1" % Test, "org.typelevel" %%% "cats-effect-testkit" % "3.6.0-RC1" % Test, "org.typelevel" %%% "cats-laws" % "2.11.0" % Test, diff --git a/core/shared/src/main/scala-3/fs2/ChunkPlatform.scala b/core/shared/src/main/scala-3/fs2/ChunkPlatform.scala index 1fb816e0e4..2d6be7db0e 100644 --- a/core/shared/src/main/scala-3/fs2/ChunkPlatform.scala +++ b/core/shared/src/main/scala-3/fs2/ChunkPlatform.scala @@ -86,7 +86,8 @@ private[fs2] trait ChunkCompanionPlatform extends ChunkCompanion213And3Compat { private[fs2] val ct: ClassTag[O] ) extends Chunk[O] { require( - offset >= 0 && offset <= values.size && length >= 0 && length <= values.size && offset + length <= values.size + offset >= 0 && offset <= values.size && length >= 0 && length <= values.size && offset + length <= values.size, + "IArraySlice out of bounds" ) def size = length diff --git a/core/shared/src/main/scala/fs2/Chunk.scala b/core/shared/src/main/scala/fs2/Chunk.scala index 7e8dc2a830..376ad325e3 100644 --- a/core/shared/src/main/scala/fs2/Chunk.scala +++ b/core/shared/src/main/scala/fs2/Chunk.scala @@ -863,7 +863,8 @@ object Chunk // ClassTag(values.getClass.getComponentType) -- we only keep it for bincompat require( - offset >= 0 && offset <= values.size && length >= 0 && length <= values.size && offset + length <= values.size + offset >= 0 && offset <= values.size && length >= 0 && length <= values.size && offset + length <= values.size, + "ArraySlice out of bounds" ) override protected def thisClassTag: ClassTag[Any] = ct.asInstanceOf[ClassTag[Any]] diff --git a/core/shared/src/main/scala/fs2/Stream.scala b/core/shared/src/main/scala/fs2/Stream.scala index b70a1372ac..a8e44af2e6 100644 --- a/core/shared/src/main/scala/fs2/Stream.scala +++ b/core/shared/src/main/scala/fs2/Stream.scala @@ -40,7 +40,7 @@ import fs2.internal._ import org.typelevel.scalaccompat.annotation._ import Pull.StreamPullOps -import java.util.concurrent.Flow.{Publisher, Subscriber} +import java.util.concurrent.Flow.{Publisher, Processor, Subscriber} /** A stream producing output of type `O` and which may evaluate `F` effects. * @@ -2853,7 +2853,7 @@ final class Stream[+F[_], +O] private[fs2] (private[fs2] val underlying: Pull[F, * @param subscriber the [[Subscriber]] that will receive the elements of the stream. */ def subscribe[F2[x] >: F[x]: Async, O2 >: O](subscriber: Subscriber[O2]): Stream[F2, Nothing] = - interop.flow.subscribeAsStream[F2, O2](this, subscriber) + interop.flow.StreamSubscription.subscribe[F2, O2](this, subscriber) /** Emits all elements of the input except the first one. * @@ -3001,7 +3001,7 @@ final class Stream[+F[_], +O] private[fs2] (private[fs2] val underlying: Pull[F, /** @see [[toPublisher]] */ def toPublisherResource[F2[x] >: F[x]: Async, O2 >: O]: Resource[F2, Publisher[O2]] = - interop.flow.toPublisher(this) + interop.flow.StreamPublisher(this) /** Translates effect type from `F` to `G` using the supplied `FunctionK`. */ @@ -3883,6 +3883,56 @@ object Stream extends StreamLowPriority { await } + /** Creates a [[Stream]] from a `subscribe` function; + * analogous to a `Publisher`, but effectual. + * + * This function is useful when you actually need to provide a subscriber to a third-party. + * + * @example {{{ + * scala> import cats.effect.IO + * scala> import java.util.concurrent.Flow.{Publisher, Subscriber} + * scala> + * scala> def thirdPartyLibrary(subscriber: Subscriber[Int]): Unit = { + * | def somePublisher: Publisher[Int] = ??? + * | somePublisher.subscribe(subscriber) + * | } + * scala> + * scala> // Interop with the third party library. + * scala> Stream.fromPublisher[IO, Int](chunkSize = 16) { subscriber => + * | IO.println("Subscribing!") >> + * | IO.delay(thirdPartyLibrary(subscriber)) >> + * | IO.println("Subscribed!") + * | } + * res0: Stream[IO, Int] = Stream(..) + * }}} + * + * @note The subscribe function will not be executed until the stream is run. + * + * @see the overload that only requires a [[Publisher]]. + * + * @param chunkSize setup the number of elements asked each time from the [[Publisher]]. + * A high number may be useful if the publisher is triggering from IO, + * like requesting elements from a database. + * A high number will also lead to more elements in memory. + * The stream will not emit new element until, + * either the `Chunk` is filled or the publisher finishes. + * @param subscribe The effectual function that will be used to initiate the consumption process, + * it receives a [[Subscriber]] that should be used to subscribe to a [[Publisher]]. + * The `subscribe` operation must be called exactly once. + */ + def fromPublisher[F[_], A]( + chunkSize: Int + )( + subscribe: Subscriber[A] => F[Unit] + )(implicit + F: Async[F] + ): Stream[F, A] = + Stream + .eval(interop.flow.StreamSubscriber[F, A](chunkSize)) + .flatMap { subscriber => + subscriber.stream(subscribe(subscriber)) + } + /** Creates a [[Stream]] from a [[Publisher]]. * * @example {{{ @@ -3900,8 +3950,6 @@ object Stream extends StreamLowPriority { * * @note The [[Publisher]] will not receive a [[Subscriber]] until the stream is run. * - * @see the `toStream` extension method added to `Publisher` - * * @param publisher The [[Publisher]] to consume. * @param chunkSize setup the number of elements asked each time from the [[Publisher]]. * A high number may be useful if the publisher is triggering from IO, @@ -3911,7 +3959,7 @@ object Stream extends StreamLowPriority { * either the `Chunk` is filled or the publisher finishes. */ def fromPublisher[F[_]]: interop.flow.syntax.FromPublisherPartiallyApplied[F] = - interop.flow.fromPublisher + new interop.flow.syntax.FromPublisherPartiallyApplied(dummy = true) /** Like `emits`, but works for any G that has a `Foldable` instance. */ @@ -4697,7 +4745,7 @@ object Stream extends StreamLowPriority { def unsafeToPublisher()(implicit runtime: IORuntime ): Publisher[A] = - interop.flow.unsafeToPublisher(self) + interop.flow.StreamPublisher.unsafe(self) } /** Projection of a `Stream` providing various ways to get a `Pull` from the `Stream`. */ @@ -5541,6 +5589,47 @@ object Stream extends StreamLowPriority { /** Transforms the right input of the given `Pipe2` using a `Pipe`. */ def attachR[I0, O2](p: Pipe2[F, I0, O, O2]): Pipe2[F, I0, I, O2] = (l, r) => p(l, self(r)) + + /** Creates a flow [[Processor]] from this [[Pipe]]. + * + * You are required to manually subscribe this [[Processor]] to an upstream [[Publisher]], and have at least one downstream [[Subscriber]] subscribe to the [[Consumer]]. + * + * Closing the [[Resource]] means not accepting new subscriptions, + * but waiting for all active ones to finish consuming. + * Canceling the [[Resource.use]] means gracefully shutting down all active subscriptions. + * Thus, no more elements will be published. + * + * @param chunkSize setup the number of elements asked each time from the upstream [[Publisher]]. + * A high number may be useful if the publisher is triggering from IO, + * like requesting elements from a database. + * A high number will also lead to more elements in memory. + */ + def toProcessor( + chunkSize: Int + )(implicit + F: Async[F] + ): Resource[F, Processor[I, O]] = + interop.flow.StreamProcessor.fromPipe(pipe = self, chunkSize) + } + + /** Provides operations on IO pipes for syntactic convenience. */ + implicit final class IOPipeOps[I, O](private val self: Pipe[IO, I, O]) extends AnyVal { + + /** Creates a [[Processor]] from this [[Pipe]]. + * + * You are required to manually subscribe this [[Processor]] to an upstream [[Publisher]], and have at least one downstream [[Subscriber]] subscribe to the [[Consumer]]. + * + * @param chunkSize setup the number of elements asked each time from the upstream [[Publisher]]. + * A high number may be useful if the publisher is triggering from IO, + * like requesting elements from a database. + * A high number will also lead to more elements in memory. + */ + def unsafeToProcessor( + chunkSize: Int + )(implicit + runtime: IORuntime + ): Processor[I, O] = + interop.flow.StreamProcessor.unsafeFromPipe(pipe = self, chunkSize) } /** Provides operations on pure pipes for syntactic convenience. */ diff --git a/core/shared/src/main/scala/fs2/concurrent/Signal.scala b/core/shared/src/main/scala/fs2/concurrent/Signal.scala index 375e105e98..6144d8bd95 100644 --- a/core/shared/src/main/scala/fs2/concurrent/Signal.scala +++ b/core/shared/src/main/scala/fs2/concurrent/Signal.scala @@ -28,8 +28,11 @@ import cats.effect.std.MapRef import cats.effect.syntax.all._ import cats.syntax.all._ import cats.{Applicative, Functor, Invariant, Monad} - +import cats.arrow.FunctionK import scala.collection.immutable.LongMap +import fs2.concurrent.SignallingRef.TransformedSignallingRef +import fs2.concurrent.Signal.TransformedSignal +import cats.data.State /** Pure holder of a single value of type `A` that can be read in the effect `F`. */ trait Signal[F[_], A] { outer => @@ -135,6 +138,11 @@ trait Signal[F[_], A] { outer => */ def waitUntil(p: A => Boolean)(implicit F: Concurrent[F]): F[Unit] = discrete.forall(a => !p(a)).compile.drain + + def mapK[G[_]]( + f: FunctionK[F, G] + ): Signal[G, A] = + new TransformedSignal(this, f) } object Signal extends SignalInstances { @@ -162,6 +170,16 @@ object Signal extends SignalInstances { def get: F[B] = Functor[F].map(fa.get)(f) } + final private class TransformedSignal[F[_], G[_], A]( + underlying: Signal[F, A], + trans: FunctionK[F, G] + ) extends Signal[G, A] { + override def get: G[A] = trans(underlying.get) + override def discrete: Stream[G, A] = underlying.discrete.translate(trans) + override def continuous: Stream[G, A] = underlying.continuous.translate(trans) + override def changes(implicit eqA: Eq[A]): Signal[G, A] = underlying.changes.mapK(trans) + } + implicit class SignalOps[F[_], A](val self: Signal[F, A]) extends AnyVal { /** Converts this signal to signal of `B` by applying `f`. @@ -196,7 +214,12 @@ object Signal extends SignalInstances { * function, in the presence of `discrete`, can return `false` and * need looping even without any other writers. */ -abstract class SignallingRef[F[_], A] extends Ref[F, A] with Signal[F, A] +abstract class SignallingRef[F[_], A] extends Ref[F, A] with Signal[F, A] { + override def mapK[G[_]]( + f: FunctionK[F, G] + )(implicit G: Functor[G], dummy: DummyImplicit): SignallingRef[G, A] = + new TransformedSignallingRef(this, f) +} object SignallingRef { @@ -222,6 +245,7 @@ object SignallingRef { * * @see [[of]] */ + def apply[F[_]]: PartiallyApplied[F] = new PartiallyApplied[F] /** Alias for `of`. */ @@ -341,7 +365,31 @@ object SignallingRef { ref: SignallingRef[F, A] )(get: A => B, set: A => B => A)(implicit F: Functor[F]): SignallingRef[F, B] = new LensSignallingRef(ref)(get, set) - + final private class TransformedSignallingRef[F[_], G[_], A]( + underlying: SignallingRef[F, A], + trans: FunctionK[F, G] + )(implicit G: Functor[G]) + extends SignallingRef[G, A] { + + // --- Ref methods: these are lifted using trans, just like in TransformedRef2 + override def get: G[A] = trans(underlying.get) + override def set(a: A): G[Unit] = trans(underlying.set(a)) + override def getAndSet(a: A): G[A] = trans(underlying.getAndSet(a)) + override def tryUpdate(f: A => A): G[Boolean] = trans(underlying.tryUpdate(f)) + override def tryModify[B](f: A => (A, B)): G[Option[B]] = trans(underlying.tryModify(f)) + override def update(f: A => A): G[Unit] = trans(underlying.update(f)) + override def modify[B](f: A => (A, B)): G[B] = trans(underlying.modify(f)) + override def tryModifyState[B](state: State[A, B]): G[Option[B]] = + trans(underlying.tryModifyState(state)) + override def modifyState[B](state: State[A, B]): G[B] = trans(underlying.modifyState(state)) + override def access: G[(A, A => G[Boolean])] = + G.compose[(A, *)].compose[A => *].map(trans(underlying.access))(trans(_)) + + // --- Signal-specific methods + override def discrete: Stream[G, A] = underlying.discrete.translate(trans) + override def continuous: Stream[G, A] = underlying.continuous.translate(trans) + override def changes(implicit eqA: Eq[A]): Signal[G, A] = underlying.changes.mapK(trans) + } private final class LensSignallingRef[F[_], A, B](underlying: SignallingRef[F, A])( lensGet: A => B, lensSet: A => B => A @@ -481,8 +529,11 @@ object SignallingMapRef { .map { case (state, ids) => def newId = ids.getAndUpdate(_ + 1) - def updateAndNotify[U](state: State, k: K, f: Option[V] => (Option[V], U)) - : (State, F[U]) = { + def updateAndNotify[U]( + state: State, + k: K, + f: Option[V] => (Option[V], U) + ): (State, F[U]) = { val keyState = state.keys.get(k) diff --git a/core/shared/src/main/scala/fs2/fs2.scala b/core/shared/src/main/scala/fs2/fs2.scala index 8588110b32..ea43cca56e 100644 --- a/core/shared/src/main/scala/fs2/fs2.scala +++ b/core/shared/src/main/scala/fs2/fs2.scala @@ -19,6 +19,9 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +import java.util.concurrent.Flow.Processor +import cats.effect.Async + package object fs2 { /** A stream transformation represented as a function from stream to stream. @@ -27,6 +30,36 @@ package object fs2 { */ type Pipe[F[_], -I, +O] = Stream[F, I] => Stream[F, O] + object Pipe { + final class FromProcessorPartiallyApplied[F[_]](private val dummy: Boolean) extends AnyVal { + def apply[I, O]( + processor: Processor[I, O], + chunkSize: Int + )(implicit + F: Async[F] + ): Pipe[F, I, O] = + new interop.flow.ProcessorPipe(processor, chunkSize) + } + + /** Creates a [[Pipe]] from the given [[Processor]]. + * + * The input stream won't be consumed until you request elements from the output stream, + * and thus the processor is not initiated until then. + * + * @note The [[Pipe]] can be reused multiple times as long as the [[Processor]] can be reused. + * Each invocation of the pipe will create and manage its own internal [[Publisher]] and [[Subscriber]], + * and use them to subscribe to and from the [[Processor]] respectively. + * + * @param [[processor]] the [[Processor]] that represents the [[Pipe]] logic. + * @param chunkSize setup the number of elements asked each time from the upstream [[Publisher]]. + * A high number may be useful if the publisher is triggering from IO, + * like requesting elements from a database. + * A high number will also lead to more elements in memory. + */ + def fromProcessor[F[_]]: FromProcessorPartiallyApplied[F] = + new FromProcessorPartiallyApplied[F](dummy = true) + } + /** A stream transformation that combines two streams in to a single stream, * represented as a function from two streams to a single stream. * diff --git a/core/shared/src/main/scala/fs2/interop/flow/ProcessorPipe.scala b/core/shared/src/main/scala/fs2/interop/flow/ProcessorPipe.scala new file mode 100644 index 0000000000..c78e2c62eb --- /dev/null +++ b/core/shared/src/main/scala/fs2/interop/flow/ProcessorPipe.scala @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package interop +package flow + +import cats.syntax.all.* + +import java.util.concurrent.Flow +import cats.effect.Async + +private[fs2] final class ProcessorPipe[F[_], I, O]( + processor: Flow.Processor[I, O], + chunkSize: Int +)(implicit + F: Async[F] +) extends Pipe[F, I, O] { + override def apply(stream: Stream[F, I]): Stream[F, O] = + ( + Stream.resource(StreamPublisher[F, I](stream)), + Stream.eval(StreamSubscriber[F, O](chunkSize)) + ).flatMapN { (publisher, subscriber) => + val initiateUpstreamProduction = F.delay(publisher.subscribe(processor)) + val initiateDownstreamConsumption = F.delay(processor.subscribe(subscriber)) + + subscriber.stream( + subscribe = initiateUpstreamProduction >> initiateDownstreamConsumption + ) + } +} diff --git a/core/shared/src/main/scala/fs2/interop/flow/StreamProcessor.scala b/core/shared/src/main/scala/fs2/interop/flow/StreamProcessor.scala new file mode 100644 index 0000000000..408772a478 --- /dev/null +++ b/core/shared/src/main/scala/fs2/interop/flow/StreamProcessor.scala @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package interop +package flow + +import java.util.concurrent.Flow +import cats.effect.{Async, IO, Resource} +import cats.effect.unsafe.IORuntime + +private[fs2] final class StreamProcessor[F[_], I, O]( + streamSubscriber: StreamSubscriber[F, I], + streamPublisher: StreamPublisher[F, O] +) extends Flow.Processor[I, O] { + override def onSubscribe(subscription: Flow.Subscription): Unit = + streamSubscriber.onSubscribe(subscription) + + override def onNext(i: I): Unit = + streamSubscriber.onNext(i) + + override def onError(ex: Throwable): Unit = + streamSubscriber.onError(ex) + + override def onComplete(): Unit = + streamSubscriber.onComplete() + + override def subscribe(subscriber: Flow.Subscriber[? >: O]): Unit = + streamPublisher.subscribe(subscriber) +} + +private[fs2] object StreamProcessor { + def fromPipe[F[_], I, O]( + pipe: Pipe[F, I, O], + chunkSize: Int + )(implicit + F: Async[F] + ): Resource[F, StreamProcessor[F, I, O]] = + for { + streamSubscriber <- Resource.eval(StreamSubscriber[F, I](chunkSize)) + inputStream = streamSubscriber.stream(subscribe = F.unit) + outputStream = pipe(inputStream) + streamPublisher <- StreamPublisher(outputStream) + } yield new StreamProcessor( + streamSubscriber, + streamPublisher + ) + + def unsafeFromPipe[I, O]( + pipe: Pipe[IO, I, O], + chunkSize: Int + )(implicit + runtime: IORuntime + ): StreamProcessor[IO, I, O] = { + val streamSubscriber = StreamSubscriber.unsafe[IO, I](chunkSize) + val inputStream = streamSubscriber.stream(subscribe = IO.unit) + val outputStream = pipe(inputStream) + val streamPublisher = StreamPublisher.unsafe(outputStream) + new StreamProcessor( + streamSubscriber, + streamPublisher + ) + } +} diff --git a/core/shared/src/main/scala/fs2/interop/flow/StreamPublisher.scala b/core/shared/src/main/scala/fs2/interop/flow/StreamPublisher.scala index b93ce35a82..be082e4dba 100644 --- a/core/shared/src/main/scala/fs2/interop/flow/StreamPublisher.scala +++ b/core/shared/src/main/scala/fs2/interop/flow/StreamPublisher.scala @@ -23,8 +23,7 @@ package fs2 package interop package flow -import cats.effect.IO -import cats.effect.kernel.{Async, Resource} +import cats.effect.{Async, IO, Resource} import cats.effect.std.Dispatcher import cats.effect.unsafe.IORuntime @@ -42,7 +41,7 @@ import scala.util.control.NoStackTrace * * @see [[https://github.com/reactive-streams/reactive-streams-jvm#1-publisher-code]] */ -private[flow] sealed abstract class StreamPublisher[F[_], A] private ( +private[fs2] sealed abstract class StreamPublisher[F[_], A] private ( stream: Stream[F, A] )(implicit F: Async[F] @@ -65,7 +64,7 @@ private[flow] sealed abstract class StreamPublisher[F[_], A] private ( } } -private[flow] object StreamPublisher { +private[fs2] object StreamPublisher { private final class DispatcherStreamPublisher[F[_], A]( stream: Stream[F, A], dispatcher: Dispatcher[F] diff --git a/core/shared/src/main/scala/fs2/interop/flow/StreamSubscriber.scala b/core/shared/src/main/scala/fs2/interop/flow/StreamSubscriber.scala index 4212bcb94a..4f2eb3dbaa 100644 --- a/core/shared/src/main/scala/fs2/interop/flow/StreamSubscriber.scala +++ b/core/shared/src/main/scala/fs2/interop/flow/StreamSubscriber.scala @@ -23,7 +23,7 @@ package fs2 package interop package flow -import cats.effect.kernel.Async +import cats.effect.Async import java.util.Objects.requireNonNull import java.util.concurrent.Flow.{Subscriber, Subscription} @@ -36,7 +36,7 @@ import scala.util.control.NoStackTrace * * @see [[https://github.com/reactive-streams/reactive-streams-jvm#2-subscriber-code]] */ -private[flow] final class StreamSubscriber[F[_], A] private ( +private[fs2] final class StreamSubscriber[F[_], A] private ( chunkSize: Int, currentState: AtomicReference[(StreamSubscriber.State, () => Unit)] )(implicit @@ -66,9 +66,9 @@ private[flow] final class StreamSubscriber[F[_], A] private ( * since they are always done on the effect run after the state update took place. * Meaning this should be correct if the Producer is well-behaved. */ - private var inOnNextLoop: Boolean = _ + private var inOnNextLoop: Boolean = false private var buffer: Array[Any] = null - private var index: Int = _ + private var index: Int = 0 /** Receives the next record from the upstream reactive-streams system. */ override final def onNext(a: A): Unit = { @@ -106,7 +106,7 @@ private[flow] final class StreamSubscriber[F[_], A] private ( // Interop API. /** Creates a downstream [[Stream]] from this [[Subscriber]]. */ - private[flow] def stream(subscribe: F[Unit]): Stream[F, A] = { + private[fs2] def stream(subscribe: F[Unit]): Stream[F, A] = { // Called when downstream has finished consuming records. val finalize = F.delay(nextState(input = Complete(canceled = true))) @@ -164,10 +164,10 @@ private[flow] final class StreamSubscriber[F[_], A] private ( state -> run { // We do the updates here, // to ensure they happen after we have secured the state. - inOnNextLoop = true - index = 1 buffer = new Array(chunkSize) buffer(0) = a + index = 1 + inOnNextLoop = true } } @@ -188,15 +188,18 @@ private[flow] final class StreamSubscriber[F[_], A] private ( Idle(s) -> run { // We do the updates here, // to ensure they happen after we have secured the state. - cb.apply(Right(Some(Chunk.array(buffer)))) + val chunk = Chunk.array(buffer) inOnNextLoop = false buffer = null + cb.apply(Right(Some(chunk))) } case state => Failed( new InvalidStateException(operation = s"Received record [${buffer.last}]", state) ) -> run { + // We do the updates here, + // to ensure they happen after we have secured the state. inOnNextLoop = false buffer = null } @@ -206,19 +209,15 @@ private[flow] final class StreamSubscriber[F[_], A] private ( case Uninitialized(Some(cb)) => Terminal -> run { cb.apply(Left(ex)) - // We do the updates here, - // to ensure they happen after we have secured the state. - inOnNextLoop = false - buffer = null } case WaitingOnUpstream(cb, _) => Terminal -> run { - cb.apply(Left(ex)) // We do the updates here, // to ensure they happen after we have secured the state. inOnNextLoop = false buffer = null + cb.apply(Left(ex)) } case _ => @@ -240,17 +239,21 @@ private[flow] final class StreamSubscriber[F[_], A] private ( case WaitingOnUpstream(cb, s) => Terminal -> run { + // We do the updates here, + // to ensure they happen after we have secured the state. if (canceled) { s.cancel() + inOnNextLoop = false + buffer = null cb.apply(Right(None)) - } else if (index == 0) { + } else if (buffer eq null) { + inOnNextLoop = false cb.apply(Right(None)) } else { - cb.apply(Right(Some(Chunk.array(buffer, offset = 0, length = index)))) - // We do the updates here, - // to ensure they happen after we have secured the state. + val chunk = Chunk.array(buffer, offset = 0, length = index) inOnNextLoop = false buffer = null + cb.apply(Right(Some(chunk))) } } @@ -268,10 +271,6 @@ private[flow] final class StreamSubscriber[F[_], A] private ( case Idle(s) => WaitingOnUpstream(cb, s) -> run { s.request(chunkSize.toLong) - // We do the updates here, - // to ensure they happen after we have secured the state. - inOnNextLoop = false - index = 0 } case state @ Uninitialized(Some(otherCB)) => @@ -283,14 +282,14 @@ private[flow] final class StreamSubscriber[F[_], A] private ( case state @ WaitingOnUpstream(otherCB, s) => Terminal -> run { - s.cancel() - val ex = Left(new InvalidStateException(operation = "Received request", state)) - otherCB.apply(ex) - cb.apply(ex) // We do the updates here, // to ensure they happen after we have secured the state. inOnNextLoop = false buffer = null + s.cancel() + val ex = Left(new InvalidStateException(operation = "Received request", state)) + otherCB.apply(ex) + cb.apply(ex) } case Failed(ex) => @@ -324,24 +323,30 @@ private[flow] final class StreamSubscriber[F[_], A] private ( } } -private[flow] object StreamSubscriber { +private[fs2] object StreamSubscriber { private final val noop = () => () /** Instantiates a new [[StreamSubscriber]] for the given buffer size. */ def apply[F[_], A]( chunkSize: Int - )(implicit F: Async[F]): F[StreamSubscriber[F, A]] = { - require(chunkSize > 0, "The buffer size MUST be positive") + )(implicit + F: Async[F] + ): F[StreamSubscriber[F, A]] = + F.delay(unsafe(chunkSize)) - F.delay { - val currentState = - new AtomicReference[(State, () => Unit)]((State.Uninitialized(cb = None), noop)) + private[fs2] def unsafe[F[_], A]( + chunkSize: Int + )(implicit + F: Async[F] + ): StreamSubscriber[F, A] = { + require(chunkSize > 0, "The buffer size MUST be positive") - new StreamSubscriber[F, A]( - chunkSize, - currentState + new StreamSubscriber[F, A]( + chunkSize, + currentState = new AtomicReference[(State, () => Unit)]( + (State.Uninitialized(cb = None), noop) ) - } + ) } private sealed abstract class StreamSubscriberException(msg: String, cause: Throwable = null) diff --git a/core/shared/src/main/scala/fs2/interop/flow/StreamSubscription.scala b/core/shared/src/main/scala/fs2/interop/flow/StreamSubscription.scala index 5e6ef43120..a62d9f3696 100644 --- a/core/shared/src/main/scala/fs2/interop/flow/StreamSubscription.scala +++ b/core/shared/src/main/scala/fs2/interop/flow/StreamSubscription.scala @@ -37,7 +37,7 @@ import java.util.concurrent.atomic.{AtomicLong, AtomicReference} * * @see [[https://github.com/reactive-streams/reactive-streams-jvm#3-subscription-code]] */ -private[flow] final class StreamSubscription[F[_], A] private ( +private[fs2] final class StreamSubscription[F[_], A] private ( stream: Stream[F, A], subscriber: Subscriber[A], requests: AtomicLong, @@ -171,7 +171,7 @@ private[flow] final class StreamSubscription[F[_], A] private ( } } -private[flow] object StreamSubscription { +private[fs2] object StreamSubscription { private final val Sentinel = () => () // UNSAFE + SIDE-EFFECTING! diff --git a/core/shared/src/main/scala/fs2/interop/flow/package.scala b/core/shared/src/main/scala/fs2/interop/flow/package.scala index 32932c5cd9..4fb25aec3d 100644 --- a/core/shared/src/main/scala/fs2/interop/flow/package.scala +++ b/core/shared/src/main/scala/fs2/interop/flow/package.scala @@ -22,8 +22,7 @@ package fs2 package interop -import cats.effect.IO -import cats.effect.kernel.{Async, Resource} +import cats.effect.{Async, IO, Resource} import cats.effect.unsafe.IORuntime import java.util.concurrent.Flow.{Publisher, Subscriber, defaultBufferSize} diff --git a/core/shared/src/main/scala/fs2/interop/flow/syntax.scala b/core/shared/src/main/scala/fs2/interop/flow/syntax.scala index ae6c367064..24536dbd26 100644 --- a/core/shared/src/main/scala/fs2/interop/flow/syntax.scala +++ b/core/shared/src/main/scala/fs2/interop/flow/syntax.scala @@ -23,7 +23,7 @@ package fs2 package interop package flow -import cats.effect.kernel.{Async, Resource} +import cats.effect.{Async, Resource} import java.util.concurrent.Flow.{Publisher, Subscriber} @@ -46,6 +46,7 @@ object syntax { flow.subscribeStream(stream, subscriber) } + // TODO: Move to the Stream companion object when removing the deprecated flow package object and syntax. final class FromPublisherPartiallyApplied[F[_]](private val dummy: Boolean) extends AnyVal { def apply[A]( publisher: Publisher[A], diff --git a/core/shared/src/test/scala/fs2/StreamPerformanceSuite.scala b/core/shared/src/test/scala/fs2/StreamPerformanceSuite.scala index 2f7e188205..b1787e10bf 100644 --- a/core/shared/src/test/scala/fs2/StreamPerformanceSuite.scala +++ b/core/shared/src/test/scala/fs2/StreamPerformanceSuite.scala @@ -198,9 +198,8 @@ class StreamPerformanceSuite extends Fs2Suite { val s: Stream[SyncIO, Int] = List .fill(N)(bracketed) - .foldLeft(Stream.raiseError[SyncIO](new Err): Stream[SyncIO, Int]) { - (acc, hd) => - acc.handleErrorWith(_ => hd) + .foldLeft(Stream.raiseError[SyncIO](new Err): Stream[SyncIO, Int]) { (acc, hd) => + acc.handleErrorWith(_ => hd) } s.compile.toList.attempt .flatMap(_ => (ok.get, open.get).tupled) diff --git a/core/shared/src/test/scala/fs2/StreamZipSuite.scala b/core/shared/src/test/scala/fs2/StreamZipSuite.scala index 8a478bfaf8..73c49076e3 100644 --- a/core/shared/src/test/scala/fs2/StreamZipSuite.scala +++ b/core/shared/src/test/scala/fs2/StreamZipSuite.scala @@ -177,9 +177,8 @@ class StreamZipSuite extends Fs2Suite { Logger[IO] .flatMap { logger => def s(tag: String) = - logger.logLifecycle(tag) >> { + logger.logLifecycle(tag) >> logger.logLifecycle(s"$tag - 1") ++ logger.logLifecycle(s"$tag - 2") - } s("a").zip(s("b")).compile.drain >> logger.get.assertEquals( diff --git a/core/shared/src/test/scala/fs2/concurrent/SignalSuite.scala b/core/shared/src/test/scala/fs2/concurrent/SignalSuite.scala index d4af8a3edb..e306ae0492 100644 --- a/core/shared/src/test/scala/fs2/concurrent/SignalSuite.scala +++ b/core/shared/src/test/scala/fs2/concurrent/SignalSuite.scala @@ -25,6 +25,7 @@ package concurrent import cats.effect.IO import cats.effect.kernel.Ref import cats.syntax.all._ +import cats.arrow.FunctionK import cats.effect.testkit.TestControl // import cats.laws.discipline.{ApplicativeTests, FunctorTests} import scala.concurrent.duration._ @@ -320,6 +321,19 @@ class SignalSuite extends Fs2Suite { TestControl.executeEmbed(prog).assertEquals(expected) } + test("SignallingRef#mapK returns a SignallingRef") { + for { + s <- SignallingRef[IO, Int](0) + nt = new FunctionK[IO, IO] { + def apply[A](fa: IO[A]): IO[A] = fa + } + transformed: SignallingRef[IO, Int] = s.mapK(nt) + } yield assert( + transformed.isInstanceOf[SignallingRef[IO, Int]], + s"Expected transformed to be a SignallingRef but got: ${transformed.getClass.getName}" + ) + } + // TODO - Port laws tests once we have a compatible version of cats-laws // /** // * This is unsafe because the Signal created cannot have multiple consumers diff --git a/core/shared/src/test/scala/fs2/interop/flow/ProcessorPipeSpec.scala b/core/shared/src/test/scala/fs2/interop/flow/ProcessorPipeSpec.scala new file mode 100644 index 0000000000..b4ca0bb568 --- /dev/null +++ b/core/shared/src/test/scala/fs2/interop/flow/ProcessorPipeSpec.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package interop +package flow + +import cats.effect.IO +import org.scalacheck.{Arbitrary, Gen} +import org.scalacheck.effect.PropF.forAllF + +final class ProcessorPipeSpec extends Fs2Suite { + test("should process upstream input and propagate results to downstream") { + forAllF(Arbitrary.arbitrary[Seq[Int]], Gen.posNum[Int]) { (ints, bufferSize) => + // Since creating a Flow.Processor is very complex, + // we will reuse our Pipe => Processor logic. + val processor = ((stream: Stream[IO, Int]) => stream.map(_ * 1)).unsafeToProcessor( + chunkSize = bufferSize + ) + + val pipe = Pipe.fromProcessor[IO]( + processor, + chunkSize = bufferSize + ) + + val inputStream = Stream.emits(ints) + val outputStream = pipe(inputStream) + val program = outputStream.compile.toVector + + program.assertEquals(ints.toVector) + } + } +} diff --git a/core/shared/src/test/scala/fs2/interop/flow/StreamProcessorSpec.scala b/core/shared/src/test/scala/fs2/interop/flow/StreamProcessorSpec.scala new file mode 100644 index 0000000000..bf2c7dd5f7 --- /dev/null +++ b/core/shared/src/test/scala/fs2/interop/flow/StreamProcessorSpec.scala @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package interop +package flow + +import cats.effect.{IO, Resource} +import org.scalacheck.{Arbitrary, Gen} +import org.scalacheck.effect.PropF.forAllF +import java.util.concurrent.Flow.Publisher + +final class StreamProcessorSpec extends Fs2Suite { + test("should process upstream input and propagate results to downstream") { + forAllF(Arbitrary.arbitrary[Seq[Int]], Gen.posNum[Int]) { (ints, bufferSize) => + val pipe = (stream: Stream[IO, Int]) => stream.map(_ * 1) + + val processor = pipe.toProcessor(chunkSize = bufferSize) + + val publisher = toPublisher( + Stream.emits(ints).covary[IO] + ) + + def subscriber(publisher: Publisher[Int]): IO[Vector[Int]] = + Stream + .fromPublisher[IO]( + publisher, + chunkSize = bufferSize + ) + .compile + .toVector + + val program = Resource.both(processor, publisher).use { case (pr, p) => + IO(p.subscribe(pr)) >> subscriber(pr) + } + + program.assertEquals(ints.toVector) + } + } +} diff --git a/flake.lock b/flake.lock index 1a8dd73cd1..cd3dcfa184 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ ] }, "locked": { - "lastModified": 1728330715, - "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=", + "lastModified": 1735644329, + "narHash": "sha256-tO3HrHriyLvipc4xr+Ewtdlo7wM1OjXNjlWRgmM7peY=", "owner": "numtide", "repo": "devshell", - "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef", + "rev": "f7795ede5b02664b57035b3b757876703e2c3eac", "type": "github" }, "original": { @@ -41,11 +41,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1731763621, - "narHash": "sha256-ddcX4lQL0X05AYkrkV2LMFgGdRvgap7Ho8kgon3iWZk=", + "lastModified": 1740019556, + "narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=", "owner": "nixos", "repo": "nixpkgs", - "rev": "c69a9bffbecde46b4b939465422ddc59493d3e4d", + "rev": "dad564433178067be1fbdfcce23b546254b6d641", "type": "github" }, "original": { @@ -90,11 +90,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1732116097, - "narHash": "sha256-m69yH9uqQ38/91vzdfoz5QP5yZbId6Rj22unoVRzgi8=", + "lastModified": 1740417560, + "narHash": "sha256-wRBD3SgqCd8Z9SyH1aJwGG+1ah9xFmDNSXu0sDzhGz4=", "owner": "typelevel", "repo": "typelevel-nix", - "rev": "1657fd774eb053167e074e7fe11e4b675a137f71", + "rev": "65e876072da230a0d7249d44550a6489624d4148", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index d563616818..ac3acf84d4 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ let pkgs = import nixpkgs { inherit system; - overlays = [ typelevel-nix.overlay ]; + overlays = [ typelevel-nix.overlays.default ]; }; in { diff --git a/integration/src/test/scala/fs2/MemoryLeakSpec.scala b/integration/src/test/scala/fs2/MemoryLeakSpec.scala index 7b72ec95d2..6adf3fb370 100644 --- a/integration/src/test/scala/fs2/MemoryLeakSpec.scala +++ b/integration/src/test/scala/fs2/MemoryLeakSpec.scala @@ -46,7 +46,7 @@ class MemoryLeakSpec extends FunSuite { warmupIterations: Int = 3, samplePeriod: FiniteDuration = 1.seconds, monitorPeriod: FiniteDuration = 10.seconds, - limitTotalBytesIncreasePerSecond: Long = 700000, + limitTotalBytesIncreasePerSecond: Long = 1400000, limitConsecutiveIncreases: Int = 10 ) diff --git a/io/jvm-native/src/main/scala/fs2/io/process/ProcessesPlatform.scala b/io/jvm-native/src/main/scala/fs2/io/process/ProcessesPlatform.scala index 315c09847d..0c9f5452bf 100644 --- a/io/jvm-native/src/main/scala/fs2/io/process/ProcessesPlatform.scala +++ b/io/jvm-native/src/main/scala/fs2/io/process/ProcessesPlatform.scala @@ -25,13 +25,14 @@ package process import cats.effect.kernel.Async import cats.effect.kernel.Resource -import cats.syntax.all._ -import fs2.io.CollectionCompat._ +import cats.syntax.all.* +import fs2.io.CollectionCompat.* import java.lang private[process] trait ProcessesCompanionPlatform { def forAsync[F[_]](implicit F: Async[F]): Processes[F] = new UnsealedProcesses[F] { + def spawn(process: ProcessBuilder): Resource[F, Process[F]] = Resource .make { @@ -53,11 +54,13 @@ private[process] trait ProcessesCompanionPlatform { } { process => F.delay(process.isAlive()) .ifM( - F.blocking { - process.destroy() - process.waitFor() - () - }, + evalOnVirtualThreadIfAvailable( + F.blocking { + process.destroy() + process.waitFor() + () + } + ), F.unit ) } @@ -66,7 +69,7 @@ private[process] trait ProcessesCompanionPlatform { def isAlive = F.delay(process.isAlive()) def exitValue = isAlive.ifM( - F.interruptible(process.waitFor()), + evalOnVirtualThreadIfAvailable(F.interruptible(process.waitFor())), F.delay(process.exitValue()) ) diff --git a/io/jvm/src/main/scala/fs2/io/ioplatform.scala b/io/jvm/src/main/scala/fs2/io/ioplatform.scala index 42e32069f1..0273aa15cf 100644 --- a/io/jvm/src/main/scala/fs2/io/ioplatform.scala +++ b/io/jvm/src/main/scala/fs2/io/ioplatform.scala @@ -34,6 +34,8 @@ import java.io.{InputStream, OutputStream} import java.nio.charset.Charset import java.nio.charset.StandardCharsets import java.util.concurrent.Executors +import scala.concurrent.ExecutionContext +import java.util.concurrent.ExecutorService private[fs2] trait ioplatform extends iojvmnative { @@ -163,4 +165,30 @@ private[fs2] trait ioplatform extends iojvmnative { } } + // Using null instead of Option because null check is faster + private lazy val vtExecutor: ExecutionContext = { + val javaVersion: Int = + System.getProperty("java.version").stripPrefix("1.").takeWhile(_.isDigit).toInt + + // From JVM 21 on we can use virtual threads + if (javaVersion >= 21) { + val virtualThreadExecutor = classOf[Executors] + .getDeclaredMethod("newVirtualThreadPerTaskExecutor") + .invoke(null) + .asInstanceOf[ExecutorService] + + ExecutionContext.fromExecutor(virtualThreadExecutor) + } else { + null + } + + } + + private[io] def evalOnVirtualThreadIfAvailable[F[_]: Async, A](fa: F[A]): F[A] = + if (vtExecutor != null) { + fa.evalOn(vtExecutor) + } else { + fa + } + } diff --git a/io/jvm/src/test/scala/fs2/io/IoPlatformSuite.scala b/io/jvm/src/test/scala/fs2/io/IoPlatformSuite.scala index 94a31f0f64..c5bc17b561 100644 --- a/io/jvm/src/test/scala/fs2/io/IoPlatformSuite.scala +++ b/io/jvm/src/test/scala/fs2/io/IoPlatformSuite.scala @@ -64,12 +64,11 @@ class IoPlatformSuite extends Fs2Suite { (bs1.length != (o1 + l1)) && // we expect that next slice will wrap same buffer ((bs2 eq bs1) && (o2 == o1 + l1)) - } || { - // if first slice buffer is 'full' - (bs2.length == (o1 + l1)) && - // we expect new buffer allocated for next slice - ((bs2 ne bs1) && (o2 == 0)) - } + } || + // if first slice buffer is 'full' + (bs2.length == (o1 + l1)) && + // we expect new buffer allocated for next slice + ((bs2 ne bs1) && (o2 == 0)) case _ => false // unexpected chunk subtype } } diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index ffa0006fc4..a1d914c523 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -171,4 +171,7 @@ private[fs2] trait ioplatform extends iojvmnative { def stdinUtf8[F[_], SourceBreakingDummy](bufSize: Int, F: Sync[F]): Stream[F, String] = stdin(bufSize, F).through(text.utf8.decode) + // Scala-native doesn't support virtual threads + private[io] def evalOnVirtualThreadIfAvailable[F[_], A](fa: F[A]): F[A] = fa + } diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index ed61d7bf22..ac895af1c9 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -225,7 +225,7 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } - test("read after timed out read not allowed on JVM or Native".ignore) { + test("read after timed out read") { val setup = for { serverSetup <- Network[IO].serverResource(Some(ip"127.0.0.1")) (bindAddress, server) = serverSetup @@ -239,22 +239,10 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { val prg = client.write(msg) *> client.readN(msg.size) *> - client.readN(msg.size).timeout(100.millis).recover { case _: TimeoutException => - Chunk.empty - } *> + (client.readN(msg.size) *> IO.raiseError(new AssertionError("didn't timeout"))) + .timeoutTo(100.millis, IO.unit) *> client.write(msg) *> - client - .readN(msg.size) - .flatMap { c => - if (isJVM) { - assertEquals(c.size, 0) - // Read again now that the pending read is no longer pending - client.readN(msg.size).map(c => assertEquals(c.size, 0)) - } else { - assertEquals(c, msg) - IO.unit - } - } + client.readN(msg.size).flatMap(c => IO(assertEquals(c, msg))) Stream.eval(prg).concurrently(echoServer) } .compile diff --git a/project/build.properties b/project/build.properties index 73df629ac1..cc68b53f1a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.7 +sbt.version=1.10.11 diff --git a/project/plugins.sbt b/project/plugins.sbt index af40679d88..ba8d2a7247 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,7 @@ -val sbtTypelevelVersion = "0.7.5" +val sbtTypelevelVersion = "0.7.7" addSbtPlugin("org.typelevel" % "sbt-typelevel" % sbtTypelevelVersion) addSbtPlugin("org.typelevel" % "sbt-typelevel-site" % sbtTypelevelVersion) -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.17.0") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.18.2") addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.17") addSbtPlugin("com.armanbilge" % "sbt-scala-native-config-brew-github-actions" % "0.3.0") addSbtPlugin("io.github.sbt-doctest" % "sbt-doctest" % "0.11.1") diff --git a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala index 3d8805b83d..e7884e304b 100644 --- a/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala +++ b/protocols/shared/src/main/scala-2/fs2/protocols/pcapng/BlockCodec.scala @@ -44,8 +44,9 @@ object BlockCodec { ("Block Total Length" | constant(length.bv) )} // format: on - def unknownByteOrder[L <: HList, LB <: HList](hexConstant: ByteVector)(f: Length => Codec[L])( - implicit + def unknownByteOrder[L <: HList, LB <: HList]( + hexConstant: ByteVector + )(f: Length => Codec[L])(implicit prepend: Prepend.Aux[L, Unit :: HNil, LB], init: Init.Aux[LB, L], last: Last.Aux[LB, Unit] diff --git a/reactive-streams/src/main/scala/fs2/interop/reactivestreams/package.scala b/reactive-streams/src/main/scala/fs2/interop/reactivestreams/package.scala index 7f76e0c072..8057fd732a 100644 --- a/reactive-streams/src/main/scala/fs2/interop/reactivestreams/package.scala +++ b/reactive-streams/src/main/scala/fs2/interop/reactivestreams/package.scala @@ -60,6 +60,7 @@ package object reactivestreams { * A high number can be useful if the publisher is triggering from IO, like requesting elements from a database. * The publisher can use this `bufferSize` to query elements in batch. * A high number will also lead to more elements in memory. + * The stream will not emit new element until, either the `Chunk` is filled or the publisher finishes. */ def fromPublisher[F[_]: Async, A](p: Publisher[A], bufferSize: Int): Stream[F, A] = Stream @@ -87,6 +88,7 @@ package object reactivestreams { * A high number can be useful if the publisher is triggering from IO, like requesting elements from a database. * The publisher can use this `bufferSize` to query elements in batch. * A high number will also lead to more elements in memory. + * The stream will not emit new element until, either the `Chunk` is filled or the publisher finishes. */ def toStreamBuffered[F[_]: Async](bufferSize: Int): Stream[F, A] = fromPublisher(publisher, bufferSize) diff --git a/scalafix/project/build.properties b/scalafix/project/build.properties index 67d27a1dfe..cc68b53f1a 100644 --- a/scalafix/project/build.properties +++ b/scalafix/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.3 +sbt.version=1.10.11 diff --git a/scalafix/project/plugins.sbt b/scalafix/project/plugins.sbt index 56a53c90c3..cb53d0ff30 100644 --- a/scalafix/project/plugins.sbt +++ b/scalafix/project/plugins.sbt @@ -1 +1 @@ -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.29") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.2") diff --git a/scalafix/rules/src/main/scala/fix/v1.scala b/scalafix/rules/src/main/scala/fix/v1.scala index e5e0d83225..47ef247be0 100644 --- a/scalafix/rules/src/main/scala/fix/v1.scala +++ b/scalafix/rules/src/main/scala/fix/v1.scala @@ -391,7 +391,7 @@ object StreamAppRules { tpl.copy( inits = tpl.inits :+ Init(Type.Name("IOApp"), Name("IOApp"), List()), stats = addProgramRun(tpl.stats)).toString() - ) + Patch.addLeft(tpl, "extends ") + ) private[this] def replaceStats(stats: List[Stat]): List[Patch] = stats.flatMap{ diff --git a/scalafix/tests/src/test/scala/fix/RuleSuite.scala b/scalafix/tests/src/test/scala/fix/RuleSuite.scala index 22fb678dac..5ca361bc67 100644 --- a/scalafix/tests/src/test/scala/fix/RuleSuite.scala +++ b/scalafix/tests/src/test/scala/fix/RuleSuite.scala @@ -1,8 +1,8 @@ package fix -import org.scalatest.FunSuiteLike +import org.scalatest.funsuite.AnyFunSuiteLike import scalafix.testkit.AbstractSemanticRuleSuite -class RuleSuite extends AbstractSemanticRuleSuite with FunSuiteLike { +class RuleSuite extends AbstractSemanticRuleSuite with AnyFunSuiteLike { runAllTests() }