From 96cbda140e80ed23dcdcc5c5711958f2203ee04f Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Mon, 25 May 2020 09:38:22 -0400 Subject: [PATCH 1/2] initial work --- .../scala/zio/test/sbt/BaseTestTask.scala | 4 +- .../main/scala/zio/test/sbt/ZTestEvent.scala | 15 +-- .../scala/zio/test/ReportingTestUtils.scala | 2 +- .../src/test/scala/zio/test/SpecSpec.scala | 4 +- .../src/test/scala/zio/test/TestUtils.scala | 18 +-- .../scala/zio/test/DefaultTestReporter.scala | 120 +++++++----------- .../main/scala/zio/test/ExecutedSpec.scala | 80 ++++++++++++ .../main/scala/zio/test/RunnableSpec.scala | 15 +-- .../main/scala/zio/test/SummaryBuilder.scala | 64 ++-------- .../main/scala/zio/test/TestExecutor.scala | 16 +-- .../src/main/scala/zio/test/package.scala | 10 -- 11 files changed, 161 insertions(+), 187 deletions(-) create mode 100644 test/shared/src/main/scala/zio/test/ExecutedSpec.scala diff --git a/test-sbt/shared/src/main/scala/zio/test/sbt/BaseTestTask.scala b/test-sbt/shared/src/main/scala/zio/test/sbt/BaseTestTask.scala index 4ded31b13248..22fef745ae16 100644 --- a/test-sbt/shared/src/main/scala/zio/test/sbt/BaseTestTask.scala +++ b/test-sbt/shared/src/main/scala/zio/test/sbt/BaseTestTask.scala @@ -27,9 +27,9 @@ abstract class BaseTestTask( protected def run(eventHandler: EventHandler): ZIO[TestLogger with Clock, Throwable, Unit] = for { spec <- specInstance.runSpec(FilteredSpec(specInstance.spec, args)) - summary <- SummaryBuilder.buildSummary(spec) + summary = SummaryBuilder.buildSummary(spec) _ <- sendSummary.provide(summary) - events <- ZTestEvent.from(spec, taskDef.fullyQualifiedName, taskDef.fingerprint) + events = ZTestEvent.from(spec, taskDef.fullyQualifiedName, taskDef.fingerprint) _ <- ZIO.foreach[Any, Throwable, ZTestEvent, Unit](events)(e => ZIO.effect(eventHandler.handle(e))) } yield () diff --git a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala index 5e55cf4db772..a42b1c44c93c 100644 --- a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala +++ b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala @@ -2,8 +2,7 @@ package zio.test.sbt import sbt.testing._ -import zio.UIO -import zio.test.{ ExecutedSpec, Spec, TestFailure, TestSuccess } +import zio.test.{ ExecutedSpec, TestFailure, TestSuccess } final case class ZTestEvent( fullyQualifiedName: String, @@ -21,14 +20,10 @@ object ZTestEvent { executedSpec: ExecutedSpec[E], fullyQualifiedName: String, fingerprint: Fingerprint - ): UIO[Seq[ZTestEvent]] = - executedSpec.fold[UIO[Seq[ZTestEvent]]] { - case Spec.SuiteCase(_, results, _) => - results.use(UIO.collectAll(_).map(_.flatten)) - case Spec.TestCase(label, result, _) => - result.map { result => - Seq(ZTestEvent(fullyQualifiedName, new TestSelector(label), toStatus(result), None, 0, fingerprint)) - } + ): Seq[ZTestEvent] = + executedSpec.flattenTests { + case ExecutedSpec.Test(label, result, _) => + ZTestEvent(fullyQualifiedName, new TestSelector(label), toStatus(result), None, 0, fingerprint) } private def toStatus[E](result: Either[TestFailure[E], TestSuccess]) = result match { diff --git a/test-tests/shared/src/test/scala/zio/test/ReportingTestUtils.scala b/test-tests/shared/src/test/scala/zio/test/ReportingTestUtils.scala index fd457349a52e..8a06c8a2cfa4 100644 --- a/test-tests/shared/src/test/scala/zio/test/ReportingTestUtils.scala +++ b/test-tests/shared/src/test/scala/zio/test/ReportingTestUtils.scala @@ -59,7 +59,7 @@ object ReportingTestUtils { .provideLayer[Nothing, TestEnvironment, TestLogger with Clock]( TestLogger.fromConsole ++ TestClock.default ) - actualSummary <- SummaryBuilder.buildSummary(results) + actualSummary = SummaryBuilder.buildSummary(results) } yield actualSummary.summary private[this] def TestTestRunner(testEnvironment: Layer[Nothing, TestEnvironment]) = diff --git a/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala b/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala index 674dcaaaeaa9..fbf3cb204dc5 100644 --- a/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala +++ b/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala @@ -69,8 +69,8 @@ object SpecSpec extends ZIOBaseSpec { ).provideLayerShared(ZLayer.succeed(43)) for { executedSpec <- execute(spec) - successes <- executedSpec.countTests(_.isRight).useNow - failures <- executedSpec.countTests(_.isLeft).useNow + successes = executedSpec.countSuccesses + failures = executedSpec.countFailures } yield assert(successes)(equalTo(1)) && assert(failures)(equalTo(2)) } ), diff --git a/test-tests/shared/src/test/scala/zio/test/TestUtils.scala b/test-tests/shared/src/test/scala/zio/test/TestUtils.scala index 93264c4424cc..d5588599a991 100644 --- a/test-tests/shared/src/test/scala/zio/test/TestUtils.scala +++ b/test-tests/shared/src/test/scala/zio/test/TestUtils.scala @@ -8,29 +8,19 @@ object TestUtils { def execute[E](spec: ZSpec[TestEnvironment, E]): UIO[ExecutedSpec[E]] = TestExecutor.default(environment.testEnvironment).run(spec, ExecutionStrategy.Sequential) - def forAllTests[E]( - execSpec: UIO[ExecutedSpec[E]] - )(f: Either[TestFailure[E], TestSuccess] => Boolean): ZIO[Any, Nothing, Boolean] = - execSpec.flatMap { results => - results.forall { - case Spec.TestCase(_, test, _) => test.map(r => f(r)) - case _ => ZIO.succeed(true) - }.useNow - } - def isIgnored[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = { val execSpec = execute(spec) - forAllTests(execSpec) { + execSpec.map(_.forAllTests { case Right(TestSuccess.Ignored) => true case _ => false - } + }) } def succeeded[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = { val execSpec = execute(spec) - forAllTests(execSpec) { + execSpec.map(_.forAllTests { case Right(TestSuccess.Succeeded(_)) => true case _ => false - } + }) } } diff --git a/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala b/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala index 69b2c646f9f2..b8f5bfb1c3d9 100644 --- a/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala +++ b/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala @@ -29,103 +29,71 @@ import zio.test.RenderedResult.Status._ import zio.test.RenderedResult.{ CaseType, Status } import zio.test.mock.Expectation import zio.test.mock.internal.{ InvalidCall, MockException } -import zio.{ Cause, Has, UIO, URIO } +import zio.{ Cause, Has } object DefaultTestReporter { def render[E]( executedSpec: ExecutedSpec[E], testAnnotationRenderer: TestAnnotationRenderer - ): UIO[Seq[RenderedResult[String]]] = { + ): Seq[RenderedResult[String]] = { def loop( executedSpec: ExecutedSpec[E], depth: Int, ancestors: List[TestAnnotationMap] - ): UIO[Seq[RenderedResult[String]]] = - executedSpec.caseValue match { - case c @ Spec.SuiteCase(label, executedSpecs, _) => - for { - specs <- executedSpecs.useNow - failures <- UIO.foreach(specs) { specs => - specs.exists { - case Spec.TestCase(_, test, _) => test.map(_.isLeft) - case _ => UIO.succeedNow(false) - }.useNow - } - annotations <- Spec(c).fold[UIO[TestAnnotationMap]] { - case Spec.SuiteCase(_, specs, _) => - specs.use(UIO.collectAll(_).map(_.foldLeft(TestAnnotationMap.empty)(_ ++ _))) - case Spec.TestCase(_, _, annotations) => UIO.succeedNow(annotations) - } - hasFailures = failures.exists(identity) - status = if (hasFailures) Failed else Passed - renderedLabel = if (specs.isEmpty) Seq.empty + ): Seq[RenderedResult[String]] = + executedSpec match { + case ExecutedSpec.Suite(label, specs) => + val hasFailures = specs.exists(_.hasFailures) + val annotations = specs.foldLeft(TestAnnotationMap.empty)(_ ++ _.annotationsMap) + val status = if (hasFailures) Failed else Passed + val renderedLabel = + if (specs.isEmpty) Seq.empty else if (hasFailures) Seq(renderFailureLabel(label, depth)) else Seq(renderSuccessLabel(label, depth)) - renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) - rest <- UIO.foreach(specs)(loop(_, depth + tabSize, annotations :: ancestors)).map(_.flatten) - } yield rendered(Suite, label, status, depth, (renderedLabel): _*) - .withAnnotations(renderedAnnotations) +: rest - case Spec.TestCase(label, result, annotations) => - result.map { result => - val renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) - val renderedResult = result match { - case Right(TestSuccess.Succeeded(_)) => - rendered(Test, label, Passed, depth, withOffset(depth)(green("+") + " " + label)) - case Right(TestSuccess.Ignored) => - rendered(Test, label, Ignored, depth) - case Left(TestFailure.Assertion(result)) => - result.fold(details => rendered(Test, label, Failed, depth, renderFailure(label, depth, details): _*))( - _ && _, - _ || _, - !_ - ) - case Left(TestFailure.Runtime(cause)) => - rendered( - Test, - label, - Failed, - depth, - (Seq(renderFailureLabel(label, depth)) ++ Seq(renderCause(cause, depth))): _* - ) - } - Seq(renderedResult.withAnnotations(renderedAnnotations)) + val renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) + val rest = specs.flatMap(loop(_, depth + tabSize, annotations :: ancestors)) + rendered(Suite, label, status, depth, (renderedLabel): _*).withAnnotations(renderedAnnotations) +: rest + case ExecutedSpec.Test(label, result, annotations) => + val renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) + val renderedResult = result match { + case Right(TestSuccess.Succeeded(_)) => + rendered(Test, label, Passed, depth, withOffset(depth)(green("+") + " " + label)) + case Right(TestSuccess.Ignored) => + rendered(Test, label, Ignored, depth) + case Left(TestFailure.Assertion(result)) => + result.fold(details => rendered(Test, label, Failed, depth, renderFailure(label, depth, details): _*))( + _ && _, + _ || _, + !_ + ) + case Left(TestFailure.Runtime(cause)) => + rendered( + Test, + label, + Failed, + depth, + (Seq(renderFailureLabel(label, depth)) ++ Seq(renderCause(cause, depth))): _* + ) } + Seq(renderedResult.withAnnotations(renderedAnnotations)) } loop(executedSpec, 0, List.empty) } def apply[E](testAnnotationRenderer: TestAnnotationRenderer): TestReporter[E] = { (duration: Duration, executedSpec: ExecutedSpec[E]) => - for { - rendered <- render(executedSpec, testAnnotationRenderer).map(_.flatMap(_.rendered)) - stats <- logStats(duration, executedSpec) - _ <- TestLogger.logLine((rendered ++ Seq(stats)).mkString("\n")) - } yield () + val rendered = render(executedSpec, testAnnotationRenderer).flatMap(_.rendered) + val stats = logStats(duration, executedSpec) + TestLogger.logLine((rendered ++ Seq(stats)).mkString("\n")) } - private def logStats[E](duration: Duration, executedSpec: ExecutedSpec[E]): URIO[TestLogger, String] = { - def loop(executedSpec: ExecutedSpec[E]): UIO[(Int, Int, Int)] = - executedSpec.caseValue match { - case Spec.SuiteCase(_, executedSpecs, _) => - for { - specs <- executedSpecs.useNow - stats <- UIO.foreach(specs)(loop) - } yield stats.foldLeft((0, 0, 0)) { - case ((x1, x2, x3), (y1, y2, y3)) => (x1 + y1, x2 + y2, x3 + y3) - } - case Spec.TestCase(_, result, _) => - result.map { - case Left(_) => (0, 0, 1) - case Right(TestSuccess.Succeeded(_)) => (1, 0, 0) - case Right(TestSuccess.Ignored) => (0, 1, 0) - } - } - for { - stats <- loop(executedSpec) - (success, ignore, failure) = stats - total = success + ignore + failure - } yield cyan( + private def logStats[E](duration: Duration, executedSpec: ExecutedSpec[E]): String = { + val success = executedSpec.countSuccesses + val ignore = executedSpec.countIgnored + val failure = executedSpec.countFailures + val total = success + ignore + failure + cyan( s"Ran $total test${if (total == 1) "" else "s"} in ${duration.render}: $success succeeded, $ignore ignored, $failure failed" ) } diff --git a/test/shared/src/main/scala/zio/test/ExecutedSpec.scala b/test/shared/src/main/scala/zio/test/ExecutedSpec.scala new file mode 100644 index 000000000000..400aaf8369a9 --- /dev/null +++ b/test/shared/src/main/scala/zio/test/ExecutedSpec.scala @@ -0,0 +1,80 @@ +package zio.test + +import zio.test.ExecutedSpec._ + +sealed trait ExecutedSpec[+E] { self => + + def annotationsMap: TestAnnotationMap = + self match { + case Suite(_, specs) => + specs.foldLeft(TestAnnotationMap.empty)((annotations, spec) => annotations ++ spec.annotationsMap) + case Test(_, _, annotations) => annotations + } + + def countFailures: Int = + self match { + case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countFailures) + case Test(_, test, _) => test.fold(_ => 1, _ => 0) + } + + def countIgnored: Int = + self match { + case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countIgnored) + case Test(_, test, _) => test.fold(_ => 0, { case TestSuccess.Ignored => 1; case _ => 0 }) + } + + def countSuccesses: Int = + self match { + case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countSuccesses) + case Test(_, test, _) => test.fold(_ => 0, { case TestSuccess.Succeeded(_) => 1; case _ => 0 }) + } + + def failures: Seq[ExecutedSpec[E]] = + self match { + case Suite(label, specs) => + val failures = specs.flatMap(_.failures) + if (failures.isEmpty) Seq.empty else Seq(Suite(label, failures)) + case c @ Test(_, test, _) => if (test.isLeft) Seq(c) else Seq.empty + } + + def flattenTests[Z](f: Test[E] => Z): Seq[Z] = + self match { + case Suite(_, specs) => specs.flatMap(_.flattenTests((f))) + case c @ Test(_, _, _) => Seq(f(c)) + } + + def forAllTests(f: Either[TestFailure[E], TestSuccess] => Boolean): Boolean = + self match { + case Suite(_, specs) => specs.forall(_.forAllTests(f)) + case Test(_, test, _) => f(test) + } + + def hasFailures: Boolean = + self match { + case Suite(_, specs) => specs.exists(_.hasFailures) + case Test(_, test, _) => test.isLeft + } + + def isEmpty: Boolean = + self match { + case Suite(_, specs) => specs.forall(_.isEmpty) + case Test(_, _, _) => false + } +} + +object ExecutedSpec { + + final case class Suite[+E](label: String, specs: Vector[ExecutedSpec[E]]) extends ExecutedSpec[E] + final case class Test[+E](label: String, test: Either[TestFailure[E], TestSuccess], annotations: TestAnnotationMap) + extends ExecutedSpec[E] + + def suite[E](label: String, specs: Vector[ExecutedSpec[E]]): ExecutedSpec[E] = + Suite(label, specs) + + def test[E]( + label: String, + test: Either[TestFailure[E], TestSuccess], + annotations: TestAnnotationMap + ): ExecutedSpec[E] = + Test(label, test, annotations) +} diff --git a/test/shared/src/main/scala/zio/test/RunnableSpec.scala b/test/shared/src/main/scala/zio/test/RunnableSpec.scala index 3541bcf7926a..9f1f7fd5b8e5 100644 --- a/test/shared/src/main/scala/zio/test/RunnableSpec.scala +++ b/test/shared/src/main/scala/zio/test/RunnableSpec.scala @@ -17,8 +17,7 @@ package zio.test import zio.clock.Clock -import zio.test.Spec.TestCase -import zio.{ Has, UIO, URIO } +import zio.{ Has, URIO } /** * A `RunnableSpec` has a main function and can be run by the JVM / Scala.js. @@ -29,14 +28,10 @@ trait RunnableSpec[R <: Has[_], E] extends AbstractRunnableSpec { private def run(spec: ZSpec[Environment, Failure]): URIO[TestLogger with Clock, Int] = for { - results <- runSpec(spec) - hasFailures <- results.exists { - case TestCase(_, test, _) => test.map(_.isLeft) - case _ => UIO.succeedNow(false) - }.useNow - summary <- SummaryBuilder.buildSummary(results) - _ <- TestLogger.logLine(summary.summary) - } yield if (hasFailures) 1 else 0 + executedSpec <- runSpec(spec) + summary = SummaryBuilder.buildSummary(executedSpec) + _ <- TestLogger.logLine(summary.summary) + } yield if (executedSpec.hasFailures) 1 else 0 /** * A simple main function that can be used to run the spec. diff --git a/test/shared/src/main/scala/zio/test/SummaryBuilder.scala b/test/shared/src/main/scala/zio/test/SummaryBuilder.scala index 5155766d5def..efac320d2b42 100644 --- a/test/shared/src/main/scala/zio/test/SummaryBuilder.scala +++ b/test/shared/src/main/scala/zio/test/SummaryBuilder.scala @@ -1,59 +1,15 @@ package zio.test -import zio.test.Spec._ -import zio.{ UIO, ZIO } - object SummaryBuilder { - def buildSummary[E](executedSpec: ExecutedSpec[E]): UIO[Summary] = - for { - success <- countTestResults(executedSpec) { - case Right(TestSuccess.Succeeded(_)) => true - case _ => false - } - fail <- countTestResults(executedSpec)(_.isLeft) - ignore <- countTestResults(executedSpec) { - case Right(TestSuccess.Ignored) => true - case _ => false - } - failures <- extractFailures(executedSpec) - rendered <- ZIO.foreach(failures)(DefaultTestReporter.render(_, TestAnnotationRenderer.silent)) - } yield Summary(success, fail, ignore, rendered.flatten.flatMap(_.rendered).mkString("\n")) - - private def countTestResults[E]( - executedSpec: ExecutedSpec[E] - )(pred: Either[TestFailure[E], TestSuccess] => Boolean): UIO[Int] = - executedSpec.fold[UIO[Int]] { - case SuiteCase(_, counts, _) => counts.use(ZIO.collectAll(_).map(_.sum)) - case TestCase(_, test, _) => - test.map(r => if (pred(r)) 1 else 0) - } - - private def extractFailures[E](executedSpec: ExecutedSpec[E]): UIO[Seq[ExecutedSpec[E]]] = { - def ifM[A](condition: UIO[Boolean])(success: UIO[A])(failure: UIO[A]): UIO[A] = - condition.flatMap(result => if (result) success else failure) - - def append[A](collection: UIO[Seq[A]], item: A): UIO[Seq[A]] = collection.map(_ :+ item) - - def hasFailures(spec: ExecutedSpec[E]): UIO[Boolean] = - spec.exists { - case Spec.TestCase(_, test, _) => test.map(_.isLeft) - case _ => UIO.succeedNow(false) - }.useNow - - def loop(current: ExecutedSpec[E], accM: UIO[Seq[ExecutedSpec[E]]]): UIO[Seq[ExecutedSpec[E]]] = - ifM(hasFailures(current)) { - current.caseValue match { - case suite @ Spec.SuiteCase(_, specs, _) => - val newSpecs = specs.use(ZIO.foreach(_)(extractFailures).map(_.flatten.toVector)) - append(accM, Spec(suite.copy(specs = newSpecs.toManaged_))) - - case Spec.TestCase(_, _, _) => - append(accM, current) - } - } { - accM - } - - loop(executedSpec, UIO.succeedNow(Vector.empty[ExecutedSpec[E]])) + def buildSummary[E](executedSpec: ExecutedSpec[E]): Summary = { + val success = executedSpec.countSuccesses + val fail = executedSpec.countFailures + val ignore = executedSpec.countIgnored + val failures = executedSpec.failures + val rendered = failures + .flatMap(DefaultTestReporter.render(_, TestAnnotationRenderer.silent)) + .flatMap(_.rendered) + .mkString("\n") + Summary(success, fail, ignore, rendered) } } diff --git a/test/shared/src/main/scala/zio/test/TestExecutor.scala b/test/shared/src/main/scala/zio/test/TestExecutor.scala index a07edecaa59c..99fd4ae40f40 100644 --- a/test/shared/src/main/scala/zio/test/TestExecutor.scala +++ b/test/shared/src/main/scala/zio/test/TestExecutor.scala @@ -43,15 +43,15 @@ object TestExecutor { case (success, annotations) => ZIO.succeedNow((Right(success), annotations)) } ) - .use(_.fold[UIO[ExecutedSpec[E]]] { - case Spec.SuiteCase(label, specs, exec) => - UIO.succeedNow(Spec.suite(label, specs.mapM(UIO.collectAll(_)).map(_.toVector), exec)) - case Spec.TestCase(label, test, annotations) => + .use(_.foldM[Any, Nothing, ExecutedSpec[E]](defExec) { + case Spec.SuiteCase(label, specs, _) => + specs.map(specs => ExecutedSpec.suite(label, specs)) + case Spec.TestCase(label, test, staticAnnotations) => test.map { - case (result, annotations1) => - Spec.test(label, UIO.succeedNow(result), annotations ++ annotations1) - } - }) + case (result, dynamicAnnotations) => + ExecutedSpec.test(label, result, staticAnnotations ++ dynamicAnnotations) + }.toManaged_ + }.useNow) val environment = env } } diff --git a/test/shared/src/main/scala/zio/test/package.scala b/test/shared/src/main/scala/zio/test/package.scala index 31aceea1cfb0..f7500cc41fa2 100644 --- a/test/shared/src/main/scala/zio/test/package.scala +++ b/test/shared/src/main/scala/zio/test/package.scala @@ -117,16 +117,6 @@ package object test extends CompileVariants { */ type ZSpec[-R, +E] = Spec[R, TestFailure[E], TestSuccess] - /** - * An `ExecutedResult[E] is either a `TestSuccess` or a `TestFailure[E]`. - */ - type ExecutedResult[+E] = Either[TestFailure[E], TestSuccess] - - /** - * An `ExecutedSpec` is a spec that has been run to produce test results. - */ - type ExecutedSpec[+E] = Spec[Any, Nothing, ExecutedResult[E]] - /** * An `Annotated[A]` contains a value of type `A` along with zero or more * test annotations. From 0c8b6becf5b52e8b310eab8aa7789f4729393b51 Mon Sep 17 00:00:00 2001 From: Adam Fraser Date: Tue, 26 May 2020 20:51:22 -0400 Subject: [PATCH 2/2] cleanup --- .../main/scala/zio/test/sbt/ZTestEvent.scala | 7 +- .../src/test/scala/zio/test/SpecSpec.scala | 10 +- .../src/test/scala/zio/test/TestUtils.scala | 36 +++-- .../scala/zio/test/DefaultTestReporter.scala | 35 +++-- .../main/scala/zio/test/ExecutedSpec.scala | 123 +++++++++++------- .../main/scala/zio/test/RunnableSpec.scala | 12 +- .../main/scala/zio/test/SummaryBuilder.scala | 30 ++++- 7 files changed, 168 insertions(+), 85 deletions(-) diff --git a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala index a42b1c44c93c..47da38157980 100644 --- a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala +++ b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala @@ -21,9 +21,10 @@ object ZTestEvent { fullyQualifiedName: String, fingerprint: Fingerprint ): Seq[ZTestEvent] = - executedSpec.flattenTests { - case ExecutedSpec.Test(label, result, _) => - ZTestEvent(fullyQualifiedName, new TestSelector(label), toStatus(result), None, 0, fingerprint) + executedSpec.fold[Seq[ZTestEvent]] { + case ExecutedSpec.SuiteCase(_, results) => results.flatten + case ExecutedSpec.TestCase(label, result, _) => + Seq(ZTestEvent(fullyQualifiedName, new TestSelector(label), toStatus(result), None, 0, fingerprint)) } private def toStatus[E](result: Either[TestFailure[E], TestSuccess]) = result match { diff --git a/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala b/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala index fbf3cb204dc5..1fe0feb75f09 100644 --- a/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala +++ b/test-tests/shared/src/test/scala/zio/test/SpecSpec.scala @@ -69,8 +69,14 @@ object SpecSpec extends ZIOBaseSpec { ).provideLayerShared(ZLayer.succeed(43)) for { executedSpec <- execute(spec) - successes = executedSpec.countSuccesses - failures = executedSpec.countFailures + successes = executedSpec.fold[Int] { + case ExecutedSpec.SuiteCase(_, counts) => counts.sum + case ExecutedSpec.TestCase(_, test, _) => if (test.isRight) 1 else 0 + } + failures = executedSpec.fold[Int] { + case ExecutedSpec.SuiteCase(_, counts) => counts.sum + case ExecutedSpec.TestCase(_, test, _) => if (test.isLeft) 1 else 0 + } } yield assert(successes)(equalTo(1)) && assert(failures)(equalTo(2)) } ), diff --git a/test-tests/shared/src/test/scala/zio/test/TestUtils.scala b/test-tests/shared/src/test/scala/zio/test/TestUtils.scala index d5588599a991..9e9d7a03e57d 100644 --- a/test-tests/shared/src/test/scala/zio/test/TestUtils.scala +++ b/test-tests/shared/src/test/scala/zio/test/TestUtils.scala @@ -8,19 +8,27 @@ object TestUtils { def execute[E](spec: ZSpec[TestEnvironment, E]): UIO[ExecutedSpec[E]] = TestExecutor.default(environment.testEnvironment).run(spec, ExecutionStrategy.Sequential) - def isIgnored[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = { - val execSpec = execute(spec) - execSpec.map(_.forAllTests { - case Right(TestSuccess.Ignored) => true - case _ => false - }) - } + def forAllTests[E]( + execSpec: ExecutedSpec[E] + )(f: Either[TestFailure[E], TestSuccess] => Boolean): Boolean = + execSpec.forall { + case ExecutedSpec.TestCase(_, test, _) => f(test) + case _ => true + } - def succeeded[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = { - val execSpec = execute(spec) - execSpec.map(_.forAllTests { - case Right(TestSuccess.Succeeded(_)) => true - case _ => false - }) - } + def isIgnored[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = + execute(spec).map { executedSpec => + forAllTests(executedSpec) { + case Right(TestSuccess.Ignored) => true + case _ => false + } + } + + def succeeded[E](spec: ZSpec[environment.TestEnvironment, E]): ZIO[Any, Nothing, Boolean] = + execute(spec).map { executedSpec => + forAllTests(executedSpec) { + case Right(TestSuccess.Succeeded(_)) => true + case _ => false + } + } } diff --git a/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala b/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala index b8f5bfb1c3d9..c3adaf417170 100644 --- a/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala +++ b/test/shared/src/main/scala/zio/test/DefaultTestReporter.scala @@ -42,11 +42,17 @@ object DefaultTestReporter { depth: Int, ancestors: List[TestAnnotationMap] ): Seq[RenderedResult[String]] = - executedSpec match { - case ExecutedSpec.Suite(label, specs) => - val hasFailures = specs.exists(_.hasFailures) - val annotations = specs.foldLeft(TestAnnotationMap.empty)(_ ++ _.annotationsMap) - val status = if (hasFailures) Failed else Passed + executedSpec.caseValue match { + case ExecutedSpec.SuiteCase(label, specs) => + val hasFailures = executedSpec.exists { + case ExecutedSpec.TestCase(_, test, _) => test.isLeft + case _ => false + } + val annotations = executedSpec.fold[TestAnnotationMap] { + case ExecutedSpec.SuiteCase(_, annotations) => annotations.foldLeft(TestAnnotationMap.empty)(_ ++ _) + case ExecutedSpec.TestCase(_, _, annotations) => annotations + } + val status = if (hasFailures) Failed else Passed val renderedLabel = if (specs.isEmpty) Seq.empty else if (hasFailures) Seq(renderFailureLabel(label, depth)) @@ -54,7 +60,7 @@ object DefaultTestReporter { val renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) val rest = specs.flatMap(loop(_, depth + tabSize, annotations :: ancestors)) rendered(Suite, label, status, depth, (renderedLabel): _*).withAnnotations(renderedAnnotations) +: rest - case ExecutedSpec.Test(label, result, annotations) => + case ExecutedSpec.TestCase(label, result, annotations) => val renderedAnnotations = testAnnotationRenderer.run(ancestors, annotations) val renderedResult = result match { case Right(TestSuccess.Succeeded(_)) => @@ -89,10 +95,19 @@ object DefaultTestReporter { } private def logStats[E](duration: Duration, executedSpec: ExecutedSpec[E]): String = { - val success = executedSpec.countSuccesses - val ignore = executedSpec.countIgnored - val failure = executedSpec.countFailures - val total = success + ignore + failure + val (success, ignore, failure) = executedSpec.fold[(Int, Int, Int)] { + case ExecutedSpec.SuiteCase(_, stats) => + stats.foldLeft((0, 0, 0)) { + case ((x1, x2, x3), (y1, y2, y3)) => (x1 + y1, x2 + y2, x3 + y3) + } + case ExecutedSpec.TestCase(_, result, _) => + result match { + case Left(_) => (0, 0, 1) + case Right(TestSuccess.Succeeded(_)) => (1, 0, 0) + case Right(TestSuccess.Ignored) => (0, 1, 0) + } + } + val total = success + ignore + failure cyan( s"Ran $total test${if (total == 1) "" else "s"} in ${duration.render}: $success succeeded, $ignore ignored, $failure failed" ) diff --git a/test/shared/src/main/scala/zio/test/ExecutedSpec.scala b/test/shared/src/main/scala/zio/test/ExecutedSpec.scala index 400aaf8369a9..9a9cb6850456 100644 --- a/test/shared/src/main/scala/zio/test/ExecutedSpec.scala +++ b/test/shared/src/main/scala/zio/test/ExecutedSpec.scala @@ -2,79 +2,106 @@ package zio.test import zio.test.ExecutedSpec._ -sealed trait ExecutedSpec[+E] { self => +/** + * An `ExecutedSpec` is a spec that has been run to produce test results. + */ +final case class ExecutedSpec[+E](caseValue: SpecCase[E, ExecutedSpec[E]]) { self => - def annotationsMap: TestAnnotationMap = - self match { - case Suite(_, specs) => - specs.foldLeft(TestAnnotationMap.empty)((annotations, spec) => annotations ++ spec.annotationsMap) - case Test(_, _, annotations) => annotations + /** + * Determines if any node in the spec is satisfied by the given predicate. + */ + def exists(f: SpecCase[E, Boolean] => Boolean): Boolean = + fold[Boolean] { + case c @ SuiteCase(_, specs) => specs.exists(identity) || f(c) + case c @ TestCase(_, _, _) => f(c) } - def countFailures: Int = - self match { - case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countFailures) - case Test(_, test, _) => test.fold(_ => 1, _ => 0) + /** + * Folds over all nodes to produce a final result. + */ + def fold[Z](f: SpecCase[E, Z] => Z): Z = + caseValue match { + case SuiteCase(label, specs) => f(SuiteCase(label, specs.map(_.fold(f)))) + case t @ TestCase(_, _, _) => f(t) } - def countIgnored: Int = - self match { - case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countIgnored) - case Test(_, test, _) => test.fold(_ => 0, { case TestSuccess.Ignored => 1; case _ => 0 }) + /** + * Determines if all nodes in the spec are satisfied by the given predicate. + */ + def forall(f: SpecCase[E, Boolean] => Boolean): Boolean = + fold[Boolean] { + case c @ SuiteCase(_, specs) => specs.forall(identity) && f(c) + case c @ TestCase(_, _, _) => f(c) } - def countSuccesses: Int = - self match { - case Suite(_, specs) => specs.foldLeft(0)((n, spec) => n + spec.countSuccesses) - case Test(_, test, _) => test.fold(_ => 0, { case TestSuccess.Succeeded(_) => 1; case _ => 0 }) + /** + * Computes the size of the spec, i.e. the number of tests in the spec. + */ + def size: Int = + fold[Int] { + case SuiteCase(_, counts) => counts.sum + case TestCase(_, _, _) => 1 } - def failures: Seq[ExecutedSpec[E]] = - self match { - case Suite(label, specs) => - val failures = specs.flatMap(_.failures) - if (failures.isEmpty) Seq.empty else Seq(Suite(label, failures)) - case c @ Test(_, test, _) => if (test.isLeft) Seq(c) else Seq.empty + /** + * Transforms the spec one layer at a time. + */ + def transform[E1](f: SpecCase[E, ExecutedSpec[E1]] => SpecCase[E1, ExecutedSpec[E1]]): ExecutedSpec[E1] = + caseValue match { + case SuiteCase(label, specs) => ExecutedSpec(f(SuiteCase(label, specs.map(_.transform(f))))) + case t @ TestCase(_, _, _) => ExecutedSpec(f(t)) } - def flattenTests[Z](f: Test[E] => Z): Seq[Z] = - self match { - case Suite(_, specs) => specs.flatMap(_.flattenTests((f))) - case c @ Test(_, _, _) => Seq(f(c)) - } + /** + * Transforms the spec statefully, one layer at a time. + */ + def transformAccum[E1, Z]( + z0: Z + )(f: (Z, SpecCase[E, ExecutedSpec[E1]]) => (Z, SpecCase[E1, ExecutedSpec[E1]])): (Z, ExecutedSpec[E1]) = + caseValue match { + case SuiteCase(label, specs) => + val (z, specs1) = + specs.foldLeft(z0 -> Vector.empty[ExecutedSpec[E1]]) { + case ((z, vector), spec) => + val (z1, spec1) = spec.transformAccum(z)(f) - def forAllTests(f: Either[TestFailure[E], TestSuccess] => Boolean): Boolean = - self match { - case Suite(_, specs) => specs.forall(_.forAllTests(f)) - case Test(_, test, _) => f(test) - } + z1 -> (vector :+ spec1) + } - def hasFailures: Boolean = - self match { - case Suite(_, specs) => specs.exists(_.hasFailures) - case Test(_, test, _) => test.isLeft - } + val (z1, caseValue) = f(z, SuiteCase(label, specs1)) - def isEmpty: Boolean = - self match { - case Suite(_, specs) => specs.forall(_.isEmpty) - case Test(_, _, _) => false + z1 -> ExecutedSpec(caseValue) + case t @ TestCase(_, _, _) => + val (z, caseValue) = f(z0, t) + z -> ExecutedSpec(caseValue) } } object ExecutedSpec { - final case class Suite[+E](label: String, specs: Vector[ExecutedSpec[E]]) extends ExecutedSpec[E] - final case class Test[+E](label: String, test: Either[TestFailure[E], TestSuccess], annotations: TestAnnotationMap) - extends ExecutedSpec[E] + trait SpecCase[+E, +A] { self => + def map[B](f: A => B): SpecCase[E, B] = + self match { + case SuiteCase(label, specs) => SuiteCase(label, specs.map(f)) + case TestCase(label, test, annotations) => TestCase(label, test, annotations) + } + } + + final case class SuiteCase[+A](label: String, specs: Vector[A]) extends SpecCase[Nothing, A] + + final case class TestCase[+E]( + label: String, + test: Either[TestFailure[E], TestSuccess], + annotations: TestAnnotationMap + ) extends SpecCase[E, Nothing] def suite[E](label: String, specs: Vector[ExecutedSpec[E]]): ExecutedSpec[E] = - Suite(label, specs) + ExecutedSpec(SuiteCase(label, specs)) def test[E]( label: String, test: Either[TestFailure[E], TestSuccess], annotations: TestAnnotationMap ): ExecutedSpec[E] = - Test(label, test, annotations) + ExecutedSpec(TestCase(label, test, annotations)) } diff --git a/test/shared/src/main/scala/zio/test/RunnableSpec.scala b/test/shared/src/main/scala/zio/test/RunnableSpec.scala index 9f1f7fd5b8e5..de3c7cb20428 100644 --- a/test/shared/src/main/scala/zio/test/RunnableSpec.scala +++ b/test/shared/src/main/scala/zio/test/RunnableSpec.scala @@ -28,10 +28,14 @@ trait RunnableSpec[R <: Has[_], E] extends AbstractRunnableSpec { private def run(spec: ZSpec[Environment, Failure]): URIO[TestLogger with Clock, Int] = for { - executedSpec <- runSpec(spec) - summary = SummaryBuilder.buildSummary(executedSpec) - _ <- TestLogger.logLine(summary.summary) - } yield if (executedSpec.hasFailures) 1 else 0 + results <- runSpec(spec) + hasFailures = results.exists { + case ExecutedSpec.TestCase(_, test, _) => test.isLeft + case _ => false + } + summary = SummaryBuilder.buildSummary(results) + _ <- TestLogger.logLine(summary.summary) + } yield if (hasFailures) 1 else 0 /** * A simple main function that can be used to run the spec. diff --git a/test/shared/src/main/scala/zio/test/SummaryBuilder.scala b/test/shared/src/main/scala/zio/test/SummaryBuilder.scala index efac320d2b42..e517ae8c1dc3 100644 --- a/test/shared/src/main/scala/zio/test/SummaryBuilder.scala +++ b/test/shared/src/main/scala/zio/test/SummaryBuilder.scala @@ -2,14 +2,36 @@ package zio.test object SummaryBuilder { def buildSummary[E](executedSpec: ExecutedSpec[E]): Summary = { - val success = executedSpec.countSuccesses - val fail = executedSpec.countFailures - val ignore = executedSpec.countIgnored - val failures = executedSpec.failures + val success = countTestResults(executedSpec) { + case Right(TestSuccess.Succeeded(_)) => true + case _ => false + } + val fail = countTestResults(executedSpec)(_.isLeft) + val ignore = countTestResults(executedSpec) { + case Right(TestSuccess.Ignored) => true + case _ => false + } + val failures = extractFailures(executedSpec) val rendered = failures .flatMap(DefaultTestReporter.render(_, TestAnnotationRenderer.silent)) .flatMap(_.rendered) .mkString("\n") Summary(success, fail, ignore, rendered) } + + private def countTestResults[E]( + executedSpec: ExecutedSpec[E] + )(pred: Either[TestFailure[E], TestSuccess] => Boolean): Int = + executedSpec.fold[Int] { + case ExecutedSpec.SuiteCase(_, counts) => counts.sum + case ExecutedSpec.TestCase(_, test, _) => if (pred(test)) 1 else 0 + } + + private def extractFailures[E](executedSpec: ExecutedSpec[E]): Seq[ExecutedSpec[E]] = + executedSpec.fold[Seq[ExecutedSpec[E]]] { + case ExecutedSpec.SuiteCase(label, specs) => + val newSpecs = specs.flatten + if (newSpecs.nonEmpty) Seq(ExecutedSpec(ExecutedSpec.SuiteCase(label, newSpecs))) else Seq.empty + case c @ ExecutedSpec.TestCase(_, test, _) => if (test.isLeft) Seq(ExecutedSpec(c)) else Seq.empty + } }