From 00bbbd1ee67e6d9e4b82313f886aaeb540a26f81 Mon Sep 17 00:00:00 2001 From: Georgi Krastev Date: Wed, 23 Feb 2022 22:55:49 +0100 Subject: [PATCH] Report error messages to sbt events which end up in JUnit XML This is a quick and dirty solution reusing `DefaultTestReporter` and subsequently stripping the ANSI colors from the result. A more elegant solution would involve writing a new reporter, or adding an option to render without colors to `DefaultTestReporter`, but this is probably a major issue for anyone running ZIO test in CI, so I think the quick win is worth it. --- .../zio/test/sbt/ZTestFrameworkSpec.scala | 36 +++++++++++++++++-- .../main/scala/zio/test/sbt/ZTestEvent.scala | 29 +++++++++++---- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/test-sbt/jvm/src/test/scala/zio/test/sbt/ZTestFrameworkSpec.scala b/test-sbt/jvm/src/test/scala/zio/test/sbt/ZTestFrameworkSpec.scala index f2ee07a18694..e0f04235c358 100644 --- a/test-sbt/jvm/src/test/scala/zio/test/sbt/ZTestFrameworkSpec.scala +++ b/test-sbt/jvm/src/test/scala/zio/test/sbt/ZTestFrameworkSpec.scala @@ -31,6 +31,7 @@ object ZTestFrameworkSpec { test("should return correct fingerprints")(testFingerprints()), test("should report events")(testReportEvents()), test("should report durations")(testReportDurations()), + test("should report errors")(testReportErrors()), test("should log messages")(testLogMessages()), test("should correctly display colorized output for multi-line strings")(testColored()), test("should test only selected test")(testTestSelection()), @@ -44,16 +45,21 @@ object ZTestFrameworkSpec { } def testReportEvents(): Unit = { - val reported = ArrayBuffer[Event]() + val reported = ArrayBuffer.empty[Event] loadAndExecute(failingSpecFQN, reported.append(_)) - val expected = Set( + val actual = reported.map { + case event: ZTestEvent => event.copy(maybeThrowable = None) + case event => event + } + + val expected = Seq( sbtEvent(failingSpecFQN, "failing test", Status.Failure), sbtEvent(failingSpecFQN, "passing test", Status.Success), sbtEvent(failingSpecFQN, "ignored test", Status.Ignored) ) - assertEquals("reported events", reported.toSet, expected) + assertEquals("reported events", actual, expected) } private def sbtEvent(fqn: String, label: String, status: Status) = @@ -66,6 +72,21 @@ object ZTestFrameworkSpec { assert(reported.forall(_.duration() > 0), s"reported events should have positive durations: $reported") } + def testReportErrors(): Unit = { + val reported = ArrayBuffer.empty[Event] + loadAndExecute(errorReportingSpecFQN, reported.append(_)) + assert(reported.forall(_.throwable.isDefined), s"reported events should have errors: $reported") + + val actual = reported.map(_.throwable.get.getMessage.split('\n').take(4).mkString("|").withNoLineNumbers) + val expected = Seq( + s"- failing assertion| true did not satisfy isFalse()| $assertLocation", + s"- failing test| Fiber failed.| A checked error was not handled.| Boom", + s"- dying test| Fiber failed.| An unchecked error was produced.| java.lang.Exception: KO" + ) + + assertEquals("reported errors", actual, expected) + } + def testLogMessages(): Unit = { val loggers = Seq.fill(3)(new MockLogger) @@ -211,6 +232,15 @@ object ZTestFrameworkSpec { } @@ TestAspect.before(Live.live(ZIO.sleep(5.millis))) @@ TestAspect.timed } + lazy val errorReportingSpecFQN = ErrorReportingSpec.getClass.getName + object ErrorReportingSpec extends DefaultRunnableSpec { + override def spec: ZSpec[Environment, Failure] = suite("error reporting")( + test("failing assertion")(zio.test.assert(true)(Assertion.isFalse)), + testM("failing test")(ZIO.fail("Boom")), + testM("dying test")(ZIO.die(new Exception("KO"))) + ) + } + lazy val multiLineSpecFQN = MultiLineSpec.getClass.getName object MultiLineSpec extends DefaultRunnableSpec { def spec: ZSpec[Environment, Failure] = test("multi-line test") { diff --git a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala index f88422642948..cfda2ec1daae 100644 --- a/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala +++ b/test-sbt/shared/src/main/scala/zio/test/sbt/ZTestEvent.scala @@ -1,7 +1,7 @@ package zio.test.sbt import sbt.testing._ -import zio.test.{ExecutedSpec, TestAnnotation, TestFailure, TestSuccess} +import zio.test.{DefaultTestReporter, ExecutedSpec, TestAnnotation, TestAnnotationRenderer, TestFailure, TestSuccess} final case class ZTestEvent( fullyQualifiedName: String, @@ -22,24 +22,24 @@ object ZTestEvent { fingerprint: Fingerprint ): Seq[ZTestEvent] = { - def loop(executedSpec: ExecutedSpec[E], labels: List[String]): Seq[ZTestEvent] = + def loop(executedSpec: ExecutedSpec[E], label: Option[String]): Seq[ZTestEvent] = executedSpec.caseValue match { - case ExecutedSpec.LabeledCase(label, spec) => loop(spec, label :: labels) - case ExecutedSpec.MultipleCase(specs) => specs.flatMap(spec => loop(spec, labels)) + case ExecutedSpec.LabeledCase(label, spec) => loop(spec, Some(label)) + case ExecutedSpec.MultipleCase(specs) => specs.flatMap(loop(_, label)) case ExecutedSpec.TestCase(result, annotations) => Seq( ZTestEvent( fullyQualifiedName, - new TestSelector(labels.headOption.getOrElse("")), + new TestSelector(label.getOrElse("")), toStatus(result), - None, + toThrowable(executedSpec, label, result), annotations.get(TestAnnotation.timing).toMillis, fingerprint ) ) } - loop(executedSpec, List.empty) + loop(executedSpec, None) } private def toStatus[E](result: Either[TestFailure[E], TestSuccess]) = result match { @@ -47,4 +47,19 @@ object ZTestEvent { case Right(TestSuccess.Succeeded(_)) => Status.Success case Right(TestSuccess.Ignored) => Status.Ignored } + + private def toThrowable[E]( + spec: ExecutedSpec[E], + label: Option[String], + result: Either[TestFailure[E], TestSuccess] + ) = result.left.toOption.map { + case TestFailure.Assertion(_) => new AssertionError(render(spec, label)) + case TestFailure.Runtime(_) => new RuntimeException(render(spec, label)) + } + + private def render[E](spec: ExecutedSpec[E], label: Option[String]) = DefaultTestReporter + .render(label.fold(spec)(ExecutedSpec.labeled(_, spec)), TestAnnotationRenderer.default, includeCause = true) + .flatMap(_.rendered) + .mkString("\n") + .replaceAll("\u001B\\[[;\\d]*m", "") }