Description
Describe the bug
When running tests with ut_tfs_junit_reporter
on suite packages containing contexts, each context is reported as a separate suite.
This doesn't seem correct as we loose ability to represent physical test suite pacakges as units of work/units of reporting
Provide version info
3.1.10
Information about client software
Not applicable.
To Reproduce
Steps to reproduce the behavior:
Create suites with context(s):
create or replace package demo_tfs_issue_0 as
--%suite(some suite name)
--%context(some context)
--%context(some nested context)
--%test(first test)
procedure test_procedure1;
--%endcontext
--%endcontext
end;
/
create or replace package body demo_tfs_issue_0 as
procedure test_procedure1 is begin ut.expect(1).to_be_not_null; end;
end;
/
create or replace package demo_tfs_issue as
--%suite(some suite name)
--%context(some context)
--%context(some nested context)
--%test(first test)
procedure test_procedure1;
--%endcontext
--%endcontext
--%context(some context)
--%context(some nested context)
--%test(second test)
procedure test_procedure2;
--%endcontext
--%test(third test)
procedure test_procedure3;
--%endcontext
--%context(second context)
--%test(fourth test)
procedure test_procedure4;
--%endcontext
--%context(third context)
--%test(fifth test)
procedure test_procedure5;
--%test(sixth test)
procedure test_procedure6;
--%endcontext
--%test(seventh test)
procedure test_procedure7;
--%test(eight test)
procedure test_procedure8;
--%test(nineth test)
procedure test_procedure9;
--%test(tenth test)
procedure test_procedure10;
end;
/
create or replace package body demo_tfs_issue as
procedure test_procedure1 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure2 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure3 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure4 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure5 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure6 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure7 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure8 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure9 is begin ut.expect(1).to_be_not_null; end;
procedure test_procedure10 is begin ut.expect(1).to_be_not_null; end;
end;
/
Run unit tests with TSF reporter:
clear screen
set serverout on
exec ut.run( utp3.ut_tfs_junit_reporter() );
See results, where each context is reported as individual item and there are no tests reported for unit test suite except the ones that live outside of context.
<?xml version="1.0"?>
<testsuites>
<testsuite tests="1" id="1" package="demo_tfs_issue.nested_context_#1.nested_context_#1" errors="0" failures="0"
name="some nested context" time=".002429" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#1.nested_context_#1" name="first test" time=".002168">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="2" package="demo_tfs_issue.nested_context_#1" errors="0" failures="0" name="some context"
time=".002719" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="3" package="demo_tfs_issue.nested_context_#2.nested_context_#1" errors="0" failures="0"
name="some nested context" time=".000899" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#2.nested_context_#1" name="second test" time=".000616">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="2" id="4" package="demo_tfs_issue.nested_context_#2" errors="0" failures="0" name="some context"
time=".001846" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#2" name="third test" time=".000632">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="5" package="demo_tfs_issue.nested_context_#3" errors="0" failures="0" name="second context"
time=".001064" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#3" name="fourth test" time=".000784">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="2" id="6" package="demo_tfs_issue.nested_context_#4" errors="0" failures="0" name="third context"
time=".001544" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#4" name="fifth test" time=".0006">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#4" name="sixth test" time=".000622">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="10" id="7" package="demo_tfs_issue" errors="0" failures="0" name="some suite name" time=".010303"
timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue" name="seventh test" time=".000592">
</testcase>
<testcase classname="demo_tfs_issue" name="eight test" time=".000602">
</testcase>
<testcase classname="demo_tfs_issue" name="nineth test" time=".000595">
</testcase>
<testcase classname="demo_tfs_issue" name="tenth test" time=".000568">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="8" package="demo_tfs_issue_0.nested_context_#1.nested_context_#1" errors="0" failures="0"
name="some nested context" time=".002722" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue_0.nested_context_#1.nested_context_#1" name="first test" time=".002462">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="9" package="demo_tfs_issue_0.nested_context_#1" errors="0" failures="0" name="some context"
time=".00298" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="10" package="demo_tfs_issue_0" errors="0" failures="0" name="some suite name"
time=".003262" timestamp="2020-06-15T08:13:06" hostname="...">
<properties/>
<system-out/>
<system-err/>
</testsuite>
</testsuites>
Expected behavior
- each unit test suite is reported as a
<testsuite>
item - contexts and nested contexts, are rolled into
<testsuite>
- test name/description is prefixed with context name/description
Example of expected output
<?xml version="1.0"?>
<testsuites>
<testsuite tests="10" id="1" package="demo_tfs_issue" errors="0" failures="0" name="some suite name" time=".009857"
timestamp="2020-06-15T08:22:24" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue.nested_context_#1.nested_context_#1"
name="some context - some nested context - first test" time=".001944">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#2.nested_context_#1"
name="some context - some nested context - second test" time=".000609">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#2" name="some context - third test" time=".000608">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#3" name="second context - fourth test" time=".000627">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#4" name="third context - fifth test" time=".000609">
</testcase>
<testcase classname="demo_tfs_issue.nested_context_#4" name="third context - sixth test" time=".00061">
</testcase>
<testcase classname="demo_tfs_issue" name="seventh test" time=".000586">
</testcase>
<testcase classname="demo_tfs_issue" name="eight test" time=".000589">
</testcase>
<testcase classname="demo_tfs_issue" name="nineth test" time=".000585">
</testcase>
<testcase classname="demo_tfs_issue" name="tenth test" time=".000619">
</testcase>
<system-out/>
<system-err/>
</testsuite>
<testsuite tests="1" id="2" package="demo_tfs_issue_0" errors="0" failures="0" name="some suite name" time=".002022"
timestamp="2020-06-15T08:22:24" hostname="...">
<properties/>
<testcase classname="demo_tfs_issue_0.nested_context_#1.nested_context_#1"
name="some context - some nested context - first test" time=".001248">
</testcase>
<system-out/>
<system-err/>
</testsuite>
</testsuites>
Additional context
The ut_tfs_junit_reporter
is reporting the "flat" suite structure as opposed to ut_junit_reporter
which supports nesting of <testsuite>
elements.