Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit d0f49d2

Browse files
zoobazooba
authored
bpo-34582: Adds JUnit XML output for regression tests (GH-9210)
1 parent cb5778f commit d0f49d2

File tree

12 files changed

+329
-33
lines changed

12 files changed

+329
-33
lines changed

‎.vsts/linux-pr.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@ steps:
7070
displayName: 'Run patchcheck.py'
7171
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
7272

73-
- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
73+
- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
7474
displayName: 'Tests'
7575
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
76+
77+
- task: PublishTestResults@2
78+
displayName: 'Publish Test Results'
79+
inputs:
80+
testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
81+
mergeTestResults: true
82+
testRunTitle: '$(system.pullRequest.targetBranch)-linux'
83+
platform: linux
84+
condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

‎.vsts/macos-pr.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,15 @@ steps:
5050
displayName: 'Display build info'
5151
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
5252

53-
- script: make buildbottest TESTOPTS="-j4 -uall,-cpu"
53+
- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
5454
displayName: 'Tests'
5555
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
56+
57+
- task: PublishTestResults@2
58+
displayName: 'Publish Test Results'
59+
inputs:
60+
testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
61+
mergeTestResults: true
62+
testRunTitle: '$(system.pullRequest.targetBranch)-macOS'
63+
platform: macOS
64+
condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

‎.vsts/windows-pr.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,17 @@ steps:
5454
displayName: 'Display build info'
5555
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
5656

57-
- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
57+
- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml"
5858
displayName: 'Tests'
5959
env:
6060
PREFIX: $(Py_OutDir)\$(outDirSuffix)
6161
condition: and(succeeded(), ne(variables['DocOnly'], 'true'))
62+
63+
- task: PublishTestResults@2
64+
displayName: 'Publish Test Results'
65+
inputs:
66+
testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml'
67+
mergeTestResults: true
68+
testRunTitle: '$(System.PullRequest.TargetBranch)-$(outDirSuffix)'
69+
platform: $(outDirSuffix)
70+
condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true'))

‎Lib/test/eintrdata/eintr_tester.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ def setUpClass(cls):
5252

5353
# Issue #25277: Use faulthandler to try to debug a hang on FreeBSD
5454
if hasattr(faulthandler, 'dump_traceback_later'):
55-
faulthandler.dump_traceback_later(10 * 60, exit=True)
55+
faulthandler.dump_traceback_later(10 * 60, exit=True,
56+
file=sys.__stderr__)
5657

5758
@classmethod
5859
def stop_alarm(cls):

‎Lib/test/libregrtest/cmdline.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,10 @@ def _create_parser():
268268
help='if a test file alters the environment, mark '
269269
'the test as failed')
270270

271+
group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
272+
help='writes JUnit-style XML results to the specified '
273+
'file')
274+
271275
return parser
272276

273277

‎Lib/test/libregrtest/main.py

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,11 @@ def __init__(self):
100100
self.next_single_test = None
101101
self.next_single_filename = None
102102

103+
# used by --junit-xml
104+
self.testsuite_xml = None
105+
103106
def accumulate_result(self, test, result):
104-
ok, test_time = result
107+
ok, test_time, xml_data = result
105108
if ok not in (CHILD_ERROR, INTERRUPTED):
106109
self.test_times.append((test_time, test))
107110
if ok == PASSED:
@@ -118,6 +121,15 @@ def accumulate_result(self, test, result):
118121
elif ok != INTERRUPTED:
119122
raise ValueError("invalid test result: %r" % ok)
120123

124+
if xml_data:
125+
import xml.etree.ElementTree as ET
126+
for e in xml_data:
127+
try:
128+
self.testsuite_xml.append(ET.fromstring(e))
129+
except ET.ParseError:
130+
print(xml_data, file=sys.__stderr__)
131+
raise
132+
121133
def display_progress(self, test_index, test):
122134
if self.ns.quiet:
123135
return
@@ -164,6 +176,9 @@ def parse_args(self, kwargs):
164176
file=sys.stderr)
165177
ns.findleaks = False
166178

179+
if ns.xmlpath:
180+
support.junit_xml_list = self.testsuite_xml = []
181+
167182
# Strip .py extensions.
168183
removepy(ns.args)
169184

@@ -384,7 +399,7 @@ def run_tests_sequential(self):
384399
result = runtest(self.ns, test)
385400
except KeyboardInterrupt:
386401
self.interrupted = True
387-
self.accumulate_result(test, (INTERRUPTED, None))
402+
self.accumulate_result(test, (INTERRUPTED, None, None))
388403
break
389404
else:
390405
self.accumulate_result(test, result)
@@ -508,6 +523,31 @@ def finalize(self):
508523
if self.ns.runleaks:
509524
os.system("leaks %d" % os.getpid())
510525

526+
def save_xml_result(self):
527+
if not self.ns.xmlpath and not self.testsuite_xml:
528+
return
529+
530+
import xml.etree.ElementTree as ET
531+
root = ET.Element("testsuites")
532+
533+
# Manually count the totals for the overall summary
534+
totals = {'tests': 0, 'errors': 0, 'failures': 0}
535+
for suite in self.testsuite_xml:
536+
root.append(suite)
537+
for k in totals:
538+
try:
539+
totals[k] += int(suite.get(k, 0))
540+
except ValueError:
541+
pass
542+
543+
for k, v in totals.items():
544+
root.set(k, str(v))
545+
546+
xmlpath = os.path.join(support.SAVEDCWD, self.ns.xmlpath)
547+
with open(xmlpath, 'wb') as f:
548+
for s in ET.tostringlist(root):
549+
f.write(s)
550+
511551
def main(self, tests=None, **kwargs):
512552
global TEMPDIR
513553

@@ -570,6 +610,9 @@ def _main(self, tests, kwargs):
570610
self.rerun_failed_tests()
571611

572612
self.finalize()
613+
614+
self.save_xml_result()
615+
573616
if self.bad:
574617
sys.exit(2)
575618
if self.interrupted:

‎Lib/test/libregrtest/runtest.py

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,18 @@ def runtest(ns, test):
8585
ns -- regrtest namespace of options
8686
test -- the name of the test
8787
88-
Returns the tuple (result, test_time), where result is one of the
89-
constants:
88+
Returns the tuple (result, test_time, xml_data), where result is one
89+
of the constants:
9090
9191
INTERRUPTED KeyboardInterrupt when run under -j
9292
RESOURCE_DENIED test skipped because resource denied
9393
SKIPPED test skipped for some other reason
9494
ENV_CHANGED test failed because it changed the execution environment
9595
FAILED test failed
9696
PASSED test passed
97+
98+
If ns.xmlpath is not None, xml_data is a list containing each
99+
generated testsuite element.
97100
"""
98101

99102
output_on_failure = ns.verbose3
@@ -106,22 +109,13 @@ def runtest(ns, test):
106109
# reset the environment_altered flag to detect if a test altered
107110
# the environment
108111
support.environment_altered = False
112+
support.junit_xml_list = xml_list = [] if ns.xmlpath else None
109113
if ns.failfast:
110114
support.failfast = True
111115
if output_on_failure:
112116
support.verbose = True
113117

114-
# Reuse the same instance to all calls to runtest(). Some
115-
# tests keep a reference to sys.stdout or sys.stderr
116-
# (eg. test_argparse).
117-
if runtest.stringio is None:
118-
stream = io.StringIO()
119-
runtest.stringio = stream
120-
else:
121-
stream = runtest.stringio
122-
stream.seek(0)
123-
stream.truncate()
124-
118+
stream = io.StringIO()
125119
orig_stdout = sys.stdout
126120
orig_stderr = sys.stderr
127121
try:
@@ -138,12 +132,18 @@ def runtest(ns, test):
138132
else:
139133
support.verbose = ns.verbose # Tell tests to be moderately quiet
140134
result = runtest_inner(ns, test, display_failure=not ns.verbose)
141-
return result
135+
136+
if xml_list:
137+
import xml.etree.ElementTree as ET
138+
xml_data = [ET.tostring(x).decode('us-ascii') for x in xml_list]
139+
else:
140+
xml_data = None
141+
return result + (xml_data,)
142142
finally:
143143
if use_timeout:
144144
faulthandler.cancel_dump_traceback_later()
145145
cleanup_test_droppings(test, ns.verbose)
146-
runtest.stringio = None
146+
support.junit_xml_list = None
147147

148148

149149
def post_test_cleanup():

‎Lib/test/libregrtest/runtest_mp.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def run_tests_worker(worker_args):
6767
try:
6868
result = runtest(ns, testname)
6969
except KeyboardInterrupt:
70-
result = INTERRUPTED, ''
70+
result = INTERRUPTED, '', None
7171
except BaseException as e:
7272
traceback.print_exc()
7373
result = CHILD_ERROR, str(e)
@@ -122,7 +122,7 @@ def _runtest(self):
122122
self.current_test = None
123123

124124
if retcode != 0:
125-
result = (CHILD_ERROR, "Exit code %s" % retcode)
125+
result = (CHILD_ERROR, "Exit code %s" % retcode, None)
126126
self.output.put((test, stdout.rstrip(), stderr.rstrip(),
127127
result))
128128
return False
@@ -133,6 +133,7 @@ def _runtest(self):
133133
return True
134134

135135
result = json.loads(result)
136+
assert len(result) == 3, f"Invalid result tuple: {result!r}"
136137
self.output.put((test, stdout.rstrip(), stderr.rstrip(),
137138
result))
138139
return False
@@ -195,7 +196,7 @@ def get_running(workers):
195196
regrtest.accumulate_result(test, result)
196197

197198
# Display progress
198-
ok, test_time = result
199+
ok, test_time, xml_data = result
199200
text = format_test_result(test, ok)
200201
if (ok not in (CHILD_ERROR, INTERRUPTED)
201202
and test_time >= PROGRESS_MIN_TIME

‎Lib/test/support/__init__.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@
66
import asyncio.events
77
import collections.abc
88
import contextlib
9+
import datetime
910
import errno
1011
import faulthandler
1112
import fnmatch
1213
import functools
1314
import gc
1415
import importlib
1516
import importlib.util
17+
import io
1618
import logging.handlers
1719
import nntplib
1820
import os
@@ -34,6 +36,8 @@
3436
import urllib.error
3537
import warnings
3638

39+
from .testresult import get_test_runner
40+
3741
try:
3842
import multiprocessing.process
3943
except ImportError:
@@ -295,6 +299,7 @@ def get_attribute(obj, name):
295299
max_memuse = 0 # Disable bigmem tests (they will still be run with
296300
# small sizes, to make sure they work.)
297301
real_max_memuse = 0
302+
junit_xml_list = None # list of testsuite XML elements
298303
failfast = False
299304

300305
# _original_stdout is meant to hold stdout at the time regrtest began.
@@ -1891,13 +1896,16 @@ def _filter_suite(suite, pred):
18911896

18921897
def _run_suite(suite):
18931898
"""Run tests from a unittest.TestSuite-derived class."""
1894-
if verbose:
1895-
runner = unittest.TextTestRunner(sys.stdout, verbosity=2,
1896-
failfast=failfast)
1897-
else:
1898-
runner = BasicTestRunner()
1899+
runner = get_test_runner(sys.stdout, verbosity=verbose)
1900+
1901+
# TODO: Remove this before merging (here for easy comparison with old impl)
1902+
#runner = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=failfast)
18991903

19001904
result = runner.run(suite)
1905+
1906+
if junit_xml_list is not None:
1907+
junit_xml_list.append(result.get_xml_element())
1908+
19011909
if not result.wasSuccessful():
19021910
if len(result.errors) == 1 and not result.failures:
19031911
err = result.errors[0][1]

0 commit comments

Comments
 (0)