Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix flaky CI tests #26680

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 4, 2023
Merged

Fix flaky CI tests #26680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5fb45b8
TST: Move locale comma test to a subprocess
QuLogic Jun 7, 2023
4bb4dc7
Use Agg backend in pickle test subprocess
QuLogic Aug 1, 2023
8530b1f
TST: Mark macosx backend in leak test as xfail
QuLogic Aug 11, 2023
02263d2
TST: Mark test_other_signal_before_sigint mac as xfail
QuLogic Aug 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions lib/matplotlib/tests/test_backends_interactive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -660,7 +660,7 @@ def _test_figure_leak():
reason="appveyor tests fail; gh-22988 suggests reworking")
@pytest.mark.parametrize("env", _get_testable_interactive_backends())
@pytest.mark.parametrize("time_mem", [(0.0, 2_000_000), (0.1, 30_000_000)])
def test_figure_leak_20490(env, time_mem):
def test_figure_leak_20490(env, time_mem, request):
pytest.importorskip("psutil", reason="psutil needed to run this test")

# We haven't yet directly identified the leaks so test with a memory growth
Expand All @@ -669,9 +669,10 @@ def test_figure_leak_20490(env, time_mem):
if env["MPLBACKEND"] == "wx":
pytest.skip("wx backend is deprecated; tests failed on appveyor")

if env["MPLBACKEND"] == "macosx" or (
env["MPLBACKEND"] == "tkagg" and sys.platform == 'darwin'
):
if env["MPLBACKEND"] == "macosx":
request.node.add_marker(pytest.mark.xfail(reason="macosx backend is leaky"))

if env["MPLBACKEND"] == "tkagg" and sys.platform == "darwin":
acceptable_memory_leakage += 11_000_000

result = _run_helper(
Expand Down Expand Up @@ -815,12 +816,12 @@ def custom_signal_handler(signum, frame):
('show', {'block': True}),
('pause', {'interval': 10})
])
def test_other_signal_before_sigint(env, target, kwargs):
def test_other_signal_before_sigint(env, target, kwargs, request):
backend = env.get("MPLBACKEND")
if not backend.startswith(("qt", "macosx")):
pytest.skip("SIGINT currently only tested on qt and macosx")
if backend == "macosx" and target == "show":
pytest.xfail("test currently failing for macosx + show()")
if backend == "macosx":
request.node.add_marker(pytest.mark.xfail(reason="macosx backend is buggy"))
proc = _WaitForStringPopen(
[sys.executable, "-c",
inspect.getsource(_test_other_signal_before_sigint_impl) +
Expand Down
2 changes: 1 addition & 1 deletion lib/matplotlib/tests/test_pickle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def test_pickle_load_from_subprocess(fig_test, fig_ref, tmp_path):
proc = subprocess_run_helper(
_pickle_load_subprocess,
timeout=60,
extra_env={'PICKLE_FILE_PATH': str(fp)}
extra_env={'PICKLE_FILE_PATH': str(fp), 'MPLBACKEND': 'Agg'}
)

loaded_fig = pickle.loads(ast.literal_eval(proc.stdout))
Expand Down
40 changes: 27 additions & 13 deletions lib/matplotlib/tests/test_ticker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1640,22 +1640,36 @@ def test_latex(self, is_latex, usetex, expected):
assert fmt.format_pct(50, 100) == expected


def test_locale_comma():
currentLocale = locale.getlocale()
def _impl_locale_comma():
try:
locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
ticks = mticker.ScalarFormatter(useMathText=True, useLocale=True)
fmt = '$\\mathdefault{%1.1f}$'
x = ticks._format_maybe_minus_and_locale(fmt, 0.5)
assert x == '$\\mathdefault{0{,}5}$'
# Do not change , in the format string
fmt = ',$\\mathdefault{,%1.1f},$'
x = ticks._format_maybe_minus_and_locale(fmt, 0.5)
assert x == ',$\\mathdefault{,0{,}5},$'
except locale.Error:
pytest.skip("Locale de_DE.UTF-8 is not supported on this machine")
finally:
locale.setlocale(locale.LC_ALL, currentLocale)
print('SKIP: Locale de_DE.UTF-8 is not supported on this machine')
return
ticks = mticker.ScalarFormatter(useMathText=True, useLocale=True)
fmt = '$\\mathdefault{%1.1f}$'
x = ticks._format_maybe_minus_and_locale(fmt, 0.5)
assert x == '$\\mathdefault{0{,}5}$'
# Do not change , in the format string
fmt = ',$\\mathdefault{,%1.1f},$'
x = ticks._format_maybe_minus_and_locale(fmt, 0.5)
assert x == ',$\\mathdefault{,0{,}5},$'


def test_locale_comma():
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain this song and dance?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it's in the commit message:

On some systems/pytest versions, the skip in an exception handler does not skip, but is treated as an exception. Namely, the ARM test machine in Cirrus and on my WSL Ubuntu.

I assume that is a bug somewhere in pytest but the subprocess avoids that. Also, changing the locale is a global modification, and it seems safer to do so in a subprocess, and not have to worry about fixing it up in the main process.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps comment the code as well? This seems an odd thing to do.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

# On some systems/pytest versions, `pytest.skip` in an exception handler
# does not skip, but is treated as an exception, so directly running this
# test can incorrectly fail instead of skip.
# Instead, run this test in a subprocess, which avoids the problem, and the
# need to fix the locale after.
proc = mpl.testing.subprocess_run_helper(_impl_locale_comma, timeout=60,
extra_env={'MPLBACKEND': 'Agg'})
skip_msg = next((line[len('SKIP:'):].strip()
for line in proc.stdout.splitlines()
if line.startswith('SKIP:')),
'')
if skip_msg:
pytest.skip(skip_msg)


def test_majformatter_type():
Expand Down