From 9c6634d7b747a66fc8eb21e1ef105bbe516d87ca Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Oct 2023 10:27:44 +0300 Subject: [PATCH 1/7] gh-110171: `libregrtest` always sets `random.seed` unless `--no-use-randseed` is provided --- Lib/test/libregrtest/cmdline.py | 24 ++++++++++++++----- Lib/test/libregrtest/main.py | 8 ++++--- Lib/test/test_regrtest.py | 16 +++++++++++++ ...-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst | 2 ++ 4 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 8562a48446b4a7..7b7d942236f328 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -27,8 +27,11 @@ Additional option details: -r randomizes test execution order. You can use --randseed=int to provide an -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. +int seed value for the randomizer. The randseed value will be used +to set seeds for all random usages in tests +(including randomizing the tests order if -r is set). +By default we always set random seed, but do not randomize test order. +Use --no-use-randseed to disable random seeding. -s On the first invocation of regrtest using -s, the first test file found or the first test file given on the command line is run, and the name of @@ -156,6 +159,7 @@ def __init__(self, **kwargs) -> None: self.list_tests = False self.single = False self.randomize = False + self.use_random_seed = True self.fromfile = None self.fail_env_changed = False self.use_resources = None @@ -229,6 +233,13 @@ def _create_parser(): more_details) group.add_argument('-p', '--python', metavar='PYTHON', help='Command to run Python test subprocesses with.') + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a global random seed') + group.add_argument('--use-randseed', + dest='use_random_seed', + action=argparse.BooleanOptionalAction, + help='control if random should be seeded') group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', @@ -249,10 +260,6 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') group.add_argument('-f', '--fromfile', metavar='FILE', help='read names of tests to run from a file.' + more_details) @@ -483,6 +490,11 @@ def _parse_args(args, **kwargs): ns.use_resources.remove(r) elif r not in ns.use_resources: ns.use_resources.append(r) + if not ns.use_random_seed: + if ns.random_seed is not None: + parser.error("--no-use-randseed and --randseed=... don't go together") + if ns.randomize: + parser.error("--no-use-randseed and --randomize don't go together") if ns.random_seed is not None: ns.randomize = True if ns.verbose: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index dcb2c5870de176..f14dff05324be8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -107,6 +107,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.forever: bool = ns.forever self.randomize: bool = ns.randomize self.random_seed: int | None = ns.random_seed + self.use_random_seed: bool = ns.use_random_seed self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout if ns.huntrleaks: @@ -208,11 +209,12 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) - if self.randomize: + if self.use_random_seed: if self.random_seed is None: self.random_seed = random.randrange(100_000_000) random.seed(self.random_seed) - random.shuffle(selected) + if self.randomize: + random.shuffle(selected) return (tuple(selected), tests) @@ -433,7 +435,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources) - if self.randomize: + if self.use_random_seed: print("Using random seed", self.random_seed) runtests = self.create_run_tests(selected) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c98b05abcea98c..c3d3dde1fb0ed5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -949,6 +949,22 @@ def test_random(self): test_random2 = int(match.group(1)) self.assertEqual(test_random2, test_random) + # check that random.seed is used by default + output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) + self.assertIsInstance(self.parse_random_seed(output), int) + + # check that --no-use-randseed disables seed + output = self.run_tests('--no-use-randseed', test, + exitcode=EXITCODE_NO_TESTS_RAN) + with self.assertRaises(AssertionError): + self.parse_random_seed(output) + + # check that --no-use-randseed and --randseed are not compatible + self.run_tests('--no-use-randseed', f'--randseed={randseed}', test, + exitcode=EXITCODE_BAD_TEST) + self.run_tests('--no-use-randseed', '-r', test, + exitcode=EXITCODE_BAD_TEST) + def test_fromfile(self): # test --fromfile tests = [self.create_test() for index in range(5)] diff --git a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst new file mode 100644 index 00000000000000..96ebd94569ecd0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst @@ -0,0 +1,2 @@ +``libregrtest`` now always sets ``random.seed`` unless ``--no-use-randseed`` +is provided. From 1bef6392ec627c3ad399772499c3dc8d6415b613 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Oct 2023 16:57:06 +0300 Subject: [PATCH 2/7] Address review --- Lib/test/libregrtest/cmdline.py | 11 ----------- Lib/test/libregrtest/main.py | 18 ++++++++---------- Lib/test/libregrtest/setup.py | 3 +-- Lib/test/test_regrtest.py | 14 +------------- ...3-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst | 3 +-- 5 files changed, 11 insertions(+), 38 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 7b7d942236f328..7cfe904cb5a535 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -31,7 +31,6 @@ to set seeds for all random usages in tests (including randomizing the tests order if -r is set). By default we always set random seed, but do not randomize test order. -Use --no-use-randseed to disable random seeding. -s On the first invocation of regrtest using -s, the first test file found or the first test file given on the command line is run, and the name of @@ -159,7 +158,6 @@ def __init__(self, **kwargs) -> None: self.list_tests = False self.single = False self.randomize = False - self.use_random_seed = True self.fromfile = None self.fail_env_changed = False self.use_resources = None @@ -236,10 +234,6 @@ def _create_parser(): group.add_argument('--randseed', metavar='SEED', dest='random_seed', type=int, help='pass a global random seed') - group.add_argument('--use-randseed', - dest='use_random_seed', - action=argparse.BooleanOptionalAction, - help='control if random should be seeded') group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', @@ -490,11 +484,6 @@ def _parse_args(args, **kwargs): ns.use_resources.remove(r) elif r not in ns.use_resources: ns.use_resources.append(r) - if not ns.use_random_seed: - if ns.random_seed is not None: - parser.error("--no-use-randseed and --randseed=... don't go together") - if ns.randomize: - parser.error("--no-use-randseed and --randomize don't go together") if ns.random_seed is not None: ns.randomize = True if ns.verbose: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f14dff05324be8..18e3cd4c7544fb 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -106,8 +106,11 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.fail_rerun: bool = ns.fail_rerun self.forever: bool = ns.forever self.randomize: bool = ns.randomize - self.random_seed: int | None = ns.random_seed - self.use_random_seed: bool = ns.use_random_seed + self.random_seed: int = ( + ns.random_seed + if ns.random_seed is not None + else random.randrange(100_000_000) + ) self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout if ns.huntrleaks: @@ -209,12 +212,8 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) - if self.use_random_seed: - if self.random_seed is None: - self.random_seed = random.randrange(100_000_000) - random.seed(self.random_seed) - if self.randomize: - random.shuffle(selected) + if self.randomize: + random.shuffle(selected) return (tuple(selected), tests) @@ -435,8 +434,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources) - if self.use_random_seed: - print("Using random seed", self.random_seed) + print("Using random seed", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index f0d8d7ebaa2fdb..cb410da5acb4c3 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -126,5 +126,4 @@ def setup_tests(runtests: RunTests): if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) - if runtests.randomize: - random.seed(runtests.random_seed) + random.seed(runtests.random_seed) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c3d3dde1fb0ed5..c1da354e40c7a0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -383,7 +383,7 @@ def check_ci_mode(self, args, use_resources, rerun=True): self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.fail_rerun) self.assertTrue(regrtest.print_slowest) @@ -953,18 +953,6 @@ def test_random(self): output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) self.assertIsInstance(self.parse_random_seed(output), int) - # check that --no-use-randseed disables seed - output = self.run_tests('--no-use-randseed', test, - exitcode=EXITCODE_NO_TESTS_RAN) - with self.assertRaises(AssertionError): - self.parse_random_seed(output) - - # check that --no-use-randseed and --randseed are not compatible - self.run_tests('--no-use-randseed', f'--randseed={randseed}', test, - exitcode=EXITCODE_BAD_TEST) - self.run_tests('--no-use-randseed', '-r', test, - exitcode=EXITCODE_BAD_TEST) - def test_fromfile(self): # test --fromfile tests = [self.create_test() for index in range(5)] diff --git a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst index 96ebd94569ecd0..52e0be0d7bd878 100644 --- a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst +++ b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst @@ -1,2 +1 @@ -``libregrtest`` now always sets ``random.seed`` unless ``--no-use-randseed`` -is provided. +``libregrtest`` now always sets ``random.seed``. From bc512ac171dc4140edba8f0e71b64da7cbdb9bee Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Oct 2023 17:21:41 +0300 Subject: [PATCH 3/7] Restore seed --- Lib/test/libregrtest/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 18e3cd4c7544fb..02b48e2e99daba 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -212,6 +212,7 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList print(f"Cannot find starting test: {self.starting_test}") sys.exit(1) + random.seed(self.random_seed) if self.randomize: random.shuffle(selected) From b193156ce1443fda99968bc167b903e8065cffcf Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Oct 2023 17:57:21 +0300 Subject: [PATCH 4/7] Address review --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 02b48e2e99daba..b1ae00cb9fb1d9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -109,7 +109,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.random_seed: int = ( ns.random_seed if ns.random_seed is not None - else random.randrange(100_000_000) + else random.Random().getrandbits(32) ) self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout From 0d53c059740948219940be441e6422daecc215a1 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 1 Oct 2023 17:58:08 +0300 Subject: [PATCH 5/7] Fix test --- Lib/test/test_regrtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c1da354e40c7a0..34f49ead05469a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -654,7 +654,7 @@ def list_regex(line_format, tests): def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) randseed = int(match.group(1)) - self.assertTrue(0 <= randseed <= 100_000_000, randseed) + self.assertTrue(0 <= randseed, randseed) return randseed def run_command(self, args, input=None, exitcode=0, **kw): From 9ba3edbd51ff895ad89a385b6b3bac3fcc01f6b5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 1 Oct 2023 22:37:47 +0200 Subject: [PATCH 6/7] Update Lib/test/libregrtest/main.py --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b1ae00cb9fb1d9..7db30994fa3f85 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -109,7 +109,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.random_seed: int = ( ns.random_seed if ns.random_seed is not None - else random.Random().getrandbits(32) + else random.getrandbits(32) ) self.output_on_failure: bool = ns.verbose3 self.timeout: float | None = ns.timeout From 7b18378e56770ae5dcc4c2b0c2d1a0270227fb84 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 3 Oct 2023 22:07:29 +0300 Subject: [PATCH 7/7] Update NEWS --- .../next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst index 52e0be0d7bd878..9b41b033bc7f2b 100644 --- a/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst +++ b/Misc/NEWS.d/next/Tests/2023-10-01-10-27-02.gh-issue-110171.ZPlo0h.rst @@ -1 +1,3 @@ -``libregrtest`` now always sets ``random.seed``. +``libregrtest`` now always sets and shows ``random.seed``, +so tests are more reproducible. Use ``--randseed`` flag +to pass the explicit random seed for tests.