From 920fb835f19f30d6b27484decf2c8aca44d7d770 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 10 Oct 2023 23:37:44 +0900 Subject: [PATCH 1/5] gh-109595: Update -Xcpu_count= cmdline to support process mode --- Doc/c-api/init_config.rst | 2 +- Doc/using/cmdline.rst | 6 +++++- Lib/os.py | 3 +++ Lib/test/test_cmd_line.py | 9 +++++++++ .../2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst | 4 ++++ Python/initconfig.c | 8 +++++++- 6 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 0240e25b6f1607..fcc87a13471aea 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -884,7 +884,7 @@ PyConfig override the return values of :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. - Configured by the :samp:`-X cpu_count={n|default}` command line + Configured by the :samp:`-X cpu_count={n|default|process}` command line flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. Default: ``-1``. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 2767b0cb15451c..6224a179ee19c1 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -551,7 +551,9 @@ Miscellaneous options *n* must be greater than or equal to 1. This option may be useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHON_CPU_COUNT`. - If *n* is ``default``, nothing is overridden. + If *n* is ``default``, nothing is overridden, if *n* is ``process``, + :func:`os.os.process_cpu_count` will be alias of :func:`os.cpu_count` + and :func:`multiprocessing.cpu_count`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1076,6 +1078,8 @@ conflict. If this variable is set to a positive integer, it overrides the return values of :func:`os.cpu_count` and :func:`os.process_cpu_count`. + if this variable is set to ``process``, :func:`os.process_cpu_count` becomes + alias of :func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. See also the :option:`-X cpu_count <-X>` command-line option. diff --git a/Lib/os.py b/Lib/os.py index a17946750ea7e7..68eefbaa7e687f 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1147,6 +1147,9 @@ def process_cpu_count(): current process. Return None if indeterminable. """ return len(sched_getaffinity(0)) + if sys._get_cpu_count_config() == -2: + # Just an alias to process_count() (same docstring) + cpu_count = process_cpu_count else: # Just an alias to cpu_count() (same docstring) process_cpu_count = cpu_count diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index eaf19aa160e860..08896face20e71 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -915,6 +915,15 @@ def test_cpu_count_default(self): es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='default') self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + def test_cpu_count_process(self): + code = "import os; print(os.cpu_count(), os.process_cpu_count())" + res = assert_python_ok('-X', 'cpu_count=process', '-c', code) + self.assertEqual(self.res2int(res), (os.process_cpu_count(), os.process_cpu_count())) + res = assert_python_ok('-X', 'cpu_count=default', '-c', code, PYTHON_CPU_COUNT='process') + self.assertEqual(self.res2int(res), (os.cpu_count(), os.process_cpu_count())) + es = assert_python_ok('-c', code, PYTHON_CPU_COUNT='process') + self.assertEqual(self.res2int(res), (os.process_cpu_count(), os.process_cpu_count())) + def res2int(self, res): out = res.out.strip().decode("utf-8") return tuple(int(i) for i in out.split()) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst new file mode 100644 index 00000000000000..b307ee5e5ecaaa --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst @@ -0,0 +1,4 @@ +Update :option:`-X cpu_count <-X>` command line option to support +``process`` mode that :func:`os.process_cpu_count` becomes an alias of +:func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. Patch by Donghee +Na. diff --git a/Python/initconfig.c b/Python/initconfig.c index f7eb8535e98a6a..d3d18ff7965f46 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -232,7 +232,7 @@ The following implementation-specific options are available:\n\ This helps avoid denial of service attacks when parsing untrusted data.\n\ The default is sys.int_info.default_max_str_digits. 0 disables.\n\ \n\ --X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ +-X cpu_count=[n|default|process]: Override the return value of os.cpu_count(),\n\ os.process_cpu_count(), and multiprocessing.cpu_count(). This can help users who need\n\ to limit resources in a container." @@ -1636,6 +1636,9 @@ config_init_cpu_count(PyConfig *config) if (strcmp(env, "default") == 0) { cpu_count = -1; } + else if(strcmp(env, "process") == 0) { + cpu_count = -2; + } else if (_Py_str_to_int(env, &cpu_count) < 0 || cpu_count < 1) { goto error; } @@ -1650,6 +1653,9 @@ config_init_cpu_count(PyConfig *config) if (wcscmp(sep + 1, L"default") == 0) { cpu_count = -1; } + else if (wcscmp(sep + 1, L"process") == 0) { + cpu_count = -2; + } else if (config_wstr_to_int(sep + 1, &cpu_count) < 0 || cpu_count < 1) { goto error; } From 4832a782b0cc7dbf43a287a384fb524ef9fe1e20 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 10 Oct 2023 23:39:47 +0900 Subject: [PATCH 2/5] nit --- Doc/using/cmdline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 6224a179ee19c1..707007aa0a84f1 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -552,7 +552,7 @@ Miscellaneous options This option may be useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHON_CPU_COUNT`. If *n* is ``default``, nothing is overridden, if *n* is ``process``, - :func:`os.os.process_cpu_count` will be alias of :func:`os.cpu_count` + :func:`os.os.process_cpu_count` becomes an alias of :func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. It also allows passing arbitrary values and retrieving them through the @@ -1079,7 +1079,7 @@ conflict. If this variable is set to a positive integer, it overrides the return values of :func:`os.cpu_count` and :func:`os.process_cpu_count`. if this variable is set to ``process``, :func:`os.process_cpu_count` becomes - alias of :func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. + an alias of :func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. See also the :option:`-X cpu_count <-X>` command-line option. From a954a560a54d9a0799d799f06a201e6196ec9aa6 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 11 Oct 2023 06:25:18 +0900 Subject: [PATCH 3/5] Update NEWS.d --- ....jPp6s9.rst => 2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/Core and Builtins/{2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst => 2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst} (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-109595.jPp6s9.rst rename to Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst From fe829b85e65b7d69c6fd82fd65d7acd73584f6bc Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 11 Oct 2023 06:26:38 +0900 Subject: [PATCH 4/5] Update whats news --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 9a24c1fabf05d5..35627be28f1b39 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -192,7 +192,7 @@ os the new environment variable :envvar:`PYTHON_CPU_COUNT` or the new command-line option :option:`-X cpu_count <-X>`. This option is useful for users who need to limit CPU resources of a container system without having to modify the container (application code). - (Contributed by Donghee Na in :gh:`109595`) + (Contributed by Donghee Na in :gh:`109595` and :gh:`110649`) pathlib ------- From 0967c522ae93e8fceb995485613f4f6ad1b8ae5d Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Wed, 11 Oct 2023 07:03:54 +0900 Subject: [PATCH 5/5] Address code review --- Doc/using/cmdline.rst | 9 ++++----- .../2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst | 5 ++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 707007aa0a84f1..4dcd99b17908ac 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -551,9 +551,8 @@ Miscellaneous options *n* must be greater than or equal to 1. This option may be useful for users who need to limit CPU resources of a container system. See also :envvar:`PYTHON_CPU_COUNT`. - If *n* is ``default``, nothing is overridden, if *n* is ``process``, - :func:`os.os.process_cpu_count` becomes an alias of :func:`os.cpu_count` - and :func:`multiprocessing.cpu_count`. + If *n* is ``default``, nothing is overridden. If *n* is ``process``, + :func:`os.cpu_count` becomes an alias of :func:`os.process_cpu_count` It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1078,8 +1077,8 @@ conflict. If this variable is set to a positive integer, it overrides the return values of :func:`os.cpu_count` and :func:`os.process_cpu_count`. - if this variable is set to ``process``, :func:`os.process_cpu_count` becomes - an alias of :func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. + If this variable is set to ``process``, :func:`os.cpu_count` becomes + an alias of :func:`os.process_cpu_count`. See also the :option:`-X cpu_count <-X>` command-line option. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst index b307ee5e5ecaaa..2d29160b6d69cf 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-10-23-37-42.gh-issue-110649.jPp6s9.rst @@ -1,4 +1,3 @@ Update :option:`-X cpu_count <-X>` command line option to support -``process`` mode that :func:`os.process_cpu_count` becomes an alias of -:func:`os.cpu_count` and :func:`multiprocessing.cpu_count`. Patch by Donghee -Na. +``process`` mode that :func:`os.cpu_count` becomes an alias of +:func:`os.process_cpu_count`. Patch by Donghee Na.