From 8987eebefd7eeb734f55b5c6c03bb7e947940124 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 12 Dec 2023 00:17:15 +0000 Subject: [PATCH 1/8] [WIP] Figuring out why sys._base_executable isn't resolved --- Lib/sysconfig/__init__.py | 2 +- Lib/test/test_venv.py | 16 ++++++++++++++++ Modules/getpath.c | 2 +- Modules/getpath.py | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 2a7fa45be079de..deb438c705f3a0 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,7 +404,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 890672c5d27eec..a468c73698d76d 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -283,6 +283,14 @@ def test_sysconfig(self): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode(), err) + for attr, expected in ( + ('executable', self.envpy()), + ('_base_executable', sys.executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd) + self.assertEqual(out.strip(), expected.encode(), err) @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') @@ -305,6 +313,14 @@ def test_sysconfig_symlinks(self): cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode(), err) + for attr, expected in ( + ('executable', self.envpy()), + ('_base_executable', sys.executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd) + self.assertEqual(out.strip(), expected.encode(), err) if sys.platform == 'win32': ENV_SUBDIRS = ( diff --git a/Modules/getpath.c b/Modules/getpath.c index 6c1078b8914522..46f3cf113efa25 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -898,7 +898,7 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !library_to_dict(dict, "library") || !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || - !funcs_to_dict(dict, config->pathconfig_warnings) || + !funcs_to_dict(dict, 1) || //config->pathconfig_warnings) || #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index 1410ffdbed8c70..dcbd0fb6dea9cb 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -358,6 +358,7 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] + warn(f"{base_executable=} {venv_prefix=}") for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -367,6 +368,7 @@ def search_up(prefix, *landmarks, test=isfile): # more accurate than assuming the executable in 'home'. try: base_executable = realpath(executable) + warn(f"{base_executable=} {executable=}") if base_executable == executable: # No change, so probably not a link. Clear it and fall back base_executable = '' From 01b1f825544a859775e0622feabd24e5ac0ff3e5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 12 Dec 2023 20:53:28 +0000 Subject: [PATCH 2/8] Implement realpath for Windows --- Modules/getpath.c | 41 ++++++++++++++++++++++++++++++++++++++++- Modules/getpath.py | 2 -- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 46f3cf113efa25..14d8547ced2015 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -502,6 +502,45 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) PyMem_Free((void *)path); PyMem_Free((void *)narrow); return r; +#elif defined(MS_WINDOWS) + HANDLE hFile; + wchar_t resolved[MAXPATHLEN+1]; + int len = 0, err; + PyObject *result; + + wchar_t *path = PyUnicode_AsWideCharString(pathobj, NULL); + if (!path) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + hFile = CreateFileW(path, 0, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (hFile != INVALID_HANDLE_VALUE) { + len = GetFinalPathNameByHandleW(hFile, resolved, MAXPATHLEN, VOLUME_NAME_DOS); + err = len ? 0 : GetLastError(); + CloseHandle(hFile); + } else { + err = GetLastError(); + } + Py_END_ALLOW_THREADS + + if (err) { + return PyErr_SetFromWindowsErr(GetLastError()); + } + if (len <= MAXPATHLEN) { + const wchar_t *p = resolved; + if (0 == wcsncmp(p, L"\\\\?\\", 4)) { + if (GetFileAttributesW(&p[4]) != INVALID_FILE_ATTRIBUTES) { + p += 4; + len -= 4; + } + } + result = PyUnicode_FromWideChar(p, len); + } else { + result = Py_NewRef(pathobj); + } + PyMem_Free(path); + return result; #endif return Py_NewRef(pathobj); @@ -898,7 +937,7 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !library_to_dict(dict, "library") || !wchar_to_dict(dict, "executable_dir", NULL) || !wchar_to_dict(dict, "py_setpath", _PyPathConfig_GetGlobalModuleSearchPath()) || - !funcs_to_dict(dict, 1) || //config->pathconfig_warnings) || + !funcs_to_dict(dict, config->pathconfig_warnings) || #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || #endif diff --git a/Modules/getpath.py b/Modules/getpath.py index dcbd0fb6dea9cb..1410ffdbed8c70 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -358,7 +358,6 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] - warn(f"{base_executable=} {venv_prefix=}") for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -368,7 +367,6 @@ def search_up(prefix, *landmarks, test=isfile): # more accurate than assuming the executable in 'home'. try: base_executable = realpath(executable) - warn(f"{base_executable=} {executable=}") if base_executable == executable: # No change, so probably not a link. Clear it and fall back base_executable = '' From c69f931f0f029a2c2efa14b811e368a50148c40e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 12 Dec 2023 20:57:20 +0000 Subject: [PATCH 3/8] Revert sysconfig change --- Lib/sysconfig/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index deb438c705f3a0..2a7fa45be079de 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,7 +404,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, "PC") else: inc_dir = _PROJECT_BASE else: From 902a093efaef3bdc1f79890d4ed22fb53ae0ddf7 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 12 Dec 2023 20:58:13 +0000 Subject: [PATCH 4/8] NEWS --- .../next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst diff --git a/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst new file mode 100644 index 00000000000000..c1d96792bdae0b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-12-12-20-58-09.gh-issue-86179.YYSk_6.rst @@ -0,0 +1 @@ +Fixes path calculations when launching Python on Windows through a symlink. From d579fa0564686036d628dd602d2ed6e5ff3db6ef Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 17:48:42 +0000 Subject: [PATCH 5/8] Add missed realpath and simplify sysconfig --- Lib/sysconfig/__init__.py | 11 +---------- Modules/getpath.py | 8 +++++++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index c60c9f3440615b..deb438c705f3a0 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -404,16 +404,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - # This ought to be as simple as dirname(sys._base_executable), but - # if a venv uses symlinks to a build in the source tree, then this - # fails. So instead we guess the subdirectory name from sys.winver - if sys.winver.endswith('-32'): - arch = 'win32' - elif sys.winver.endswith('-arm64'): - arch = 'arm64' - else: - arch = 'amd64' - inc_dir = os.path.join(_PROJECT_BASE, 'PCbuild', arch) + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: diff --git a/Modules/getpath.py b/Modules/getpath.py index 1410ffdbed8c70..59a98deb377b7b 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -403,7 +403,13 @@ def search_up(prefix, *landmarks, test=isfile): # ****************************************************************************** if not base_executable: - base_executable = executable or real_executable or '' + if executable: + try: + base_executable = realpath(executable) + except OSError: + base_executable = executable + else: + base_executable = real_executable or '' if not real_executable: real_executable = base_executable From 6f77da362156f04423652d5aeded913c5eb7f188 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 19:41:20 +0000 Subject: [PATCH 6/8] Alternate fix --- Lib/test/test_venv.py | 3 ++- Modules/getpath.py | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 8caf6422c8a553..26158be138c6f8 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -46,7 +46,8 @@ def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: if verbose and err: diff --git a/Modules/getpath.py b/Modules/getpath.py index 59a98deb377b7b..1410ffdbed8c70 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -403,13 +403,7 @@ def search_up(prefix, *landmarks, test=isfile): # ****************************************************************************** if not base_executable: - if executable: - try: - base_executable = realpath(executable) - except OSError: - base_executable = executable - else: - base_executable = real_executable or '' + base_executable = executable or real_executable or '' if not real_executable: real_executable = base_executable From 823d0ddc3310bd8232da81a1ae1b06387850bb81 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 20:58:55 +0000 Subject: [PATCH 7/8] Update test --- Lib/test/test_venv.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 26158be138c6f8..8ecb23ff384362 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -290,7 +290,9 @@ def test_sysconfig(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - ('_base_executable', sys.executable), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' @@ -320,7 +322,9 @@ def test_sysconfig_symlinks(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - ('_base_executable', sys.executable), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' From 11b229c8517461ae8628c296b2391bc847b771a5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 13 Dec 2023 23:19:16 +0000 Subject: [PATCH 8/8] Fix error handling --- Modules/getpath.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 14d8547ced2015..422056b1fb6de4 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -525,9 +525,9 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) Py_END_ALLOW_THREADS if (err) { - return PyErr_SetFromWindowsErr(GetLastError()); - } - if (len <= MAXPATHLEN) { + PyErr_SetFromWindowsErr(err); + result = NULL; + } else if (len <= MAXPATHLEN) { const wchar_t *p = resolved; if (0 == wcsncmp(p, L"\\\\?\\", 4)) { if (GetFileAttributesW(&p[4]) != INVALID_FILE_ATTRIBUTES) {