Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 17c6104

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45506: Normalize _PyPathConfig.stdlib_dir when calculated. (#29040)
The recently added PyConfig.stdlib_dir was being set with ".." entries. When __file__ was added for from modules this caused a problem on out-of-tree builds. This PR fixes that by normalizing "stdlib_dir" when it is calculated in getpath.c. https://bugs.python.org/issue45506
1 parent f30ad65 commit 17c6104

6 files changed

Lines changed: 236 additions & 24 deletions

File tree

‎Include/internal/pycore_fileutils.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ extern int _Py_add_relfile(wchar_t *dirname,
8080
const wchar_t *relfile,
8181
size_t bufsize);
8282
extern size_t _Py_find_basename(const wchar_t *filename);
83+
PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path,
84+
wchar_t *buf, const size_t buf_len);
85+
8386

8487
// Macros to protect CRT calls against instant termination when passed an
8588
// invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler.

‎Lib/test/test_fileutils.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Run tests for functions in Python/fileutils.c.
2+
3+
import os
4+
import os.path
5+
import unittest
6+
from test.support import import_helper
7+
8+
# Skip this test if the _testcapi module isn't available.
9+
_testcapi = import_helper.import_module('_testinternalcapi')
10+
11+
12+
class PathTests(unittest.TestCase):
13+
14+
def test_capi_normalize_path(self):
15+
if os.name == 'nt':
16+
raise unittest.SkipTest('Windows has its own helper for this')
17+
else:
18+
from .test_posixpath import PosixPathTest as posixdata
19+
tests = posixdata.NORMPATH_CASES
20+
for filename, expected in tests:
21+
if not os.path.isabs(filename):
22+
continue
23+
with self.subTest(filename):
24+
result = _testcapi.normalize_path(filename)
25+
self.assertEqual(result, expected,
26+
msg=f'input: {filename!r} expected output: {expected!r}')
27+
28+
29+
if __name__ == "__main__":
30+
unittest.main()

‎Lib/test/test_posixpath.py‎

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -304,25 +304,51 @@ def test_expanduser_pwd(self):
304304
for path in ('~', '~/.local', '~vstinner/'):
305305
self.assertEqual(posixpath.expanduser(path), path)
306306

307+
NORMPATH_CASES = [
308+
("", "."),
309+
("/", "/"),
310+
("/.", "/"),
311+
("/./", "/"),
312+
("/.//.", "/"),
313+
("/foo", "/foo"),
314+
("/foo/bar", "/foo/bar"),
315+
("//", "//"),
316+
("///", "/"),
317+
("///foo/.//bar//", "/foo/bar"),
318+
("///foo/.//bar//.//..//.//baz///", "/foo/baz"),
319+
("///..//./foo/.//bar", "/foo/bar"),
320+
(".", "."),
321+
(".//.", "."),
322+
("..", ".."),
323+
("../", ".."),
324+
("../foo", "../foo"),
325+
("../../foo", "../../foo"),
326+
("../foo/../bar", "../bar"),
327+
("../../foo/../bar/./baz/boom/..", "../../bar/baz"),
328+
("/..", "/"),
329+
("/..", "/"),
330+
("/../", "/"),
331+
("/..//", "/"),
332+
("//..", "//"),
333+
("/../foo", "/foo"),
334+
("/../../foo", "/foo"),
335+
("/../foo/../", "/"),
336+
("/../foo/../bar", "/bar"),
337+
("/../../foo/../bar/./baz/boom/..", "/bar/baz"),
338+
("/../../foo/../bar/./baz/boom/.", "/bar/baz/boom"),
339+
]
340+
307341
def test_normpath(self):
308-
self.assertEqual(posixpath.normpath(""), ".")
309-
self.assertEqual(posixpath.normpath("/"), "/")
310-
self.assertEqual(posixpath.normpath("//"), "//")
311-
self.assertEqual(posixpath.normpath("///"), "/")
312-
self.assertEqual(posixpath.normpath("///foo/.//bar//"), "/foo/bar")
313-
self.assertEqual(posixpath.normpath("///foo/.//bar//.//..//.//baz"),
314-
"/foo/baz")
315-
self.assertEqual(posixpath.normpath("///..//./foo/.//bar"), "/foo/bar")
316-
317-
self.assertEqual(posixpath.normpath(b""), b".")
318-
self.assertEqual(posixpath.normpath(b"/"), b"/")
319-
self.assertEqual(posixpath.normpath(b"//"), b"//")
320-
self.assertEqual(posixpath.normpath(b"///"), b"/")
321-
self.assertEqual(posixpath.normpath(b"///foo/.//bar//"), b"/foo/bar")
322-
self.assertEqual(posixpath.normpath(b"///foo/.//bar//.//..//.//baz"),
323-
b"/foo/baz")
324-
self.assertEqual(posixpath.normpath(b"///..//./foo/.//bar"),
325-
b"/foo/bar")
342+
for path, expected in self.NORMPATH_CASES:
343+
with self.subTest(path):
344+
result = posixpath.normpath(path)
345+
self.assertEqual(result, expected)
346+
347+
path = path.encode('utf-8')
348+
expected = expected.encode('utf-8')
349+
with self.subTest(path, type=bytes):
350+
result = posixpath.normpath(path)
351+
self.assertEqual(result, expected)
326352

327353
@skip_if_ABSTFN_contains_backslash
328354
def test_realpath_curdir(self):

‎Modules/_testinternalcapi.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@
1414
#include "Python.h"
1515
#include "pycore_atomic_funcs.h" // _Py_atomic_int_get()
1616
#include "pycore_bitutils.h" // _Py_bswap32()
17+
#include "pycore_fileutils.h" // _Py_normalize_path
1718
#include "pycore_gc.h" // PyGC_Head
1819
#include "pycore_hashtable.h" // _Py_hashtable_new()
1920
#include "pycore_initconfig.h" // _Py_GetConfigsAsDict()
2021
#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy()
2122
#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost()
2223
#include "pycore_pystate.h" // _PyThreadState_GET()
24+
#include "osdefs.h" // MAXPATHLEN
2325

2426

2527
static PyObject *
@@ -366,6 +368,27 @@ test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args))
366368
}
367369

368370

371+
static PyObject *
372+
normalize_path(PyObject *self, PyObject *filename)
373+
{
374+
Py_ssize_t size = -1;
375+
wchar_t *encoded = PyUnicode_AsWideCharString(filename, &size);
376+
if (encoded == NULL) {
377+
return NULL;
378+
}
379+
380+
wchar_t buf[MAXPATHLEN + 1];
381+
int res = _Py_normalize_path(encoded, buf, Py_ARRAY_LENGTH(buf));
382+
PyMem_Free(encoded);
383+
if (res != 0) {
384+
PyErr_SetString(PyExc_ValueError, "string too long");
385+
return NULL;
386+
}
387+
388+
return PyUnicode_FromWideChar(buf, -1);
389+
}
390+
391+
369392
static PyMethodDef TestMethods[] = {
370393
{"get_configs", get_configs, METH_NOARGS},
371394
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -377,6 +400,7 @@ static PyMethodDef TestMethods[] = {
377400
{"set_config", test_set_config, METH_O},
378401
{"test_atomic_funcs", test_atomic_funcs, METH_NOARGS},
379402
{"test_edit_cost", test_edit_cost, METH_NOARGS},
403+
{"normalize_path", normalize_path, METH_O, NULL},
380404
{NULL, NULL} /* sentinel */
381405
};
382406

‎Modules/getpath.c‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,42 @@ search_for_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig,
519519
}
520520

521521

522+
static PyStatus
523+
calculate_set_stdlib_dir(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
524+
{
525+
// Note that, unlike calculate_set_prefix(), here we allow a negative
526+
// prefix_found. That means the source tree Lib dir gets used.
527+
if (!calculate->prefix_found) {
528+
return _PyStatus_OK();
529+
}
530+
PyStatus status;
531+
wchar_t *prefix = calculate->prefix;
532+
if (!_Py_isabs(prefix)) {
533+
prefix = _PyMem_RawWcsdup(prefix);
534+
if (prefix == NULL) {
535+
return _PyStatus_NO_MEMORY();
536+
}
537+
status = absolutize(&prefix);
538+
if (_PyStatus_EXCEPTION(status)) {
539+
return status;
540+
}
541+
}
542+
wchar_t buf[MAXPATHLEN + 1];
543+
int res = _Py_normalize_path(prefix, buf, Py_ARRAY_LENGTH(buf));
544+
if (prefix != calculate->prefix) {
545+
PyMem_RawFree(prefix);
546+
}
547+
if (res < 0) {
548+
return PATHLEN_ERR();
549+
}
550+
pathconfig->stdlib_dir = _PyMem_RawWcsdup(buf);
551+
if (pathconfig->stdlib_dir == NULL) {
552+
return _PyStatus_NO_MEMORY();
553+
}
554+
return _PyStatus_OK();
555+
}
556+
557+
522558
static PyStatus
523559
calculate_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
524560
{
@@ -1494,12 +1530,10 @@ calculate_path(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
14941530
}
14951531

14961532
if (pathconfig->stdlib_dir == NULL) {
1497-
if (calculate->prefix_found) {
1498-
/* This must be done *before* calculate_set_prefix() is called. */
1499-
pathconfig->stdlib_dir = _PyMem_RawWcsdup(calculate->prefix);
1500-
if (pathconfig->stdlib_dir == NULL) {
1501-
return _PyStatus_NO_MEMORY();
1502-
}
1533+
/* This must be done *before* calculate_set_prefix() is called. */
1534+
status = calculate_set_stdlib_dir(calculate, pathconfig);
1535+
if (_PyStatus_EXCEPTION(status)) {
1536+
return status;
15031537
}
15041538
}
15051539

‎Python/fileutils.c‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2181,6 +2181,101 @@ _Py_find_basename(const wchar_t *filename)
21812181
}
21822182

21832183

2184+
/* Remove navigation elements such as "." and "..".
2185+
2186+
This is mostly a C implementation of posixpath.normpath().
2187+
Return 0 on success. Return -1 if "orig" is too big for the buffer. */
2188+
int
2189+
_Py_normalize_path(const wchar_t *path, wchar_t *buf, const size_t buf_len)
2190+
{
2191+
assert(path && *path != L'\0');
2192+
assert(*path == SEP); // an absolute path
2193+
if (wcslen(path) + 1 >= buf_len) {
2194+
return -1;
2195+
}
2196+
2197+
int dots = -1;
2198+
int check_leading = 1;
2199+
const wchar_t *buf_start = buf;
2200+
wchar_t *buf_next = buf;
2201+
// The resulting filename will never be longer than path.
2202+
for (const wchar_t *remainder = path; *remainder != L'\0'; remainder++) {
2203+
wchar_t c = *remainder;
2204+
buf_next[0] = c;
2205+
buf_next++;
2206+
if (c == SEP) {
2207+
assert(dots <= 2);
2208+
if (dots == 2) {
2209+
// Turn "/x/y/../z" into "/x/z".
2210+
buf_next -= 4; // "/../"
2211+
assert(*buf_next == SEP);
2212+
// We cap it off at the root, so "/../spam" becomes "/spam".
2213+
if (buf_next == buf_start) {
2214+
buf_next++;
2215+
}
2216+
else {
2217+
// Move to the previous SEP in the buffer.
2218+
while (*(buf_next - 1) != SEP) {
2219+
assert(buf_next != buf_start);
2220+
buf_next--;
2221+
}
2222+
}
2223+
}
2224+
else if (dots == 1) {
2225+
// Turn "/./" into "/".
2226+
buf_next -= 2; // "./"
2227+
assert(*(buf_next - 1) == SEP);
2228+
}
2229+
else if (dots == 0) {
2230+
// Turn "//" into "/".
2231+
buf_next--;
2232+
assert(*(buf_next - 1) == SEP);
2233+
if (check_leading) {
2234+
if (buf_next - 1 == buf && *(remainder + 1) != SEP) {
2235+
// Leave a leading "//" alone, unless "///...".
2236+
buf_next++;
2237+
buf_start++;
2238+
}
2239+
check_leading = 0;
2240+
}
2241+
}
2242+
dots = 0;
2243+
}
2244+
else {
2245+
check_leading = 0;
2246+
if (dots >= 0) {
2247+
if (c == L'.' && dots < 2) {
2248+
dots++;
2249+
}
2250+
else {
2251+
dots = -1;
2252+
}
2253+
}
2254+
}
2255+
}
2256+
if (dots >= 0) {
2257+
// Strip any trailing dots and trailing slash.
2258+
buf_next -= dots + 1; // "/" or "/." or "/.."
2259+
assert(*buf_next == SEP);
2260+
if (buf_next == buf_start) {
2261+
// Leave the leading slash for root.
2262+
buf_next++;
2263+
}
2264+
else {
2265+
if (dots == 2) {
2266+
// Move to the previous SEP in the buffer.
2267+
do {
2268+
assert(buf_next != buf_start);
2269+
buf_next--;
2270+
} while (*(buf_next) != SEP);
2271+
}
2272+
}
2273+
}
2274+
*buf_next = L'\0';
2275+
return 0;
2276+
}
2277+
2278+
21842279
/* Get the current directory. buflen is the buffer size in wide characters
21852280
including the null character. Decode the path from the locale encoding.
21862281

0 commit comments

Comments
 (0)