From f6fe5958c9434aed61c758ba00c6e4a856cc33a1 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 21 Dec 2023 14:18:35 +0100 Subject: [PATCH 1/5] gh-113356: Ignore errors in "._ABC.pth" On macOS the system can create a "._" file next to a regular file when the system needs to store metadata that is not supported by the filesystem (such as storing extended attributes on an exFAT filesystem). Those files are binary data and cause startup failures when the base file is a ".pth" file. --- Lib/site.py | 55 +++++++++++-------- Lib/test/test_site.py | 14 +++++ ...-12-21-14-18-24.gh-issue-113356.Vz2osH.rst | 3 + 3 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-12-21-14-18-24.gh-issue-113356.Vz2osH.rst diff --git a/Lib/site.py b/Lib/site.py index 2517b7e5f1d22a..10deb3c794e703 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -176,29 +176,40 @@ def addpackage(sitedir, name, known_paths): except OSError: return with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": - continue - try: - if line.startswith(("import ", "import\t")): - exec(line) + try: + for n, line in enumerate(f): + if line.startswith("#"): continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print("Error processing line {:d} of {}:\n".format(n+1, fullname), + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break + except UnicodeDecodeError: + # MacOS can create files with a "._" prefix in the name + # next to the regular file when the system needs to store + # metadata (such as extended attributes) that the filesystem + # cannot store natively. + # + # Ignore errors when trying to parse these files. + if name.startswith("._") and os.path.exists(os.path.join(sitedir, name[2:])): + return + raise if reset: known_paths = None return known_paths diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 33d0975bda8eaa..d7ec8158bb05ca 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -181,6 +181,20 @@ def test_addpackage_import_bad_pth_file(self): if isinstance(path, str): self.assertNotIn("abc\x00def", path) + def test_addpackage_macOS_resources(self): + # GH-113356 + pth_dir, pth_fn = self.make_pth("dummy\n") + resource_fork = os.path.join(pth_dir, "._" + pth_fn) + with open(resource_fork, "wb") as fp: + self.addCleanup(lambda: os.remove(resource_fork)) + # Some random that that isn't valid UTF-8 + fp.write(b"\xff\xff\xff") + + with captured_stderr() as err_out: + self.assertFalse(site.addpackage(pth_dir, "._" + pth_fn, set())) + self.assertEqual(err_out.getvalue(), "") + + def test_addsitedir(self): # Same tests for test_addpackage since addsitedir() essentially just # calls addpackage() for every .pth file in the directory diff --git a/Misc/NEWS.d/next/macOS/2023-12-21-14-18-24.gh-issue-113356.Vz2osH.rst b/Misc/NEWS.d/next/macOS/2023-12-21-14-18-24.gh-issue-113356.Vz2osH.rst new file mode 100644 index 00000000000000..472e8a2b8bb7d9 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-12-21-14-18-24.gh-issue-113356.Vz2osH.rst @@ -0,0 +1,3 @@ +Ignore "._" prefixed pth files when those cannot be parsed. These files are +created on macOS when the system tries store metadata that is not supported +by the filesystem (such as extended attributes on exFAT) From 805dab6a93b71e95b34d82e04933eaf263d5e12e Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 21 Dec 2023 14:29:45 +0100 Subject: [PATCH 2/5] a more targetted approach to detecting the error --- Lib/site.py | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 10deb3c794e703..aeb2fb8e6e96b8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -177,29 +177,8 @@ def addpackage(sitedir, name, known_paths): return with f: try: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": - continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + f.readline() + f.seek(0) except UnicodeDecodeError: # MacOS can create files with a "._" prefix in the name # next to the regular file when the system needs to store @@ -210,6 +189,30 @@ def addpackage(sitedir, name, known_paths): if name.startswith("._") and os.path.exists(os.path.join(sitedir, name[2:])): return raise + + for n, line in enumerate(f): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) + continue + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if not dircase in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print("Error processing line {:d} of {}:\n".format(n+1, fullname), + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths From eb4074790114db4dae09df0fb85c19b2ba4c1346 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 21 Dec 2023 14:31:18 +0100 Subject: [PATCH 3/5] a more targetted approach to detecting the error --- Lib/site.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index aeb2fb8e6e96b8..34246b253b55c2 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -176,19 +176,20 @@ def addpackage(sitedir, name, known_paths): except OSError: return with f: - try: - f.readline() - f.seek(0) - except UnicodeDecodeError: - # MacOS can create files with a "._" prefix in the name - # next to the regular file when the system needs to store - # metadata (such as extended attributes) that the filesystem - # cannot store natively. - # - # Ignore errors when trying to parse these files. - if name.startswith("._") and os.path.exists(os.path.join(sitedir, name[2:])): - return - raise + if name.startswith*("._"): + try: + f.readline() + f.seek(0) + except UnicodeDecodeError: + # MacOS can create files with a "._" prefix in the name + # next to the regular file when the system needs to store + # metadata (such as extended attributes) that the filesystem + # cannot store natively. + # + # Ignore errors when trying to parse these files. + if os.path.exists(os.path.join(sitedir, name[2:])): + return + raise for n, line in enumerate(f): if line.startswith("#"): From cc3ced72331cbddea248b8b338b9c9b12dd26517 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 21 Dec 2023 14:32:02 +0100 Subject: [PATCH 4/5] Fix previous commmit --- Lib/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index 34246b253b55c2..9b5accb6e100bd 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -176,7 +176,7 @@ def addpackage(sitedir, name, known_paths): except OSError: return with f: - if name.startswith*("._"): + if name.startswith("._"): try: f.readline() f.seek(0) From 7f7666db886475c13ad870fcaa78372a0f7f1b90 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 24 Dec 2023 13:02:01 +0100 Subject: [PATCH 5/5] Slightly different approach to the targetted workaround --- Lib/site.py | 21 +++++++++------------ Lib/test/test_site.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 9b5accb6e100bd..e289fba74b4663 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -177,19 +177,16 @@ def addpackage(sitedir, name, known_paths): return with f: if name.startswith("._"): - try: - f.readline() - f.seek(0) - except UnicodeDecodeError: - # MacOS can create files with a "._" prefix in the name - # next to the regular file when the system needs to store - # metadata (such as extended attributes) that the filesystem - # cannot store natively. - # - # Ignore errors when trying to parse these files. - if os.path.exists(os.path.join(sitedir, name[2:])): + # MacOS will create "._" files next to a regular file + # when a filesystem driver needs to store metadata that + # cannot be stored natively. Such files are encoded + # in AppleDouble format. + # The test looks for the magic marker at the start of such + # files. + if f.buffer.read(4) == b"\x00\x05\x16\x07" \ + and os.path.exists(os.path.join(sitedir, name[2:])): return - raise + f.seek(0) for n, line in enumerate(f): if line.startswith("#"): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index d7ec8158bb05ca..b194693dad4c3f 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -187,14 +187,21 @@ def test_addpackage_macOS_resources(self): resource_fork = os.path.join(pth_dir, "._" + pth_fn) with open(resource_fork, "wb") as fp: self.addCleanup(lambda: os.remove(resource_fork)) - # Some random that that isn't valid UTF-8 - fp.write(b"\xff\xff\xff") + + # The bytes below were generated on macOS 14 using an + # exFAT filesystem. Command to write an xattr: + # `xattr -w key value test.txt`. These bytes are not + # the complete AppleDouble file, but just a significant + # prefix. + fp.write(b'\x00\x05\x16\x07\x00\x02\x00\x00Mac OS X ') + fp.write(b'\x00\x02\x00\x00\x00\t\x00\x00\x002\x00\x00\x0e') + fp.write(b'\xb0\x00\x00\x00\x02\x00\x00\x0e\xe2\x00\x00') + fp.write(b'\x01\x1e') with captured_stderr() as err_out: self.assertFalse(site.addpackage(pth_dir, "._" + pth_fn, set())) self.assertEqual(err_out.getvalue(), "") - def test_addsitedir(self): # Same tests for test_addpackage since addsitedir() essentially just # calls addpackage() for every .pth file in the directory