Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 984b11f

Browse files
ericvsmithericvsmith
committed
issue 14660: Implement PEP 420, namespace packages.
1 parent fa52cbd commit 984b11f

25 files changed

Lines changed: 4425 additions & 3382 deletions

File tree

‎Lib/importlib/_bootstrap.py‎

Lines changed: 139 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,10 @@ class BuiltinImporter:
467467
468468
"""
469469

470+
@classmethod
471+
def module_repr(cls, module):
472+
return "<module '{}' (built-in)>".format(module.__name__)
473+
470474
@classmethod
471475
def find_module(cls, fullname, path=None):
472476
"""Find the built-in module.
@@ -520,6 +524,10 @@ class FrozenImporter:
520524
521525
"""
522526

527+
@classmethod
528+
def module_repr(cls, m):
529+
return "<module '{}' (frozen)>".format(m.__name__)
530+
523531
@classmethod
524532
def find_module(cls, fullname, path=None):
525533
"""Find a frozen module."""
@@ -533,7 +541,10 @@ def load_module(cls, fullname):
533541
"""Load a frozen module."""
534542
is_reload = fullname in sys.modules
535543
try:
536-
return _imp.init_frozen(fullname)
544+
m = _imp.init_frozen(fullname)
545+
# Let our own module_repr() method produce a suitable repr.
546+
del m.__file__
547+
return m
537548
except:
538549
if not is_reload and fullname in sys.modules:
539550
del sys.modules[fullname]
@@ -875,6 +886,79 @@ def get_source(self, fullname):
875886
return None
876887

877888

889+
class _NamespacePath:
890+
"""Represents a namespace package's path. It uses the module name
891+
to find its parent module, and from there it looks up the parent's
892+
__path__. When this changes, the module's own path is recomputed,
893+
using path_finder. For top-leve modules, the parent module's path
894+
is sys.path."""
895+
896+
def __init__(self, name, path, path_finder):
897+
self._name = name
898+
self._path = path
899+
self._last_parent_path = tuple(self._get_parent_path())
900+
self._path_finder = path_finder
901+
902+
def _find_parent_path_names(self):
903+
"""Returns a tuple of (parent-module-name, parent-path-attr-name)"""
904+
parent, dot, me = self._name.rpartition('.')
905+
if dot == '':
906+
# This is a top-level module. sys.path contains the parent path.
907+
return 'sys', 'path'
908+
# Not a top-level module. parent-module.__path__ contains the
909+
# parent path.
910+
return parent, '__path__'
911+
912+
def _get_parent_path(self):
913+
parent_module_name, path_attr_name = self._find_parent_path_names()
914+
return getattr(sys.modules[parent_module_name], path_attr_name)
915+
916+
def _recalculate(self):
917+
# If the parent's path has changed, recalculate _path
918+
parent_path = tuple(self._get_parent_path()) # Make a copy
919+
if parent_path != self._last_parent_path:
920+
loader, new_path = self._path_finder(self._name, parent_path)
921+
# Note that no changes are made if a loader is returned, but we
922+
# do remember the new parent path
923+
if loader is None:
924+
self._path = new_path
925+
self._last_parent_path = parent_path # Save the copy
926+
return self._path
927+
928+
def __iter__(self):
929+
return iter(self._recalculate())
930+
931+
def __len__(self):
932+
return len(self._recalculate())
933+
934+
def __repr__(self):
935+
return "_NamespacePath({0!r})".format(self._path)
936+
937+
def __contains__(self, item):
938+
return item in self._recalculate()
939+
940+
def append(self, item):
941+
self._path.append(item)
942+
943+
944+
class NamespaceLoader:
945+
def __init__(self, name, path, path_finder):
946+
self._path = _NamespacePath(name, path, path_finder)
947+
948+
@classmethod
949+
def module_repr(cls, module):
950+
return "<module '{}' (namespace)>".format(module.__name__)
951+
952+
@set_package
953+
@set_loader
954+
@module_for_loader
955+
def load_module(self, module):
956+
"""Load a namespace module."""
957+
_verbose_message('namespace module loaded with path {!r}', self._path)
958+
module.__path__ = self._path
959+
return module
960+
961+
878962
# Finders #####################################################################
879963

880964
class PathFinder:
@@ -915,20 +999,47 @@ def _path_importer_cache(cls, path):
915999
sys.path_importer_cache[path] = finder
9161000
return finder
9171001

1002+
@classmethod
1003+
def _get_loader(cls, fullname, path):
1004+
"""Find the loader or namespace_path for this module/package name."""
1005+
# If this ends up being a namespace package, namespace_path is
1006+
# the list of paths that will become its __path__
1007+
namespace_path = []
1008+
for entry in path:
1009+
finder = cls._path_importer_cache(entry)
1010+
if finder is not None:
1011+
if hasattr(finder, 'find_loader'):
1012+
loader, portions = finder.find_loader(fullname)
1013+
else:
1014+
loader = finder.find_module(fullname)
1015+
portions = []
1016+
if loader is not None:
1017+
# We found a loader: return it immediately.
1018+
return (loader, namespace_path)
1019+
# This is possibly part of a namespace package.
1020+
# Remember these path entries (if any) for when we
1021+
# create a namespace package, and continue iterating
1022+
# on path.
1023+
namespace_path.extend(portions)
1024+
else:
1025+
return (None, namespace_path)
1026+
9181027
@classmethod
9191028
def find_module(cls, fullname, path=None):
9201029
"""Find the module on sys.path or 'path' based on sys.path_hooks and
9211030
sys.path_importer_cache."""
9221031
if path is None:
9231032
path = sys.path
924-
for entry in path:
925-
finder = cls._path_importer_cache(entry)
926-
if finder is not None:
927-
loader = finder.find_module(fullname)
928-
if loader:
929-
return loader
1033+
loader, namespace_path = cls._get_loader(fullname, path)
1034+
if loader is not None:
1035+
return loader
9301036
else:
931-
return None
1037+
if namespace_path:
1038+
# We found at least one namespace path. Return a
1039+
# loader which can create the namespace package.
1040+
return NamespaceLoader(fullname, namespace_path, cls._get_loader)
1041+
else:
1042+
return None
9321043

9331044

9341045
class FileFinder:
@@ -942,8 +1053,8 @@ class FileFinder:
9421053

9431054
def __init__(self, path, *details):
9441055
"""Initialize with the path to search on and a variable number of
945-
3-tuples containing the loader, file suffixes the loader recognizes, and
946-
a boolean of whether the loader handles packages."""
1056+
3-tuples containing the loader, file suffixes the loader recognizes,
1057+
and a boolean of whether the loader handles packages."""
9471058
packages = []
9481059
modules = []
9491060
for loader, suffixes, supports_packages in details:
@@ -964,6 +1075,19 @@ def invalidate_caches(self):
9641075

9651076
def find_module(self, fullname):
9661077
"""Try to find a loader for the specified module."""
1078+
# Call find_loader(). If it returns a string (indicating this
1079+
# is a namespace package portion), generate a warning and
1080+
# return None.
1081+
loader, portions = self.find_loader(fullname)
1082+
assert len(portions) in [0, 1]
1083+
if loader is None and len(portions):
1084+
msg = "Not importing directory {}: missing __init__"
1085+
_warnings.warn(msg.format(portions[0]), ImportWarning)
1086+
return loader
1087+
1088+
def find_loader(self, fullname):
1089+
"""Try to find a loader for the specified module, or the namespace
1090+
package portions. Returns (loader, list-of-portions)."""
9671091
tail_module = fullname.rpartition('.')[2]
9681092
try:
9691093
mtime = _os.stat(self.path).st_mtime
@@ -987,17 +1111,17 @@ def find_module(self, fullname):
9871111
init_filename = '__init__' + suffix
9881112
full_path = _path_join(base_path, init_filename)
9891113
if _path_isfile(full_path):
990-
return loader(fullname, full_path)
1114+
return (loader(fullname, full_path), [base_path])
9911115
else:
992-
msg = "Not importing directory {}: missing __init__"
993-
_warnings.warn(msg.format(base_path), ImportWarning)
1116+
# A namespace package, return the path
1117+
return (None, [base_path])
9941118
# Check for a file w/ a proper suffix exists.
9951119
for suffix, loader in self.modules:
9961120
if cache_module + suffix in cache:
9971121
full_path = _path_join(self.path, tail_module + suffix)
9981122
if _path_isfile(full_path):
999-
return loader(fullname, full_path)
1000-
return None
1123+
return (loader(fullname, full_path), [])
1124+
return (None, [])
10011125

10021126
def _fill_cache(self):
10031127
"""Fill the cache of potential modules and packages for this directory."""

‎Lib/importlib/test/frozen/test_loader.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,38 +10,46 @@ class LoaderTests(abc.LoaderTests):
1010
def test_module(self):
1111
with util.uncache('__hello__'), captured_stdout() as stdout:
1212
module = machinery.FrozenImporter.load_module('__hello__')
13-
check = {'__name__': '__hello__', '__file__': '<frozen>',
14-
'__package__': '', '__loader__': machinery.FrozenImporter}
13+
check = {'__name__': '__hello__',
14+
'__package__': '',
15+
'__loader__': machinery.FrozenImporter,
16+
}
1517
for attr, value in check.items():
1618
self.assertEqual(getattr(module, attr), value)
1719
self.assertEqual(stdout.getvalue(), 'Hello world!\n')
20+
self.assertFalse(hasattr(module, '__file__'))
1821

1922
def test_package(self):
2023
with util.uncache('__phello__'), captured_stdout() as stdout:
2124
module = machinery.FrozenImporter.load_module('__phello__')
22-
check = {'__name__': '__phello__', '__file__': '<frozen>',
23-
'__package__': '__phello__', '__path__': ['__phello__'],
24-
'__loader__': machinery.FrozenImporter}
25+
check = {'__name__': '__phello__',
26+
'__package__': '__phello__',
27+
'__path__': ['__phello__'],
28+
'__loader__': machinery.FrozenImporter,
29+
}
2530
for attr, value in check.items():
2631
attr_value = getattr(module, attr)
2732
self.assertEqual(attr_value, value,
2833
"for __phello__.%s, %r != %r" %
2934
(attr, attr_value, value))
3035
self.assertEqual(stdout.getvalue(), 'Hello world!\n')
36+
self.assertFalse(hasattr(module, '__file__'))
3137

3238
def test_lacking_parent(self):
3339
with util.uncache('__phello__', '__phello__.spam'), \
3440
captured_stdout() as stdout:
3541
module = machinery.FrozenImporter.load_module('__phello__.spam')
36-
check = {'__name__': '__phello__.spam', '__file__': '<frozen>',
42+
check = {'__name__': '__phello__.spam',
3743
'__package__': '__phello__',
38-
'__loader__': machinery.FrozenImporter}
44+
'__loader__': machinery.FrozenImporter,
45+
}
3946
for attr, value in check.items():
4047
attr_value = getattr(module, attr)
4148
self.assertEqual(attr_value, value,
4249
"for __phello__.spam.%s, %r != %r" %
4350
(attr, attr_value, value))
4451
self.assertEqual(stdout.getvalue(), 'Hello world!\n')
52+
self.assertFalse(hasattr(module, '__file__'))
4553

4654
def test_module_reuse(self):
4755
with util.uncache('__hello__'), captured_stdout() as stdout:
@@ -51,6 +59,12 @@ def test_module_reuse(self):
5159
self.assertEqual(stdout.getvalue(),
5260
'Hello world!\nHello world!\n')
5361

62+
def test_module_repr(self):
63+
with util.uncache('__hello__'), captured_stdout():
64+
module = machinery.FrozenImporter.load_module('__hello__')
65+
self.assertEqual(repr(module),
66+
"<module '__hello__' (frozen)>")
67+
5468
def test_state_after_failure(self):
5569
# No way to trigger an error in a frozen module.
5670
pass

‎Lib/importlib/test/source/test_finder.py‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -106,36 +106,17 @@ def test_package_in_package(self):
106106
loader = self.import_(pkg_dir, 'pkg.sub')
107107
self.assertTrue(hasattr(loader, 'load_module'))
108108

109-
# [sub empty]
110-
def test_empty_sub_directory(self):
111-
context = source_util.create_modules('pkg.__init__', 'pkg.sub.__init__')
112-
with warnings.catch_warnings():
113-
warnings.simplefilter("error", ImportWarning)
114-
with context as mapping:
115-
os.unlink(mapping['pkg.sub.__init__'])
116-
pkg_dir = os.path.dirname(mapping['pkg.__init__'])
117-
with self.assertRaises(ImportWarning):
118-
self.import_(pkg_dir, 'pkg.sub')
119-
120109
# [package over modules]
121110
def test_package_over_module(self):
122111
name = '_temp'
123112
loader = self.run_test(name, {'{0}.__init__'.format(name), name})
124113
self.assertTrue('__init__' in loader.get_filename(name))
125114

126-
127115
def test_failure(self):
128116
with source_util.create_modules('blah') as mapping:
129117
nothing = self.import_(mapping['.root'], 'sdfsadsadf')
130118
self.assertTrue(nothing is None)
131119

132-
# [empty dir]
133-
def test_empty_dir(self):
134-
with warnings.catch_warnings():
135-
warnings.simplefilter("error", ImportWarning)
136-
with self.assertRaises(ImportWarning):
137-
self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'})
138-
139120
def test_empty_string_for_dir(self):
140121
# The empty string from sys.path means to search in the cwd.
141122
finder = machinery.FileFinder('', (machinery.SourceFileLoader,

‎Lib/pkgutil.py‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -515,19 +515,29 @@ def extend_path(path, name):
515515

516516
pname = os.path.join(*name.split('.')) # Reconstitute as relative path
517517
sname_pkg = name + ".pkg"
518-
init_py = "__init__.py"
519518

520519
path = path[:] # Start with a copy of the existing path
521520

522521
for dir in sys.path:
523-
if not isinstance(dir, str) or not os.path.isdir(dir):
522+
if not isinstance(dir, str):
524523
continue
525-
subdir = os.path.join(dir, pname)
526-
# XXX This may still add duplicate entries to path on
527-
# case-insensitive filesystems
528-
initfile = os.path.join(subdir, init_py)
529-
if subdir not in path and os.path.isfile(initfile):
530-
path.append(subdir)
524+
525+
finder = get_importer(dir)
526+
if finder is not None:
527+
# Is this finder PEP 420 compliant?
528+
if hasattr(finder, 'find_loader'):
529+
loader, portions = finder.find_loader(name)
530+
else:
531+
# No, no need to call it
532+
loader = None
533+
portions = []
534+
535+
for portion in portions:
536+
# XXX This may still add duplicate entries to path on
537+
# case-insensitive filesystems
538+
if portion not in path:
539+
path.append(portion)
540+
531541
# XXX Is this the right thing for subpackages like zope.app?
532542
# It looks for a file named "zope.app.pkg"
533543
pkgfile = os.path.join(dir, sname_pkg)

‎Lib/test/namespace_pkgs/both_portions/foo/one.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
attr = 'both_portions foo one'

‎Lib/test/namespace_pkgs/both_portions/foo/two.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
attr = 'both_portions foo two'

‎Lib/test/namespace_pkgs/missing_directory.zip‎

515 Bytes
Binary file not shown.

‎Lib/test/namespace_pkgs/nested_portion1.zip‎

556 Bytes
Binary file not shown.

‎Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/__init__.py‎

Whitespace-only changes.

‎Lib/test/namespace_pkgs/not_a_namespace_pkg/foo/one.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
attr = 'portion1 foo one'

0 commit comments

Comments
 (0)