Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b2fc549

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-101758: Clean Up Uses of Import State (gh-101919)
This change is almost entirely moving code around and hiding import state behind internal API. We introduce no changes to behavior, nor to non-internal API. (Since there was already going to be a lot of churn, I took this as an opportunity to re-organize import.c into topically-grouped sections of code.) The motivation is to simplify a number of upcoming changes. Specific changes: * move existing import-related code to import.c, wherever possible * add internal API for interacting with import state (both global and per-interpreter) * use only API outside of import.c (to limit churn there when changing the location, etc.) * consolidate the import-related state of PyInterpreterState into a single struct field (this changes layout slightly) * add macros for import state in import.c (to simplify changing the location) * group code in import.c into sections *remove _PyState_AddModule() #101758
1 parent c1ce0d1 commit b2fc549

20 files changed

+1586
-1182
lines changed

‎Include/internal/pycore_import.h

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,112 @@ struct _import_runtime_state {
3636
const char * pkgcontext;
3737
};
3838

39+
struct _import_state {
40+
/* cached sys.modules dictionary */
41+
PyObject *modules;
42+
/* This is the list of module objects for all legacy (single-phase init)
43+
extension modules ever loaded in this process (i.e. imported
44+
in this interpreter or in any other). Py_None stands in for
45+
modules that haven't actually been imported in this interpreter.
46+
47+
A module's index (PyModuleDef.m_base.m_index) is used to look up
48+
the corresponding module object for this interpreter, if any.
49+
(See PyState_FindModule().) When any extension module
50+
is initialized during import, its moduledef gets initialized by
51+
PyModuleDef_Init(), and the first time that happens for each
52+
PyModuleDef, its index gets set to the current value of
53+
a global counter (see _PyRuntimeState.imports.last_module_index).
54+
The entry for that index in this interpreter remains unset until
55+
the module is actually imported here. (Py_None is used as
56+
a placeholder.) Note that multi-phase init modules always get
57+
an index for which there will never be a module set.
58+
59+
This is initialized lazily in PyState_AddModule(), which is also
60+
where modules get added. */
61+
PyObject *modules_by_index;
62+
/* importlib module._bootstrap */
63+
PyObject *importlib;
64+
/* override for config->use_frozen_modules (for tests)
65+
(-1: "off", 1: "on", 0: no override) */
66+
int override_frozen_modules;
67+
#ifdef HAVE_DLOPEN
68+
int dlopenflags;
69+
#endif
70+
PyObject *import_func;
71+
};
72+
73+
#ifdef HAVE_DLOPEN
74+
# include <dlfcn.h>
75+
# if HAVE_DECL_RTLD_NOW
76+
# define _Py_DLOPEN_FLAGS RTLD_NOW
77+
# else
78+
# define _Py_DLOPEN_FLAGS RTLD_LAZY
79+
# endif
80+
# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
81+
#else
82+
# define _Py_DLOPEN_FLAGS 0
83+
# define DLOPENFLAGS_INIT
84+
#endif
85+
86+
#define IMPORTS_INIT \
87+
{ \
88+
.override_frozen_modules = 0, \
89+
DLOPENFLAGS_INIT \
90+
}
91+
92+
extern void _PyImport_ClearCore(PyInterpreterState *interp);
93+
94+
extern Py_ssize_t _PyImport_GetNextModuleIndex(void);
95+
extern const char * _PyImport_ResolveNameWithPackageContext(const char *name);
96+
extern const char * _PyImport_SwapPackageContext(const char *newcontext);
97+
98+
extern int _PyImport_GetDLOpenFlags(PyInterpreterState *interp);
99+
extern void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val);
100+
101+
extern PyObject * _PyImport_InitModules(PyInterpreterState *interp);
102+
extern PyObject * _PyImport_GetModules(PyInterpreterState *interp);
103+
extern void _PyImport_ClearModules(PyInterpreterState *interp);
104+
105+
extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
106+
107+
extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
108+
extern int _PyImport_IsDefaultImportFunc(
109+
PyInterpreterState *interp,
110+
PyObject *func);
111+
112+
extern PyObject * _PyImport_GetImportlibLoader(
113+
PyInterpreterState *interp,
114+
const char *loader_name);
115+
extern PyObject * _PyImport_GetImportlibExternalLoader(
116+
PyInterpreterState *interp,
117+
const char *loader_name);
118+
extern PyObject * _PyImport_BlessMyLoader(
119+
PyInterpreterState *interp,
120+
PyObject *module_globals);
121+
extern PyObject * _PyImport_ImportlibModuleRepr(
122+
PyInterpreterState *interp,
123+
PyObject *module);
124+
125+
126+
extern PyStatus _PyImport_Init(void);
127+
extern void _PyImport_Fini(void);
128+
extern void _PyImport_Fini2(void);
129+
130+
extern PyStatus _PyImport_InitCore(
131+
PyThreadState *tstate,
132+
PyObject *sysmod,
133+
int importlib);
134+
extern PyStatus _PyImport_InitExternal(PyThreadState *tstate);
135+
extern void _PyImport_FiniCore(PyInterpreterState *interp);
136+
extern void _PyImport_FiniExternal(PyInterpreterState *interp);
137+
39138

40139
#ifdef HAVE_FORK
41140
extern PyStatus _PyImport_ReInitLock(void);
42141
#endif
43-
extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate);
142+
143+
144+
extern PyObject* _PyImport_GetBuiltinModuleNames(void);
44145

45146
struct _module_alias {
46147
const char *name; /* ASCII encoded string */

‎Include/internal/pycore_interp.h

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ extern "C" {
2121
#include "pycore_function.h" // FUNC_MAX_WATCHERS
2222
#include "pycore_genobject.h" // struct _Py_async_gen_state
2323
#include "pycore_gc.h" // struct _gc_runtime_state
24+
#include "pycore_import.h" // struct _import_state
2425
#include "pycore_list.h" // struct _Py_list_state
2526
#include "pycore_global_objects.h" // struct _Py_interp_static_objects
2627
#include "pycore_tuple.h" // struct _Py_tuple_state
@@ -92,53 +93,24 @@ struct _is {
9293
struct _ceval_state ceval;
9394
struct _gc_runtime_state gc;
9495

95-
// sys.modules dictionary
96-
PyObject *modules;
97-
/* This is the list of module objects for all legacy (single-phase init)
98-
extension modules ever loaded in this process (i.e. imported
99-
in this interpreter or in any other). Py_None stands in for
100-
modules that haven't actually been imported in this interpreter.
101-
102-
A module's index (PyModuleDef.m_base.m_index) is used to look up
103-
the corresponding module object for this interpreter, if any.
104-
(See PyState_FindModule().) When any extension module
105-
is initialized during import, its moduledef gets initialized by
106-
PyModuleDef_Init(), and the first time that happens for each
107-
PyModuleDef, its index gets set to the current value of
108-
a global counter (see _PyRuntimeState.imports.last_module_index).
109-
The entry for that index in this interpreter remains unset until
110-
the module is actually imported here. (Py_None is used as
111-
a placeholder.) Note that multi-phase init modules always get
112-
an index for which there will never be a module set.
113-
114-
This is initialized lazily in _PyState_AddModule(), which is also
115-
where modules get added. */
116-
PyObject *modules_by_index;
96+
struct _import_state imports;
97+
11798
// Dictionary of the sys module
11899
PyObject *sysdict;
119100
// Dictionary of the builtins module
120101
PyObject *builtins;
121-
// importlib module
122-
PyObject *importlib;
123-
// override for config->use_frozen_modules (for tests)
124-
// (-1: "off", 1: "on", 0: no override)
125-
int override_frozen_modules;
126102

127103
PyObject *codec_search_path;
128104
PyObject *codec_search_cache;
129105
PyObject *codec_error_registry;
130106
int codecs_initialized;
131107

132108
PyConfig config;
133-
#ifdef HAVE_DLOPEN
134-
int dlopenflags;
135-
#endif
136109
unsigned long feature_flags;
137110

138111
PyObject *dict; /* Stores per-interpreter state */
139112

140113
PyObject *builtins_copy;
141-
PyObject *import_func;
142114
// Initialized to _PyEval_EvalFrameDefault().
143115
_PyFrameEvalFunction eval_frame;
144116

@@ -205,7 +177,6 @@ struct _is {
205177

206178
/* other API */
207179

208-
extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
209180
extern void _PyInterpreterState_Clear(PyThreadState *tstate);
210181

211182

‎Include/internal/pycore_pylifecycle.h

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc);
3030
/* Various one-time initializers */
3131

3232
extern void _Py_InitVersion(void);
33-
extern PyStatus _PyImport_Init(void);
3433
extern PyStatus _PyFaulthandler_Init(int enable);
3534
extern int _PyTraceMalloc_Init(int enable);
3635
extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
@@ -45,7 +44,6 @@ extern int _PyBuiltins_AddExceptions(PyObject * bltinmod);
4544
extern PyStatus _Py_HashRandomization_Init(const PyConfig *);
4645

4746
extern PyStatus _PyTime_Init(void);
48-
extern PyStatus _PyImportZip_Init(PyThreadState *tstate);
4947
extern PyStatus _PyGC_Init(PyInterpreterState *interp);
5048
extern PyStatus _PyAtExit_Init(PyInterpreterState *interp);
5149
extern int _Py_Deepfreeze_Init(void);
@@ -55,8 +53,6 @@ extern int _Py_Deepfreeze_Init(void);
5553
extern int _PySignal_Init(int install_signal_handlers);
5654
extern void _PySignal_Fini(void);
5755

58-
extern void _PyImport_Fini(void);
59-
extern void _PyImport_Fini2(void);
6056
extern void _PyGC_Fini(PyInterpreterState *interp);
6157
extern void _Py_HashRandomization_Fini(void);
6258
extern void _PyFaulthandler_Fini(void);

‎Include/internal/pycore_pystate.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,12 +152,6 @@ extern void _PySignal_AfterFork(void);
152152
#endif
153153

154154

155-
PyAPI_FUNC(int) _PyState_AddModule(
156-
PyThreadState *tstate,
157-
PyObject* module,
158-
PyModuleDef* def);
159-
160-
161155
PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
162156

163157
#define HEAD_LOCK(runtime) \

‎Include/internal/pycore_runtime_init.h

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,23 +97,10 @@ extern "C" {
9797
._main_interpreter = _PyInterpreterState_INIT, \
9898
}
9999

100-
#ifdef HAVE_DLOPEN
101-
# include <dlfcn.h>
102-
# if HAVE_DECL_RTLD_NOW
103-
# define _Py_DLOPEN_FLAGS RTLD_NOW
104-
# else
105-
# define _Py_DLOPEN_FLAGS RTLD_LAZY
106-
# endif
107-
# define DLOPENFLAGS_INIT .dlopenflags = _Py_DLOPEN_FLAGS,
108-
#else
109-
# define _Py_DLOPEN_FLAGS 0
110-
# define DLOPENFLAGS_INIT
111-
#endif
112-
113100
#define _PyInterpreterState_INIT \
114101
{ \
115102
.id_refcount = -1, \
116-
DLOPENFLAGS_INIT \
103+
.imports = IMPORTS_INIT, \
117104
.ceval = { \
118105
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
119106
}, \

‎Include/internal/pycore_sysmodule.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ extern void _PySys_ClearAuditHooks(PyThreadState *tstate);
2020

2121
PyAPI_FUNC(int) _PySys_SetAttr(PyObject *, PyObject *);
2222

23+
extern int _PySys_ClearAttrString(PyInterpreterState *interp,
24+
const char *name, int verbose);
25+
2326
#ifdef __cplusplus
2427
}
2528
#endif

‎Lib/test/test_imp.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized,
387387
check_basic_reloaded(mod, lookedup, initialized, init_count,
388388
before, reloaded)
389389

390-
# Currently _PyState_AddModule() always replaces the cached module.
390+
# Currently PyState_AddModule() always replaces the cached module.
391391
self.assertIs(basic.look_up_self(), mod)
392392
self.assertEqual(basic.initialized_count(), expected_init_count)
393393

‎Lib/test/test_stable_abi_ctypes.py

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Misc/stable_abi.toml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1684,9 +1684,6 @@
16841684
[function._PyObject_NewVar]
16851685
added = '3.2'
16861686
abi_only = true
1687-
[function._PyState_AddModule]
1688-
added = '3.2'
1689-
abi_only = true
16901687
[function._PyThreadState_Init]
16911688
added = '3.2'
16921689
abi_only = true

‎Objects/moduleobject.c

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,9 @@ PyModuleDef_Init(PyModuleDef* def)
4242
{
4343
assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY);
4444
if (def->m_base.m_index == 0) {
45-
_PyRuntime.imports.last_module_index++;
4645
Py_SET_REFCNT(def, 1);
4746
Py_SET_TYPE(def, &PyModuleDef_Type);
48-
def->m_base.m_index = _PyRuntime.imports.last_module_index;
47+
def->m_base.m_index = _PyImport_GetNextModuleIndex();
4948
}
5049
return (PyObject*)def;
5150
}
@@ -209,24 +208,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version)
209208
"module %s: PyModule_Create is incompatible with m_slots", name);
210209
return NULL;
211210
}
212-
/* Make sure name is fully qualified.
213-
214-
This is a bit of a hack: when the shared library is loaded,
215-
the module name is "package.module", but the module calls
216-
PyModule_Create*() with just "module" for the name. The shared
217-
library loader squirrels away the true name of the module in
218-
_Py_PackageContext, and PyModule_Create*() will substitute this
219-
(if the name actually matches).
220-
*/
221-
#define _Py_PackageContext (_PyRuntime.imports.pkgcontext)
222-
if (_Py_PackageContext != NULL) {
223-
const char *p = strrchr(_Py_PackageContext, '.');
224-
if (p != NULL && strcmp(module->m_name, p+1) == 0) {
225-
name = _Py_PackageContext;
226-
_Py_PackageContext = NULL;
227-
}
228-
}
229-
#undef _Py_PackageContext
211+
name = _PyImport_ResolveNameWithPackageContext(name);
230212
if ((m = (PyModuleObject*)PyModule_New(name)) == NULL)
231213
return NULL;
232214

@@ -710,8 +692,7 @@ static PyObject *
710692
module_repr(PyModuleObject *m)
711693
{
712694
PyInterpreterState *interp = _PyInterpreterState_GET();
713-
714-
return PyObject_CallMethod(interp->importlib, "_module_repr", "O", m);
695+
return _PyImport_ImportlibModuleRepr(interp, (PyObject *)m);
715696
}
716697

717698
/* Check if the "_initializing" attribute of the module spec is set to true.

‎PC/python3dll.c

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/_warnings.c

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import)
214214
gone, then we can't even use PyImport_GetModule without triggering
215215
an interpreter abort.
216216
*/
217-
if (!interp->modules) {
217+
if (!_PyImport_GetModules(interp)) {
218218
return NULL;
219219
}
220220
warnings_module = PyImport_GetModule(&_Py_ID(warnings));
@@ -1050,7 +1050,6 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category,
10501050
static PyObject *
10511051
get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno)
10521052
{
1053-
PyObject *external;
10541053
PyObject *loader;
10551054
PyObject *module_name;
10561055
PyObject *get_source;
@@ -1059,13 +1058,7 @@ get_source_line(PyInterpreterState *interp, PyObject *module_globals, int lineno
10591058
PyObject *source_line;
10601059

10611060
/* stolen from import.c */
1062-
external = PyObject_GetAttrString(interp->importlib, "_bootstrap_external");
1063-
if (external == NULL) {
1064-
return NULL;
1065-
}
1066-
1067-
loader = PyObject_CallMethod(external, "_bless_my_loader", "O", module_globals, NULL);
1068-
Py_DECREF(external);
1061+
loader = _PyImport_BlessMyLoader(interp, module_globals);
10691062
if (loader == NULL) {
10701063
return NULL;
10711064
}

‎Python/ceval.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2688,7 +2688,7 @@ import_name(PyThreadState *tstate, _PyInterpreterFrame *frame,
26882688
}
26892689
PyObject *locals = frame->f_locals;
26902690
/* Fast path for not overloaded __import__. */
2691-
if (import_func == tstate->interp->import_func) {
2691+
if (_PyImport_IsDefaultImportFunc(tstate->interp, import_func)) {
26922692
int ilevel = _PyLong_AsInt(level);
26932693
if (ilevel == -1 && _PyErr_Occurred(tstate)) {
26942694
return NULL;

‎Python/dynload_shlib.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ _PyImport_FindSharedFuncptr(const char *prefix,
7575
return NULL;
7676
}
7777

78-
dlopenflags = _PyInterpreterState_GET()->dlopenflags;
78+
dlopenflags = _PyImport_GetDLOpenFlags(_PyInterpreterState_GET());
7979

8080
handle = dlopen(pathname, dlopenflags);
8181

0 commit comments

Comments
 (0)