/* Module definition and import implementation */

#include "Python.h"

#include "pycore_ceval.h"

#include "pycore_hashtable.h" // _Py_hashtable_new_full()

#include "pycore_import.h" // _PyImport_BootstrapImp()

#include "pycore_initconfig.h" // _PyStatus_OK()

#include "pycore_interp.h" // struct _import_runtime_state

#include "pycore_namespace.h" // _PyNamespace_Type

#include "pycore_object.h" // _Py_SetImmortal()

#include "pycore_pyerrors.h" // _PyErr_SetString()

#include "pycore_pyhash.h" // _Py_KeyedHash()

#include "pycore_pylifecycle.h"

#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_sysmodule.h" // _PySys_Audit()

#include "pycore_time.h" // _PyTime_AsMicroseconds()

#include "pycore_weakref.h" // _PyWeakref_GET_REF()

#include "marshal.h" // PyMarshal_ReadObjectFromString()

#include "pycore_importdl.h" // _PyImport_DynLoadFiletab

#include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()

#include <stdbool.h> // bool

#ifdef HAVE_FCNTL_H

#include <fcntl.h>

#endif

/*[clinic input]

module _imp

[clinic start generated code]*/

/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/

#include "clinic/import.c.h"

#ifndef NDEBUG

static bool

is_interpreter_isolated(PyInterpreterState *interp)

{

return !_Py_IsMainInterpreter(interp)

&& !(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC)

&& interp->ceval.own_gil;

}

#endif

/*******************************/

/* process-global import state */

/*******************************/

/* This table is defined in config.c: */

extern struct _inittab _PyImport_Inittab[];

// This is not used after Py_Initialize() is called.

// (See _PyRuntimeState.imports.inittab.)

struct _inittab *PyImport_Inittab = _PyImport_Inittab;

// When we dynamically allocate a larger table for PyImport_ExtendInittab(),

// we track the pointer here so we can deallocate it during finalization.

static struct _inittab *inittab_copy = NULL;

/*******************************/

/* runtime-global import state */

/*******************************/

#define INITTAB _PyRuntime.imports.inittab

#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index

#define EXTENSIONS _PyRuntime.imports.extensions

#define PKGCONTEXT (_PyRuntime.imports.pkgcontext)

/*******************************/

/* interpreter import state */

/*******************************/

#define MODULES(interp) \

(interp)->imports.modules

#define MODULES_BY_INDEX(interp) \

(interp)->imports.modules_by_index

#define IMPORTLIB(interp) \

(interp)->imports.importlib

#define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \

(interp)->imports.override_multi_interp_extensions_check

#define OVERRIDE_FROZEN_MODULES(interp) \

(interp)->imports.override_frozen_modules

#ifdef HAVE_DLOPEN

# define DLOPENFLAGS(interp) \

(interp)->imports.dlopenflags

#endif

#define IMPORT_FUNC(interp) \

(interp)->imports.import_func

#define IMPORT_LOCK(interp) \

(interp)->imports.lock

#define FIND_AND_LOAD(interp) \

(interp)->imports.find_and_load

/*******************/

/* the import lock */

/*******************/

/* Locking primitives to prevent parallel imports of the same module

in different threads to return with a partially loaded module.

These calls are serialized by the global interpreter lock. */

void

_PyImport_AcquireLock(PyInterpreterState *interp)

{

_PyRecursiveMutex_Lock(&IMPORT_LOCK(interp));

}

void

_PyImport_ReleaseLock(PyInterpreterState *interp)

{

_PyRecursiveMutex_Unlock(&IMPORT_LOCK(interp));

}

/***************/

/* sys.modules */

/***************/

PyObject *

_PyImport_InitModules(PyInterpreterState *interp)

{

assert(MODULES(interp) == NULL);

MODULES(interp) = PyDict_New();

if (MODULES(interp) == NULL) {

return NULL;

}

return MODULES(interp);

}

PyObject *

_PyImport_GetModules(PyInterpreterState *interp)

{

return MODULES(interp);

}

void

_PyImport_ClearModules(PyInterpreterState *interp)

{

Py_SETREF(MODULES(interp), NULL);

}

static inline PyObject *

get_modules_dict(PyThreadState *tstate, bool fatal)

{

/* Technically, it would make sense to incref the dict,

* since sys.modules could be swapped out and decref'ed to 0

* before the caller is done using it. However, that is highly

* unlikely, especially since we can rely on a global lock

* (i.e. the GIL) for thread-safety. */

PyObject *modules = MODULES(tstate->interp);

if (modules == NULL) {

if (fatal) {

Py_FatalError("interpreter has no modules dictionary");

}

_PyErr_SetString(tstate, PyExc_RuntimeError,

"unable to get sys.modules");

return NULL;

}

return modules;

}

PyObject *

PyImport_GetModuleDict(void)

{

PyThreadState *tstate = _PyThreadState_GET();

return get_modules_dict(tstate, true);

}

int

_PyImport_SetModule(PyObject *name, PyObject *m)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *modules = get_modules_dict(tstate, true);

return PyObject_SetItem(modules, name, m);

}

int

_PyImport_SetModuleString(const char *name, PyObject *m)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *modules = get_modules_dict(tstate, true);

return PyMapping_SetItemString(modules, name, m);

}

static PyObject *

import_get_module(PyThreadState *tstate, PyObject *name)

{

PyObject *modules = get_modules_dict(tstate, false);

if (modules == NULL) {

return NULL;

}

PyObject *m;

Py_INCREF(modules);

(void)PyMapping_GetOptionalItem(modules, name, &m);

Py_DECREF(modules);

return m;

}

static int

import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name)

{

PyObject *spec;

/* Optimization: only call _bootstrap._lock_unlock_module() if

__spec__._initializing is true.

NOTE: because of this, initializing must be set *before*

stuffing the new module in sys.modules.

*/

int rc = PyObject_GetOptionalAttr(mod, &_Py_ID(__spec__), &spec);

if (rc > 0) {

rc = _PyModuleSpec_IsInitializing(spec);

Py_DECREF(spec);

}

if (rc <= 0) {

return rc;

}

/* Wait until module is done importing. */

PyObject *value = PyObject_CallMethodOneArg(

IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name);

if (value == NULL) {

return -1;

}

Py_DECREF(value);

return 0;

}

static void remove_importlib_frames(PyThreadState *tstate);

PyObject *

PyImport_GetModule(PyObject *name)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *mod;

mod = import_get_module(tstate, name);

if (mod != NULL && mod != Py_None) {

if (import_ensure_initialized(tstate->interp, mod, name) < 0) {

Py_DECREF(mod);

remove_importlib_frames(tstate);

return NULL;

}

}

return mod;

}

/* Get the module object corresponding to a module name.

First check the modules dictionary if there's one there,

if not, create a new one and insert it in the modules dictionary. */

static PyObject *

import_add_module(PyThreadState *tstate, PyObject *name)

{

PyObject *modules = get_modules_dict(tstate, false);

if (modules == NULL) {

return NULL;

}

PyObject *m;

if (PyMapping_GetOptionalItem(modules, name, &m) < 0) {

return NULL;

}

if (m != NULL && PyModule_Check(m)) {

return m;

}

Py_XDECREF(m);

m = PyModule_NewObject(name);

if (m == NULL)

return NULL;

if (PyObject_SetItem(modules, name, m) != 0) {

Py_DECREF(m);

return NULL;

}

return m;

}

PyObject *

PyImport_AddModuleRef(const char *name)

{

PyObject *name_obj = PyUnicode_FromString(name);

if (name_obj == NULL) {

return NULL;

}

PyThreadState *tstate = _PyThreadState_GET();

PyObject *module = import_add_module(tstate, name_obj);

Py_DECREF(name_obj);

return module;

}

PyObject *

PyImport_AddModuleObject(PyObject *name)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *mod = import_add_module(tstate, name);

if (!mod) {

return NULL;

}

// gh-86160: PyImport_AddModuleObject() returns a borrowed reference.

// Create a weak reference to produce a borrowed reference, since it can

// become NULL. sys.modules type can be different than dict and it is not

// guaranteed that it keeps a strong reference to the module. It can be a

// custom mapping with __getitem__() which returns a new object or removes

// returned object, or __setitem__ which does nothing. There is so much

// unknown. With weakref we can be sure that we get either a reference to

// live object or NULL.

//

// Use PyImport_AddModuleRef() to avoid these issues.

PyObject *ref = PyWeakref_NewRef(mod, NULL);

Py_DECREF(mod);

if (ref == NULL) {

return NULL;

}

mod = _PyWeakref_GET_REF(ref);

Py_DECREF(ref);

Py_XDECREF(mod);

if (mod == NULL && !PyErr_Occurred()) {

PyErr_SetString(PyExc_RuntimeError,

"sys.modules does not hold a strong reference "

"to the module");

}

return mod; /* borrowed reference */

}

PyObject *

PyImport_AddModule(const char *name)

{

PyObject *nameobj = PyUnicode_FromString(name);

if (nameobj == NULL) {

return NULL;

}

PyObject *module = PyImport_AddModuleObject(nameobj);

Py_DECREF(nameobj);

return module;

}

/* Remove name from sys.modules, if it's there.

* Can be called with an exception raised.

* If fail to remove name a new exception will be chained with the old

* exception, otherwise the old exception is preserved.

*/

static void

remove_module(PyThreadState *tstate, PyObject *name)

{

PyObject *exc = _PyErr_GetRaisedException(tstate);

PyObject *modules = get_modules_dict(tstate, true);

if (PyDict_CheckExact(modules)) {

// Error is reported to the caller

(void)PyDict_Pop(modules, name, NULL);

}

else if (PyMapping_DelItem(modules, name) < 0) {

if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {

_PyErr_Clear(tstate);

}

}

_PyErr_ChainExceptions1(exc);

}

/************************************/

/* per-interpreter modules-by-index */

/************************************/

Py_ssize_t

_PyImport_GetNextModuleIndex(void)

{

return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1;

}

#ifndef NDEBUG

struct extensions_cache_value;

static struct extensions_cache_value * _find_cached_def(PyModuleDef *);

static Py_ssize_t _get_cached_module_index(struct extensions_cache_value *);

#endif

static Py_ssize_t

_get_module_index_from_def(PyModuleDef *def)

{

Py_ssize_t index = def->m_base.m_index;

#ifndef NDEBUG

struct extensions_cache_value *cached = _find_cached_def(def);

assert(cached == NULL || index == _get_cached_module_index(cached));

#endif

return index;

}

static void

_set_module_index(PyModuleDef *def, Py_ssize_t index)

{

assert(index > 0);

if (index == def->m_base.m_index) {

/* There's nothing to do. */

}

else if (def->m_base.m_index == 0) {

/* It should have been initialized by PyModuleDef_Init().

* We assert here to catch this in dev, but keep going otherwise. */

assert(def->m_base.m_index != 0);

def->m_base.m_index = index;

}

else {

/* It was already set for a different module.

* We replace the old value. */

assert(def->m_base.m_index > 0);

def->m_base.m_index = index;

}

}

static const char *

_modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index)

{

if (index <= 0) {

return "invalid module index";

}

if (MODULES_BY_INDEX(interp) == NULL) {

return "Interpreters module-list not accessible.";

}

if (index >= PyList_GET_SIZE(MODULES_BY_INDEX(interp))) {

return "Module index out of bounds.";

}

return NULL;

}

static PyObject *

_modules_by_index_get(PyInterpreterState *interp, Py_ssize_t index)

{

if (_modules_by_index_check(interp, index) != NULL) {

return NULL;

}

PyObject *res = PyList_GET_ITEM(MODULES_BY_INDEX(interp), index);

return res==Py_None ? NULL : res;

}

static int

_modules_by_index_set(PyInterpreterState *interp,

Py_ssize_t index, PyObject *module)

{

assert(index > 0);

if (MODULES_BY_INDEX(interp) == NULL) {

MODULES_BY_INDEX(interp) = PyList_New(0);

if (MODULES_BY_INDEX(interp) == NULL) {

return -1;

}

}

while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) {

if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) {

return -1;

}

}

return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(module));

}

static int

_modules_by_index_clear_one(PyInterpreterState *interp, Py_ssize_t index)

{

const char *err = _modules_by_index_check(interp, index);

if (err != NULL) {

Py_FatalError(err);

return -1;

}

return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(Py_None));

}

PyObject*

PyState_FindModule(PyModuleDef* module)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

if (module->m_slots) {

return NULL;

}

Py_ssize_t index = _get_module_index_from_def(module);

return _modules_by_index_get(interp, index);

}

/* _PyState_AddModule() has been completely removed from the C-API

(and was removed from the limited API in 3.6). However, we're

playing it safe and keeping it around for any stable ABI extensions

built against 3.2-3.5. */

int

_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def)

{

if (!def) {

assert(_PyErr_Occurred(tstate));

return -1;

}

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_AddModule called on module with slots");

return -1;

}

assert(def->m_slots == NULL);

Py_ssize_t index = _get_module_index_from_def(def);

return _modules_by_index_set(tstate->interp, index, module);

}

int

PyState_AddModule(PyObject* module, PyModuleDef* def)

{

if (!def) {

Py_FatalError("module definition is NULL");

return -1;

}

PyThreadState *tstate = _PyThreadState_GET();

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_AddModule called on module with slots");

return -1;

}

PyInterpreterState *interp = tstate->interp;

Py_ssize_t index = _get_module_index_from_def(def);

if (MODULES_BY_INDEX(interp) &&

index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) &&

module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index))

{

_Py_FatalErrorFormat(__func__, "module %p already added", module);

return -1;

}

assert(def->m_slots == NULL);

return _modules_by_index_set(interp, index, module);

}

int

PyState_RemoveModule(PyModuleDef* def)

{

PyThreadState *tstate = _PyThreadState_GET();

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_RemoveModule called on module with slots");

return -1;

}

Py_ssize_t index = _get_module_index_from_def(def);

return _modules_by_index_clear_one(tstate->interp, index);

}

// Used by finalize_modules()

void

_PyImport_ClearModulesByIndex(PyInterpreterState *interp)

{

if (!MODULES_BY_INDEX(interp)) {

return;

}

Py_ssize_t i;

for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) {

PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i);

if (PyModule_Check(m)) {

/* cleanup the saved copy of module dicts */

PyModuleDef *md = PyModule_GetDef(m);

if (md) {

// XXX Do this more carefully. The dict might be owned

// by another interpreter.

Py_CLEAR(md->m_base.m_copy);

}

}

}

/* Setting modules_by_index to NULL could be dangerous, so we

clear the list instead. */

if (PyList_SetSlice(MODULES_BY_INDEX(interp),

0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)),

NULL)) {

PyErr_FormatUnraisable("Exception ignored on clearing interpreters module list");

}

}

/*********************/

/* extension modules */

/*********************/

/*

It may help to have a big picture view of what happens

when an extension is loaded. This includes when it is imported

for the first time.

Here's a summary, using importlib._bootstrap._load() as a starting point.

1. importlib._bootstrap._load()

2. _load(): acquire import lock

3. _load() -> importlib._bootstrap._load_unlocked()

4. _load_unlocked() -> importlib._bootstrap.module_from_spec()

5. module_from_spec() -> ExtensionFileLoader.create_module()

6. create_module() -> _imp.create_dynamic()

(see below)

7. module_from_spec() -> importlib._bootstrap._init_module_attrs()

8. _load_unlocked(): sys.modules[name] = module

9. _load_unlocked() -> ExtensionFileLoader.exec_module()

10. exec_module() -> _imp.exec_dynamic()

(see below)

11. _load(): release import lock

...for single-phase init modules, where m_size == -1:

(6). first time (not found in _PyRuntime.imports.extensions):

A. _imp_create_dynamic_impl() -> import_find_extension()

B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc()

C. _PyImport_GetModInitFunc(): load <module init func>

D. _imp_create_dynamic_impl() -> import_run_extension()

E. import_run_extension() -> _PyImport_RunModInitFunc()

F. _PyImport_RunModInitFunc(): call <module init func>

G. <module init func> -> PyModule_Create() -> PyModule_Create2()

-> PyModule_CreateInitialized()

H. PyModule_CreateInitialized() -> PyModule_New()

I. PyModule_CreateInitialized(): allocate mod->md_state

J. PyModule_CreateInitialized() -> PyModule_AddFunctions()

K. PyModule_CreateInitialized() -> PyModule_SetDocString()

L. PyModule_CreateInitialized(): set mod->md_def

M. <module init func>: initialize the module, etc.

N. import_run_extension()

-> _PyImport_CheckSubinterpIncompatibleExtensionAllowed()

O. import_run_extension(): set __file__

P. import_run_extension() -> update_global_state_for_extension()

Q. update_global_state_for_extension():

copy __dict__ into def->m_base.m_copy

R. update_global_state_for_extension():

add it to _PyRuntime.imports.extensions

S. import_run_extension() -> finish_singlephase_extension()

T. finish_singlephase_extension():

add it to interp->imports.modules_by_index

U. finish_singlephase_extension(): add it to sys.modules

Step (Q) is skipped for core modules (sys/builtins).

(6). subsequent times (found in _PyRuntime.imports.extensions):

A. _imp_create_dynamic_impl() -> import_find_extension()

B. import_find_extension() -> reload_singlephase_extension()

C. reload_singlephase_extension()

-> _PyImport_CheckSubinterpIncompatibleExtensionAllowed()

D. reload_singlephase_extension() -> import_add_module()

E. if name in sys.modules: use that module

F. else:

1. import_add_module() -> PyModule_NewObject()

2. import_add_module(): set it on sys.modules

G. reload_singlephase_extension(): copy the "m_copy" dict into __dict__

H. reload_singlephase_extension(): add to modules_by_index

(10). (every time):

A. noop

...for single-phase init modules, where m_size >= 0:

(6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions):

A-P. (same as for m_size == -1)

Q. _PyImport_RunModInitFunc(): set def->m_base.m_init

R. (skipped)

S-U. (same as for m_size == -1)

(6). main interpreter - first time (not found in _PyRuntime.imports.extensions):

A-P. (same as for m_size == -1)

Q. _PyImport_RunModInitFunc(): set def->m_base.m_init

R-U. (same as for m_size == -1)

(6). subsequent times (found in _PyRuntime.imports.extensions):

A. _imp_create_dynamic_impl() -> import_find_extension()

B. import_find_extension() -> reload_singlephase_extension()

C. reload_singlephase_extension()

-> _PyImport_CheckSubinterpIncompatibleExtensionAllowed()

D. reload_singlephase_extension(): call def->m_base.m_init (see above)

E. reload_singlephase_extension(): add the module to sys.modules

F. reload_singlephase_extension(): add to modules_by_index

(10). every time:

A. noop

...for multi-phase init modules:

(6). every time:

A. _imp_create_dynamic_impl() -> import_find_extension() (not found)

B. _imp_create_dynamic_impl() -> _PyImport_GetModInitFunc()

C. _PyImport_GetModInitFunc(): load <module init func>

D. _imp_create_dynamic_impl() -> import_run_extension()

E. import_run_extension() -> _PyImport_RunModInitFunc()

F. _PyImport_RunModInitFunc(): call <module init func>

G. import_run_extension() -> PyModule_FromDefAndSpec()

H. PyModule_FromDefAndSpec(): gather/check moduledef slots

I. if there's a Py_mod_create slot:

1. PyModule_FromDefAndSpec(): call its function

J. else:

1. PyModule_FromDefAndSpec() -> PyModule_NewObject()

K: PyModule_FromDefAndSpec(): set mod->md_def

L. PyModule_FromDefAndSpec() -> _add_methods_to_object()

M. PyModule_FromDefAndSpec() -> PyModule_SetDocString()

(10). every time:

A. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic()

B. if mod->md_state == NULL (including if m_size == 0):

1. exec_builtin_or_dynamic() -> PyModule_ExecDef()

2. PyModule_ExecDef(): allocate mod->md_state

3. if there's a Py_mod_exec slot:

1. PyModule_ExecDef(): call its function

*/

/* Make sure name is fully qualified.

This is a bit of a hack: when the shared library is loaded,

the module name is "package.module", but the module calls

PyModule_Create*() with just "module" for the name. The shared

library loader squirrels away the true name of the module in

_PyRuntime.imports.pkgcontext, and PyModule_Create*() will

substitute this (if the name actually matches).

*/

#ifdef HAVE_THREAD_LOCAL

_Py_thread_local const char *pkgcontext = NULL;

# undef PKGCONTEXT

# define PKGCONTEXT pkgcontext

#endif

const char *

_PyImport_ResolveNameWithPackageContext(const char *name)

{

#ifndef HAVE_THREAD_LOCAL

PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);

#endif

if (PKGCONTEXT != NULL) {

const char *p = strrchr(PKGCONTEXT, '.');

if (p != NULL && strcmp(name, p+1) == 0) {

name = PKGCONTEXT;

PKGCONTEXT = NULL;

}

}

#ifndef HAVE_THREAD_LOCAL

PyThread_release_lock(EXTENSIONS.mutex);

#endif

return name;

}

const char *

_PyImport_SwapPackageContext(const char *newcontext)

{

#ifndef HAVE_THREAD_LOCAL

PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);

#endif

const char *oldcontext = PKGCONTEXT;

PKGCONTEXT = newcontext;

#ifndef HAVE_THREAD_LOCAL

PyThread_release_lock(EXTENSIONS.mutex);

#endif

return oldcontext;

}

#ifdef HAVE_DLOPEN

int

_PyImport_GetDLOpenFlags(PyInterpreterState *interp)

{

return DLOPENFLAGS(interp);

}

void

_PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val)

{

DLOPENFLAGS(interp) = new_val;

}

#endif // HAVE_DLOPEN

/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */

static int

exec_builtin_or_dynamic(PyObject *mod) {

PyModuleDef *def;

void *state;

if (!PyModule_Check(mod)) {

return 0;

}

def = PyModule_GetDef(mod);

if (def == NULL) {

return 0;

}

state = PyModule_GetState(mod);

if (state) {

/* Already initialized; skip reload */

return 0;

}

return PyModule_ExecDef(mod, def);

}

static int clear_singlephase_extension(PyInterpreterState *interp,

PyObject *name, PyObject *filename);

// Currently, this is only used for testing.

// (See _testinternalcapi.clear_extension().)

// If adding another use, be careful about modules that import themselves

// recursively (see gh-123880).

int

_PyImport_ClearExtension(PyObject *name, PyObject *filename)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

/* Clearing a module's C globals is up to the module. */

if (clear_singlephase_extension(interp, name, filename) < 0) {

return -1;

}

// In the future we'll probably also make sure the extension's

// file handle (and DL handle) is closed (requires saving it).

return 0;

}

/*****************************/

/* single-phase init modules */

/*****************************/

/*

We support a number of kinds of single-phase init builtin/extension modules:

* "basic"

* no module state (PyModuleDef.m_size == -1)

* does not support repeated init (we use PyModuleDef.m_base.m_copy)

* may have process-global state

* the module's def is cached in _PyRuntime.imports.extensions,

by (name, filename)

* "reinit"

* no module state (PyModuleDef.m_size == 0)

* supports repeated init (m_copy is never used)

* should not have any process-global state

* its def is never cached in _PyRuntime.imports.extensions

(except, currently, under the main interpreter, for some reason)

* "with state" (almost the same as reinit)

* has module state (PyModuleDef.m_size > 0)

* supports repeated init (m_copy is never used)

* should not have any process-global state

* its def is never cached in _PyRuntime.imports.extensions

(except, currently, under the main interpreter, for some reason)

There are also variants within those classes:

* two or more modules share a PyModuleDef

* a module's init func uses another module's PyModuleDef

* a module's init func calls another's module's init func

* a module's init "func" is actually a variable statically initialized

to another module's init func

* two or modules share "methods"

* a module's init func copies another module's PyModuleDef

(with a different name)

* (basic-only) two or modules share process-global state

In the first case, where modules share a PyModuleDef, the following

notable weirdness happens:

* the module's __name__ matches the def, not the requested name

* the last module (with the same def) to be imported for the first time wins

* returned by PyState_Find_Module() (via interp->modules_by_index)

* (non-basic-only) its init func is used when re-loading any of them

(via the def's m_init)

* (basic-only) the copy of its __dict__ is used when re-loading any of them

(via the def's m_copy)

However, the following happens as expected:

* a new module object (with its own __dict__) is created for each request

* the module's __spec__ has the requested name

* the loaded module is cached in sys.modules under the requested name

* the m_index field of the shared def is not changed,

so at least PyState_FindModule() will always look in the same place

For "basic" modules there are other quirks:

* (whether sharing a def or not) when loaded the first time,

m_copy is set before _init_module_attrs() is called

in importlib._bootstrap.module_from_spec(),

so when the module is re-loaded, the previous value

for __wpec__ (and others) is reset, possibly unexpectedly.

Generally, when multiple interpreters are involved, some of the above

gets even messier.

*/

static inline void

extensions_lock_acquire(void)

{

PyMutex_Lock(&_PyRuntime.imports.extensions.mutex);

}

static inline void

extensions_lock_release(void)

{

PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex);

}

/* Magic for extension modules (built-in as well as dynamically

loaded). To prevent initializing an extension module more than

once, we keep a static dictionary 'extensions' keyed by the tuple

(module name, module name) (for built-in modules) or by

(filename, module name) (for dynamically loaded modules), containing these

modules. A copy of the module's dictionary is stored by calling

fix_up_extension() immediately after the module initialization

function succeeds. A copy can be retrieved from there by calling

import_find_extension().

Modules which do support multiple initialization set their m_size

field to a non-negative number (indicating the size of the

module-specific state). They are still recorded in the extensions

dictionary, to avoid loading shared libraries twice.

*/

typedef struct cached_m_dict {

/* A shallow copy of the original module's __dict__. */

PyObject *copied;

/* The interpreter that owns the copy. */

int64_t interpid;

} *cached_m_dict_t;

struct extensions_cache_value {

PyModuleDef *def;

/* The function used to re-initialize the module.

This is only set for legacy (single-phase init) extension modules

and only used for those that support multiple initializations

(m_size >= 0).

It is set by update_global_state_for_extension(). */

PyModInitFunction m_init;

/* The module's index into its interpreter's modules_by_index cache.

This is set for all extension modules but only used for legacy ones.

(See PyInterpreterState.modules_by_index for more info.) */

Py_ssize_t m_index;

/* A copy of the module's __dict__ after the first time it was loaded.

This is only set/used for legacy modules that do not support

multiple initializations.

It is set exclusively by fixup_cached_def(). */

cached_m_dict_t m_dict;

struct cached_m_dict _m_dict;

_Py_ext_module_origin origin;

#ifdef Py_GIL_DISABLED

/* The module's md_gil slot, for legacy modules that are reinitialized from

m_dict rather than calling their initialization function again. */

void *md_gil;

#endif

};

static struct extensions_cache_value *

alloc_extensions_cache_value(void)

{

struct extensions_cache_value *value

= PyMem_RawMalloc(sizeof(struct extensions_cache_value));

if (value == NULL) {

PyErr_NoMemory();

return NULL;

}

*value = (struct extensions_cache_value){0};

return value;

}

static void

free_extensions_cache_value(struct extensions_cache_value *value)

{

PyMem_RawFree(value);

}

static Py_ssize_t

_get_cached_module_index(struct extensions_cache_value *cached)

{

assert(cached->m_index > 0);

return cached->m_index;

}

static void

fixup_cached_def(struct extensions_cache_value *value)

{