/* Module definition and import implementation */

#include "Python.h"

#include "pycore_dict.h" // _PyDict_Pop()

#include "pycore_hashtable.h" // _Py_hashtable_new_full()

#include "pycore_import.h" // _PyImport_BootstrapImp()

#include "pycore_initconfig.h" // _PyStatus_OK()

#include "pycore_interp.h" // struct _import_runtime_state

#include "pycore_namespace.h" // _PyNamespace_Type

#include "pycore_object.h" // _Py_SetImmortal()

#include "pycore_pyerrors.h" // _PyErr_SetString()

#include "pycore_pyhash.h" // _Py_KeyedHash()

#include "pycore_pylifecycle.h"

#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_sysmodule.h" // _PySys_Audit()

#include "pycore_weakref.h" // _PyWeakref_GET_REF()

#include "marshal.h" // PyMarshal_ReadObjectFromString()

#include "importdl.h" // _PyImport_DynLoadFiletab

#include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()

#include <stdbool.h> // bool

#ifdef HAVE_FCNTL_H

#include <fcntl.h>

#endif

/*[clinic input]

module _imp

[clinic start generated code]*/

/*[clinic end generated code: output=da39a3ee5e6b4b0d input=9c332475d8686284]*/

#include "clinic/import.c.h"

/*******************************/

/* process-global import state */

/*******************************/

/* This table is defined in config.c: */

extern struct _inittab _PyImport_Inittab[];

// This is not used after Py_Initialize() is called.

// (See _PyRuntimeState.imports.inittab.)

struct _inittab *PyImport_Inittab = _PyImport_Inittab;

// When we dynamically allocate a larger table for PyImport_ExtendInittab(),

// we track the pointer here so we can deallocate it during finalization.

static struct _inittab *inittab_copy = NULL;

/*******************************/

/* runtime-global import state */

/*******************************/

#define INITTAB _PyRuntime.imports.inittab

#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index

#define EXTENSIONS _PyRuntime.imports.extensions

#define PKGCONTEXT (_PyRuntime.imports.pkgcontext)

/*******************************/

/* interpreter import state */

/*******************************/

#define MODULES(interp) \

(interp)->imports.modules

#define MODULES_BY_INDEX(interp) \

(interp)->imports.modules_by_index

#define IMPORTLIB(interp) \

(interp)->imports.importlib

#define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \

(interp)->imports.override_multi_interp_extensions_check

#define OVERRIDE_FROZEN_MODULES(interp) \

(interp)->imports.override_frozen_modules

#ifdef HAVE_DLOPEN

# define DLOPENFLAGS(interp) \

(interp)->imports.dlopenflags

#endif

#define IMPORT_FUNC(interp) \

(interp)->imports.import_func

#define IMPORT_LOCK(interp) \

(interp)->imports.lock.mutex

#define IMPORT_LOCK_THREAD(interp) \

(interp)->imports.lock.thread

#define IMPORT_LOCK_LEVEL(interp) \

(interp)->imports.lock.level

#define FIND_AND_LOAD(interp) \

(interp)->imports.find_and_load

/*******************/

/* the import lock */

/*******************/

/* Locking primitives to prevent parallel imports of the same module

in different threads to return with a partially loaded module.

These calls are serialized by the global interpreter lock. */

void

_PyImport_AcquireLock(PyInterpreterState *interp)

{

unsigned long me = PyThread_get_thread_ident();

if (me == PYTHREAD_INVALID_THREAD_ID)

return; /* Too bad */

if (IMPORT_LOCK(interp) == NULL) {

IMPORT_LOCK(interp) = PyThread_allocate_lock();

if (IMPORT_LOCK(interp) == NULL)

return; /* Nothing much we can do. */

}

if (IMPORT_LOCK_THREAD(interp) == me) {

IMPORT_LOCK_LEVEL(interp)++;

return;

}

if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID ||

!PyThread_acquire_lock(IMPORT_LOCK(interp), 0))

{

PyThreadState *tstate = PyEval_SaveThread();

PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);

PyEval_RestoreThread(tstate);

}

assert(IMPORT_LOCK_LEVEL(interp) == 0);

IMPORT_LOCK_THREAD(interp) = me;

IMPORT_LOCK_LEVEL(interp) = 1;

}

int

_PyImport_ReleaseLock(PyInterpreterState *interp)

{

unsigned long me = PyThread_get_thread_ident();

if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL)

return 0; /* Too bad */

if (IMPORT_LOCK_THREAD(interp) != me)

return -1;

IMPORT_LOCK_LEVEL(interp)--;

assert(IMPORT_LOCK_LEVEL(interp) >= 0);

if (IMPORT_LOCK_LEVEL(interp) == 0) {

IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;

PyThread_release_lock(IMPORT_LOCK(interp));

}

return 1;

}

#ifdef HAVE_FORK

/* This function is called from PyOS_AfterFork_Child() to ensure that newly

created child processes do not share locks with the parent.

We now acquire the import lock around fork() calls but on some platforms

(Solaris 9 and earlier? see isue7242) that still left us with problems. */

PyStatus

_PyImport_ReInitLock(PyInterpreterState *interp)

{

if (IMPORT_LOCK(interp) != NULL) {

if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) {

return _PyStatus_ERR("failed to create a new lock");

}

}

if (IMPORT_LOCK_LEVEL(interp) > 1) {

/* Forked as a side effect of import */

unsigned long me = PyThread_get_thread_ident();

PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);

IMPORT_LOCK_THREAD(interp) = me;

IMPORT_LOCK_LEVEL(interp)--;

} else {

IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;

IMPORT_LOCK_LEVEL(interp) = 0;

}

return _PyStatus_OK();

}

#endif

/***************/

/* sys.modules */

/***************/

PyObject *

_PyImport_InitModules(PyInterpreterState *interp)

{

assert(MODULES(interp) == NULL);

MODULES(interp) = PyDict_New();

if (MODULES(interp) == NULL) {

return NULL;

}

return MODULES(interp);

}

PyObject *

_PyImport_GetModules(PyInterpreterState *interp)

{

return MODULES(interp);

}

void

_PyImport_ClearModules(PyInterpreterState *interp)

{

Py_SETREF(MODULES(interp), NULL);

}

PyObject *

PyImport_GetModuleDict(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

if (MODULES(interp) == NULL) {

Py_FatalError("interpreter has no modules dictionary");

}

return MODULES(interp);

}

int

_PyImport_SetModule(PyObject *name, PyObject *m)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

PyObject *modules = MODULES(interp);

return PyObject_SetItem(modules, name, m);

}

int

_PyImport_SetModuleString(const char *name, PyObject *m)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

PyObject *modules = MODULES(interp);

return PyMapping_SetItemString(modules, name, m);

}

static PyObject *

import_get_module(PyThreadState *tstate, PyObject *name)

{

PyObject *modules = MODULES(tstate->interp);

if (modules == NULL) {

_PyErr_SetString(tstate, PyExc_RuntimeError,

"unable to get sys.modules");

return NULL;

}

PyObject *m;

Py_INCREF(modules);

(void)PyMapping_GetOptionalItem(modules, name, &m);

Py_DECREF(modules);

return m;

}

static int

import_ensure_initialized(PyInterpreterState *interp, PyObject *mod, PyObject *name)

{

PyObject *spec;

/* Optimization: only call _bootstrap._lock_unlock_module() if

__spec__._initializing is true.

NOTE: because of this, initializing must be set *before*

stuffing the new module in sys.modules.

*/

spec = PyObject_GetAttr(mod, &_Py_ID(__spec__));

int busy = _PyModuleSpec_IsInitializing(spec);

Py_XDECREF(spec);

if (busy) {

/* Wait until module is done importing. */

PyObject *value = PyObject_CallMethodOneArg(

IMPORTLIB(interp), &_Py_ID(_lock_unlock_module), name);

if (value == NULL) {

return -1;

}

Py_DECREF(value);

}

return 0;

}

static void remove_importlib_frames(PyThreadState *tstate);

PyObject *

PyImport_GetModule(PyObject *name)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *mod;

mod = import_get_module(tstate, name);

if (mod != NULL && mod != Py_None) {

if (import_ensure_initialized(tstate->interp, mod, name) < 0) {

Py_DECREF(mod);

remove_importlib_frames(tstate);

return NULL;

}

}

return mod;

}

/* Get the module object corresponding to a module name.

First check the modules dictionary if there's one there,

if not, create a new one and insert it in the modules dictionary. */

static PyObject *

import_add_module(PyThreadState *tstate, PyObject *name)

{

PyObject *modules = MODULES(tstate->interp);

if (modules == NULL) {

_PyErr_SetString(tstate, PyExc_RuntimeError,

"no import module dictionary");

return NULL;

}

PyObject *m;

if (PyMapping_GetOptionalItem(modules, name, &m) < 0) {

return NULL;

}

if (m != NULL && PyModule_Check(m)) {

return m;

}

Py_XDECREF(m);

m = PyModule_NewObject(name);

if (m == NULL)

return NULL;

if (PyObject_SetItem(modules, name, m) != 0) {

Py_DECREF(m);

return NULL;

}

return m;

}

PyObject *

PyImport_AddModuleRef(const char *name)

{

PyObject *name_obj = PyUnicode_FromString(name);

if (name_obj == NULL) {

return NULL;

}

PyThreadState *tstate = _PyThreadState_GET();

PyObject *module = import_add_module(tstate, name_obj);

Py_DECREF(name_obj);

return module;

}

PyObject *

PyImport_AddModuleObject(PyObject *name)

{

PyThreadState *tstate = _PyThreadState_GET();

PyObject *mod = import_add_module(tstate, name);

if (!mod) {

return NULL;

}

// gh-86160: PyImport_AddModuleObject() returns a borrowed reference.

// Create a weak reference to produce a borrowed reference, since it can

// become NULL. sys.modules type can be different than dict and it is not

// guaranteed that it keeps a strong reference to the module. It can be a

// custom mapping with __getitem__() which returns a new object or removes

// returned object, or __setitem__ which does nothing. There is so much

// unknown. With weakref we can be sure that we get either a reference to

// live object or NULL.

//

// Use PyImport_AddModuleRef() to avoid these issues.

PyObject *ref = PyWeakref_NewRef(mod, NULL);

Py_DECREF(mod);

if (ref == NULL) {

return NULL;

}

mod = _PyWeakref_GET_REF(ref);

Py_DECREF(ref);

Py_XDECREF(mod);

if (mod == NULL && !PyErr_Occurred()) {

PyErr_SetString(PyExc_RuntimeError,

"sys.modules does not hold a strong reference "

"to the module");

}

return mod; /* borrowed reference */

}

PyObject *

PyImport_AddModule(const char *name)

{

PyObject *nameobj = PyUnicode_FromString(name);

if (nameobj == NULL) {

return NULL;

}

PyObject *module = PyImport_AddModuleObject(nameobj);

Py_DECREF(nameobj);

return module;

}

/* Remove name from sys.modules, if it's there.

* Can be called with an exception raised.

* If fail to remove name a new exception will be chained with the old

* exception, otherwise the old exception is preserved.

*/

static void

remove_module(PyThreadState *tstate, PyObject *name)

{

PyObject *exc = _PyErr_GetRaisedException(tstate);

PyObject *modules = MODULES(tstate->interp);

if (PyDict_CheckExact(modules)) {

PyObject *mod = _PyDict_Pop(modules, name, Py_None);

Py_XDECREF(mod);

}

else if (PyMapping_DelItem(modules, name) < 0) {

if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {

_PyErr_Clear(tstate);

}

}

_PyErr_ChainExceptions1(exc);

}

/************************************/

/* per-interpreter modules-by-index */

/************************************/

Py_ssize_t

_PyImport_GetNextModuleIndex(void)

{

PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);

LAST_MODULE_INDEX++;

Py_ssize_t index = LAST_MODULE_INDEX;

PyThread_release_lock(EXTENSIONS.mutex);

return index;

}

static const char *

_modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index)

{

if (index == 0) {

return "invalid module index";

}

if (MODULES_BY_INDEX(interp) == NULL) {

return "Interpreters module-list not accessible.";

}

if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) {

return "Module index out of bounds.";

}

return NULL;

}

static PyObject *

_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def)

{

Py_ssize_t index = def->m_base.m_index;

if (_modules_by_index_check(interp, index) != NULL) {

return NULL;

}

PyObject *res = PyList_GET_ITEM(MODULES_BY_INDEX(interp), index);

return res==Py_None ? NULL : res;

}

static int

_modules_by_index_set(PyInterpreterState *interp,

PyModuleDef *def, PyObject *module)

{

assert(def != NULL);

assert(def->m_slots == NULL);

assert(def->m_base.m_index > 0);

if (MODULES_BY_INDEX(interp) == NULL) {

MODULES_BY_INDEX(interp) = PyList_New(0);

if (MODULES_BY_INDEX(interp) == NULL) {

return -1;

}

}

Py_ssize_t index = def->m_base.m_index;

while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) {

if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) {

return -1;

}

}

return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(module));

}

static int

_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def)

{

Py_ssize_t index = def->m_base.m_index;

const char *err = _modules_by_index_check(interp, index);

if (err != NULL) {

Py_FatalError(err);

return -1;

}

return PyList_SetItem(MODULES_BY_INDEX(interp), index, Py_NewRef(Py_None));

}

PyObject*

PyState_FindModule(PyModuleDef* module)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

if (module->m_slots) {

return NULL;

}

return _modules_by_index_get(interp, module);

}

/* _PyState_AddModule() has been completely removed from the C-API

(and was removed from the limited API in 3.6). However, we're

playing it safe and keeping it around for any stable ABI extensions

built against 3.2-3.5. */

int

_PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def)

{

if (!def) {

assert(_PyErr_Occurred(tstate));

return -1;

}

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_AddModule called on module with slots");

return -1;

}

return _modules_by_index_set(tstate->interp, def, module);

}

int

PyState_AddModule(PyObject* module, PyModuleDef* def)

{

if (!def) {

Py_FatalError("module definition is NULL");

return -1;

}

PyThreadState *tstate = _PyThreadState_GET();

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_AddModule called on module with slots");

return -1;

}

PyInterpreterState *interp = tstate->interp;

Py_ssize_t index = def->m_base.m_index;

if (MODULES_BY_INDEX(interp) &&

index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) &&

module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index))

{

_Py_FatalErrorFormat(__func__, "module %p already added", module);

return -1;

}

return _modules_by_index_set(interp, def, module);

}

int

PyState_RemoveModule(PyModuleDef* def)

{

PyThreadState *tstate = _PyThreadState_GET();

if (def->m_slots) {

_PyErr_SetString(tstate,

PyExc_SystemError,

"PyState_RemoveModule called on module with slots");

return -1;

}

return _modules_by_index_clear_one(tstate->interp, def);

}

// Used by finalize_modules()

void

_PyImport_ClearModulesByIndex(PyInterpreterState *interp)

{

if (!MODULES_BY_INDEX(interp)) {

return;

}

Py_ssize_t i;

for (i = 0; i < PyList_GET_SIZE(MODULES_BY_INDEX(interp)); i++) {

PyObject *m = PyList_GET_ITEM(MODULES_BY_INDEX(interp), i);

if (PyModule_Check(m)) {

/* cleanup the saved copy of module dicts */

PyModuleDef *md = PyModule_GetDef(m);

if (md) {

Py_CLEAR(md->m_base.m_copy);

}

}

}

/* Setting modules_by_index to NULL could be dangerous, so we

clear the list instead. */

if (PyList_SetSlice(MODULES_BY_INDEX(interp),

0, PyList_GET_SIZE(MODULES_BY_INDEX(interp)),

NULL)) {

PyErr_WriteUnraisable(MODULES_BY_INDEX(interp));

}

}

/*********************/

/* extension modules */

/*********************/

/*

It may help to have a big picture view of what happens

when an extension is loaded. This includes when it is imported

for the first time.

Here's a summary, using importlib._bootstrap._load() as a starting point.

1. importlib._bootstrap._load()

2. _load(): acquire import lock

3. _load() -> importlib._bootstrap._load_unlocked()

4. _load_unlocked() -> importlib._bootstrap.module_from_spec()

5. module_from_spec() -> ExtensionFileLoader.create_module()

6. create_module() -> _imp.create_dynamic()

(see below)

7. module_from_spec() -> importlib._bootstrap._init_module_attrs()

8. _load_unlocked(): sys.modules[name] = module

9. _load_unlocked() -> ExtensionFileLoader.exec_module()

10. exec_module() -> _imp.exec_dynamic()

(see below)

11. _load(): release import lock

...for single-phase init modules, where m_size == -1:

(6). first time (not found in _PyRuntime.imports.extensions):

1. _imp_create_dynamic_impl() -> import_find_extension()

2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec()

3. _PyImport_LoadDynamicModuleWithSpec(): load <module init func>

4. _PyImport_LoadDynamicModuleWithSpec(): call <module init func>

5. <module init func> -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized()

6. PyModule_CreateInitialized() -> PyModule_New()

7. PyModule_CreateInitialized(): allocate mod->md_state

8. PyModule_CreateInitialized() -> PyModule_AddFunctions()

9. PyModule_CreateInitialized() -> PyModule_SetDocString()

10. PyModule_CreateInitialized(): set mod->md_def

11. <module init func>: initialize the module

12. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed()

13. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init

14. _PyImport_LoadDynamicModuleWithSpec(): set __file__

15. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject()

16. _PyImport_FixupExtensionObject(): add it to interp->imports.modules_by_index

17. _PyImport_FixupExtensionObject(): copy __dict__ into def->m_base.m_copy

18. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions

(6). subsequent times (found in _PyRuntime.imports.extensions):

1. _imp_create_dynamic_impl() -> import_find_extension()

2. import_find_extension() -> import_add_module()

3. if name in sys.modules: use that module

4. else:

1. import_add_module() -> PyModule_NewObject()

2. import_add_module(): set it on sys.modules

5. import_find_extension(): copy the "m_copy" dict into __dict__

6. _imp_create_dynamic_impl() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed()

(10). (every time):

1. noop

...for single-phase init modules, where m_size >= 0:

(6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions):

1-16. (same as for m_size == -1)

(6). main interpreter - first time (not found in _PyRuntime.imports.extensions):

1-16. (same as for m_size == -1)

17. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions

(6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions):

1. _imp_create_dynamic_impl() -> import_find_extension()

2. import_find_extension(): call def->m_base.m_init

3. import_find_extension(): add the module to sys.modules

(10). every time:

1. noop

...for multi-phase init modules:

(6). every time:

1. _imp_create_dynamic_impl() -> import_find_extension() (not found)

2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec()

3. _PyImport_LoadDynamicModuleWithSpec(): load module init func

4. _PyImport_LoadDynamicModuleWithSpec(): call module init func

5. _PyImport_LoadDynamicModuleWithSpec() -> PyModule_FromDefAndSpec()

6. PyModule_FromDefAndSpec(): gather/check moduledef slots

7. if there's a Py_mod_create slot:

1. PyModule_FromDefAndSpec(): call its function

8. else:

1. PyModule_FromDefAndSpec() -> PyModule_NewObject()

9: PyModule_FromDefAndSpec(): set mod->md_def

10. PyModule_FromDefAndSpec() -> _add_methods_to_object()

11. PyModule_FromDefAndSpec() -> PyModule_SetDocString()

(10). every time:

1. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic()

2. if mod->md_state == NULL (including if m_size == 0):

1. exec_builtin_or_dynamic() -> PyModule_ExecDef()

2. PyModule_ExecDef(): allocate mod->md_state

3. if there's a Py_mod_exec slot:

1. PyModule_ExecDef(): call its function

*/

/* Make sure name is fully qualified.

This is a bit of a hack: when the shared library is loaded,

the module name is "package.module", but the module calls

PyModule_Create*() with just "module" for the name. The shared

library loader squirrels away the true name of the module in

_PyRuntime.imports.pkgcontext, and PyModule_Create*() will

substitute this (if the name actually matches).

*/

#ifdef HAVE_THREAD_LOCAL

_Py_thread_local const char *pkgcontext = NULL;

# undef PKGCONTEXT

# define PKGCONTEXT pkgcontext

#endif

const char *

_PyImport_ResolveNameWithPackageContext(const char *name)

{

#ifndef HAVE_THREAD_LOCAL

PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);

#endif

if (PKGCONTEXT != NULL) {

const char *p = strrchr(PKGCONTEXT, '.');

if (p != NULL && strcmp(name, p+1) == 0) {

name = PKGCONTEXT;

PKGCONTEXT = NULL;

}

}

#ifndef HAVE_THREAD_LOCAL

PyThread_release_lock(EXTENSIONS.mutex);

#endif

return name;

}

const char *

_PyImport_SwapPackageContext(const char *newcontext)

{

#ifndef HAVE_THREAD_LOCAL

PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK);

#endif

const char *oldcontext = PKGCONTEXT;

PKGCONTEXT = newcontext;

#ifndef HAVE_THREAD_LOCAL

PyThread_release_lock(EXTENSIONS.mutex);

#endif

return oldcontext;

}

#ifdef HAVE_DLOPEN

int

_PyImport_GetDLOpenFlags(PyInterpreterState *interp)

{

return DLOPENFLAGS(interp);

}

void

_PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val)

{

DLOPENFLAGS(interp) = new_val;

}

#endif // HAVE_DLOPEN

/* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */

static int

exec_builtin_or_dynamic(PyObject *mod) {

PyModuleDef *def;

void *state;

if (!PyModule_Check(mod)) {

return 0;

}

def = PyModule_GetDef(mod);

if (def == NULL) {

return 0;

}

state = PyModule_GetState(mod);

if (state) {

/* Already initialized; skip reload */

return 0;

}

return PyModule_ExecDef(mod, def);

}

static int clear_singlephase_extension(PyInterpreterState *interp,

PyObject *name, PyObject *filename);

// Currently, this is only used for testing.

// (See _testinternalcapi.clear_extension().)

int

_PyImport_ClearExtension(PyObject *name, PyObject *filename)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

/* Clearing a module's C globals is up to the module. */

if (clear_singlephase_extension(interp, name, filename) < 0) {

return -1;

}

// In the future we'll probably also make sure the extension's

// file handle (and DL handle) is closed (requires saving it).

return 0;

}

/*****************************/

/* single-phase init modules */

/*****************************/

/*

We support a number of kinds of single-phase init builtin/extension modules:

* "basic"

* no module state (PyModuleDef.m_size == -1)

* does not support repeated init (we use PyModuleDef.m_base.m_copy)

* may have process-global state

* the module's def is cached in _PyRuntime.imports.extensions,

by (name, filename)

* "reinit"

* no module state (PyModuleDef.m_size == 0)

* supports repeated init (m_copy is never used)

* should not have any process-global state

* its def is never cached in _PyRuntime.imports.extensions

(except, currently, under the main interpreter, for some reason)

* "with state" (almost the same as reinit)

* has module state (PyModuleDef.m_size > 0)

* supports repeated init (m_copy is never used)

* should not have any process-global state

* its def is never cached in _PyRuntime.imports.extensions

(except, currently, under the main interpreter, for some reason)

There are also variants within those classes:

* two or more modules share a PyModuleDef

* a module's init func uses another module's PyModuleDef

* a module's init func calls another's module's init func

* a module's init "func" is actually a variable statically initialized

to another module's init func

* two or modules share "methods"

* a module's init func copies another module's PyModuleDef

(with a different name)

* (basic-only) two or modules share process-global state

In the first case, where modules share a PyModuleDef, the following

notable weirdness happens:

* the module's __name__ matches the def, not the requested name

* the last module (with the same def) to be imported for the first time wins

* returned by PyState_Find_Module() (via interp->modules_by_index)

* (non-basic-only) its init func is used when re-loading any of them

(via the def's m_init)

* (basic-only) the copy of its __dict__ is used when re-loading any of them

(via the def's m_copy)

However, the following happens as expected:

* a new module object (with its own __dict__) is created for each request

* the module's __spec__ has the requested name

* the loaded module is cached in sys.modules under the requested name

* the m_index field of the shared def is not changed,

so at least PyState_FindModule() will always look in the same place

For "basic" modules there are other quirks:

* (whether sharing a def or not) when loaded the first time,

m_copy is set before _init_module_attrs() is called

in importlib._bootstrap.module_from_spec(),

so when the module is re-loaded, the previous value

for __wpec__ (and others) is reset, possibly unexpectedly.

Generally, when multiple interpreters are involved, some of the above

gets even messier.

*/

static inline void

extensions_lock_acquire(void)

{

PyThread_acquire_lock(_PyRuntime.imports.extensions.mutex, WAIT_LOCK);

}

static inline void

extensions_lock_release(void)

{

PyThread_release_lock(_PyRuntime.imports.extensions.mutex);

}

/* Magic for extension modules (built-in as well as dynamically

loaded). To prevent initializing an extension module more than

once, we keep a static dictionary 'extensions' keyed by the tuple

(module name, module name) (for built-in modules) or by

(filename, module name) (for dynamically loaded modules), containing these

modules. A copy of the module's dictionary is stored by calling

_PyImport_FixupExtensionObject() immediately after the module initialization

function succeeds. A copy can be retrieved from there by calling

import_find_extension().

Modules which do support multiple initialization set their m_size

field to a non-negative number (indicating the size of the

module-specific state). They are still recorded in the extensions

dictionary, to avoid loading shared libraries twice.

*/

static void *

hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep)

{

Py_ssize_t str1_len, str2_len;

const char *str1_data = PyUnicode_AsUTF8AndSize(str1, &str1_len);

const char *str2_data = PyUnicode_AsUTF8AndSize(str2, &str2_len);

if (str1_data == NULL || str2_data == NULL) {

return NULL;

}

/* Make sure sep and the NULL byte won't cause an overflow. */

assert(SIZE_MAX - str1_len - str2_len > 2);

size_t size = str1_len + 1 + str2_len + 1;

char *key = PyMem_RawMalloc(size);

if (key == NULL) {

PyErr_NoMemory();

return NULL;

}

strncpy(key, str1_data, str1_len);

key[str1_len] = sep;

strncpy(key + str1_len + 1, str2_data, str2_len + 1);

assert(strlen(key) == size - 1);

return key;

}

static Py_uhash_t

hashtable_hash_str(const void *key)

{

return _Py_HashBytes(key, strlen((const char *)key));

}

static int

hashtable_compare_str(const void *key1, const void *key2)

{

return strcmp((const char *)key1, (const char *)key2) == 0;

}

static void

hashtable_destroy_str(void *ptr)

{

PyMem_RawFree(ptr);

}

#define HTSEP ':'

static PyModuleDef *

_extensions_cache_get(PyObject *filename, PyObject *name)

{

PyModuleDef *def = NULL;

void *key = NULL;

extensions_lock_acquire();

if (EXTENSIONS.hashtable == NULL) {

goto finally;

}

key = hashtable_key_from_2_strings(filename, name, HTSEP);

if (key == NULL) {

goto finally;

}

_Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(

EXTENSIONS.hashtable, key);

if (entry == NULL) {

goto finally;

}

def = (PyModuleDef *)entry->value;

finally:

extensions_lock_release();

if (key != NULL) {

PyMem_RawFree(key);

}

return def;

}

static int

_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def)

{

int res = -1;

extensions_lock_acquire();

if (EXTENSIONS.hashtable == NULL) {

_Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree};

EXTENSIONS.hashtable = _Py_hashtable_new_full(

hashtable_hash_str,

hashtable_compare_str,

hashtable_destroy_str, // key

/* There's no need to decref the def since it's immortal. */

NULL, // value

&alloc

);

if (EXTENSIONS.hashtable == NULL) {