Backport gh-157384: Set package context in PyImport_CreateModuleFromInitfunc()

Summary:
Backport of the upstream fix (python/cpython#157387, 3.15 backport python/cpython#159316), folded into the `05-native-python-api` patch that carries our 3.14 backport of `PyImport_CreateModuleFromInitfunc`.

A single-phase init function that creates its module using only the last component of a dotted name (as pybind11 2.13 does for `PYBIND11_MODULE` extensions inside a package) got a module whose `__name__` was the short name, so `def_submodule` registered submodules under the wrong `sys.modules` keys and `from pkg.ext.sub import x` failed with "`pkg.ext` is not a package". 571 failures in V66 of the py3.14 canary (D82323768) carry this signature.

Reviewed By: alexmalyshev

Differential Revision: D119701167

fbshipit-source-id: dc50c5ece93752c069548588545eee6526181cad

Author: itamaro

Doc/c-api/import.rst

@@ -367,6 +367,10 @@ Importing Modules
    ``PyInit_<name>`` function and *spec* is the
    :class:`~importlib.machinery.ModuleSpec` describing the module.
 
+   If the spec's name is dotted and *initfunc* creates the module using only
+   the last component of the name, the module's :attr:`~module.__name__`
+   is set to the full name from the spec.
+
 
 .. _lazy-imports:
 

Lib/test/test_embed.py

@@ -246,6 +246,21 @@ def test_repeated_init_and_inittab(self):
         lines = "\n".join(lines) + "\n"
         self.assertEqual(out, lines)
 
+    def test_create_module_from_initfunc(self):
+        out, err = self.run_embedded_interpreter("test_create_module_from_initfunc")
+        self.assertEqual(self._nogil_filtered_err(err, "embedded_ext"), "")
+        self.assertEqual(out,
+                         "<module 'my_test_extension' (static-extension)>\n"
+                         "my_test_extension.executed='yes'\n"
+                         "my_test_extension.exec_slot_ran='yes'\n"
+                         "<module 'embedded_ext' (static-extension)>\n"
+                         "embedded_ext.executed='yes'\n"
+                         "<module 'sp_pkg.sp_submod' (static-extension)>\n"
+                         "sp_pkg.sp_submod.__name__='sp_pkg.sp_submod'\n"
+                         "sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=True\n"
+                         "\"sp_submod\" in sys.modules=False\n"
+                         )
+
     def test_inittab_submodule_multiphase(self):
         out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase")
         self.assertEqual(err, "")

Programs/_testembed.c

@@ -214,6 +214,8 @@ static PyModuleDef embedded_ext = {
 static PyObject*
 PyInit_embedded_ext(void)
 {
+    // keep this as a single-phase initialization module;
+    // see test_create_module_from_initfunc
     return PyModule_Create(&embedded_ext);
 }
 
@@ -1980,8 +1982,16 @@ static int test_initconfig_exit(void)
 }
 
 
+int
+extension_module_exec(PyObject *mod)
+{
+    return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes");
+}
+
+
 static PyModuleDef_Slot extension_slots[] = {
     {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+    {Py_mod_exec, extension_module_exec},
     {0, NULL}
 };
 
@@ -2380,6 +2390,160 @@ static int test_repeated_init_and_inittab(void)
     return 0;
 }
 
+// A single-phase init submodule that only knows its short name,
+// like pybind11's PYBIND11_MODULE(sp_submod, m) would.
+static PyModuleDef cmfi_sp_submod_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "sp_submod",
+    .m_size = -1,
+};
+
+static PyObject*
+PyInit_cmfi_sp_submod(void)
+{
+    return PyModule_Create(&cmfi_sp_submod_def);
+}
+
+// A multi-phase init package for the submodule above.
+static PyModuleDef_Slot cmfi_sp_pkg_slots[] = {
+    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
+    {0, NULL},
+};
+
+static PyModuleDef cmfi_sp_pkg_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "sp_pkg",
+    .m_size = 0,
+    .m_slots = cmfi_sp_pkg_slots,
+};
+
+static PyObject*
+PyInit_cmfi_sp_pkg(void)
+{
+    return PyModuleDef_Init(&cmfi_sp_pkg_def);
+}
+
+static PyObject*
+create_module(PyObject* self, PyObject* spec)
+{
+    PyObject *name = PyObject_GetAttrString(spec, "name");
+    if (!name) {
+        return NULL;
+    }
+    if (PyUnicode_EqualToUTF8(name, "my_test_extension")) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension);
+    }
+    if (PyUnicode_EqualToUTF8(name, "embedded_ext")) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext);
+    }
+    if (PyUnicode_EqualToUTF8(name, "sp_pkg")) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_sp_pkg);
+    }
+    if (PyUnicode_EqualToUTF8(name, "sp_pkg.sp_submod")) {
+        Py_DECREF(name);
+        return PyImport_CreateModuleFromInitfunc(spec, PyInit_cmfi_sp_submod);
+    }
+    PyErr_Format(PyExc_LookupError, "static module %R not found", name);
+    Py_DECREF(name);
+    return NULL;
+}
+
+static PyObject*
+exec_module(PyObject* self, PyObject* mod)
+{
+    PyModuleDef *def = PyModule_GetDef(mod);
+    if (def == NULL) {
+        return NULL;
+    }
+    if (PyModule_ExecDef(mod, def) < 0) {
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef create_static_module_methods[] = {
+    {"create_module", create_module, METH_O, NULL},
+    {"exec_module", exec_module, METH_O, NULL},
+    {NULL}
+};
+
+static struct PyModuleDef create_static_module_def = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "create_static_module",
+    .m_size = 0,
+    .m_methods = create_static_module_methods,
+    .m_slots = extension_slots,
+};
+
+PyMODINIT_FUNC PyInit_create_static_module(void) {
+    return PyModuleDef_Init(&create_static_module_def);
+}
+
+static int
+test_create_module_from_initfunc(void)
+{
+    wchar_t* argv[] = {
+        PROGRAM_NAME,
+        L"-c",
+        // Multi-phase initialization
+        L"import my_test_extension;"
+        L"print(my_test_extension);"
+        L"print(f'{my_test_extension.executed=}');"
+        L"print(f'{my_test_extension.exec_slot_ran=}');"
+        // Single-phase initialization
+        L"import embedded_ext;"
+        L"print(embedded_ext);"
+        L"print(f'{embedded_ext.executed=}');"
+        // Single-phase submodule whose init only knows its short name
+        L"import sp_pkg.sp_submod;"
+        L"print(sp_pkg.sp_submod);"
+        L"print(f'{sp_pkg.sp_submod.__name__=}');"
+        L"print(f'{sys.modules[\"sp_pkg.sp_submod\"] is sp_pkg.sp_submod=}');"
+        L"print(f'{\"sp_submod\" in sys.modules=}');"
+    };
+    PyConfig config;
+    if (PyImport_AppendInittab("create_static_module",
+                               &PyInit_create_static_module) != 0) {
+        fprintf(stderr, "PyImport_AppendInittab() failed\n");
+        return 1;
+    }
+    PyConfig_InitPythonConfig(&config);
+    config.isolated = 1;
+    config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv);
+    init_from_config_clear(&config);
+    int result = PyRun_SimpleString(
+        "import sys\n"
+        "from importlib.util import spec_from_loader\n"
+        "import create_static_module\n"
+        "class StaticExtensionImporter:\n"
+        "   _ORIGIN = \"static-extension\"\n"
+        "   @classmethod\n"
+        "   def find_spec(cls, fullname, path, target=None):\n"
+        "       if fullname in {'my_test_extension', 'embedded_ext',\n"
+        "                       'sp_pkg', 'sp_pkg.sp_submod'}:\n"
+        "           return spec_from_loader(fullname, cls, origin=cls._ORIGIN,\n"
+        "                                   is_package=(fullname == 'sp_pkg'))\n"
+        "       return None\n"
+        "   @staticmethod\n"
+        "   def create_module(spec):\n"
+        "       return create_static_module.create_module(spec)\n"
+        "   @staticmethod\n"
+        "   def exec_module(module):\n"
+        "       create_static_module.exec_module(module)\n"
+        "       module.executed = 'yes'\n"
+        "sys.meta_path.append(StaticExtensionImporter)\n"
+    );
+    if (result < 0) {
+        fprintf(stderr, "PyRun_SimpleString() failed\n");
+        return 1;
+    }
+    return Py_RunMain();
+}
+
 /// Multi-phase initialization package & submodule ///
 
 int
@@ -2714,6 +2878,7 @@ static struct TestCase TestCases[] = {
     {"test_get_incomplete_frame", test_get_incomplete_frame},
     {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase},
     {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase},
+    {"test_create_module_from_initfunc", test_create_module_from_initfunc},
     {NULL, NULL}
 };
 

Python/import.c

@@ -2439,6 +2439,19 @@ create_builtin(
     if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) {
         return NULL;
     }
+    if (initfunc != NULL) {
+        /* An explicitly provided init function (see
+         * PyImport_CreateModuleFromInitfunc()) may belong to a submodule
+         * whose single-phase init creates the module using only the last
+         * component of the name (as e.g. pybind11 does).  Set the package
+         * context so that PyModule_Create() resolves the full name, the
+         * same as for dynamically loaded extensions. */
+        info.newcontext = PyUnicode_AsUTF8(info.name);
+        if (info.newcontext == NULL) {
+            _Py_ext_module_loader_info_clear(&info);
+            return NULL;
+        }
+    }
 
     struct extensions_cache_value *cached = NULL;
     PyObject *mod = import_find_extension(tstate, &info, &cached);