diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index f426ae0e103b9c..a436adfc1dd822 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -114,6 +114,14 @@ PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); // Export for special main.c string compiling with source tracebacks int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); +struct pyfinalize_args { + const char *caller; + int verbose; +}; + +// Export for _testembed +PyAPI_FUNC(int) _Py_Finalize(_PyRuntimeState *, struct pyfinalize_args *); + /* interpreter config */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 634513ec7a5812..a8d0ebfa4b4c0d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -93,6 +93,8 @@ def setUp(self): def tearDown(self): os.chdir(self.oldcwd) + ANY_FAILURE = object() + def run_embedded_interpreter(self, *args, env=None, timeout=None, returncode=0, input=None, cwd=None): @@ -117,6 +119,8 @@ def run_embedded_interpreter(self, *args, env=None, p.terminate() p.wait() raise + if returncode is self.ANY_FAILURE: + returncode = 1 if p.returncode == 0 else p.returncode if p.returncode != returncode and support.verbose: print(f"--- {cmd} failed ---") print(f"stdout:\n{out}") @@ -226,6 +230,46 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_replace_main_tstate(self): + for reuse in (True, False): + with self.subTest(reuse=reuse, exec=False): + self.run_embedded_interpreter( + 'test_replace_main_tstate', + # At the moment, this fails because main_tstate gets broken. + returncode=self.ANY_FAILURE, + ) + with self.subTest(reuse=reuse, exec=True): + out, rc = self.run_embedded_interpreter( + 'test_replace_main_tstate', + 'print("spam!")', + # At the moment, this fails because main_tstate gets broken. + returncode=self.ANY_FAILURE, + ) + if rc == 0: + self.assertEqual(out.strip(), 'spam!') + + def test_fini_in_subthread(self): + self.run_embedded_interpreter( + 'test_fini_in_subthread', + # At the moment, this actually succeeds on all platforms, + # except for Windows (STATUS_ACCESS_VIOLATION). + returncode=self.ANY_FAILURE if MS_WINDOWS else 0, + ) + + def test_fini_in_main_thread_with_other_tstate(self): + self.run_embedded_interpreter( + 'test_fini_in_main_thread_with_other_tstate', + # At the moment, this actually succeeds on all platforms. + returncode=0, + ) + + def test_fini_in_main_thread_with_subinterpreter(self): + self.run_embedded_interpreter( + 'test_fini_in_main_thread_with_subinterpreter', + # At the moment, this actually succeeds on all platforms. + returncode=0, + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d149b6a0c5cd21..51ca1c8f435b3e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -7,6 +7,8 @@ #include #include "pycore_initconfig.h" // _PyConfig_InitCompatConfig() +#include "pycore_pylifecycle.h" // _Py_Finalize() +#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_runtime.h" // _PyRuntime #include "pycore_import.h" // _PyImport_FrozenBootstrap #include @@ -207,6 +209,193 @@ static int test_repeated_simple_init(void) } +/**************************************************************************** + * Test discarding the initial main tstate + ***************************************************************************/ + +static int test_replace_main_tstate(void) +{ + int err = 0; + + int reuse = 0; + int hascode = 0; + for (int i = 2; i < main_argc; i++) { + if (strcmp(main_argv[i], "--reuse") == 0) { + reuse = 1; + } + else if (strncmp(main_argv[i], "--", 2) == 0) { + fprintf(stderr, "ERROR: unsupported arg %s\n", main_argv[i]); + err = 1; + } + else { + hascode = 1; + } + if (err) { + fprintf(stderr, + "usage: %s test_replace_main_tstate --reuse [CODE ...]\n", + PROGRAM); + return 1; + } + } + + _testembed_Py_InitializeFromConfig(); + PyThreadState *main_tstate = PyThreadState_Get(); + PyInterpreterState *main_interp = main_tstate->interp; + assert(_Py_IsMainInterpreter(main_interp)); + + // If the initial tstate is reused then there are slightly + // different possible failure paths through the code than if we got + // a completely new one. + + PyThreadState *tstate = NULL; + if (!reuse) { + // The initial thread state is still alive, + // so this will create a completely new one, + // with its own distinct pointer. + tstate = PyThreadState_New(main_interp); + assert(tstate != NULL); + assert(tstate != main_tstate); + } + PyThreadState_Clear(main_tstate); + PyThreadState_DeleteCurrent(); + assert(reuse == (PyInterpreterState_ThreadHead(main_interp) == NULL)); + if (reuse) { + // The initial thread state has already been "destroyed", + // so this will re-use the statically allocated tstate + // (along with reinitializing it). + tstate = PyThreadState_New(main_interp); + assert(tstate != NULL); + assert(tstate == main_tstate); + } + assert(_PyThreadState_GET() == NULL); + + (void)PyThreadState_Swap(tstate); + + if (hascode) { + for (int i = 2; i < main_argc; i++) { + const char *code = main_argv[i]; + if (PyRun_SimpleString(code) != 0) { + err = 1; + break; + } + } + } + + assert(PyThreadState_Get() == tstate); + struct pyfinalize_args args = { + .caller = "_testembed.test_replace_main_tstate", + .verbose = 1, + }; + if (_Py_Finalize(&_PyRuntime, &args) != 0 && !err) { + err = 1; + } + + return err; +} + + +/**************************************************************************** + * Test (mostly) unsupported Py_Finalize() scenarios + ***************************************************************************/ + +struct fini_subthread_args { + struct pyfinalize_args *fini_args; + PyThreadState *main_tstate; + PyInterpreterState *interp; + PyMutex done; + int rc; +}; + +static void fini_with_new_tstate(void *arg) +{ + struct fini_subthread_args *args = (struct fini_subthread_args *)arg; + + assert(!_Py_IsMainThread()); + assert(_PyThreadState_GET() == NULL); + + PyThreadState *tstate = PyThreadState_New(args->interp); + assert(tstate != NULL); + assert(tstate != args->main_tstate); + (void)PyThreadState_Swap(tstate); + + assert(PyThreadState_Get() != args->main_tstate); + if (_Py_Finalize(&_PyRuntime, args->fini_args) != 0) { + args->rc = 1; + } + + PyMutex_Unlock(&args->done); +} + +static int test_fini_in_subthread(void) +{ + _testembed_Py_InitializeFromConfig(); + PyThreadState *main_tstate = PyThreadState_Get(); + + struct pyfinalize_args fini_args = { + .caller = "_testembed.test_fini_in_subthread", + .verbose = 1, + }; + struct fini_subthread_args args = { + .fini_args = &fini_args, + .main_tstate = main_tstate, + .interp = main_tstate->interp, + }; + PyMutex_Lock(&args.done); + (void)PyThread_start_new_thread(fini_with_new_tstate, &args); + + // Wait for fini to finish. + PyMutex_Lock(&args.done); + PyMutex_Unlock(&args.done); + + return args.rc; +} + +static int test_fini_in_main_thread_with_other_tstate(void) +{ + _testembed_Py_InitializeFromConfig(); + PyThreadState *main_tstate = PyThreadState_Get(); + + PyThreadState *tstate = PyThreadState_New(main_tstate->interp); + (void)PyThreadState_Swap(tstate); + + assert(PyThreadState_Get() != main_tstate); + struct pyfinalize_args args = { + .caller = "_testembed.test_fini_in_main_thread_with_other_tstate", + .verbose = 1, + }; + if (_Py_Finalize(&_PyRuntime, &args) != 0) { + return 1; + } + + return 0; +} + +static int test_fini_in_main_thread_with_subinterpreter(void) +{ + _testembed_Py_InitializeFromConfig(); + PyThreadState *main_tstate = PyThreadState_Get(); + + PyThreadState *substate = Py_NewInterpreter(); + assert(substate != main_tstate); +#ifndef NDEBUG + (void)main_tstate; + (void)substate; +#endif + + // The subinterpreter's tstate is still current. + assert(PyThreadState_Get() == substate); + struct pyfinalize_args args = { + .caller = "_testembed.test_fini_in_main_thread_with_subinterpreter", + .verbose = 1, + }; + if (_Py_Finalize(&_PyRuntime, &args) != 0) { + return 1; + } + + return 0; +} + + /***************************************************** * Test forcing a particular IO encoding *****************************************************/ @@ -1379,11 +1568,20 @@ static int test_audit_subinterpreter(void) Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_subinterpreter_hook, NULL); _testembed_Py_InitializeFromConfig(); + PyThreadState *save_tstate = PyThreadState_Get(); + + PyThreadState *substate1 = Py_NewInterpreter(); + PyThreadState *substate2 = Py_NewInterpreter(); + PyThreadState *substate3 = Py_NewInterpreter(); - Py_NewInterpreter(); - Py_NewInterpreter(); - Py_NewInterpreter(); + (void)PyThreadState_Swap(substate3); + Py_EndInterpreter(substate3); + (void)PyThreadState_Swap(substate2); + Py_EndInterpreter(substate2); + (void)PyThreadState_Swap(substate1); + Py_EndInterpreter(substate1); + (void)PyThreadState_Swap(save_tstate); Py_Finalize(); switch (_audit_subinterpreter_interpreter_count) { @@ -2161,6 +2359,12 @@ static struct TestCase TestCases[] = { {"test_forced_io_encoding", test_forced_io_encoding}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, + {"test_replace_main_tstate", test_replace_main_tstate}, + {"test_fini_in_subthread", test_fini_in_subthread}, + {"test_fini_in_main_thread_with_other_tstate", + test_fini_in_main_thread_with_other_tstate}, + {"test_fini_in_main_thread_with_subinterpreter", + test_fini_in_main_thread_with_subinterpreter}, {"test_pre_initialization_api", test_pre_initialization_api}, {"test_pre_initialization_sys_options", test_pre_initialization_sys_options}, {"test_bpo20891", test_bpo20891}, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9bbfa831db317b..bd4f164272780b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1917,23 +1917,69 @@ finalize_interp_delete(PyInterpreterState *interp) we do our best here to accommodate that possibility. */ static PyThreadState * -resolve_final_tstate(_PyRuntimeState *runtime) +resolve_final_tstate(_PyRuntimeState *runtime, struct pyfinalize_args *args) { - PyThreadState *main_tstate = runtime->main_tstate; - assert(main_tstate != NULL); - assert(main_tstate->thread_id == runtime->main_thread); + PyThreadState *tstate = _PyThreadState_GET(); + assert(tstate->interp->runtime == runtime); + assert(tstate->thread_id == PyThread_get_thread_ident()); PyInterpreterState *main_interp = _PyInterpreterState_Main(); - assert(main_tstate->interp == main_interp); + PyThreadState *main_tstate = runtime->main_tstate; - PyThreadState *tstate = _PyThreadState_GET(); - if (_Py_IsMainThread()) { + /* First we report unexpected Py_Finalize() usage. */ + +#define PRINT_ERROR(msg) \ + if (args->verbose) { \ + fprintf(stderr, "%s: %s\n", args->caller, msg); \ + } + + /* The main tstate is set by Py_Initialize(), but can be unset + * or even replaced in unlikely cases. */ + if (main_tstate == NULL) { + PRINT_ERROR("main thread state not set"); + /* Ideally, we would make sure a main tstate is set. + For now we leave it unset. */ + } + else { + assert(main_tstate->thread_id == runtime->main_thread); + assert(main_tstate->interp == main_interp); if (tstate != main_tstate) { + /* Code running in the main thread could swap out the main tstate, + which ends up being a headache. */ + PRINT_ERROR("using different thread state than Py_Initialize()"); + } + else { + assert(main_tstate->interp != NULL); + assert(main_tstate->interp == main_interp); + } + } + + if (!_Py_IsMainThread()) { + PRINT_ERROR("expected to be in the main thread"); + } + + if (tstate->interp != main_interp) { + PRINT_ERROR("expected main interpreter to be active"); + } + +#undef PRINT_ERROR + + /* Then we decide the thread state we should use for finalization. */ + + PyThreadState *final_tstate = tstate; + if (_Py_IsMainThread()) { + if (main_tstate == NULL) { + if (tstate->interp != main_interp) { + /* We will swap in a tstate for the main interpreter. */ + final_tstate = NULL; + } + } + else if (tstate != main_tstate) { /* This implies that Py_Finalize() was called while a non-main interpreter was active or while the main tstate was temporarily swapped out with another. Neither case should be allowed, but, until we get around to fixing that (and Py_Exit()), we're letting it go. */ - (void)PyThreadState_Swap(main_tstate); + final_tstate = main_tstate; } } else { @@ -1941,30 +1987,32 @@ resolve_final_tstate(_PyRuntimeState *runtime) called when it shouldn't have been. We can't simply switch over to the main thread. At the least, however, we can make sure the main interpreter is active. */ - if (!_Py_IsMainInterpreter(tstate->interp)) { - /* We don't go to the trouble of updating runtime->main_tstate - since it will be dead soon anyway. */ - main_tstate = - _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); - if (main_tstate != NULL) { - _PyThreadState_Bind(main_tstate); - (void)PyThreadState_Swap(main_tstate); - } - else { - /* Fall back to the current tstate. It's better than nothing. */ - main_tstate = tstate; - } + if (tstate->interp != main_interp) { + final_tstate = NULL; + } + } + if (final_tstate == NULL) { + /* We don't go to the trouble of updating runtime->main_tstate + since it will be dead soon anyway. */ + final_tstate = + _PyThreadState_New(main_interp, _PyThreadState_WHENCE_FINI); + if (final_tstate == NULL) { + /* Fall back to the current tstate. It's better than nothing. */ + final_tstate = tstate; + } + else { + _PyThreadState_Bind(final_tstate); } } - assert(main_tstate != NULL); + assert(final_tstate->interp == main_interp); - /* We might want to warn if main_tstate->current_frame != NULL. */ + /* We might want to warn if final_tstate->current_frame != NULL. */ - return main_tstate; + return final_tstate; } -static int -_Py_Finalize(_PyRuntimeState *runtime) +int +_Py_Finalize(_PyRuntimeState *runtime, struct pyfinalize_args *args) { int status = 0; @@ -1973,8 +2021,11 @@ _Py_Finalize(_PyRuntimeState *runtime) return status; } - /* Get final thread state pointer. */ - PyThreadState *tstate = resolve_final_tstate(runtime); + /* Get/attach the final thread state pointer. */ + PyThreadState *tstate = resolve_final_tstate(runtime, args); + if (tstate != _PyThreadState_GET()) { + (void)PyThreadState_Swap(tstate); + } // Block some operations. tstate->interp->finalizing = 1; @@ -2202,13 +2253,21 @@ _Py_Finalize(_PyRuntimeState *runtime) int Py_FinalizeEx(void) { - return _Py_Finalize(&_PyRuntime); + struct pyfinalize_args args = { + .caller = "Py_FinalizeEx", + .verbose = 1, + }; + return _Py_Finalize(&_PyRuntime, &args); } void Py_Finalize(void) { - (void)_Py_Finalize(&_PyRuntime); + struct pyfinalize_args args = { + .caller = "Py_Finalize", + .verbose = 1, + }; + (void)_Py_Finalize(&_PyRuntime, &args); } @@ -3354,7 +3413,13 @@ Py_Exit(int sts) if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { _PyInterpreterState_SetNotRunningMain(tstate->interp); } - if (_Py_Finalize(&_PyRuntime) < 0) { + struct pyfinalize_args args = { + .caller = "Py_Exit", +#ifdef Py_DEBUG + .verbose = 1, +#endif + }; + if (_Py_Finalize(&_PyRuntime, &args) < 0) { sts = 120; }