Merge branch 'main' into pythongh-109823

Lib/test/_test_multiprocessing.py

@@ -79,7 +79,7 @@
 
 
 if support.check_sanitizer(address=True):
-    # bpo-45200: Skip multiprocessing tests if Python is built with ASAN to
+    # gh-89363: Skip multiprocessing tests if Python is built with ASAN to
     # work around a libasan race condition: dead lock in pthread_create().
     raise unittest.SkipTest("libasan has a pthread_create() dead lock")
 

Lib/test/libregrtest/results.py

@@ -25,14 +25,19 @@ def __init__(self):
         self.env_changed: TestList = []
         self.run_no_tests: TestList = []
         self.rerun: TestList = []
-        self.bad_results: list[TestResult] = []
+        self.rerun_results: list[TestResult] = []
 
         self.interrupted: bool = False
         self.test_times: list[tuple[float, TestName]] = []
         self.stats = TestStats()
         # used by --junit-xml
         self.testsuite_xml: list[str] = []
 
+    def is_all_good(self):
+        return (not self.bad
+                and not self.skipped
+                and not self.interrupted)
+
     def get_executed(self):
         return (set(self.good) | set(self.bad) | set(self.skipped)
                 | set(self.resource_denied) | set(self.env_changed)
@@ -82,6 +87,7 @@ def accumulate_result(self, result: TestResult, runtests: RunTests):
                 self.good.append(test_name)
             case State.ENV_CHANGED:
                 self.env_changed.append(test_name)
+                self.rerun_results.append(result)
             case State.SKIPPED:
                 self.skipped.append(test_name)
             case State.RESOURCE_DENIED:
@@ -93,7 +99,7 @@ def accumulate_result(self, result: TestResult, runtests: RunTests):
             case _:
                 if result.is_failed(fail_env_changed):
                     self.bad.append(test_name)
-                    self.bad_results.append(result)
+                    self.rerun_results.append(result)
                 else:
                     raise ValueError(f"invalid test state: {result.state!r}")
 
@@ -109,12 +115,12 @@ def accumulate_result(self, result: TestResult, runtests: RunTests):
             self.add_junit(xml_data)
 
     def need_rerun(self):
-        return bool(self.bad_results)
+        return bool(self.rerun_results)
 
     def prepare_rerun(self) -> tuple[TestTuple, FilterDict]:
         tests: TestList = []
         match_tests_dict = {}
-        for result in self.bad_results:
+        for result in self.rerun_results:
             tests.append(result.test_name)
 
             match_tests = result.get_rerun_match_tests()
@@ -125,7 +131,8 @@ def prepare_rerun(self) -> tuple[TestTuple, FilterDict]:
         # Clear previously failed tests
         self.rerun_bad.extend(self.bad)
         self.bad.clear()
-        self.bad_results.clear()
+        self.env_changed.clear()
+        self.rerun_results.clear()
 
         return (tuple(tests), match_tests_dict)
 
@@ -164,61 +171,47 @@ def write_junit(self, filename: StrPath):
                 f.write(s)
 
     def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool):
-        if self.interrupted:
-            print("Test suite interrupted by signal SIGINT.")
-
         omitted = set(tests) - self.get_executed()
         if omitted:
             print()
             print(count(len(omitted), "test"), "omitted:")
             printlist(omitted)
 
-        if self.good and not quiet:
-            print()
-            if (not self.bad
-                and not self.skipped
-                and not self.interrupted
-                and len(self.good) > 1):
-                print("All", end=' ')
-            print(count(len(self.good), "test"), "OK.")
-
         if print_slowest:
             self.test_times.sort(reverse=True)
             print()
             print("10 slowest tests:")
             for test_time, test in self.test_times[:10]:
                 print("- %s: %s" % (test, format_duration(test_time)))
 
-        if self.bad:
-            print()
-            print(count(len(self.bad), "test"), "failed:")
-            printlist(self.bad)
-
-        if self.env_changed:
-            print()
-            print("{} altered the execution environment:".format(
-                     count(len(self.env_changed), "test")))
-            printlist(self.env_changed)
-
-        if self.skipped and not quiet:
-            print()
-            print(count(len(self.skipped), "test"), "skipped:")
-            printlist(self.skipped)
-
-        if self.resource_denied and not quiet:
-            print()
-            print(count(len(self.resource_denied), "test"), "skipped (resource denied):")
-            printlist(self.resource_denied)
+        all_tests = [
+            (self.bad, "test", "{} failed:"),
+            (self.env_changed, "test", "{} altered the execution environment (env changed):"),
+        ]
+        if not quiet:
+            all_tests.append((self.skipped, "test", "{} skipped:"))
+            all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):"))
+        all_tests.append((self.rerun, "re-run test", "{}:"))
+        all_tests.append((self.run_no_tests, "test", "{} run no tests:"))
+
+        for tests_list, count_text, title_format in all_tests:
+            if tests_list:
+                print()
+                count_text = count(len(tests_list), count_text)
+                print(title_format.format(count_text))
+                printlist(tests_list)
 
-        if self.rerun:
+        if self.good and not quiet:
             print()
-            print("%s:" % count(len(self.rerun), "re-run test"))
-            printlist(self.rerun)
+            text = count(len(self.good), "test")
+            text = f"{text} OK."
+            if (self.is_all_good() and len(self.good) > 1):
+                text = f"All {text}"
+            print(text)
 
-        if self.run_no_tests:
+        if self.interrupted:
             print()
-            print(count(len(self.run_no_tests), "test"), "run no tests:")
-            printlist(self.run_no_tests)
+            print("Test suite interrupted by signal SIGINT.")
 
     def display_summary(self, first_runtests: RunTests, filtered: bool):
         # Total tests

Lib/test/test_asyncio/test_waitfor.py

@@ -1,6 +1,7 @@
 import asyncio
 import unittest
 import time
+from test import support
 
 
 def tearDownModule():
@@ -130,7 +131,7 @@ async def foo():
             nonlocal foo_running
             foo_running = True
             try:
-                await asyncio.sleep(10)
+                await asyncio.sleep(support.LONG_TIMEOUT)
             finally:
                 foo_running = False
             return 'done'
@@ -144,7 +145,7 @@ async def foo():
         self.assertTrue(fut.done())
         # it should have been cancelled due to the timeout
         self.assertTrue(fut.cancelled())
-        self.assertLess(t1 - t0, 0.5)
+        self.assertLess(t1 - t0, support.SHORT_TIMEOUT)
         self.assertEqual(foo_running, False)
 
     async def test_wait_for_blocking(self):

Lib/test/test_regrtest.py

@@ -463,7 +463,7 @@ def check_executed_tests(self, output, tests, *, stats,
             randomize = True
 
         rerun_failed = []
-        if rerun is not None:
+        if rerun is not None and not env_changed:
             failed = [rerun.name]
             if not rerun.success:
                 rerun_failed.append(rerun.name)
@@ -500,7 +500,8 @@ def list_regex(line_format, tests):
             self.check_line(output, regex)
 
         if env_changed:
-            regex = list_regex('%s test%s altered the execution environment',
+            regex = list_regex(r'%s test%s altered the execution environment '
+                               r'\(env changed\)',
                                env_changed)
             self.check_line(output, regex)
 
@@ -590,7 +591,7 @@ def list_regex(line_format, tests):
         state = ', '.join(state)
         if rerun is not None:
             new_state = 'SUCCESS' if rerun.success else 'FAILURE'
-            state = 'FAILURE then ' + new_state
+            state = f'{state} then {new_state}'
         self.check_line(output, f'Result: {state}', full=True)
 
     def parse_random_seed(self, output):
@@ -1228,6 +1229,15 @@ def test_env_changed(self):
         self.check_executed_tests(output, [testname], env_changed=testname,
                                   fail_env_changed=True, stats=1)
 
+        # rerun
+        output = self.run_tests("--rerun", testname)
+        self.check_executed_tests(output, [testname],
+                                  env_changed=testname,
+                                  rerun=Rerun(testname,
+                                              match=None,
+                                              success=True),
+                                  stats=2)
+
     def test_rerun_fail(self):
         # FAILURE then FAILURE
         code = textwrap.dedent("""

Lib/test/test_threading.py

@@ -572,6 +572,10 @@ def background_thread(evt):
         self.assertEqual(err, b'')
 
     @support.requires_fork()
+    # gh-89363: Skip multiprocessing tests if Python is built with ASAN to
+    # work around a libasan race condition: dead lock in pthread_create().
+    @support.skip_if_sanitizer("libasan has a pthread_create() dead lock",
+                               address=True)
     def test_is_alive_after_fork(self):
         # Try hard to trigger #18418: is_alive() could sometimes be True on
         # threads that vanished after a fork.

Lib/turtle.py

@@ -874,7 +874,7 @@ def __init__(self, type_, data=None):
             if isinstance(data, str):
                 if data.lower().endswith(".gif") and isfile(data):
                     data = TurtleScreen._image(data)
-                # else data assumed to be Photoimage
+                # else data assumed to be PhotoImage
         elif type_ == "compound":
             data = []
         else:

Misc/NEWS.d/next/Tests/2023-09-25-14-41-18.gh-issue-109276.uC_cWo.rst

@@ -0,0 +1,3 @@
+regrtest: When a test fails with "env changed" and the --rerun option is
+used, the test is now re-run in verbose mode in a fresh process. Patch by
+Victor Stinner.

Modules/_testcapi/vectorcall_limited.c

@@ -1,7 +1,13 @@
+/* Test Vectorcall in the limited API */
+
 #define Py_LIMITED_API 0x030c0000 // 3.12
 #include "parts.h"
+#include "clinic/vectorcall_limited.c.h"
 
-/* Test Vectorcall in the limited API */
+/*[clinic input]
+module _testcapi
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/
 
 static PyObject *
 LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) {
@@ -28,8 +34,16 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw)
     return self;
 }
 
+/*[clinic input]
+_testcapi.call_vectorcall
+
+    callable: object
+    /
+[clinic start generated code]*/
+
 static PyObject *
-call_vectorcall(PyObject* self, PyObject *callable)
+_testcapi_call_vectorcall(PyObject *module, PyObject *callable)
+/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/
 {
     PyObject *args[3] = { NULL, NULL, NULL };
     PyObject *kwname = NULL, *kwnames = NULL, *result = NULL;
@@ -73,8 +87,16 @@ call_vectorcall(PyObject* self, PyObject *callable)
     return result;
 }
 
+/*[clinic input]
+_testcapi.call_vectorcall_method
+
+    callable: object
+    /
+[clinic start generated code]*/
+
 static PyObject *
-call_vectorcall_method(PyObject* self, PyObject *callable)
+_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable)
+/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/
 {
     PyObject *args[3] = { NULL, NULL, NULL };
     PyObject *name = NULL, *kwname = NULL,
@@ -149,8 +171,8 @@ static PyType_Spec LimitedVectorCallClass_spec = {
 };
 
 static PyMethodDef TestMethods[] = {
-    {"call_vectorcall", call_vectorcall, METH_O},
-    {"call_vectorcall_method", call_vectorcall_method, METH_O},
+    _TESTCAPI_CALL_VECTORCALL_METHODDEF
+    _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF
     {NULL},
 };
 

Modules/_testclinic_limited.c

@@ -1,3 +1,9 @@
+// _testclinic_limited can built with the Py_BUILD_CORE_BUILTIN macro defined
+// if one of the Modules/Setup files asks to build it as "static" (gh-109723).
+#undef Py_BUILD_CORE
+#undef Py_BUILD_CORE_MODULE
+#undef Py_BUILD_CORE_BUILTIN
+
 // For now, only limited C API 3.13 is supported
 #define Py_LIMITED_API 0x030d0000
 

Modules/_threadmodule.c

@@ -1066,7 +1066,7 @@ thread_bootstate_free(struct bootstate *boot, int decref)
         Py_DECREF(boot->args);
         Py_XDECREF(boot->kwargs);
     }
-    PyMem_Free(boot);
+    PyMem_RawFree(boot);
 }
 
 
@@ -1184,13 +1184,16 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
         return NULL;
     }
 
-    struct bootstate *boot = PyMem_NEW(struct bootstate, 1);
+    // gh-109795: Use PyMem_RawMalloc() instead of PyMem_Malloc(),
+    // because it should be possible to call thread_bootstate_free()
+    // without holding the GIL.
+    struct bootstate *boot = PyMem_RawMalloc(sizeof(struct bootstate));
     if (boot == NULL) {
         return PyErr_NoMemory();
     }
     boot->tstate = _PyThreadState_New(interp);
     if (boot->tstate == NULL) {
-        PyMem_Free(boot);
+        PyMem_RawFree(boot);
         if (!PyErr_Occurred()) {
             return PyErr_NoMemory();
         }

Tools/unicode/genmap_japanese.py

@@ -2,7 +2,7 @@
 # genmap_ja_codecs.py: Japanese Codecs Map Generator
 #
 # Original Author:  Hye-Shik Chang <[email protected]>
-# Modified Author:  Dong-hee Na <[email protected]>
+# Modified Author:  Donghee Na <[email protected]>
 #
 import os
 

Tools/unicode/genmap_korean.py

@@ -2,7 +2,7 @@
 # genmap_korean.py: Korean Codecs Map Generator
 #
 # Original Author:  Hye-Shik Chang <[email protected]>
-# Modified Author:  Dong-hee Na <[email protected]>
+# Modified Author:  Donghee Na <[email protected]>
 #
 import os
 

Tools/unicode/genmap_schinese.py

@@ -2,7 +2,7 @@
 # genmap_schinese.py: Simplified Chinese Codecs Map Generator
 #
 # Original Author:  Hye-Shik Chang <[email protected]>
-# Modified Author:  Dong-hee Na <[email protected]>
+# Modified Author:  Donghee Na <[email protected]>
 #
 import os
 import re