python · vstinner · Sep 12, 2023 · Sep 12, 2023 · Sep 12, 2023 · Sep 12, 2023
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -117,6 +117,7 @@ def __init__(self, ns: Namespace):
             self.python_cmd: tuple[str] = tuple(ns.python)
         else:
             self.python_cmd = None
+        print("main process python_cmd:", self.python_cmd)
         self.coverage: bool = ns.trace
         self.coverage_dir: StrPath | None = ns.coverdir
         self.tmp_dir: StrPath | None = ns.tempdir
@@ -406,7 +407,7 @@ def create_run_tests(self, tests: TestTuple):
             python_cmd=self.python_cmd,
             randomize=self.randomize,
             random_seed=self.random_seed,
-            json_fd=None,
+            json_file=None,
         )
 
     def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
@@ -507,5 +508,10 @@ def main(self, tests: TestList | None = None):
 
 def main(tests=None, **kwargs):
     """Run the Python suite."""
+
+    print("main process start sys.platform:", sys.platform)
+    print("main process start is_emscripten:", support.is_emscripten)
+    print("main process start is_wasi:", support.is_wasi)
+
     ns = _parse_args(sys.argv[1:], **kwargs)
     Regrtest(ns).main(tests=tests)
diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py
@@ -17,10 +17,10 @@
 from .logger import Logger
 from .result import TestResult, State
 from .results import TestResults
-from .runtests import RunTests
+from .runtests import RunTests, JsonFileType
 from .single import PROGRESS_MIN_TIME
 from .utils import (
-    StrPath, StrJSON, TestName, MS_WINDOWS,
+    StrPath, StrJSON, TestName, MS_WINDOWS, TMP_PREFIX,
     format_duration, print_warning, count, plural)
 from .worker import create_worker_process, USE_PROCESS_GROUP
 
@@ -155,10 +155,11 @@ def mp_result_error(
     ) -> MultiprocessResult:
         return MultiprocessResult(test_result, stdout, err_msg)
 
-    def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int,
+    def _run_process(self, runtests: RunTests, output_fd: int,
+                     json_file: JsonFileType,
                      tmp_dir: StrPath | None = None) -> int:
         try:
-            popen = create_worker_process(runtests, output_fd, json_fd,
+            popen = create_worker_process(runtests, output_fd, json_file,
                                           tmp_dir)
 
             self._killed = False
@@ -226,21 +227,38 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
             match_tests = None
         err_msg = None
 
+        stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)
+
+        json_file_use_filename = self.runtests.json_file_use_filename()
+        print("main process: json_file_use_filename?", json_file_use_filename)
+        if json_file_use_filename:
+            prefix = TMP_PREFIX + 'json_'
+            json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8',
+                                                       prefix=prefix)
+            print("main process: create NamedTemporaryFile", json_tmpfile.name)
+        else:
+            json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
+            print("main process: create TemporaryFile")
+
         # gh-94026: Write stdout+stderr to a tempfile as workaround for
         # non-blocking pipes on Emscripten with NodeJS.
-        with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file,
-              tempfile.TemporaryFile('w+', encoding='utf8') as json_file):
+        with (stdout_file, json_tmpfile):
             stdout_fd = stdout_file.fileno()
-            json_fd = json_file.fileno()
-            if MS_WINDOWS:
-                json_fd = msvcrt.get_osfhandle(json_fd)
+            if json_file_use_filename:
+                json_file = json_tmpfile.name
+            else:
+                json_file = json_tmpfile.fileno()
+                if MS_WINDOWS:
+                    json_file = msvcrt.get_osfhandle(json_file)
+            print("main process json_file type:", type(json_file))
+            print("main process json_file:", json_file)
 
             kwargs = {}
             if match_tests:
                 kwargs['match_tests'] = match_tests
             worker_runtests = self.runtests.copy(
                 tests=tests,
-                json_fd=json_fd,
+                json_file=json_file,
                 **kwargs)
 
             # gh-93353: Check for leaked temporary files in the parent process,
@@ -254,13 +272,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
                 tmp_dir = os.path.abspath(tmp_dir)
                 try:
                     retcode = self._run_process(worker_runtests,
-                                                stdout_fd, json_fd, tmp_dir)
+                                                stdout_fd, json_file, tmp_dir)
                 finally:
                     tmp_files = os.listdir(tmp_dir)
                     os_helper.rmtree(tmp_dir)
             else:
                 retcode = self._run_process(worker_runtests,
-                                            stdout_fd, json_fd)
+                                            stdout_fd, json_file)
                 tmp_files = ()
             stdout_file.seek(0)
 
@@ -275,8 +293,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
 
             try:
                 # deserialize run_tests_worker() output
-                json_file.seek(0)
-                worker_json: StrJSON = json_file.read()
+                json_tmpfile.seek(0)
+                worker_json: StrJSON = json_tmpfile.read()
                 if worker_json:
                     result = TestResult.from_json(worker_json)
                 else:

diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py
@@ -2,10 +2,20 @@
 import json
 from typing import Any
 
+from test import support
+
 from .utils import (
     StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
 
 
+# See RunTests.json_file_use_filename()
+JsonFileType = int | StrPath
+
+import os
+print(os.getpid(), "JsonFileType:", JsonFileType)
+
+
+
 @dataclasses.dataclass(slots=True, frozen=True)
 class HuntRefleak:
     warmups: int
@@ -38,9 +48,7 @@ class RunTests:
     python_cmd: tuple[str] | None
     randomize: bool
     random_seed: int | None
-    # On Unix, it's a file descriptor.
-    # On Windows, it's a handle.
-    json_fd: int | None
+    json_file: JsonFileType | None
 
     def copy(self, **override):
         state = dataclasses.asdict(self)
@@ -74,6 +82,18 @@ def as_json(self) -> StrJSON:
     def from_json(worker_json: StrJSON) -> 'RunTests':
         return json.loads(worker_json, object_hook=_decode_runtests)
 
+    def json_file_use_filename(self):
+        # On Unix, it's a file descriptor.
+        # On Windows, it's a handle.
+        # On Emscripten/WASI, it's a filename. Passing a file descriptor to a
+        # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
+        # worker process.
+        return (
+            self.python_cmd
+            or support.is_emscripten
+            or support.is_wasi
+        )
+
 
 class _EncodeRunTests(json.JSONEncoder):
     def default(self, o: Any) -> dict[str, Any]:

diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
@@ -17,8 +17,12 @@
 
 
 MS_WINDOWS = (sys.platform == 'win32')
-WORK_DIR_PREFIX = 'test_python_'
-WORKER_WORK_DIR_PREFIX = f'{WORK_DIR_PREFIX}worker_'
+
+# All temporary files and temporary directories created by libregrtest should
+# use TMP_PREFIX so cleanup_temp_dir() can remove them all.
+TMP_PREFIX = 'test_python_'
+WORK_DIR_PREFIX = TMP_PREFIX
+WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_'
 
 # bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
 # Used to protect against threading._shutdown() hang.
@@ -387,7 +391,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath:
     # testing (see the -j option).
     # Emscripten and WASI have stubbed getpid(), Emscripten has only
     # milisecond clock resolution. Use randint() instead.
-    if sys.platform in {"emscripten", "wasi"}:
+    if support.is_emscripten or support.is_wasi:
         nounce = random.randint(0, 1_000_000)
     else:
         nounce = os.getpid()
@@ -580,7 +584,7 @@ def display_header():
 def cleanup_temp_dir(tmp_dir: StrPath):
     import glob
 
-    path = os.path.join(glob.escape(tmp_dir), WORK_DIR_PREFIX + '*')
+    path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*')
     print("Cleanup %s directory" % tmp_dir)
     for name in glob.glob(path):
         if os.path.isdir(name):

diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py
@@ -7,7 +7,7 @@
 from test.support import os_helper
 
 from .setup import setup_process, setup_test_dir
-from .runtests import RunTests
+from .runtests import RunTests, JsonFileType
 from .single import run_single_test
 from .utils import (
     StrPath, StrJSON, FilterTuple, MS_WINDOWS,
@@ -18,7 +18,7 @@
 
 
 def create_worker_process(runtests: RunTests,
-                          output_fd: int, json_fd: int,
+                          output_fd: int, json_file: JsonFileType,
                           tmp_dir: StrPath | None = None) -> subprocess.Popen:
     python_cmd = runtests.python_cmd
     worker_json = runtests.as_json()
@@ -27,10 +27,12 @@ def create_worker_process(runtests: RunTests,
         executable = python_cmd
     else:
         executable = [sys.executable]
+    print("main process executable:", executable)
     cmd = [*executable, *support.args_from_interpreter_flags(),
            '-u',    # Unbuffered stdout and stderr
            '-m', 'test.libregrtest.worker',
            worker_json]
+    print("main process worker cmd:", cmd)
 
     env = dict(os.environ)
     if tmp_dir is not None:
@@ -55,33 +57,50 @@ def create_worker_process(runtests: RunTests,
         close_fds=True,
         cwd=work_dir,
     )
-    if not MS_WINDOWS:
-        kwargs['pass_fds'] = [json_fd]
-    else:
+
+    # Pass json_file to the worker process
+    if isinstance(json_file, str):
+        # Filename: nothing to do to
+        print("create_worker_process() json_file: filename")
+        pass
+    elif MS_WINDOWS:
+        # Windows handle
+        print("create_worker_process() json_file: Windows handle")
         startupinfo = subprocess.STARTUPINFO()
-        startupinfo.lpAttributeList = {"handle_list": [json_fd]}
+        startupinfo.lpAttributeList = {"handle_list": [json_file]}
         kwargs['startupinfo'] = startupinfo
+    else:
+        # Unix file descriptor
+        print("create_worker_process() json_file: Unix fd")
+        kwargs['pass_fds'] = [json_file]
+
     if USE_PROCESS_GROUP:
         kwargs['start_new_session'] = True
 
     if MS_WINDOWS:
-        os.set_handle_inheritable(json_fd, True)
+        os.set_handle_inheritable(json_file, True)
     try:
         return subprocess.Popen(cmd, **kwargs)
     finally:
         if MS_WINDOWS:
-            os.set_handle_inheritable(json_fd, False)
+            os.set_handle_inheritable(json_file, False)
 
 
 def worker_process(worker_json: StrJSON) -> NoReturn:
     runtests = RunTests.from_json(worker_json)
     test_name = runtests.tests[0]
     match_tests: FilterTuple | None = runtests.match_tests
-    json_fd: int = runtests.json_fd
+    # On Unix, it's a file descriptor.
+    # On Windows, it's a handle.
+    # On Emscripten/WASI, it's a filename.
+    json_file: JsonFileType = runtests.json_file
+    print("worker: json_file type:", type(json_file))
+    print("worker: json_file:", json_file)
 
     if MS_WINDOWS:
         import msvcrt
-        json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY)
+        # Create a file descriptor from the handle
+        json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY)
 
 
     setup_test_dir(runtests.test_dir)
@@ -96,8 +115,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
 
     result = run_single_test(test_name, runtests)
 
-    with open(json_fd, 'w', encoding='utf-8') as json_file:
-        result.write_json_into(json_file)
+    with open(json_file, 'w', encoding='utf-8') as fp:
+        result.write_json_into(fp)
 
     sys.exit(0)
 
@@ -111,6 +130,10 @@ def main():
     tmp_dir = get_temp_dir()
     work_dir = get_work_dir(tmp_dir, worker=True)
 
+    print("worker process sys.platform:", sys.platform)
+    print("worker process is_emscripten:", support.is_emscripten)
+    print("worker process is_wasi:", support.is_wasi)
+
     with exit_timeout():
         with os_helper.temp_cwd(work_dir, quiet=True):
             worker_process(worker_json)