Fix parking lot hash table after fork (#6963) · RustPython/RustPython@0355885 · GitHub
Skip to content

Commit 0355885

Browse files
authored
Fix parking lot hash table after fork (#6963)
* Use patched parking_lot_core with fork-safe HASHTABLE reset parking_lot_core's global HASHTABLE retains stale ThreadData after fork(), causing segfaults when contended locks enter park(). Use the patched version from youknowone/parking_lot (rustpython branch) which registers a pthread_atfork handler to reset the hash table. Unskip test_asyncio TestFork. Add Manager+fork integration test. * Unskip fork-related flaky tests after parking_lot fix With parking_lot_core's HASHTABLE now properly reset via pthread_atfork, fork-related segfaults and connection errors in multiprocessing tests should be resolved. Remove skip/expectedFailure markers from: - test_concurrent_futures/test_wait.py (6 tests) - test_concurrent_futures/test_process_pool.py (1 test) - test_multiprocessing_fork/test_manager.py (all WithManagerTest*) - test_multiprocessing_fork/test_misc.py (5 tests) - test_multiprocessing_fork/test_threads.py (2 tests) - _test_multiprocessing.py (2 shared_memory tests) Keep test_repr_rlock skipped (flaky thread start latency, not fork-related).
1 parent e645761 commit 0355885

10 files changed

Lines changed: 152 additions & 68 deletions

File tree

Cargo.lock

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions

Lib/test/_test_multiprocessing.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1459,7 +1459,7 @@ def _acquire_release(lock, timeout, l=None, n=1):
14591459
for _ in range(n):
14601460
lock.release()
14611461

1462-
@unittest.skip("TODO: RUSTPYTHON; flaky timeout")
1462+
@unittest.skip("TODO: RUSTPYTHON; flaky timeout - thread start latency")
14631463
def test_repr_rlock(self):
14641464
if self.TYPE != 'processes':
14651465
self.skipTest('test not appropriate for {}'.format(self.TYPE))
@@ -4415,7 +4415,6 @@ def test_shared_memory_across_processes(self):
44154415

44164416
sms.close()
44174417

4418-
@unittest.skip("TODO: RUSTPYTHON; flaky")
44194418
@unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms")
44204419
def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
44214420
# bpo-36368: protect SharedMemoryManager server process from
@@ -4440,7 +4439,6 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
44404439

44414440
smm.shutdown()
44424441

4443-
@unittest.skip("TODO: RUSTPYTHON: sem_unlink cleanup race causes spurious stderr output")
44444442
@unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
44454443
@resource_tracker_format_subtests
44464444
def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self):

Lib/test/test_asyncio/test_unix_events.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1179,8 +1179,6 @@ async def runner():
11791179
wsock.close()
11801180

11811181

1182-
# TODO: RUSTPYTHON, fork() segfaults due to stale parking_lot global state
1183-
@unittest.skip("TODO: RUSTPYTHON")
11841182
@support.requires_fork()
11851183
class TestFork(unittest.TestCase):
11861184

Lib/test/test_concurrent_futures/test_process_pool.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ def test_traceback(self):
8585
self.assertIn('raise RuntimeError(123) # some comment',
8686
f1.getvalue())
8787

88-
@unittest.skip('TODO: RUSTPYTHON flaky EOFError')
8988
@hashlib_helper.requires_hashdigest('md5')
9089
def test_ressources_gced_in_workers(self):
9190
# Ensure that argument for a job are correctly gc-ed after the job

Lib/test/test_concurrent_futures/test_wait.py

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -200,20 +200,5 @@ def future_func():
200200
def setUpModule():
201201
setup_module()
202202

203-
class ProcessPoolForkWaitTest(ProcessPoolForkWaitTest): # TODO: RUSTPYTHON
204-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
205-
def test_first_completed(self): super().test_first_completed() # TODO: RUSTPYTHON
206-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON Fatal Python error: Segmentation fault")
207-
def test_first_completed_some_already_completed(self): super().test_first_completed_some_already_completed() # TODO: RUSTPYTHON
208-
@unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON flaky")
209-
def test_first_exception(self): super().test_first_exception() # TODO: RUSTPYTHON
210-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
211-
def test_first_exception_one_already_failed(self): super().test_first_exception_one_already_failed() # TODO: RUSTPYTHON
212-
@unittest.skipIf(sys.platform != 'win32', "TODO: RUSTPYTHON flaky")
213-
def test_first_exception_some_already_complete(self): super().test_first_exception_some_already_complete() # TODO: RUSTPYTHON
214-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON Fatal Python error: Segmentation fault")
215-
def test_timeout(self): super().test_timeout() # TODO: RUSTPYTHON
216-
217-
218203
if __name__ == "__main__":
219204
unittest.main()

Lib/test/test_multiprocessing_fork/test_manager.py

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,5 @@
33

44
install_tests_in_module_dict(globals(), 'fork', only_type="manager")
55

6-
import sys # TODO: RUSTPYTHON
7-
class WithManagerTestCondition(WithManagerTestCondition): # TODO: RUSTPYTHON
8-
@unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON, times out')
9-
def test_notify_all(self): super().test_notify_all() # TODO: RUSTPYTHON
10-
11-
class WithManagerTestQueue(WithManagerTestQueue): # TODO: RUSTPYTHON
12-
@unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON, times out')
13-
def test_fork(self): super().test_fork() # TODO: RUSTPYTHON
14-
15-
local_globs = globals().copy() # TODO: RUSTPYTHON
16-
for name, base in local_globs.items(): # TODO: RUSTPYTHON
17-
if name.startswith('WithManagerTest') and issubclass(base, unittest.TestCase): # TODO: RUSTPYTHON
18-
base = unittest.skipIf( # TODO: RUSTPYTHON
19-
sys.platform == 'linux', # TODO: RUSTPYTHON
20-
'TODO: RUSTPYTHON flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError'
21-
)(base) # TODO: RUSTPYTHON
22-
236
if __name__ == '__main__':
247
unittest.main()

Lib/test/test_multiprocessing_fork/test_misc.py

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,5 @@
33

44
install_tests_in_module_dict(globals(), 'fork', exclude_types=True)
55

6-
import sys # TODO: RUSTPYTHON
7-
class TestManagerExceptions(TestManagerExceptions): # TODO: RUSTPYTHON
8-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
9-
def test_queue_get(self): super().test_queue_get() # TODO: RUSTPYTHON
10-
11-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
12-
class TestInitializers(TestInitializers): pass # TODO: RUSTPYTHON
13-
14-
class TestStartMethod(TestStartMethod): # TODO: RUSTPYTHON
15-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
16-
def test_nested_startmethod(self): super().test_nested_startmethod() # TODO: RUSTPYTHON
17-
18-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
19-
class TestSyncManagerTypes(TestSyncManagerTypes): pass # TODO: RUSTPYTHON
20-
21-
class MiscTestCase(MiscTestCase): # TODO: RUSTPYTHON
22-
@unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON flaky")
23-
def test_forked_thread_not_started(self): super().test_forked_thread_not_started() # TODO: RUSTPYTHON
24-
256
if __name__ == '__main__':
267
unittest.main()

Lib/test/test_multiprocessing_fork/test_threads.py

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,5 @@
33

44
install_tests_in_module_dict(globals(), 'fork', only_type="threads")
55

6-
import os, sys # TODO: RUSTPYTHON
7-
class WithThreadsTestPool(WithThreadsTestPool): # TODO: RUSTPYTHON
8-
@unittest.skip("TODO: RUSTPYTHON; flaky environment pollution when running rustpython -m test --fail-env-changed due to unknown reason")
9-
def test_terminate(self): super().test_terminate() # TODO: RUSTPYTHON
10-
11-
class WithThreadsTestManagerRestart(WithThreadsTestManagerRestart): # TODO: RUSTPYTHON
12-
@unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON flaky flaky BrokenPipeError, flaky ConnectionRefusedError, flaky ConnectionResetError, flaky EOFError')
13-
def test_rapid_restart(self): super().test_rapid_restart() # TODO: RUSTPYTHON
14-
156
if __name__ == '__main__':
167
unittest.main()
Lines changed: 149 additions & 0 deletions

0 commit comments

Comments
 (0)