44
55import contextlib
66import math
7- import multiprocessing .pool
87import sys
98
109import concurrent .futures
@@ -79,12 +78,12 @@ def partition(cmd, varargs, target_concurrency, _max_length=None):
7978
8079
8180@contextlib .contextmanager
82- def _threadpool ( size ):
83- pool = multiprocessing . pool . ThreadPool ( size )
84- try :
85- yield pool
86- finally :
87- pool . terminate ()
81+ def _thread_mapper ( maxsize ):
82+ if maxsize == 1 :
83+ yield map
84+ else :
85+ with concurrent . futures . ThreadPoolExecutor ( maxsize ) as ex :
86+ yield ex . map
8887
8988
9089def xargs (cmd , varargs , ** kwargs ):
@@ -109,22 +108,24 @@ def xargs(cmd, varargs, **kwargs):
109108 def run_cmd_partition (run_cmd ):
110109 return cmd_output (* run_cmd , encoding = None , retcode = None )
111110
112- with _threadpool (min (len (partitions ), target_concurrency )) as pool :
113- results = pool .map (run_cmd_partition , partitions )
114-
115- for proc_retcode , proc_out , proc_err in results :
116- # This is *slightly* too clever so I'll explain it.
117- # First the xor boolean table:
118- # T | F |
119- # +-------+
120- # T | F | T |
121- # --+-------+
122- # F | T | F |
123- # --+-------+
124- # When negate is True, it has the effect of flipping the return code
125- # Otherwise, the retuncode is unchanged
126- retcode |= bool (proc_retcode ) ^ negate
127- stdout += proc_out
128- stderr += proc_err
111+ with _thread_mapper (
112+ min (len (partitions ), target_concurrency ),
113+ ) as thread_map :
114+ results = thread_map (run_cmd_partition , partitions )
115+
116+ for proc_retcode , proc_out , proc_err in results :
117+ # This is *slightly* too clever so I'll explain it.
118+ # First the xor boolean table:
119+ # T | F |
120+ # +-------+
121+ # T | F | T |
122+ # --+-------+
123+ # F | T | F |
124+ # --+-------+
125+ # When negate is True, it has the effect of flipping the return
126+ # code. Otherwise, the returncode is unchanged.
127+ retcode |= bool (proc_retcode ) ^ negate
128+ stdout += proc_out
129+ stderr += proc_err
129130
130131 return retcode , stdout , stderr
0 commit comments