@@ -96,10 +96,11 @@ def get_module(name, path):
9696
9797class ProgressIndicator (object ):
9898
99- def __init__ (self , cases , flaky_tests_mode ):
99+ def __init__ (self , cases , flaky_tests_mode , measure_flakiness ):
100100 self .cases = cases
101101 self .serial_id = 0
102102 self .flaky_tests_mode = flaky_tests_mode
103+ self .measure_flakiness = measure_flakiness
103104 self .parallel_queue = Queue (len (cases ))
104105 self .sequential_queue = Queue (len (cases ))
105106 for case in cases :
@@ -211,10 +212,22 @@ def RunSingle(self, parallel, thread_id):
211212 if output .UnexpectedOutput ():
212213 if FLAKY in output .test .outcomes and self .flaky_tests_mode == DONTCARE :
213214 self .flaky_failed .append (output )
215+ elif FLAKY in output .test .outcomes and self .flaky_tests_mode == KEEP_RETRYING :
216+ for _ in range (99 ):
217+ if not case .Run ().UnexpectedOutput ():
218+ self .flaky_failed .append (output )
219+ break
220+ else :
221+ # If after 100 tries, the test is not passing, it's not flaky.
222+ self .failed .append (output )
214223 else :
215224 self .failed .append (output )
216225 if output .HasCrashed ():
217226 self .crashed += 1
227+ if self .measure_flakiness :
228+ outputs = [case .Run () for _ in range (self .measure_flakiness )]
229+ # +1s are there because the test already failed once at this point.
230+ print (" failed {} out of {}" .format (len ([i for i in outputs if i .UnexpectedOutput ()]) + 1 , self .measure_flakiness + 1 ))
218231 else :
219232 self .succeeded += 1
220233 self .remaining -= 1
@@ -436,8 +449,8 @@ def Done(self):
436449
437450class CompactProgressIndicator (ProgressIndicator ):
438451
439- def __init__ (self , cases , flaky_tests_mode , templates ):
440- super (CompactProgressIndicator , self ).__init__ (cases , flaky_tests_mode )
452+ def __init__ (self , cases , flaky_tests_mode , measure_flakiness , templates ):
453+ super (CompactProgressIndicator , self ).__init__ (cases , flaky_tests_mode , measure_flakiness )
441454 self .templates = templates
442455 self .last_status_length = 0
443456 self .start_time = time .time ()
@@ -492,29 +505,29 @@ def PrintProgress(self, name):
492505
493506class ColorProgressIndicator (CompactProgressIndicator ):
494507
495- def __init__ (self , cases , flaky_tests_mode ):
508+ def __init__ (self , cases , flaky_tests_mode , measure_flakiness ):
496509 templates = {
497510 'status_line' : "[%(mins)02i:%(secs)02i|\033 [34m%%%(remaining) 4d\033 [0m|\033 [32m+%(passed) 4d\033 [0m|\033 [31m-%(failed) 4d\033 [0m]: %(test)s" ,
498511 'stdout' : "\033 [1m%s\033 [0m" ,
499512 'stderr' : "\033 [31m%s\033 [0m" ,
500513 }
501- super (ColorProgressIndicator , self ).__init__ (cases , flaky_tests_mode , templates )
514+ super (ColorProgressIndicator , self ).__init__ (cases , flaky_tests_mode , measure_flakiness , templates )
502515
503516 def ClearLine (self , last_line_length ):
504517 print ("\033 [1K\r " , end = '' )
505518
506519
507520class MonochromeProgressIndicator (CompactProgressIndicator ):
508521
509- def __init__ (self , cases , flaky_tests_mode ):
522+ def __init__ (self , cases , flaky_tests_mode , measure_flakiness ):
510523 templates = {
511524 'status_line' : "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s" ,
512525 'stdout' : '%s' ,
513526 'stderr' : '%s' ,
514527 'clear' : lambda last_line_length : ("\r " + (" " * last_line_length ) + "\r " ),
515528 'max_length' : 78
516529 }
517- super (MonochromeProgressIndicator , self ).__init__ (cases , flaky_tests_mode , templates )
530+ super (MonochromeProgressIndicator , self ).__init__ (cases , flaky_tests_mode , measure_flakiness , templates )
518531
519532 def ClearLine (self , last_line_length ):
520533 print (("\r " + (" " * last_line_length ) + "\r " ), end = '' )
@@ -946,8 +959,8 @@ def GetVm(self, arch, mode):
946959 def GetTimeout (self , mode ):
947960 return self .timeout * TIMEOUT_SCALEFACTOR [ARCH_GUESS or 'ia32' ][mode ]
948961
949- def RunTestCases (cases_to_run , progress , tasks , flaky_tests_mode ):
950- progress = PROGRESS_INDICATORS [progress ](cases_to_run , flaky_tests_mode )
962+ def RunTestCases (cases_to_run , progress , tasks , flaky_tests_mode , measure_flakiness ):
963+ progress = PROGRESS_INDICATORS [progress ](cases_to_run , flaky_tests_mode , measure_flakiness )
951964 return progress .Run (tasks )
952965
953966# -------------------------------------------
@@ -965,6 +978,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
965978SLOW = 'slow'
966979FLAKY = 'flaky'
967980DONTCARE = 'dontcare'
981+ KEEP_RETRYING = 'keep_retrying'
968982
969983class Expression (object ):
970984 pass
@@ -1353,8 +1367,11 @@ def BuildOptions():
13531367 result .add_option ("--cat" , help = "Print the source of the tests" ,
13541368 default = False , action = "store_true" )
13551369 result .add_option ("--flaky-tests" ,
1356- help = "Regard tests marked as flaky (run|skip|dontcare)" ,
1370+ help = "Regard tests marked as flaky (run|skip|dontcare|keep_retrying )" ,
13571371 default = "run" )
1372+ result .add_option ("--measure-flakiness" ,
1373+ help = "When a test fails, re-run it x number of times" ,
1374+ default = 0 , type = "int" )
13581375 result .add_option ("--skip-tests" ,
13591376 help = "Tests that should not be executed (comma-separated)" ,
13601377 default = "" )
@@ -1426,7 +1443,7 @@ def ProcessOptions(options):
14261443 # tends to exaggerate the number of available cpus/cores.
14271444 cores = os .environ .get ('JOBS' )
14281445 options .j = int (cores ) if cores is not None else multiprocessing .cpu_count ()
1429- if options .flaky_tests not in RUN , SKIP , DONTCARE ]:
1446+ if options .flaky_tests not in RUN , SKIP , DONTCARE , KEEP_RETRYING ]:
14301447 print ("Unknown flaky-tests mode %s" % options .flaky_tests )
14311448 return False
14321449 return True
@@ -1726,7 +1743,7 @@ def should_keep(case):
17261743 else :
17271744 try :
17281745 start = time .time ()
1729- if RunTestCases (cases_to_run , options .progress , options .j , options .flaky_tests ):
1746+ if RunTestCases (cases_to_run , options .progress , options .j , options .flaky_tests , options . measure_flakiness ):
17301747 result = 0
17311748 else :
17321749 result = 1
0 commit comments