commits = GIT.git_command(

repo_path,

['log', '--first-parent', '--format=format:%H', '{}^..{}'.format(good, bad)],

abortOnError=True,

).splitlines()

if not commits:

raise RuntimeError("No merge commits found in the range. Did you swap good and bad?")

downstream_repo_path = DOWNSTREAM_REPO_MAPPING.get(repo_path)

results = [None] * len(commits)

if good_result is None and bad_result is None:

bad_index = 0

good_index = len(commits) - 1

bad_result = results[bad_index] = callback(repo_path, bad)

downstream_bad = get_commit(downstream_repo_path)

good_result = results[good_index] = callback(repo_path, good)

downstream_good = get_commit(downstream_repo_path)

if not good_result.bound_is_valid(bad_result):

raise RuntimeError(

"Didn't detect a regression: "

f"'good' value ({good_result}) is worse than or same as than 'bad' value ({bad_result})"

)

if not good_result.bound_is_significant(bad_result, 0.03):

raise RuntimeError(

"Didn't detect a regression: "

f"less that 3% difference between 'good' value ({good_result}) and 'bad' value ({bad_result})"

)

else:

bad_index = -1

good_index = len(commits)

downstream_bad = None

downstream_good = None

while True:

index = bad_index + ((good_index - bad_index) // 2)

if index in [bad_index, good_index]:

assert good_index - bad_index == 1

break

commit = commits[index]

result = results[index] = callback(repo_path, commit)

if result.is_good(good_result, bad_result):

good_index = index

downstream_good = get_commit(downstream_repo_path)

else:

bad_index = index

downstream_bad = get_commit(downstream_repo_path)

subresults = {}

if downstream_bad and downstream_good and downstream_bad != downstream_good:

GIT.update_to_branch(repo_path, commits[good_index])

subresult = run_bisect_benchmark(downstream_repo_path, downstream_bad, downstream_good, callback, good_result,

bad_result)

subresults[bad_index] = subresult

def __init__(self, repo_path: Path, commits, results, good_index, bad_index, dependency_results):

self.repo_path = repo_path

self.commits = commits

self.results = results

self.good_index = good_index

self.bad_index = bad_index

self.dependency_results = dependency_results

@property

def repo_name(self):

return self.repo_path.name

@property

def good_commit(self):

if 0 <= self.good_index < len(self.commits):

return self.commits[self.good_index]

@property

def bad_commit(self):

if 0 <= self.bad_index < len(self.commits):

return self.commits[self.bad_index]

def visualize(self, level=1):

level_marker = '=' * level

out = ["{} {}".format(level_marker, self.repo_name)]

for index, (commit, value) in enumerate(zip(self.commits, self.results)):

if value is not None:

out.append(f"{level_marker} {commit} {value} {get_message(self.repo_path, commit)}")

if self.dependency_results and index in self.dependency_results:

out.append(self.dependency_results[index].visualize(level + 1))

return '\n'.join(out)

def summarize(self):

if self.bad_commit and self.good_commit:

for dependency_result in self.dependency_results.values():

summary = dependency_result.summarize()

if summary:

return summary

return ("Detected bad commit in {} repository:\n{} {}"

.format(self.repo_name, self.bad_commit, get_message(self.repo_path, self.bad_commit)))

return ''

def to_dict(self):

return {

'repo_name': self.repo_name,

'repo_path': str(self.repo_path),

'commits': self.commits,

'results': [result.to_dict() if result is not None else None for result in self.results],

'good_index': self.good_index,

'bad_index': self.bad_index,

'good_commit': self.good_commit,

'bad_commit': self.bad_commit,

'dependency_results': {

str(index): dependency_result.to_dict()

for index, dependency_result in self.dependency_results.items()

},

def __init__(self, value, unit=None):

self.value = value

self.unit = unit

def __str__(self):

return f'{self.value:6.6} {self.unit}'

def bound_is_valid(self, bad_result):

return self.is_good(self, bad_result) or not bad_result.is_good(self, bad_result)

def bound_is_significant(self, bad_result, epsilon):

avg = (self.value + bad_result.value) / 2

if not avg:

return False

diff = abs(self.value - bad_result.value)

return diff / avg >= epsilon

@abc.abstractmethod

def is_good(self, good_result, bad_result):

pass

def to_dict(self):

return {

'kind': type(self).__name__,

'value': self.value,

'unit': self.unit,

'display': str(self),

def is_good(self, good_result, bad_result):

return self.value == 0

def bound_is_significant(self, bad_result, epsilon):

return True

def __str__(self):

exact_docs = [x for x in docs if x['benchmark'] == benchmark_name]

if exact_docs:

return exact_docs

default_metric = 'time'

if 'BISECT_BENCHMARK_CONFIG' in os.environ:

import configparser

cp = configparser.ConfigParser()

cp.read(os.environ['BISECT_BENCHMARK_CONFIG'])

sec = cp['bisect-benchmark']

args = types.SimpleNamespace()

args.bad = sec['bad']

args.good = sec['good']

args.build_command = sec['build_command']

args.benchmark_command = sec['benchmark_command']

args.benchmark_metric = sec.get('benchmark_metric', default_metric)

args.benchmark_name = sec.get('benchmark_name', None)

args.enterprise = sec.getboolean('enterprise', False)

args.no_clean = sec.getboolean('no_clean', False)

args.rerun_with_commands = sec.get('rerun_with_commands')

else:

parser = argparse.ArgumentParser()

parser.add_argument('bad', help="Bad commit for bisection")

parser.add_argument('good', help="Good commit for bisection")

parser.add_argument('build_command', help="Command to run in order to build the configuration")

parser.add_argument('benchmark_command',

help="Command to run in order to run the benchmark. Output needs to be in mx's format")

parser.add_argument('benchmark_name',

help="Filters the results to choose only benchmarks of given name. "

"Useful if the benchmark command runs multiple benchmarks.")

parser.add_argument('--rerun-with-commands',

help="Re-run the bad and good commits with this benchmark command(s) "

"(multiple commands separated by ';')")

parser.add_argument(

'--benchmark-metric', default=default_metric,

help=(

"Which result metric should be used for comparisons (metric.name in the result json). "

"A special value 'WORKS' can be used to consider only the success of the benchmark command."

),

)

parser.add_argument('--enterprise', action='store_true', help="Whether to checkout graal-enterprise")

parser.add_argument('--no-clean', action='store_true', help="Do not run 'mx clean' between runs")

args = parser.parse_args(argv)

def checkout(repo_path: Path, commit):

if 'CI' in os.environ:

# discard any changes made by the previous build before checkout

GIT.run(['git', 'reset', '--hard', 'HEAD'], nonZeroIsFatal=True, cwd=repo_path)

GIT.update_to_branch(repo_path, commit)

if repo_path == DIR:

mx.run_mx(['sforceimports'], suite=str(DIR))

if args.enterprise:

# Keep the Graal revision selected by the current bisection point. The

# enterprise suite imports Graal, so its sforceimports would otherwise

# replace that revision with the one recorded in graal-enterprise.

graal_commit = get_commit(GRAAL_DIR)

if repo_path.name != 'graal-enterprise':

mx.run_mx(['--quiet', 'checkout-downstream', 'vm', 'vm-enterprise', '--no-fetch'],

suite=str(VM_ENTERPRISE_DIR))

mx.run_mx(['--dy', 'substratevm-enterprise-gcs', 'sforceimports'], suite=str(VM_ENTERPRISE_DIR))

GIT.update_to_branch(GRAAL_DIR, graal_commit)

debug_str = f"debug: {SUITE.name}={get_commit(SUITE.vc_dir)} graal={get_commit(GRAAL_DIR)}"

if args.enterprise:

debug_str += f" graal-enterprise={get_commit(GRAAL_ENTERPRISE_DIR)}"

print(debug_str)

def fetch_jdk():

import mx_fetchjdk

if args.enterprise:

fetch_args = [

'--configuration', str(GRAAL_ENTERPRISE_DIR / 'common.json'),

'--jdk-binaries', str(GRAAL_ENTERPRISE_DIR / 'ci' / 'jdk-binaries.json'),

'labsjdk-ee-latest',

]

else:

fetch_args = [

'--configuration', str(GRAAL_DIR / 'common.json'),

'labsjdk-ce-latest',

]

# Awkward way to suppress the confirmation prompt

ci = 'CI' in os.environ

if not ci:

os.environ['CI'] = '1'

try:

return mx_fetchjdk.fetch_jdk(fetch_args)

finally:

if not ci:

del os.environ['CI']

def checkout_and_build(repo_path, commit):

checkout(repo_path, commit)

os.environ['JAVA_HOME'] = fetch_jdk()

build_command = shlex.split(args.build_command)

if not args.no_clean:

try:

clean_command = build_command[:build_command.index('build')] + ['clean', '--all', '--aggressive']

retcode = mx.run(clean_command, nonZeroIsFatal=False)

if retcode:

print("Warning: clean command failed")

except ValueError:

pass

retcode = mx.run(build_command, nonZeroIsFatal=False)

if retcode:

raise RuntimeError("Failed to execute the build command for {}".format(commit))

def benchmark_callback(repo_path: Path, commit, bench_command=args.benchmark_command):

checkout_and_build(repo_path, commit)

retcode = mx.run(shlex.split(bench_command), nonZeroIsFatal=False)

if args.benchmark_metric == 'WORKS':

return WorksResult(retcode)

if retcode:

raise RuntimeError("Failed to execute benchmark for {}".format(commit))

with open('bench-results.json') as f:

data = json.load(f)

docs = [x for x in data['queries'] if x['metric.name'] == args.benchmark_metric]

if args.benchmark_name:

docs = filter_benchmark_results(docs, args.benchmark_name)

if not docs:

raise RuntimeError(f"Couldn't find specified metric {args.benchmark_metric!r} in the results")

if len(docs) > 1:

print("WARNING: found multiple results for the metric, picking the last")

names = set([x['benchmark'] for x in docs])

if len(names) != 1:

print(f"WARNING: found multiple results with different benchmark name attributes: {names}. "

"Use benchmark_name option to filter specific benchmark name.")

doc = docs[-1]

result_class = HigherIsBetterResult if doc.get('metric.better', 'lower') == 'higher' else LowerIsBetterResult

return result_class(doc['metric.value'], doc['metric.unit'])

bad = get_commit(DIR, args.bad)

good = get_commit(DIR, args.good)

result = run_bisect_benchmark(DIR, bad, good, benchmark_callback)

visualization = result.visualize()

summary = result.summarize()

print()

print(visualization)

print()

print(summary)

with open(RESULTS_JSON_PATH, 'w', encoding='utf-8') as result_file:

json.dump({

'bisect_id': bisect_id,

'summary': summary,

'visualization': visualization,

'result': result.to_dict(),

'build_url': os.environ.get('BUILD_URL'),

}, result_file, indent=2, sort_keys=True)

result_file.write('\n')

if args.rerun_with_commands:

print('\n\nRerunning the good and bad commits with extra benchmark commands:')

repo_path = DIR

current_result = result

while current_result.subresults and current_result.bad_index in current_result.subresults:

downstream_repo_path = DOWNSTREAM_REPO_MAPPING.get(repo_path)

next_result = current_result.subresults[current_result.bad_index]

if not next_result.good_commit or not next_result.bad_commit:

print(f"Next downstream repo {downstream_repo_path.name} does not have both good and bad commits")

break

print(f"Recursing to downstream repo: {downstream_repo_path.name}, commit: {current_result.bad_commit}")

checkout(downstream_repo_path, current_result.bad_commit)

current_result = next_result

repo_path = downstream_repo_path

for commit in [current_result.good_commit, current_result.bad_commit]:

print_line(80)

print("Commit: {}".format(commit))

checkout_and_build(repo_path, commit)

for cmd in args.rerun_with_commands.split(";"):

print_line(40)

initial_branch = GIT.git_command(DIR, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()

initial_commit = GIT.git_command(DIR, ['log', '--format=%s', '-n', '1']).strip()

bisect_id = f'{initial_branch}: {initial_commit}'

try:

_bisect_benchmark(argv, bisect_id)

finally: