non_internal = []

additional_dists = []

for arg in args:

# Class path extensions

if arg.startswith('-add-dist='):

additional_dists += [arg[10:]]

else:

non_internal += [arg]

if not SUITE_COMPILER:

if must_be_jvmci:

mx.abort('** Error ** : graal compiler was not found!')

sys.exit(1)

if vm_warning:

if SUITE_COMPILER:

tag = 'jvmci'

else:

tag = None

"""run a Python program or shell"""

if '--python.WithJavaStacktrace' not in args:

args.insert(0, '--python.WithJavaStacktrace')

if not env:

env = os.environ

check_vm_env = env.get('GRAALPYTHON_MUST_USE_GRAAL', False)

if check_vm_env:

if check_vm_env == '1':

check_vm(must_be_jvmci=True)

elif check_vm_env == '0':

check_vm()

dists = ['GRAALPYTHON', 'TRUFFLE_NFI', 'SULONG']

vm_args, graalpython_args = mx.extract_VM_args(args, useDoubleDash=True, defaultAllVMArgs=False)

graalpython_args, additional_dists = _extract_graalpython_internal_options(graalpython_args)

dists += additional_dists

if mx.suite("sulong-managed", fatalIfMissing=False):

dists.append('SULONG_MANAGED')

# Try eagerly to include tools for convenience when running Python

if not mx.suite("tools", fatalIfMissing=False):

SUITE.import_suite("tools", version=None, urlinfos=None, in_subdir=True)

if mx.suite("tools", fatalIfMissing=False):

if os.path.exists(mx.suite("tools").dependency("CHROMEINSPECTOR").path):

# CHROMEINSPECTOR was built, put it on the classpath

dists.append('CHROMEINSPECTOR')

graalpython_args.insert(0, "--llvm.enableLVI=true")

else:

mx.logv("CHROMEINSPECTOR was not built, not including it automatically")

graalpython_args.insert(0, '--experimental-options=true')

vm_args += mx.get_runtime_jvm_args(dists, jdk=jdk)

if not jdk:

jdk = get_jdk()

# default: assertion checking is enabled

if extra_vm_args is None or '-da' not in extra_vm_args:

vm_args += ['-ea', '-esa']

if extra_vm_args:

vm_args += extra_vm_args

vm_args.append("com.oracle.graal.python.shell.GraalPythonMain")

if '--regex' not in args:

args += ['--regex', r'(graal\.python)|(com\.oracle\.truffle\.tck\.tests)']

args += ["-Dgraal.TruffleCompilationExceptionsAreFatal=false",

"-Dgraal.TruffleCompilationExceptionsArePrinted=true",

"-Dgraal.TrufflePerformanceWarningsAreFatal=false"]

junit = 'python-junit'

unittest = 'python-unittest'

unittest_sandboxed = 'python-unittest-sandboxed'

tagged = 'python-tagged-unittest'

svmunit = 'python-svm-unittest'

svmunit_sandboxed = 'python-svm-unittest-sandboxed'

shared_object = 'python-so'

shared_object_sandboxed = 'python-so-sandboxed'

graalvm = 'python-graalvm'

graalvm_sandboxed = 'python-graalvm-sandboxed'

svm = 'python-svm'

native_image_embedder = 'python-native-image-embedder'

if not os.environ.get("JDT"):

find_jdt()

if not os.environ.get("ECLIPSE_EXE"):

find_eclipse()

if "--tags" not in args:

args += ["--tags"]

tags = ["style"]

include_sandboxed = mx.suite("sulong-managed", fatalIfMissing=False) is not None

for x in dir(GraalPythonTags):

v = getattr(GraalPythonTags, x)

if isinstance(v, str) and v.startswith("python-"):

if include_sandboxed and "sandboxed" in v:

tags.append(v)

elif not include_sandboxed and "sandboxed" not in v:

tags.append(v)

args.append(",".join(tags))

mx.log("Running mx python-gate " + " ".join(args))

pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))

for f in [os.path.join(SUITE.dir, f) for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:

if os.path.basename(f).startswith("ecj-") and os.path.basename(f).endswith(".jar"):

mx.log("Automatically choosing %s for JDT" % f)

os.environ["JDT"] = f

pardir = os.path.abspath(os.path.join(SUITE.dir, ".."))

for f in [os.path.join(SUITE.dir, f) for f in os.listdir(SUITE.dir)] + [os.path.join(pardir, f) for f in os.listdir(pardir)]:

if os.path.basename(f) == "eclipse" and os.path.isdir(f):

mx.log("Automatically choosing %s for Eclipse" % f)

eclipse_exe = os.path.join(f, "eclipse")

if os.path.exists(eclipse_exe):

os.environ["ECLIPSE_EXE"] = eclipse_exe

"Temporarily set the process environment variables"

old_environ = dict(os.environ)

os.environ.update(environ)

try:

yield

finally:

os.environ.clear()

dy = "/vm,/tools,/substratevm"

if "sandboxed" in args:

args.remove("sandboxed")

dy += ",/sulong-managed"

dy = ["--dynamicimports", dy]

mx.run_mx(dy + ["build"])

out = mx.OutputCapture()

mx.run_mx(dy + ["graalvm-home"], out=mx.TeeOutputCapture(out))

launcher = os.path.join(out.data.strip(), "bin", "graalpython").split("\n")[-1].strip()

mx.log(launcher)

if args:

mx.run([launcher] + args)

args = args or []

args = ["--experimental-options=true", "--python.CatchAllExceptions=true"] + args

exclude = exclude or []

paths = paths or [_graalpytest_root()]

# list of excluded tests

if aot_compatible:

exclude += AOT_INCOMPATIBLE_TESTS

def is_included(path):

if path.endswith(".py"):

basename = os.path.basename(path)

return basename.startswith("test_") and basename not in exclude

return False

# list all 1st-level tests and exclude the SVM-incompatible ones

testfiles = []

for path in paths:

if not os.path.exists(path):

# allow paths relative to the test root

path = os.path.join(_graalpytest_root(), path)

if os.path.isfile(path):

testfiles.append(path)

else:

for testfile in glob.glob(os.path.join(path, "**/test_*.py")):

if is_included(testfile):

testfiles.append(testfile)

for testfile in glob.glob(os.path.join(path, "test_*.py")):

if is_included(testfile):

testfiles.append(testfile)

args += [_graalpytest_driver(), "-v"]

args += testfiles

# JUnit tests

with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit]) as task:

if task:

punittest(['--verbose'])

# Unittests on JVM

with Task('GraalPython Python unittests', tasks, tags=[GraalPythonTags.unittest]) as task:

if task:

if platform.system() != 'Darwin':

# TODO: drop condition when python3 is available on darwin

mx.log("Running tests with CPython")

test_args = [_graalpytest_driver(), "-v", _graalpytest_root()]

mx.run(["python3"] + test_args, nonZeroIsFatal=True)

run_python_unittests(python_gvm())

with Task('GraalPython sandboxed tests', tasks, tags=[GraalPythonTags.unittest_sandboxed]) as task:

if task:

run_python_unittests(python_gvm(["sandboxed"]), args=["--llvm.managed"])

with Task('GraalPython Python tests', tasks, tags=[GraalPythonTags.tagged]) as task:

if task:

with set_env(ENABLE_CPYTHON_TAGGED_UNITTESTS="true", ENABLE_THREADED_GRAALPYTEST="true"):

run_python_unittests(python_gvm(), args=["--python.WithThread=true"], paths=["test_tagged_unittests.py"])

# Unittests on SVM

with Task('GraalPython tests on SVM', tasks, tags=[GraalPythonTags.svmunit]) as task:

if task:

run_python_unittests(python_svm())

with Task('GraalPython sandboxed tests on SVM', tasks, tags=[GraalPythonTags.svmunit_sandboxed]) as task:

if task:

run_python_unittests(python_svm(["sandboxed"]), args=["--llvm.managed"])

with Task('GraalPython license header update', tasks, tags=[GraalPythonTags.license]) as task:

if task:

python_checkcopyrights([])

with Task('GraalPython GraalVM shared-library build', tasks, tags=[GraalPythonTags.shared_object, GraalPythonTags.graalvm]) as task:

if task:

run_shared_lib_test()

with Task('GraalPython GraalVM sandboxed shared-library build', tasks, tags=[GraalPythonTags.shared_object_sandboxed, GraalPythonTags.graalvm_sandboxed]) as task:

if task:

run_shared_lib_test(["sandboxed"])

with Task('GraalPython GraalVM build', tasks, tags=[GraalPythonTags.svm, GraalPythonTags.graalvm]) as task:

if task:

svm_image = python_svm(["--version"])

benchmark = os.path.join(PATH_MESO, "image-magix.py")

out = mx.OutputCapture()

mx.run([svm_image, "-v", "-S", "--log.python.level=FINEST", benchmark], nonZeroIsFatal=True, out=mx.TeeOutputCapture(out))

success = "\n".join([

"[0, 0, 0, 0, 0, 0, 10, 10, 10, 0, 0, 10, 3, 10, 0, 0, 10, 10, 10, 0, 0, 0, 0, 0, 0]",

])

if success not in out.data:

mx.abort('Output from generated SVM image "' + svm_image + '" did not match success pattern:\n' + success)

# Test that stdlib paths are not cached on packages

out = mx.OutputCapture()

mx.run([svm_image, "-v", "-S", "--log.python.level=FINEST", "--python.StdLibHome=/foobar", "-c", "import encodings; print(encodings.__path__)"], out=mx.TeeOutputCapture(out))

if "/foobar" not in out.data:

mx.abort('Output from generated SVM image "' + svm_image + '" did not have patched std lib path "/foobar"')

# Test that stdlib paths are not cached on modules

out = mx.OutputCapture()

mx.run([svm_image, "-v", "-S", "--log.python.level=FINEST", "--python.StdLibHome=/foobar", "-c", "import encodings; print(encodings.__file__)"], out=mx.TeeOutputCapture(out))

if "/foobar" not in out.data:

mx.abort('Output from generated SVM image "' + svm_image + '" did not have patched std lib path "/foobar"')

# Finally, test that we can start even if the graalvm was moved

out = mx.OutputCapture()

graalvm_home = svm_image.replace(os.path.sep.join(["", "bin", "graalpython"]), "")

new_graalvm_home = graalvm_home + "_new"

shutil.move(graalvm_home, new_graalvm_home)

launcher = os.path.join(new_graalvm_home, "bin", "graalpython")

mx.log(launcher)

mx.run([launcher, "--log.python.level=FINE", "-S", "-c", "print(b'abc'.decode('ascii'))"], out=mx.TeeOutputCapture(out), err=mx.TeeOutputCapture(out))

assert "Using preinitialized context." in out.data

with Task('GraalPython GraalVM native embedding', tasks, tags=[GraalPythonTags.svm, GraalPythonTags.graalvm, GraalPythonTags.native_image_embedder]) as task:

if task:

"""

with mx.TempDirCwd(os.getcwd()) as dirname:

python_launcher = python_gvm()

graalvm_javac = os.path.join(os.path.dirname(python_launcher), "javac")

graalvm_native_image = os.path.join(os.path.dirname(python_launcher), "native-image")

filename = os.path.join(dirname, "HelloWorld.java")

with open(filename, "w") as f:

f.write("""

""")

out = mx.OutputCapture()

mx.run([graalvm_javac, filename])

mx.run([graalvm_native_image, "-H:+ReportExceptionStackTraces", "--initialize-at-build-time", "--language:python", "HelloWorld"])

mx.run(["./helloworld"], out=mx.TeeOutputCapture(out))

assert "abc" in out.data

if args is None:

args = []

launcher = python_so(args)

svm_lib_path = os.path.abspath(os.path.join(launcher, "..", "..", "jre", "lib", "polyglot"))

fd = name = progname = None

try:

fd, name = tempfile.mkstemp(suffix='.c')

os.write(fd, """

""" % ("1" if "sandboxed" in args else "0"))

os.close(fd)

progname = os.path.join(SUITE.dir, "graalpython-embedded-tool")

mx.log("".join(["Running ", "'clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o", progname, "-lpolyglot"]))

mx.run(["clang", "-I%s" % svm_lib_path, "-L%s" % svm_lib_path, name, "-o%s" % progname, "-lpolyglot"], nonZeroIsFatal=True)

mx.log("Running " + progname + " with LD_LIBRARY_PATH " + svm_lib_path)

mx.run(["ls", "-l", progname])

mx.run(["ls", "-l", svm_lib_path])

run_env = {"LD_LIBRARY_PATH": svm_lib_path, "GRAAL_PYTHONHOME": os.environ["GRAAL_PYTHONHOME"]}

mx.log(repr(run_env))

mx.run([progname], env=run_env)

finally:

try:

os.unlink(progname)

except:

pass

try:

os.close(fd)

except:

pass

try:

os.unlink(name)

except:

def __init__(self, suite, name, deps, workingSets, theLicense, **args):

super(ArchiveProject, self).__init__(suite, name, deps, workingSets, theLicense)

def output_dir(self):

if hasattr(self, "outputFile"):

self.outputFile = mx_subst.path_substitutions.substitute(self.outputFile)

return os.path.dirname(os.path.join(self.dir, self.outputFile))

else:

assert hasattr(self, "outputDir")

self.outputDir = mx_subst.path_substitutions.substitute(self.outputDir)

return os.path.join(self.dir, self.outputDir)

def archive_prefix(self):

return getattr(self, "prefix", "")

def getResults(self):

if hasattr(self, "outputFile"):

return [os.path.join(self.dir, self.outputFile)]

else:

ignore_regexps = [re.compile(s) for s in getattr(self, "ignorePatterns", [])]

results = []

for root, _, files in os.walk(self.output_dir()):

for name in files:

path = os.path.join(root, name)

if not any(r.search(path) for r in ignore_regexps):

results.append(path)

"""if the active branch is 'master', deploy binaries for the primary suite to remote maven repository."""

master_branch = 'master'

active_branch = mx.VC.get_vc(SUITE.dir).active_branch(SUITE.dir)

if active_branch == master_branch:

if sys.platform == "darwin":

args.insert(0, "--platform-dependent")

return mx.command_function('deploy-binary')(args)

else:

mx.log('The active branch is "%s". Binaries are deployed only if the active branch is "%s".' % (

active_branch, master_branch))

for suite in mx.suites():

for p in suite.projects:

if p.name == projectname:

if len(p.source_dirs()) > 0:

return p.source_dirs()[0]

else:

return p.dir

"""

if str(SUITE.dir).endswith("testdownstream/graalpython"):

primary = mx.primary_suite()

dep_dir = mx.suite(name).vc_dir

vc = mx.VC.get_vc(dep_dir)

vc.pull(dep_dir, update=False)

vc.update(dep_dir, rev=rev)

tip = str(vc.tip(dep_dir)).strip()

contents = None

suitefile = os.path.join(primary.dir, "mx." + primary.name, "suite.py")

with open(suitefile, 'r') as f:

contents = f.read()

dep_re = re.compile(r"['\"]name['\"]:\s+['\"]%s['\"],\s+['\"]version['\"]:\s+['\"]([a-z0-9]+)['\"]" % name, re.MULTILINE)

dep_match = dep_re.search(contents)

if dep_match:

start = dep_match.start(1)

end = dep_match.end(1)

assert end - start == len(tip)

mx.update_file(suitefile, "".join([contents[:start], tip, contents[end:]]), showDiff=True)

if callback:

callback()

else:

"""Update our mx imports"""

if not args:

args = ["truffle"]

if "sulong" in args:

args.append("regex")

if "regex" in args:

args.append("sulong")

if "truffle" in args:

args.remove("truffle")

args += ["sulong", "regex"]

if "sulong" in args:

join = os.path.join

callback = lambda: shutil.copy(

join(mx.dependency("SULONG_LEGACY").output, "include", "truffle.h"),

join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext", "include", "truffle.h")

) and shutil.copy(

join(mx.dependency("SULONG_HOME").output, "include", "polyglot.h"),

join(SUITE.dir, "graalpython", "com.oracle.graal.python.cext", "include", "polyglot.h")

)

else:

callback = None

for name in set(args):

"Check (and fix where possible) copyrights, eclipse formatting, and spotbugs"

python_checkcopyrights(["--fix"] if "--fix" in args else [])

if not os.environ.get("ECLIPSE_EXE"):

find_eclipse()

if os.environ.get("ECLIPSE_EXE"):

mx.command_function("eclipseformat")([])

# we wan't to ignore lib-python/3, because that's just crazy

listfilename = tempfile.mktemp()

with open(listfilename, "w") as listfile:

mx.run(["git", "ls-tree", "-r", "HEAD", "--name-only"], out=listfile)

with open(listfilename, "r") as listfile:

content = listfile.read()

with open(listfilename, "w") as listfile:

for line in content.split("\n"):

if "lib-python/3" in line:

pass

elif os.path.splitext(line)[1] in [".py", ".java", ".c", ".h", ".sh"]:

listfile.write(line)

listfile.write("\n")

try:

r = mx.command_function("checkcopyrights")(["--primary", "--", "--file-list", listfilename] + args)

if r != 0:

mx.abort("copyrights check failed")

finally:

"Update the inlined files from PyPy and CPython"

# mappings for files that are renamed

mapping = {

"_memoryview.c": "memoryobject.c",

"_cpython_sre.c": "_sre.c",

"_cpython_unicodedata.c": "unicodedata.c",

"_bz2.c": "_bz2module.c",

"_mmap.c": "mmapmodule.c",

}

extra_pypy_files = [

"graalpython/lib-python/3/_md5.py",

"graalpython/lib-python/3/_sha1.py",

"graalpython/lib-python/3/_sha256.py",

"graalpython/lib-python/3/_sha512.py",

]

parser = ArgumentParser(prog='mx python-src-import')

parser.add_argument('--cpython', action='store', help='Path to CPython sources', required=True)

parser.add_argument('--pypy', action='store', help='Path to PyPy sources', required=True)

parser.add_argument('--msg', action='store', help='Message for import update commit', required=True)

args = parser.parse_args(args)

python_sources = args.cpython

pypy_sources = args.pypy

import_version = args.msg

print("""

""".format(mapping))

raw_input("Got it?")

cpy_files = []

pypy_files = []

with open(os.path.join(os.path.dirname(__file__), "copyrights", "overrides")) as f:

cpy_files = [line.split(",")[0] for line in f.read().split("\n") if len(line.split(",")) > 1 and line.split(",")[1] == "python.copyright"]

pypy_files = [line.split(",")[0] for line in f.read().split("\n") if len(line.split(",")) > 1 and line.split(",")[1] == "pypy.copyright"]

# move to orphaned branch with sources

if SUITE.vc.isDirty(SUITE.dir):

mx.abort("Working dir must be clean")

tip = SUITE.vc.tip(SUITE.dir).strip()

SUITE.vc.git_command(SUITE.dir, ["checkout", "python-import"])

SUITE.vc.git_command(SUITE.dir, ["clean", "-fdx"])

shutil.rmtree("graalpython")

# re-copy lib-python

shutil.copytree(os.path.join(python_sources, "Lib"), _get_stdlib_home())

for inlined_file in pypy_files + extra_pypy_files:

original_file = None

name = os.path.basename(inlined_file)

name = mapping.get(name, name)

if inlined_file.endswith(".py"):

# these files don't need to be updated, they inline some unittest code only

if name.startswith("test_") or name.endswith("_tests.py"):

original_file = inlined_file

else:

for root, _, files in os.walk(pypy_sources):

if os.path.basename(name) in files:

original_file = os.path.join(root, name)

try:

os.makedirs(os.path.dirname(inlined_file))

except:

pass

shutil.copy(original_file, inlined_file)

break

if original_file is None:

mx.warn("Could not update %s - original file not found" % inlined_file)

for inlined_file in cpy_files:

# C files are mostly just copied

original_file = None

name = os.path.basename(inlined_file)

name = mapping.get(name, name)

if inlined_file.endswith(".h") or inlined_file.endswith(".c"):

for root, _, files in os.walk(python_sources):

if os.path.basename(name) in files:

original_file = os.path.join(root, name)

try:

os.makedirs(os.path.dirname(inlined_file))

except:

pass

shutil.copy(original_file, inlined_file)

break

elif inlined_file.endswith(".py"):

# these files don't need to be updated, they inline some unittest code only

if name.startswith("test_") or name.endswith("_tests.py"):

original_file = inlined_file

if original_file is None:

mx.warn("Could not update %s - original file not found" % inlined_file)

# commit and check back

SUITE.vc.git_command(SUITE.dir, ["add", "."])

raw_input("Check that the updated files look as intended, then press RETURN...")

SUITE.vc.commit(SUITE.dir, "Update Python inlined files: %s" % import_version)

answer = raw_input("Should we push python-import (y/N)? ")

if answer and answer in "Yy":

SUITE.vc.git_command(SUITE.dir, ["push", "origin", "python-import:python-import"])

SUITE.vc.update(SUITE.dir, rev=tip)

"""Verify CI configuration"""

base_suite = SUITE

assert isinstance(base_suite, mx.SourceSuite)

ci_files = mx.suite_ci_files(SUITE, common_ci_dir, extension=ext)

mx.log("CI setup checking common file(s): \n\t{0}".format('\n\t'.join(map(str, ci_files))))

# cpython

python_vm_registry.add_vm(CPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE)

# pypy

python_vm_registry.add_vm(PyPyVm(config_name=CONFIGURATION_DEFAULT), SUITE)

# graalpython

python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE, 10)

python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI, extra_polyglot_args=[

'--experimental-options', '-multi-context',

]), SUITE, 10)

python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_SANDBOXED, extra_polyglot_args=[

'--llvm.managed',