Merge branch 'master' of https://github.com/graalvm/mx into has_doc

mx.py

@@ -227,6 +227,17 @@ def __exit__(self, tpe, value, traceback):
 
 currently_loading_suite = DynamicVar(None)
 
+def relpath_or_absolute(path, start, prefix=""):
+    """
+    Finds a relative path and joins it to 'prefix', or otherwise tries to use 'path' as an absolute path.
+    If 'path' is not an absolute path, an error is thrown.
+    """
+    try:
+        return join(prefix, os.path.relpath(path, start))
+    except ValueError:
+        if not os.path.isabs(path):
+            raise ValueError('can not find a relative path to dependency and path is not absolute: ' + path)
+        return path
 
 # Support for Python 2.6
 def check_output(*popenargs, **kwargs):
@@ -10776,11 +10787,12 @@ def _find_jdk(versionCheck=None, versionDescription=None):
         candidateJdks.append(os.environ.get('JAVA_HOME'))
         source = 'JAVA_HOME'
 
-    result = _find_jdk_in_candidates(candidateJdks, versionCheck, warn=True, source=source)
-    if result:
-        if source == '--java-home' and os.environ.get('JAVA_HOME'):
-            os.environ['JAVA_HOME'] = _opts.java_home
-        return result
+    if candidateJdks:
+        result = _filtered_jdk_configs(candidateJdks, versionCheck, missingIsError=True, source=source)
+        if result:
+            if source == '--java-home' and os.environ.get('JAVA_HOME'):
+                os.environ['JAVA_HOME'] = _opts.java_home
+            return result[0]
 
     candidateJdks = []
     if _opts.extra_java_homes:
@@ -10790,7 +10802,22 @@ def _find_jdk(versionCheck=None, versionDescription=None):
         candidateJdks += os.environ.get('EXTRA_JAVA_HOMES').split(os.pathsep)
         source = 'EXTRA_JAVA_HOMES'
 
-    return _find_jdk_in_candidates(candidateJdks, versionCheck, warn=True, source=source)
+    result = _filtered_jdk_configs(candidateJdks, versionCheck, missingIsError=False, source=source)
+    result_paths = [x.home for x in result]
+    if result_paths != candidateJdks and source:
+        # Update the source with the filtered result
+        if source == 'EXTRA_JAVA_HOMES':
+            os.environ['EXTRA_JAVA_HOMES'] = os.pathsep.join(result_paths)
+        elif source == '--extra-java-homes':
+            _opts.extra_java_homes = os.pathsep.join(result_paths)
+            if os.environ.get('EXTRA_JAVA_HOMES'):
+                del os.environ['EXTRA_JAVA_HOMES']
+        else:
+            abort('Unknown source ' + source)
+
+    if result:
+        return result[0]
+    return None
 
 def _sorted_unique_jdk_configs(configs):
     path_seen = set()
@@ -10827,29 +10854,27 @@ def _probe_JDK(home):
         _probed_JDKs[home] = res
     return res
 
-def _filtered_jdk_configs(candidates, versionCheck, warnInvalidJDK=False, source=None):
+def _filtered_jdk_configs(candidates, versionCheck, missingIsError=False, source=None):
     filtered = []
     for candidate in candidates:
         jdk = _probe_JDK(candidate)
         if isinstance(jdk, JDKConfigException):
-            if warnInvalidJDK and source:
+            if source:
                 message = 'Path in ' + source + ' is not pointing to a JDK (' + jdk.message + '): ' + candidate
                 if get_os() == 'darwin':
                     candidate = join(candidate, 'Contents', 'Home')
                     if not isinstance(_probe_JDK(candidate), JDKConfigException):
                         message += '. Set ' + source + ' to ' + candidate + ' instead.'
-                warn(message)
+
+                if missingIsError:
+                    abort(message)
+                else:
+                    warn(message)
         else:
             if not versionCheck or versionCheck(jdk.version):
                 filtered.append(jdk)
     return filtered
 
-def _find_jdk_in_candidates(candidates, versionCheck, warn=False, source=None):
-    filtered = _filtered_jdk_configs(candidates, versionCheck, warn, source)
-    if filtered:
-        return filtered[0]
-    return None
-
 def find_classpath_arg(vmArgs):
     """
     Searches for the last class path argument in `vmArgs` and returns its
@@ -15403,34 +15428,34 @@ def intellij_read_sdks():
         warn("IntelliJ SDK definitions not found")
         return sdks
 
-    verRE = re.compile(r'^.*/\.?(IntelliJIdea|IdeaIC)([^/]+)/.*$')
+    verRE = re.compile(r'^.*[/\\]\.?(IntelliJIdea|IdeaIC)([^/\\]+)[/\\].*$')
     def verSort(path):
         match = verRE.match(path)
         return match.group(2) + (".a" if match.group(1) == "IntellijIC" else ".b")
 
     xmlSdks.sort(key=verSort)
     xmlSdk = xmlSdks[-1]  # Pick the most recent IntelliJ version, preferring Ultimate over Community edition.
     log("Using SDK definitions from {}".format(xmlSdk))
-    jvVerRE = re.compile(r'^java\s+version\s+"([^"]+)"$')
-    pyVerRE = re.compile(r'^Python\s+(.+)$')
-    rbVerRE = re.compile(r'^ver\.([^\s]+)\s+.*$')
+
+    versionRegexes = {}
+    versionRegexes[intellij_java_sdk_type] = re.compile(r'^java\s+version\s+"([^"]+)"$')
+    versionRegexes[intellij_python_sdk_type] = re.compile(r'^Python\s+(.+)$')
+    versionRegexes[intellij_ruby_sdk_type] = re.compile(r'^ver\.([^\s]+)\s+.*$')
+
     for sdk in etreeParse(xmlSdk).getroot().findall("component[@name='ProjectJdkTable']/jdk[@version='2']"):
         name = sdk.find("name").get("value")
         kind = sdk.find("type").get("value")
         home = realpath(os.path.expanduser(sdk.find("homePath").get("value").replace('$USER_HOME$', '~')))
-        rawVer = sdk.find("version").get("value")
-        version = None
-        if kind == intellij_java_sdk_type:
-            version = jvVerRE.match(rawVer)
-        elif kind == intellij_python_sdk_type:
-            version = pyVerRE.match(rawVer)
-        elif kind == intellij_ruby_sdk_type:
-            version = rbVerRE.match(rawVer)
-        if version:
-            version = version.group(1)
-        else:
-            version = rawVer
+        if home.find('$APPLICATION_HOME_DIR$') != -1:
+            # Don't know how to convert this into a real path so ignore it
+            continue
+        versionRE = versionRegexes.get(kind)
+        if not versionRE:
+            # ignore unknown kinds
+            continue
+        version = versionRE.match(sdk.find("version").get("value")).group(1)
         sdks[home] = {'name': name, 'type': kind, 'version': version}
+        logv("Found sdk {} with values {}".format(home, sdks[home]))
     return sdks
 
 def intellij_get_java_sdk_name(sdks, jdk):
@@ -15773,21 +15798,24 @@ def _mx_projects_suite(visited_suite, suite_import):
 
         ensure_dir_exists(librariesDirectory)
 
-        def make_library(name, path, source_path):
+        def make_library(name, path, source_path, suite_dir):
             libraryXml = XMLDoc()
 
             libraryXml.open('component', attributes={'name': 'libraryTable'})
             libraryXml.open('library', attributes={'name': name})
             libraryXml.open('CLASSES')
-            libraryXml.element('root', attributes={'url': 'jar://$PROJECT_DIR$/' + path + '!/'})
+            pathX = relpath_or_absolute(path, suite_dir, prefix='$PROJECT_DIR$')
+            libraryXml.element('root', attributes={'url': 'jar://' + pathX + '!/'})
             libraryXml.close('CLASSES')
             libraryXml.element('JAVADOC')
             if sourcePath:
                 libraryXml.open('SOURCES')
                 if os.path.isdir(sourcePath):
-                    libraryXml.element('root', attributes={'url': 'file://$PROJECT_DIR$/' + sourcePath})
+                    sourcePathX = relpath_or_absolute(sourcePath, suite_dir, prefix='$PROJECT_DIR$')
+                    libraryXml.element('root', attributes={'url': 'file://' + sourcePathX})
                 else:
-                    libraryXml.element('root', attributes={'url': 'jar://$PROJECT_DIR$/' + source_path + '!/'})
+                    source_pathX = relpath_or_absolute(source_path, suite_dir, prefix='$PROJECT_DIR$')
+                    libraryXml.element('root', attributes={'url': 'jar://' + source_pathX + '!/'})
                 libraryXml.close('SOURCES')
             else:
                 libraryXml.element('SOURCES')
@@ -15801,25 +15829,25 @@ def make_library(name, path, source_path):
         for library in libraries:
             sourcePath = None
             if library.isLibrary():
-                path = os.path.relpath(library.get_path(True), s.dir)
+                path = library.get_path(True)
                 if library.sourcePath:
-                    sourcePath = os.path.relpath(library.get_source_path(True), s.dir)
+                    sourcePath = library.get_source_path(True)
             elif library.isMavenProject():
-                path = os.path.relpath(library.get_path(True), s.dir)
-                sourcePath = os.path.relpath(library.get_source_path(True), s.dir)
+                path = library.get_path(True)
+                sourcePath = library.get_source_path(True)
             elif library.isJARDistribution():
-                path = os.path.relpath(library.path, s.dir)
+                path = library.path
                 if library.sourcesPath:
-                    sourcePath = os.path.relpath(library.sourcesPath, s.dir)
+                    sourcePath = library.sourcesPath
             else:
                 abort('Dependency not supported: {} ({})'.format(library.name, library.__class__.__name__))
-            make_library(library.name, path, sourcePath)
+            make_library(library.name, path, sourcePath, s.dir)
 
         jdk = get_jdk()
         updated = False
         for library in jdk_libraries:
             if library.classpath_repr(jdk) is not None:
-                if make_library(library.name, os.path.relpath(library.classpath_repr(jdk), s.dir), os.path.relpath(library.get_source_path(jdk), s.dir)):
+                if make_library(library.name, library.classpath_repr(jdk), library.get_source_path(jdk), s.dir):
                     updated = True
         if jdk_libraries and updated:
             log("Setting up JDK libraries using {0}".format(jdk))
@@ -15850,9 +15878,9 @@ def make_library(name, path, source_path):
                 for apDep in processors:
                     def processApDep(dep, edge):
                         if dep.isLibrary() or dep.isJARDistribution():
-                            compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(dep.path, s.dir)})
+                            compilerXml.element('entry', attributes={'name': relpath_or_absolute(dep.path, s.dir, prefix='$PROJECT_DIR$')})
                         elif dep.isProject():
-                            compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(dep.output_dir(), s.dir)})
+                            compilerXml.element('entry', attributes={'name': relpath_or_absolute(dep.output_dir(), s.dir, prefix='$PROJECT_DIR$')})
                     apDep.walk_deps(visit=processApDep)
                 compilerXml.close('processorPath')
                 for module in modules:
@@ -18745,6 +18773,7 @@ def _visit_and_find_jmh_dep(dst, edge):
                 d.set_archiveparticipant(JMHArchiveParticipant(d))
 
     command = commandAndArgs[0]
+    mx_gate._mx_command_and_args = commandAndArgs
     mx_gate._mx_args = sys.argv[1:sys.argv.index(command)]
     command_args = commandAndArgs[1:]
 
@@ -18793,7 +18822,7 @@ def alarm_handler(signum, frame):
 
 
 # The comment after VersionSpec should be changed in a random manner for every bump to force merge conflicts!
-version = VersionSpec("5.192.4")  # GR-12412
+version = VersionSpec("5.192.11")  # safe relpath
 
 currentUmask = None
 _mx_start_datetime = datetime.utcnow()

mx_benchmark.py

@@ -939,12 +939,22 @@ def addAverageAcrossLatestResults(self, results):
             if warmupResults:
                 lastIteration = max((result["metric.iteration"] for result in warmupResults))
                 resultIterations = self.getExtraIterationCount(lastIteration + 1)
-                totalTimeForAverage = 0.0
-                for i in range(lastIteration - resultIterations + 1, lastIteration + 1):
-                    result = next(result for result in warmupResults if result["metric.iteration"] == i)
-                    totalTimeForAverage += result["metric.value"]
-                averageResult = next(result for result in warmupResults if result["metric.iteration"] == 0).copy()
-                averageResult["metric.value"] = totalTimeForAverage / resultIterations
+
+                warmupResultsToAverage = [result for result in warmupResults if result["metric.iteration"] >= lastIteration - resultIterations + 1]
+
+                if len(set([result["metric.iteration"] for result in warmupResults])) != len(warmupResults):
+                    mx.warn("Inconsistent number of iterations ! Duplicate iteration number found.")
+                    mx.warn("Iteration results : {}".format(warmupResults))
+
+                if len(warmupResultsToAverage) != resultIterations:
+                    mx.warn("Inconsistent number of iterations !")
+                    mx.warn("Expecting {} iterations, but got {} instead.".format(len(warmupResultsToAverage), resultIterations))
+                    mx.warn("Iteration results : {}".format(warmupResults))
+
+                scoresToAverage = [result["metric.value"] for result in warmupResultsToAverage]
+
+                averageResult = min(warmupResults, key=lambda result: result["metric.iteration"]).copy()
+                averageResult["metric.value"] = sum(scoresToAverage) / len(scoresToAverage)
                 averageResult["metric.name"] = "time"
                 averageResult["metric.average-over"] = resultIterations
                 results.append(averageResult)
@@ -1840,7 +1850,7 @@ def benchmark(self, mxBenchmarkArgs, bmSuiteArgs):
                 suite.validateEnvironment()
                 fork_count = 1
                 if benchnames and len(benchnames) == 1 and fork_counts:
-                    fork_count = fork_counts.get(benchnames[0], 1)
+                    fork_count = fork_counts.get("{}:{}".format(suite.name(), benchnames[0]), fork_counts.get(benchnames[0], 1))
                 elif fork_counts:
                     mx.abort("The fork-count feature is only supported when the suite is asked to run a single benchmark within a fork.")
                 for _ in range(0, fork_count):

mx_gate.py

@@ -188,6 +188,7 @@ def __repr__(self):
 _pre_gate_runners = []
 _extra_gate_arguments = []
 _mx_args = []
+_mx_command_and_args = []
 
 def add_gate_argument(*args, **kwargs):
     """
@@ -383,21 +384,26 @@ def mx_command_left(_, *__, **___):
         _command_level = _command_level - 1
 
     def print_commands_on_failure():
-        message_color = 'red'
-        mx.log(mx.colorize('\nThe sequence of mx commands that were executed until the failure follows:\n', color=message_color))
+        sys.stdout.flush()
+        sys.stderr.flush()
+
+        mx.log_error('\nThe sequence of mx commands that were executed until the failure follows:\n')
         for command, command_args, kwargs in all_commands:
-            message = command_in_gate_message(command, command_args, kwargs)
+            mx.log_error(command_in_gate_message(command, command_args, kwargs))
+
+        mx.log_error('\nIf the previous sequence is incomplete or some commands were executed programmatically use:\n')
+        mx.log_error('mx' + shell_quoted_args(_mx_args + _mx_command_and_args) + '\n')
 
-            mx.log(mx.colorize(message, color=message_color))
+        sys.stderr.flush()
 
     def command_in_gate_message(command, command_args, kwargs):
         one_list = len(command_args) == 1 and isinstance(command_args[0], (list,))
         kwargs_absent = len(kwargs) == 0
         if one_list and kwargs_absent:  # gate command reproducible on the command line
             quoted_args = (' '.join([pipes.quote(str(arg)) for arg in command_args[0]]))
-            message = 'mx ' + command + ' ' + quoted_args
+            message = 'mx' + shell_quoted_args(_mx_args) + ' ' + command + ' ' + quoted_args
         else:
-            args_message = '(Programatically executed. '
+            args_message = '(Programmatically executed. '
             if not one_list:
                 args_message += 'Args: ' + str(command_args)
             if not kwargs_absent: