[MERGE #5273 @Penguinwizzard] Add arm/arm64 and chakrafull support to runtests.py

Merge pull request #5273 from Penguinwizzard:runtestspy_arm64_full

This allows running jshost on the core unit tests easily from the python runner, as well as running core tests on arm and arm64.
Note: hold this change, and re-target to release/1.10 when ready.

Author: Penguinwizzard

test/runtests.py

@@ -50,6 +50,8 @@
                     help='use debug build');
 parser.add_argument('-t', '--test', '--test-build', action='store_true',
                     help='use test build')
+parser.add_argument('-f', '--full', '--chakrafull', action='store_true',
+                    help='test chakrafull instead of chakracore')
 parser.add_argument('--static', action='store_true',
                     help='mark that we are testing a static build')
 parser.add_argument('--variants', metavar='variant', nargs='+',
@@ -78,12 +80,18 @@
                     help='use x86 build')
 parser.add_argument('--x64', action='store_true',
                     help='use x64 build')
+parser.add_argument('--arm', action='store_true',
+                    help='use arm build')
+parser.add_argument('--arm64', action='store_true',
+                    help='use arm64 build')
 parser.add_argument('-j', '--processcount', metavar='processcount', type=int,
                     help='number of parallel threads to use')
 parser.add_argument('--warn-on-timeout', action='store_true',
                     help='warn when a test times out instead of labelling it as an error immediately')
 parser.add_argument('--override-test-root', type=str,
                     help='change the base directory for the tests (where rlexedirs will be sought)')
+parser.add_argument('--extra-flags', type=str,
+                    help='add extra flags to all executed tests')
 args = parser.parse_args()
 
 test_root = os.path.dirname(os.path.realpath(__file__))
@@ -93,8 +101,17 @@
 if args.override_test_root:
     test_root = os.path.realpath(args.override_test_root)
 
-# arch: x86, x64
-arch = 'x86' if args.x86 else ('x64' if args.x64 else None)
+# arch: x86, x64, arm, arm64
+arch = None
+if args.x86:
+    arch = 'x86'
+elif args.x64:
+    arch = 'x64'
+elif args.arm:
+    arch = 'arm'
+elif args.arm64:
+    arch = 'arm64'
+
 if arch == None:
     arch = os.environ.get('_BuildArch', 'x86')
 if sys.platform != 'win32':
@@ -110,26 +127,46 @@
     sys.exit(1)
 flavor_alias = 'chk' if flavor == 'Debug' else 'fre'
 
+# handling for extra flags
+extra_flags = []
+if args.extra_flags:
+    extra_flags = args.extra_flags.split()
+
 # test variants
 if not args.variants:
     args.variants = ['interpreted', 'dynapogo']
 
+# target binary variants
+binary_name_noext = "ch"
+if args.full:
+    binary_name_noext = "jshost"
+    repo_root = os.path.dirname(repo_root)
+    # we need this to have consistent error message formatting with ch
+    extra_flags.append("-bvt")
+else:
+    extra_flags.append('-WERExceptionSupport')
+
+# append exe to the binary name on windows
+binary_name = binary_name_noext
+if sys.platform == 'win32':
+    binary_name = binary_name + ".exe"
+
 # binary: full ch path
 binary = args.binary
 if binary == None:
     if sys.platform == 'win32':
         build = "VcBuild.SWB" if args.swb else "VcBuild"
-        binary = 'Build\\' + build + '\\bin\\{}_{}\\ch.exe'.format(arch, flavor)
+        binary = os.path.join(repo_root, 'Build', build, 'bin', '{}_{}'.format(arch, flavor), binary_name)
     else:
-        binary = 'out/{0}/ch'.format(flavor)
-    binary = os.path.join(repo_root, binary)
+        binary = os.path.join(repo_root, 'out', flavor, binary_name)
+
 if not os.path.isfile(binary):
     print('{} not found. Did you run ./build.sh already?'.format(binary))
     sys.exit(1)
 
 # global tags/not_tags
 tags = set(args.tag or [])
-not_tags = set(args.not_tag or []).union(['fail', 'exclude_' + arch, 'exclude_' + flavor, 'exclude_ch'])
+not_tags = set(args.not_tag or []).union(['fail', 'exclude_' + arch, 'exclude_' + flavor])
 
 if arch_alias:
     not_tags.add('exclude_' + arch_alias)
@@ -158,6 +195,14 @@
 else:
     not_tags.add('exclude_windows')
 
+# exclude tests that depend on features not supported on a platform
+if arch == 'arm' or arch == 'arm64':
+    not_tags.add('require_asmjs')
+
+# exclude tests that exclude the current binary
+not_tags.add('exclude_' + binary_name_noext)
+
+# exclude tests known to fail under certain sanitizers
 if args.sanitize != None:
     not_tags.add('exclude_sanitize_'+args.sanitize)
 
@@ -204,7 +249,7 @@ def __init__(self, log_file_path = None):
             # Set up the log file paths
             # Make sure the right directory exists and the log file doesn't
             log_file_name = "testrun.{0}{1}.log".format(arch, flavor)
-            log_file_directory = os.path.join(repo_root, "test", "logs")
+            log_file_directory = os.path.join(test_root, "logs")
 
             if not os.path.exists(log_file_directory):
                 os.mkdir(log_file_directory)
@@ -306,7 +351,7 @@ class TestVariant(object):
     def __init__(self, name, compile_flags=[], variant_not_tags=[]):
         self.name = name
         self.compile_flags = \
-            ['-WERExceptionSupport', '-ExtendedErrorStackForTestHost',
+            ['-ExtendedErrorStackForTestHost',
              '-BaselineMode'] + compile_flags
         self._compile_flags_has_expansion = self._has_expansion(compile_flags)
         self.tags = tags.copy()
@@ -318,6 +363,11 @@ def __init__(self, name, compile_flags=[], variant_not_tags=[]):
         self.test_count = 0
         self._print_lines = [] # _print lines buffer
         self._last_len = 0
+        if verbose:
+            print("Added variant {0}:".format(name))
+            print("Flags: " + ", ".join(self.compile_flags))
+            print("Tags: " + ", ".join(self.tags))
+            print("NotTags: " + ", ".join(self.not_tags))
 
     @staticmethod
     def _has_expansion(flags):
@@ -506,7 +556,7 @@ def timeout_func(timeout_data):
             return self._show_failed(timedout=True, **fail_args)
 
         # check ch failed
-        if exit_code != 0:
+        if exit_code != 0 and binary_name_noext == 'ch':
             return self._show_failed(**fail_args)
 
         # check output
@@ -706,19 +756,19 @@ def main():
 
     # test variants
     variants = [x for x in [
-        TestVariant('interpreted', [
+        TestVariant('interpreted', extra_flags + [
                 '-maxInterpretCount:1', '-maxSimpleJitRunCount:1', '-bgjit-',
                 '-dynamicprofilecache:profile.dpl.${id}'
             ], [
                 'require_disable_jit'
             ]),
-        TestVariant('dynapogo', [
+        TestVariant('dynapogo', extra_flags + [
                 '-forceNative', '-off:simpleJit', '-bgJitDelay:0',
                 '-dynamicprofileinput:profile.dpl.${id}'
             ], [
                 'require_disable_jit'
             ]),
-        TestVariant('disable_jit', [
+        TestVariant('disable_jit', extra_flags + [
                 '-nonative'
             ], [
                 'exclude_interpreted', 'fails_interpreted', 'require_backend'