Enable wasm64 tests to run under latest version of node (#18075)

Now that we have a way to test in CI use a recent version of node we can
define `required_wasm64` predicate that requires with d8 or a node >=
16.

.circleci/config.yml

@@ -10,6 +10,9 @@ executors:
   linux-python:
     docker:
       - image: cimg/python:3.10.7
+    environment:
+      # Currently pinned due to https://github.com/emscripten-core/emscripten/issues/17846
+      V8_VERSION: "10.7.186"
   bionic:
     docker:
       - image: emscripten/emscripten-ci
@@ -66,6 +69,34 @@ commands:
       - run:
           name: pip install
           command: << parameters.python >> -m pip install -r requirements-dev.txt
+  setup-latest-node:
+    description: "setup latest node"
+    steps:
+      - run:
+          name: setup latest node
+          command: |
+            cd $HOME
+            wget https://nodejs.org/dist/v19.0.0/node-v19.0.0-linux-x64.tar.xz
+            tar xf node-v19.0.0-linux-x64.tar.xz
+            echo "NODE_JS = [os.path.expanduser('~/node-v19.0.0-linux-x64/bin/node')]" >> ~/emsdk/.emscripten
+            echo "JS_ENGINES = [NODE_JS]" >> ~/emsdk/.emscripten
+            echo "if os.path.exists(V8_ENGINE[0]): JS_ENGINES.append(V8_ENGINE)" >> ~/emsdk/.emscripten
+            cat ~/emsdk/.emscripten
+            echo "export PATH=\"$HOME/node-v19.0.0-linux-x64/bin:\$PATH\"" >> $BASH_ENV
+  install-v8:
+    description: "install v8 using jsvu"
+    steps:
+      - run:
+          name: get v8
+          command: |
+            # We can't use a more recent version of node here because the linux
+            # image we are using doesn't have a recent enough glibc.
+            cd $HOME
+            wget https://nodejs.org/dist/v15.14.0/node-v15.14.0-linux-x64.tar.xz
+            tar -xf node-v15.14.0-linux-x64.tar.xz
+            export PATH="`pwd`/node-v15.14.0-linux-x64/bin:${PATH}"
+            npm install jsvu -g
+            jsvu --os=default --engines=v8 v8@${V8_VERSION}
   build:
     description: "Install emsdk"
     steps:
@@ -161,7 +192,6 @@ commands:
       - emsdk-env
       - npm-install
       - pip-install
-      - run: apt-get install ninja-build
   upload-test-results:
     description: "Upload test results"
     steps:
@@ -373,17 +403,7 @@ jobs:
             mkdir ~/wasi-sdk
             tar xvf libclang_rt.builtins-wasm32-wasi-11.0.tar.gz -C ~/wasi-sdk
             tar xvf wasi-sysroot-11.0.tar.gz -C ~/wasi-sdk/
-      - run:
-          name: get v8
-          command: |
-            # We can't use a more recent version of node here because the linux
-            # image we are using doesn't have a recent enough glibc.
-            wget https://nodejs.org/dist/v15.14.0/node-v15.14.0-linux-x64.tar.xz
-            tar -xf node-v15.14.0-linux-x64.tar.xz
-            export PATH="`pwd`/node-v15.14.0-linux-x64/bin:${PATH}"
-            npm install jsvu -g
-            export PATH="${HOME}/.jsvu:${PATH}"
-            jsvu --os=default --engines=v8 v8@${V8_VERSION}
+      - install-v8
       - build
       - build-libs
       - persist
@@ -478,15 +498,26 @@ jobs:
       - run-tests-linux:
           test_targets: "wasm2js1"
   test-wasm64:
-    executor: bionic
+    # We don't use `bionic` here since its tool old to run recent node versions:
+    # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
+    executor: linux-python
     steps:
-      - run-tests-linux:
-          frozen_cache: false
-          test_targets: "wasm64"
+      - prepare-for-tests
+      # The linux-python image uses /home/circleci rather than /root and jsvu
+      # hardcodes /root into its launcher scripts do we need to reinstall v8.
+      - run: rm -rf $HOME/.jsvu
+      - install-v8
+      - run-tests:
+          test_targets: "wasm64_v8"
       - run-tests:
           test_targets: "wasm64l"
+      - setup-latest-node
+      - run-tests:
+          test_targets: "wasm64"
       - upload-test-results
   test-latest-node:
+    # We don't use `bionic` here since its tool old to run recent node versions:
+    # `/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found`
     executor: linux-python
     steps:
       - checkout
@@ -495,15 +526,7 @@ jobs:
           command: git submodule update --init
       - pip-install
       - build
-      - run:
-          name: get node
-          command: |
-            cd $HOME
-            wget https://nodejs.org/dist/v19.0.0/node-v19.0.0-linux-x64.tar.xz
-            tar xf node-v19.0.0-linux-x64.tar.xz
-            echo "export EM_JS_ENGINES=$HOME/node-v19.0.0-linux-x64/bin/node" >> $BASH_ENV
-            echo "export EM_NODE_JS=$HOME/node-v19.0.0-linux-x64/bin/node" >> $BASH_ENV
-            echo "export PATH=\"$HOME/node-v19.0.0-linux-x64/bin:\$PATH\"" >> $BASH_ENV
+      - setup-latest-node
       - run-tests:
           # Run tests that on older versions of node would require flags, but
           # those flags should not be injected on newer versions.
@@ -512,6 +535,7 @@ jobs:
   test-other:
     executor: bionic
     steps:
+      - run: apt-get install ninja-build
       - run-tests-linux:
           # some native-dependent tests fail because of the lack of native
           # headers on emsdk-bundled clang
@@ -604,6 +628,7 @@ jobs:
     executor: mac
     environment:
       EMTEST_SKIP_V8: "1"
+      EMTEST_SKIP_WASM64: "1"
       EMCC_SKIP_SANITY_CHECK: "1"
     steps:
       - run:

test/common.py

@@ -183,6 +183,16 @@ def decorated(self, *args, **kwargs):
   return decorated
 
 
+def requires_wasm64(func):
+  assert callable(func)
+
+  def decorated(self, *args, **kwargs):
+    self.require_wasm64()
+    return func(self, *args, **kwargs)
+
+  return decorated
+
+
 def requires_v8(func):
   assert callable(func)
 
@@ -435,7 +445,7 @@ def require_v8(self):
         self.skipTest('test requires v8 and EMTEST_SKIP_V8 is set')
       else:
         self.fail('d8 required to run this test.  Use EMTEST_SKIP_V8 to skip')
-    self.js_engines = [config.V8_ENGINE]
+    self.require_engine(config.V8_ENGINE)
     self.emcc_args.append('-sENVIRONMENT=shell')
 
   def require_node(self):
@@ -444,9 +454,32 @@ def require_node(self):
         self.skipTest('test requires node and EMTEST_SKIP_NODE is set')
       else:
         self.fail('node required to run this test.  Use EMTEST_SKIP_NODE to skip')
-    if self.get_setting('MEMORY64') == 1:
-      self.skipTest("MEMORY64=1 tests don't yet run under node")
-    self.js_engines = [config.NODE_JS]
+    self.require_engine(config.NODE_JS)
+
+  def require_engine(self, engine):
+    if self.required_engine and self.required_engine != engine:
+      self.skipTest(f'Skipping test that requires `{engine}` when `{self.required_engine}` was previously required')
+    self.required_engine = engine
+    self.js_engines = [engine]
+
+  def require_wasm64(self):
+    if config.NODE_JS and config.NODE_JS in self.js_engines:
+      version = shared.check_node_version()
+      if version >= (16, 0, 0):
+        self.js_engines = [config.NODE_JS]
+        self.node_args += ['--experimental-wasm-memory64']
+        return
+
+    if config.V8_ENGINE and config.V8_ENGINE in self.js_engines:
+      self.emcc_args.append('-sENVIRONMENT=shell')
+      self.js_engines = [config.V8_ENGINE]
+      self.v8_args += ['--experimental-wasm-memory64']
+      return
+
+    if 'EMTEST_SKIP_WASM64' in os.environ:
+      self.skipTest('test requires node >= 16 or d8 (and EMTEST_SKIP_WASM64 is set)')
+    else:
+      self.fail('either d8 or node >= 16 required to run wasm64 tests.  Use EMTEST_SKIP_WASM64 to skip')
 
   def setup_node_pthreads(self):
     self.require_node()
@@ -500,6 +533,7 @@ def setUp(self):
     self.temp_files_before_run = []
     self.uses_es6 = False
     self.js_engines = config.JS_ENGINES.copy()
+    self.required_engine = None
     self.wasm_engines = config.WASM_ENGINES.copy()
     self.banned_js_engines = []
     self.use_all_engines = EMTEST_ALL_ENGINES

test/test_core.py

@@ -6349,7 +6349,7 @@ def test_js_libraries(self):
     self.emcc_args += ['--js-library', 'mylib1.js', '--js-library', 'mylib2.js']
     self.do_runf('main.cpp', 'hello from lib!\n*32*\n')
 
-  @with_env_modify({'LC_CTYPE': 'latin-1', 'PYTHONUTF8': '0', 'PYTHONCOERCECLOCALE': '0'})
+  @with_env_modify({'LC_ALL': 'latin-1', 'PYTHONUTF8': '0', 'PYTHONCOERCECLOCALE': '0'})
   def test_unicode_js_library(self):
     create_file('main.c', '''
       #include <stdio.h>
@@ -9655,7 +9655,10 @@ def test_native_promise(self):
 
 
 # Generate tests for everything
-def make_run(name, emcc_args, settings=None, env=None, node_args=None, require_v8=False, v8_args=None):
+def make_run(name, emcc_args, settings=None, env=None,
+             require_v8=False, v8_args=None,
+             require_node=False, node_args=None,
+             require_wasm64=False):
   if env is None:
     env = {}
   if settings is None:
@@ -9697,6 +9700,11 @@ def setUp(self):
 
     if require_v8:
       self.require_v8()
+    elif require_node:
+      self.require_node()
+
+    if require_wasm64:
+      self.require_wasm64()
 
   TT.setUp = setUp
 
@@ -9721,8 +9729,9 @@ def setUp(self):
 
 # MEMORY64=1
 wasm64 = make_run('wasm64', emcc_args=['-Wno-experimental', '--profiling-funcs'],
-                  settings={'MEMORY64': 1},
-                  require_v8=True, v8_args=['--experimental-wasm-memory64'])
+                  settings={'MEMORY64': 1}, require_wasm64=True, require_node=True)
+wasm64_v8 = make_run('wasm64_v8', emcc_args=['-Wno-experimental', '--profiling-funcs'],
+                     settings={'MEMORY64': 1}, require_wasm64=True, require_v8=True)
 # MEMORY64=2, or "lowered"
 wasm64l = make_run('wasm64l', emcc_args=['-Wno-experimental', '--profiling-funcs'],
                    settings={'MEMORY64': 2},

test/test_other.py

@@ -33,7 +33,7 @@
 from common import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable
 from common import env_modify, no_mac, no_windows, requires_native_clang, with_env_modify
 from common import create_file, parameterized, NON_ZERO, node_pthreads, TEST_ROOT, test_file
-from common import compiler_for, EMBUILDER, requires_v8, requires_node
+from common import compiler_for, EMBUILDER, requires_v8, requires_node, requires_wasm64
 from common import also_with_minimal_runtime, also_with_wasm_bigint, EMTEST_BUILD_VERBOSE, PYTHON
 from tools import shared, building, utils, deps_info, response_file, cache
 from tools.utils import read_file, write_file, delete_file, read_binary
@@ -12488,9 +12488,8 @@ def test_extended_const(self):
 
   # Smoketest for MEMORY64 setting.  Most of the testing of MEMORY64 is by way of the wasm64
   # variant of the core test suite.
-  @requires_v8
+  @requires_wasm64
   def test_memory64(self):
-    self.v8_args += ['--experimental-wasm-memory64']
     for opt in ['-O0', '-O1', '-O2', '-O3']:
       self.do_runf(test_file('hello_world.c'), 'hello, world', emcc_args=['-sMEMORY64', '-Wno-experimental', opt])
 

tools/check_clean.py

@@ -18,7 +18,8 @@ def main():
   print('')
 
   here = os.path.dirname(__file__)
-  output = subprocess.check_output(['git', 'status', '--short'], cwd=here)
+  root = os.path.dirname(here)
+  output = subprocess.check_output(['git', 'status', '--short'], cwd=root)
   output = output.decode('utf-8').strip()
   if not output:
     print('Tree is clean.')