From 2fb091cf29ac21464362b3fac0fb4d078bb9ff49 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 10 Sep 2024 15:26:24 +0800 Subject: [PATCH] Update rev, remove whitelist and change default plan Updated the `ruby` repo. It enabled more test cases that were disabled before. Removed the whitelist. Now we rely on the exclusion list in `test/.excludes-mmtk` in the `ruby` repo. Changed the default plan to StickyImmix since it has been stable so far. We also run bootstraptest/test_thread.rb with a fixed heap size in debug build because it allocates too aggressively and may cause timeout in CI. --- .github/scripts/ci-btest.sh | 24 ++++-- .github/scripts/ci-test-all.sh | 2 - .github/scripts/common.sh | 2 +- README.md | 29 +++++--- mmtk/Cargo.lock | 4 +- mmtk/Cargo.toml | 2 +- ruby-test-cases.txt | 130 --------------------------------- 7 files changed, 37 insertions(+), 156 deletions(-) delete mode 100644 ruby-test-cases.txt diff --git a/.github/scripts/ci-btest.sh b/.github/scripts/ci-btest.sh index 137edde..1d69417 100755 --- a/.github/scripts/ci-btest.sh +++ b/.github/scripts/ci-btest.sh @@ -10,13 +10,21 @@ echo "============ Bootstrape tests (btest) ($DEBUG_LEVEL) =============" if test "$DEBUG_LEVEL" == "vanilla"; then make btest TESTOPTS="-v -j${CI_JOBS}" else - if test "$DEBUG_LEVEL" == "debug"; then - # One test case in this file is taking too much time to run in debug mode, resulting in timeout. - # It is because the default dynamic heap size does not work well with that pathological use case. - # We simply run it with a fixed heap size and exclude it from the rest of the btests. - TIMEOUT_TEST=../bootstraptest/test_eval.rb - MMTK_GC_TRIGGER=FixedHeapSize:100m make btest RUN_OPTS="--mmtk-plan=$CHOSEN_PLAN" TESTOPTS="-v $TIMEOUT_TEST" - rm $TIMEOUT_TEST - fi + # Some test cases take too much time to run in debug mode, resulting in timeout. It is + # because the default GC trigger (dynamic heap size) does not scale the heap size fast + # enough to keep up with the allocation. We simply run those tests with a fixed heap size + # and exclude them from the rest of the btests. + test_and_delete() { + TEST_NAME=$1 + HEAP_SIZE=$2 + TEST_PATH=../bootstraptest/test_${TEST_NAME}.rb + MMTK_GC_TRIGGER=FixedHeapSize:${HEAP_SIZE} make btest RUN_OPTS="--mmtk-plan=$CHOSEN_PLAN" TESTOPTS="-v $TEST_PATH" + rm $TEST_PATH + } + + test_and_delete eval 100m + test_and_delete thread 400m + + # Run other btests in the regular way. make btest RUN_OPTS="--mmtk-plan=$CHOSEN_PLAN" TESTOPTS="-v -j${CI_JOBS}" fi diff --git a/.github/scripts/ci-test-all.sh b/.github/scripts/ci-test-all.sh index 3903b0f..6378aef 100755 --- a/.github/scripts/ci-test-all.sh +++ b/.github/scripts/ci-test-all.sh @@ -12,9 +12,7 @@ case $DEBUG_LEVEL in echo "Skipping test-all for $DEBUG_LEVEL..." ;; release) - TEST_CASES=$(cat $BINDING_PATH/ruby-test-cases.txt | grep -v '#' | ruby -ne 'puts "../#{$_}"' | xargs) make test-all \ - TESTS="$TEST_CASES" \ RUN_OPTS="--mmtk-plan=$CHOSEN_PLAN" \ TESTOPTS="-v --excludes-dir=../test/.excludes-mmtk -j${CI_JOBS}" ;; diff --git a/.github/scripts/common.sh b/.github/scripts/common.sh index 27d6294..778f338 100644 --- a/.github/scripts/common.sh +++ b/.github/scripts/common.sh @@ -3,7 +3,7 @@ RUBY_PATH=$BINDING_PATH/../ruby RUBY_BUILD_PATH=$RUBY_PATH/build RUBY_INSTALL_PATH=$RUBY_BUILD_PATH/install RUSTUP_TOOLCHAIN=`cat $BINDING_PATH/mmtk/rust-toolchain` -DEFAULT_PLAN=Immix +DEFAULT_PLAN=StickyImmix CI_JOBS=4 diff --git a/README.md b/README.md index fe8a090..2accd85 100644 --- a/README.md +++ b/README.md @@ -251,28 +251,33 @@ make btest RUN_OPTS="--mmtk-plan=StickyImmix" ### All tests -Note that currently it is not our priority to support Ractor. Some tests -involving Ractors are not enabled. You can get a list of enabled tests in the -file `ruby-test-cases.txt`. +We excluded some tests when testing against MMTk. Those test cases are listed +in `test/.excludes-mmtk` in the [`mmtk/ruby`](https://github.com/mmtk/ruby.git) +repository. -To run the tests +- Test cases that involve Ractors are excluded because it is currently not a + priority to support Ractors. +- Test cases that involve YJIT are excluded because we have not started + working on YJIT support, yet. +- Some tests involve implementation details of CRuby's default GC, such as + compaction and memsize. Those test cases are excluded, too. +- Other excluded tests involve things that are not yet implemented properly in + the MMTk binding. -```bash -TEST_CASES=$(grep -v '#' ../../mmtk-ruby/ruby-test-cases.txt | awk '{print("../"$1)}' | xargs) -make test-all TESTS="$TEST_CASES" RUN_OPTS="--mmtk-plan=StickyImmix" -``` - -Or in one line: +To run the tests, run the following command. ```bash -make test-all TESTS="$(grep -v '#' ../../mmtk-ruby/ruby-test-cases.txt | awk '{print("../"$1)}' | xargs)" RUN_OPTS="--mmtk-plan=StickyImmix" +make test-all RUN_OPTS="--mmtk-plan=StickyImmix" TESTOPTS="-v --excludes-dir=../test/.excludes-mmtk" ``` +That assumes you are in the `build-debug` or `build-release` directory. Adjust +the path `../test/.excludes-mmtk` if you run it in a different directory. + ## Current status Known working: - Supports MarkSweep, Immix and StickyImmix GC algorithms - - All test cases in `make btest` + - Most test cases in `make btest` - Most test cases in `make test-all` - Liquid benchmark (https://github.com/Shopify/liquid/blob/master/performance/benchmark.rb) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5b67b17..e538a49 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.28" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3c5357d..7ddd662 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -12,7 +12,7 @@ edition = "2021" # Metadata for the Ruby repository [package.metadata.ci-repos.ruby] repo = "mmtk/ruby" # This is used by actions/checkout, so the format is "owner/repo", not URL. -rev = "8590762fa53459a8f20e26bd8dd19e1760618f6b" +rev = "8c96eaf5cf4bb39610fc03797a42870f7eb8dca2" [lib] name = "mmtk_ruby" diff --git a/ruby-test-cases.txt b/ruby-test-cases.txt deleted file mode 100644 index 1337784..0000000 --- a/ruby-test-cases.txt +++ /dev/null @@ -1,130 +0,0 @@ -test/ruby/test_alias.rb -test/ruby/test_argf.rb -test/ruby/test_arithmetic_sequence.rb -test/ruby/test_arity.rb -test/ruby/test_array.rb -test/ruby/test_assignment.rb -test/ruby/test_ast.rb -test/ruby/test_autoload.rb -test/ruby/test_backtrace.rb -test/ruby/test_basicinstructions.rb -test/ruby/test_beginendblock.rb -test/ruby/test_bignum.rb -test/ruby/test_call.rb -test/ruby/test_case.rb -test/ruby/test_class.rb -test/ruby/test_clone.rb -test/ruby/test_comparable.rb -test/ruby/test_complex2.rb -test/ruby/test_complexrational.rb -test/ruby/test_complex.rb -test/ruby/test_condition.rb -test/ruby/test_const.rb -test/ruby/test_continuation.rb -test/ruby/test_default_gems.rb -test/ruby/test_defined.rb -test/ruby/test_dir_m17n.rb -test/ruby/test_dir.rb -test/ruby/test_dup.rb -test/ruby/test_econv.rb -test/ruby/test_encoding.rb -test/ruby/test_enumerator.rb -test/ruby/test_enum.rb -test/ruby/test_env.rb -test/ruby/test_eval.rb -test/ruby/test_exception.rb -test/ruby/test_fiber.rb -# test/ruby/test_file_exhaustive.rb XXXX leaks -# test/ruby/test_file.rb XXXX leaks -test/ruby/test_fixnum.rb -test/ruby/test_flip.rb -test/ruby/test_float.rb -test/ruby/test_fnmatch.rb -test/ruby/test_frozen_error.rb -test/ruby/test_frozen.rb -# test/ruby/test_gc_compact.rb XXXX hard crash -# test/ruby/test_gc.rb XXXX failures -test/ruby/test_hash.rb -test/ruby/test_ifunless.rb -test/ruby/test_inlinecache.rb -test/ruby/test_insns_leaf.rb -test/ruby/test_integer_comb.rb -test/ruby/test_integer.rb -test/ruby/test_io_buffer.rb -test/ruby/test_io_m17n.rb -# test/ruby/test_io.rb XXXXX leaks? -test/ruby/test_iseq.rb -test/ruby/test_iterator.rb -test/ruby/test_key_error.rb -test/ruby/test_keyword.rb -test/ruby/test_lambda.rb -test/ruby/test_lazy_enumerator.rb -test/ruby/test_literal.rb -test/ruby/test_m17n_comb.rb -test/ruby/test_m17n.rb -test/ruby/test_marshal.rb -test/ruby/test_math.rb -test/ruby/test_memory_view.rb -test/ruby/test_metaclass.rb -test/ruby/test_method_cache.rb -# test/ruby/test_method.rb XXXX failure -test/ruby/test_mixed_unicode_escapes.rb -test/ruby/test_module.rb -test/ruby/test_name_error.rb -test/ruby/test_nomethod_error.rb -test/ruby/test_not.rb -test/ruby/test_numeric.rb -test/ruby/test_object.rb -test/ruby/test_objectspace.rb -test/ruby/test_optimization.rb -test/ruby/test_pack.rb -test/ruby/test_parse.rb -test/ruby/test_path.rb -test/ruby/test_pattern_matching.rb -test/ruby/test_pipe.rb -test/ruby/test_primitive.rb -test/ruby/test_process.rb -test/ruby/test_proc.rb -test/ruby/test_random_formatter.rb -test/ruby/test_rand.rb -test/ruby/test_range.rb -test/ruby/test_rational2.rb -test/ruby/test_rational.rb -test/ruby/test_readpartial.rb -test/ruby/test_refinement.rb -test/ruby/test_regexp.rb -test/ruby/test_require_lib.rb -test/ruby/test_require.rb -# test/ruby/test_rubyoptions.rb XXXX failures and crashes -# test/ruby/test_rubyvm_mjit.rb XXXXX query -test/ruby/test_rubyvm.rb -test/ruby/test_settracefunc.rb -test/ruby/test_signal.rb -test/ruby/test_sleep.rb -test/ruby/test_sprintf_comb.rb -test/ruby/test_sprintf.rb -test/ruby/test_stack.rb -test/ruby/test_stringchar.rb -test/ruby/test_string.rb -test/ruby/test_struct.rb -test/ruby/test_super.rb -test/ruby/test_symbol.rb -test/ruby/test_syntax.rb -test/ruby/test_system.rb -test/ruby/test_thread_cv.rb -test/ruby/test_threadgroup.rb -test/ruby/test_thread_queue.rb -test/ruby/test_thread.rb -# test/ruby/test_time.rb XXXXX hard crash -test/ruby/test_time_tz.rb -test/ruby/test_trace.rb -test/ruby/test_transcode.rb -test/ruby/test_undef.rb -test/ruby/test_unicode_escape.rb -test/ruby/test_variable.rb -test/ruby/test_vm_dump.rb -# test/ruby/test_weakmap.rb XXXXX failures -test/ruby/test_whileuntil.rb -test/ruby/test_yield.rb -test/ruby/test_yjit_exit_locations.rb -test/ruby/test_yjit.rb