test: move benchmark tests out of main test suite

Move benchmark tests (which are slow) out of the main test suite. We can
hopefully add them to node-daily-master so that they are still run daily
on CI.

Backport-PR-URL: #25049
PR-URL: #24265
Reviewed-By: Richard Lau <riclau@uk.ibm.com>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: Trott

Makefile

@@ -270,7 +270,7 @@ v8:
 	tools/make-v8.sh $(V8_ARCH).$(BUILDTYPE_LOWER) $(V8_BUILD_OPTIONS)
 
 .PHONY: jstest
-jstest: build-addons build-addons-napi bench-addons-build ## Runs addon tests and JS tests
+jstest: build-addons build-addons-napi ## Runs addon tests and JS tests
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) \
 		--skip-tests=$(CI_SKIP_TESTS) \
 		$(CI_JS_SUITES) \
@@ -414,7 +414,7 @@ clear-stalled:
 		echo $${PS_OUT} | xargs kill -9; \
 	fi
 
-test-build: | all build-addons build-addons-napi bench-addons-build
+test-build: | all build-addons build-addons-napi
 
 test-build-addons-napi: all build-addons-napi
 
@@ -455,7 +455,7 @@ test-ci-js: | clear-stalled
 .PHONY: test-ci
 # Related CI jobs: most CI tests, excluding node-test-commit-arm-fanned
 test-ci: LOGLEVEL := info
-test-ci: | clear-stalled build-addons build-addons-napi doc-only bench-addons-build
+test-ci: | clear-stalled build-addons build-addons-napi doc-only
 	out/Release/cctest --gtest_output=tap:cctest.tap
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) -p tap --logfile test.tap \
 		--mode=$(BUILDTYPE_LOWER) --flaky-tests=$(FLAKY_TESTS) \
@@ -496,7 +496,7 @@ test-debug: test-build
 test-message: test-build
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) message
 
-test-simple: | cctest bench-addons-build  # Depends on 'all'.
+test-simple: | cctest # Depends on 'all'.
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential
 
 test-pummel: all
@@ -509,6 +509,9 @@ test-node-inspect: $(NODE_EXE)
 	USE_EMBEDDED_NODE_INSPECT=1 $(NODE) tools/test-npm-package \
 		--install deps/node-inspect test
 
+test-benchmark: | bench-addons-build
+	$(PYTHON) tools/test.py $(PARALLEL_ARGS) benchmark
+
 test-tick-processor: all
 	$(PYTHON) tools/test.py $(PARALLEL_ARGS) tick-processor
 

test/README.md

@@ -19,6 +19,7 @@ GitHub with the `autocrlf` git config flag set to true.
 | `addons`            | Yes             | Tests for [addon](https://nodejs.org/api/addons.html) functionality along with some tests that require an addon to function properly. |
 | `addons-napi`        | Yes            | Tests for [n-api](https://nodejs.org/api/n-api.html) functionality. |
 | `async-hooks`        | Yes            | Tests for [async_hooks](https://nodejs.org/api/async_hooks.html) functionality. |
+| `benchmark`          | No             | Test minimal functionality of benchmarks. |
 | `cctest`             | Yes            | C++ tests that are run as part of the build process. |
 | `code-cache`         | No             | Tests for a Node.js binary compiled with V8 code cache. |
 | `common`             |                | Common modules shared among many tests. [Documentation](./common/README.md) |

test/benchmark/benchmark.status

@@ -0,0 +1,21 @@
+prefix benchmark
+
+# To mark a test as flaky, list the test name in the appropriate section
+# below, without ".js", followed by ": PASS,FLAKY". Example:
+# sample-test                       : PASS,FLAKY
+
+[true] # This section applies to all platforms
+
+[$system==win32]
+
+[$system==linux]
+
+[$system==macos]
+
+[$system==solaris] # Also applies to SmartOS
+
+[$system==freebsd]
+
+[$system==aix]
+
+[$arch==arm]